libexec/discovery-merger.py

   1 #!/usr/bin/env python
   2 #Copyright (C) 2009-2010 :
   3 #    Gabes Jean, naparuba@gmail.com
   4 #    Gerhard Lausser, Gerhard.Lausser@consol.de
   5 #    Gregory Starck, g.starck@gmail.com
   6 #
   7 #This file is part of Shinken.
   8 #
   9 #Shinken is free software: you can redistribute it and/or modify
  10 #it under the terms of the GNU Affero General Public License as published by
  11 #the Free Software Foundation, either version 3 of the License, or
  12 #(at your option) any later version.
  13 #
  14 #Shinken is distributed in the hope that it will be useful,
  15 #but WITHOUT ANY WARRANTY; without even the implied warranty of
  16 #MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17 #GNU Affero General Public License for more details.
  18 #
  19 #You should have received a copy of the GNU Affero General Public License
  20 #along with Shinken.  If not, see <http://www.gnu.org/licenses/>.
  21
  22 #sudo nmap 192.168.0.1 -T4 -O --traceroute -oX toto.xml
  23
  24 import optparse
  25 import sys
  26 import cPickle
  27 import os
  28 import re
  29 import time
  30
  31 try:
  32     import shinken
  33 except ImportError:
  34     # If importing shinken fails, try to load from current directory
  35     # or parent directory to support running without installation.
  36     # Submodules will then be loaded from there, too.
  37     import imp
  38     if not hasattr(os, "getuid") or os.getuid() != 0:
  39         imp.load_module('shinken', *imp.find_module('shinken', [".", ".."]))
  40
  41 from shinken.log import logger
  42 from shinken.objects import *
  43 from shinken.macroresolver import MacroResolver
  44
  45
  46 VERSION = '0.1'
  47
  48 parser = optparse.OptionParser(
  49     "%prog [options] -c discovery_config -o config_output -m list of macros",
  50     version="%prog " + VERSION)
  51 parser.add_option('-c', '--cfg-input',
  52                   dest="cfg_input", help=('Discovery configuration file (discovery.cfg)'))
  53 parser.add_option('-o', '--dir-output', dest="output_dir",
  54                   help="Directory output for results")
  55 parser.add_option('-w', '--overright', dest="overright", action='store_true',
  56                   help="Allow overwithing xisting file (disabled by default)")
  57 parser.add_option('-r', '--runners', dest="runners",
  58                   help="List of runners you allow to run, (like nmap,vsphere)")
  59 parser.add_option('-m', '--macros', dest="macros",
  60                   help="List of macros (like NMAPTARGETS=192.168.0.0/24). Should be the last argument")
  61
  62
  63
  64 opts, args = parser.parse_args()
  65
  66 raw_macros = []
  67 macros=[]
  68
  69 if not opts.cfg_input:
  70     parser.error("Requires a discovery configuration file, dsicovery.cfg (option -c/--cfg-input")
  71 if not opts.output_dir:
  72     parser.error("Requires one output directory (option -o/--dir-output")
  73 if not opts.overright:
  74     overright = False
  75 else:
  76     overright = opts.overright
  77 if not opts.runners:
  78     runners = ['*']
  79 else:
  80     runners = opts.runners.split(',')
  81 if opts.macros:
  82     raw_macros.append(opts.macros)
  83 if args:
  84     raw_macros.extend(args)
  85
  86 print "Macros", raw_macros
  87
  88 for m in raw_macros:
  89     elts = m.split('=')
  90     if len(elts) < 2:
  91         print "The macro '%s' is malformed. I bail out" % m
  92         sys.exit(2)
  93     macros.append( (elts[0], '='.join(elts[1:])))
  94
  95 print "Got macros", macros
  96
  97 # Says if a host is up or not
  98 def is_up(h):
  99     status = h.find('status')
 100     state = status.attrib['state']
 101     return state == 'up'
 102
 103
 104 class ConfigurationManager:
 105     def __init__(self, h, path, criticity):
 106         self.h = h
 107         self.hosts_path = os.path.join(path, 'hosts')
 108         self.srvs_path = os.path.join(path, 'services')
 109         self.templates = ['generic-host']
 110         self.services = []
 111         self.criticity = criticity
 112         self.parents = []
 113
 114
 115     # We search if our potential parent is present in the
 116     # other detected hosts. If so, set it as my parent
 117     def look_for_parent(self, all_hosts):
 118         parent = self.h.parent
 119         print "Look for my parent", self.h.get_name(), "->", parent
 120         # Ok, we didn't find any parent
 121         # we bail out
 122         if parent == '':
 123             return
 124         for h in all_hosts:
 125             print "Is it you?", h.get_name()
 126             if h.get_name() == parent:
 127                 print "Houray, we find our parent", self.h.get_name(), "->", h.get_name()
 128                 self.parents.append(h.get_name())
 129
 130
 131
 132     def fill_system_conf(self):
 133         ios = self.h.os
 134
 135         #Ok, unknown os... not good
 136         if ios == ('', ''):
 137             return
 138
 139         map = {('Windows', '2000') : 'windows',
 140                ('Windows', '2003') : 'windows',
 141                ('Windows', '7') : 'windows',
 142                ('Windows', 'XP') : 'windows',
 143                # ME? you are a stupid moron!
 144                ('Windows', 'Me') : 'windows',
 145                ('Windows', '2008') : 'windows',
 146                # that's a good boy :)
 147                ('Linux', '2.6.X') : 'linux',
 148                ('Linux', '2.4.X') : 'linux',
 149                # HPUX? I think you didn't choose...
 150                ('HP-UX', '11.X') : 'hpux',
 151                ('HP-UX', '10.X') : 'hpux',
 152                }
 153
 154         if ios not in map:
 155             print "Unknown OS:", ios
 156             return
 157
 158         t = map[ios]
 159         self.templates.append(t)
 160
 161         # Look for VMWare VM or hosts
 162         if self.h.is_vmware_vm():
 163             self.templates.append('vmware-vm')
 164         # Now is an host?
 165         if self.h.is_vmware_esx():
 166             self.templates.append('vmware-host')
 167
 168
 169     def get_cfg_for_host(self):
 170         props = {}
 171         props['host_name'] = self.h.get_name()
 172         props['criticity'] = self.criticity
 173
 174         # Parents if we got some
 175         if self.parents != []:
 176             props['parents'] = ','.join(self.parents)
 177
 178         # Now template
 179         props['use'] = ','.join(self.templates)
 180
 181         print "Want to write", props
 182         s = 'define host {\n'
 183         for k in props:
 184             v = props[k]
 185             s += '  %s    %s\n' % (k, v)
 186         s += '}\n'
 187         print s
 188         return s
 189
 190
 191     def get_cfg_for_services(self):
 192         r = {}
 193         print "And now services:"
 194         for srv in self.services:
 195             desc = srv['service_description']
 196             s = 'define service {\n'
 197             for k in srv:
 198                 v = srv[k]
 199                 s += '  %s    %s\n' % (k, v)
 200             s += '}\n'
 201             print s
 202             r[desc] = s
 203         return r
 204
 205
 206
 207     def fill_ports_services(self):
 208         for p in self.h.open_ports:
 209             print "The port", p, " is open"
 210             f = getattr(self, 'gen_srv_'+str(p), None)
 211             if f:
 212                 f()
 213
 214     def fill_system_services(self):
 215         for t in self.templates:
 216             print "Registering services for the template", t
 217             # Python functions cannot be -, so we change it by _
 218             t = t.replace('-','_')
 219             print "Search for", 'gen_srv_'+str(t)
 220             f = getattr(self, 'gen_srv_'+str(t), None)
 221             if f:
 222                 f()
 223
 224
 225     def generate_service(self, desc, check):
 226         srv = {'use' : 'generic-service', 'service_description' : desc, 'check_command' : check, 'host_name' : self.h.get_name()}
 227         self.services.append(srv)
 228
 229
 230     ######### For network ones
 231
 232     # HTTP
 233     def gen_srv_80(self):
 234         self.generate_service('HTTP', 'check_http')
 235
 236     # SSH
 237     def gen_srv_22(self):
 238         self.generate_service('SSH', 'check_ssh')
 239
 240     # HTTPS + certificate
 241     def gen_srv_443(self):
 242         self.generate_service('HTTPS', 'check_https')
 243         self.generate_service('HTTPS-CERT', 'check_https_certificate')
 244
 245     # FTP
 246     def gen_srv_21(self):
 247         self.generate_service('FTP', 'check_ftp')
 248
 249     # DNS
 250     def gen_srv_53(self):
 251         self.generate_service('DNS', 'check_dig!$HOSTADDRESS$')
 252
 253     # Oracle Listener
 254     def gen_srv_1521(self):
 255         self.generate_service('Oracle-Listener', 'check_oracle_listener')
 256
 257     # Now for MSSQL
 258     def gen_srv_1433(self):
 259         self.generate_service('MSSQL-Connexion', 'check_mssql_connexion')
 260         print "I will need check_mssql_health from http://labs.consol.de/nagios/check_mssql_health/"
 261
 262     # SMTP
 263     def gen_srv_25(self):
 264         self.generate_service('SMTP', 'check_smtp')
 265
 266     # And the SMTPS too
 267     def gen_srv_465(self):
 268         self.generate_service('SMTPS', 'check_smtps')
 269
 270     # LDAP
 271     def gen_srv_389(self):
 272         self.generate_service('Ldap', 'check_ldap')
 273
 274     # LDAPS
 275     def gen_srv_636(self):
 276         self.generate_service('Ldaps', 'check_ldaps')
 277
 278     #Mysql
 279     def gen_srv_3306(self):
 280         self.generate_service('Mysql', 'check_mysql_connexion')
 281
 282
 283     ####
 284     #      For system ones
 285     ####
 286     def gen_srv_linux(self):
 287         print "You want a Linux check, but I don't know what to propose, sorry..."
 288
 289     def gen_srv_windows(self):
 290         print "You want a Windows check, but I don't know what to propose, sorry..."
 291
 292     #For a VM we can add cpu, io, mem and net
 293     def gen_srv_vmware_vm(self):
 294         self.generate_service('VM-Cpu', "check_esx_vm!cpu")
 295         self.generate_service('VM-IO', "check_esx_vm!io")
 296         self.generate_service('VM-Memory', "check_esx_vm!mem")
 297         self.generate_service('VM-Network', "check_esx_vm!net")
 298         print "I will need the check_esx3.pl from http://www.op5.org/community/plugin-inventory/op5-projects/op5-plugins"
 299
 300     # Quite the same for the host
 301     def gen_srv_vmware_host(self):
 302         self.generate_service('ESX-host-Cpu', "check_esx_host!cpu")
 303         self.generate_service('ESX-host-IO', "check_esx_host!io")
 304         self.generate_service('ESX-host-Memory', "check_esx_host!mem")
 305         self.generate_service('ESX-host-Network', "check_esx_host!net")
 306         print "I will need the check_esx3.pl from http://www.op5.org/community/plugin-inventory/op5-projects/op5-plugins"
 307
 308
 309
 310     # Write the host cfg file
 311     def write_host_configuration(self):
 312         name = self.h.get_name()
 313         # If the host is bad, get out
 314         if not name:
 315             return
 316
 317         # Write the directory with the host config
 318         p = os.path.join(self.hosts_path, name)
 319         print "I want to create", p
 320         try:
 321             os.mkdir(p)
 322         except OSError, exp:
 323             # If directory already exists, it's not a problem
 324             if not exp.errno != '17':
 325                 print "Cannot create the directory '%s' : '%s'" % (p, exp)
 326                 return
 327         cfg_p = os.path.join(p, name+'.cfg')
 328         print "I want to write", cfg_p
 329         s = self.get_cfg_for_host()
 330         try:
 331             fd = open(cfg_p, 'w')
 332         except OSError, exp:
 333             print "Cannot create the file '%s' : '%s'" % (cfg_p, exp)
 334             return
 335         fd.write(s)
 336         fd.close()
 337
 338
 339     def write_services_configuration(self):
 340         name = self.h.get_name()
 341         # If the host is bad, get out
 342         if not name:
 343             return
 344
 345         # Write the directory with the host config
 346         p = os.path.join(self.srvs_path, name)
 347         print "I want to create", p
 348         try:
 349             os.mkdir(p)
 350         except OSError, exp:
 351             # If directory already exist, it's not a problem
 352             if not exp.errno != '17':
 353                 print "Cannot create the directory '%s' : '%s'" % (p, exp)
 354                 return
 355         # Ok now we get the services to create
 356         r = c.get_cfg_for_services()
 357         for s in r:
 358             cfg_p = os.path.join(p, name+'-'+s+'.cfg')
 359             print "I want to write", cfg_p
 360             buf = r[s]
 361             try:
 362                 fd = open(cfg_p, 'w')
 363             except OSError, exp:
 364                 print "Cannot create the file '%s' : '%s'" % (cfg_p, exp)
 365                 return
 366             fd.write(buf)
 367             fd.close()
 368
 369
 370
 371 class DiscoveryMerger:
 372     def __init__(self, path, output_dir, macros, overright, runners):
 373         # i am arbiter-like
 374         self.log = logger
 375         self.overright = overright
 376         self.runners = runners
 377         self.output_dir = output_dir
 378         self.log.load_obj(self)
 379         self.config_files = [path]
 380         self.conf = Config()
 381         self.conf.read_config(self.config_files)
 382         buf = self.conf.read_config(self.config_files)
 383
 384         # Add macros on the end of the buf so they will
 385         # overright the resource.cfg ones
 386         for (m, v) in macros:
 387             buf += '\n$%s$=%s\n' % (m, v)
 388
 389         raw_objects = self.conf.read_config_buf(buf)
 390         self.conf.create_objects_for_type(raw_objects, 'arbiter')
 391         self.conf.create_objects_for_type(raw_objects, 'module')
 392         self.conf.early_arbiter_linking()
 393         self.conf.create_objects(raw_objects)
 394         #self.conf.instance_id = 0
 395         #self.conf.instance_name = ''
 396         self.conf.linkify_templates()
 397         self.conf.apply_inheritance()
 398         self.conf.explode()
 399         self.conf.create_reversed_list()
 400         self.conf.remove_twins()
 401         self.conf.apply_implicit_inheritance()
 402         self.conf.fill_default()
 403         self.conf.clean_useless()
 404         self.conf.pythonize()
 405         self.conf.linkify()
 406         self.conf.apply_dependancies()
 407         #self.conf.explode_global_conf()
 408         #self.conf.propagate_timezone_option()
 409         #self.conf.create_business_rules()
 410         #self.conf.create_business_rules_dependencies()
 411         self.conf.is_correct()
 412
 413         self.discoveryrules = self.conf.discoveryrules
 414         self.discoveryruns = self.conf.discoveryruns
 415
 416         #self.confs = self.conf.cut_into_parts()
 417         #self.dispatcher = Dispatcher(self.conf, self.me)
 418
 419         #scheddaemon = Shinken(None, False, False, False, None)
 420         #self.sched = Scheduler(scheddaemon)
 421
 422         #scheddaemon.sched = self.sched
 423
 424         m = MacroResolver()
 425         m.init(self.conf)
 426         #self.sched.load_conf(self.conf)
 427         #e = ExternalCommandManager(self.conf, 'applyer')
 428         #self.sched.external_command = e
 429         #e.load_scheduler(self.sched)
 430         #e2 = ExternalCommandManager(self.conf, 'dispatcher')
 431         #e2.load_arbiter(self)
 432         #self.external_command_dispatcher = e2
 433         #self.sched.schedule()
 434
 435         # Hash = name, and in it (key, value)
 436         self.disco_data = {}
 437         # Hash = name, and in it rules that apply
 438         self.disco_matches = {}
 439
 440
 441     def add(self, obj):
 442         pass
 443
 444     # Look if the name is a IPV4 address or not
 445     def is_ipv4_addr(self, name):
 446         p = r"^([01]?\d\d?|2[0-4]\d|25[0-5])\.([01]?\d\d?|2[0-4]\d|25[0-5])\.([01]?\d\d?|2[0-4]\d|25[0-5])\.([01]?\d\d?|2[0-4]\d|25[0-5])$"
 447         return (re.match(p, name) is not None)
 448
 449
 450     def read_disco_buf(self):
 451         buf = self.raw_disco_data
 452         for l in buf.split('\n'):
 453             #print ""
 454             # If it's not a disco line, bypass it
 455             if not re.search('::', l):
 456                 continue
 457             #print "line", l
 458             elts = l.split('::')
 459             if len(elts) <= 1:
 460                 #print "Bad discovery data"
 461                 continue
 462             name = elts[0].strip()
 463
 464             # We can choose to keep only the basename
 465             # of the nameid, so strip the fqdn
 466             # But not if it's a plain ipv4 addr
 467             if self.conf.strip_idname_fqdn:
 468                 if not self.is_ipv4_addr(name):
 469                     name = name.split('.', 1)[0]
 470
 471             data = '::'.join(elts[1:])
 472
 473             #print "Name", name
 474             #print "data", data
 475             # Register the name
 476             if not name in self.disco_data:
 477                 self.disco_data[name] = []
 478             # Now get key,values
 479             if not '=' in data:
 480                 #print "Bad discovery data"
 481                 continue
 482             elts = data.split('=')
 483             if len(elts) <= 1:
 484                 #print "Bad discovery data"
 485                 continue
 486             key = elts[0].strip()
 487             value = elts[1].strip()
 488             print "-->", name, key, value
 489             self.disco_data[name].append((key, value))
 490
 491
 492     def match_rules(self):
 493         for name in self.disco_data:
 494             datas = self.disco_data[name]
 495             for r in self.discoveryrules:
 496                 if r.is_matching_disco_datas(datas):
 497                     if name not in self.disco_matches:
 498                         self.disco_matches[name] = []
 499                     self.disco_matches[name].append(r)
 500                     print "Generating", name, r.writing_properties
 501
 502
 503     def is_allowing_runners(self, name):
 504         name = name.strip()
 505
 506         # If we got no value, it's * by default
 507         if '*' in self.runners:
 508             return True
 509
 510         #print self.runners
 511         #If we match the name, ok
 512         for r in self.runners:
 513             r_name = r.strip()
 514             #            print "Look", r_name, name
 515             if r_name == name:
 516                 return True
 517
 518         # Not good, so not run this!
 519         return False
 520
 521     def allowed_runners(self):
 522         return [r for r in self.discoveryruns if self.is_allowing_runners(r.get_name())]
 523
 524     def launch_runners(self):
 525         for r in self.allowed_runners():
 526             print "I'm launching", r.get_name()
 527             print r.discoveryrun_command
 528             r.launch()
 529
 530
 531     def wait_for_runners_ends(self):
 532         all_ok = False
 533         while not all_ok:
 534             all_ok = True
 535             for r in self.allowed_runners():
 536                 if not r.is_finished():
 537                     #print "Check finished of", r.get_name()
 538                     r.check_finished()
 539                 b = r.is_finished()
 540                 if not b:
 541                     #print r.get_name(), "is not finished"
 542                     all_ok = False
 543             time.sleep(0.1)
 544
 545
 546     def get_runners_outputs(self):
 547         self.raw_disco_data = '\n'.join(r.get_output() for r in self.allowed_runners() if r.is_finished())
 548         print "Got Raw disco data", self.raw_disco_data
 549
 550
 551     # Write all configuration we've got
 552     def write_config(self):
 553         for name in self.disco_data:
 554             print "Writing", name, "configuration"
 555             self.write_host_config(name)
 556             self.write_service_config(name)
 557
 558
 559     # We search for all rules of type host, and we merge them
 560     def write_host_config(self, host):
 561         host_rules = []
 562         for (name, rules) in self.disco_matches.items():
 563             if name != host:
 564                 continue
 565             rs = [r for r in rules if r.creation_type == 'host']
 566             host_rules.extend(rs)
 567
 568         # If no rule, bail out
 569         if len(host_rules) == 0:
 570             return
 571
 572         # now merge them
 573         d = {'host_name' : host}
 574         for r in host_rules:
 575             d.update(r.writing_properties)
 576         print "Will generate", d
 577         self.write_host_config_to_file(host, d)
 578
 579
 580     # Will wrote all properties/values of d for the host
 581     # in the file
 582     def write_host_config_to_file(self, host, d):
 583         p = os.path.join(self.output_dir, host)
 584         print "Want to create host path", p
 585         try:
 586             os.mkdir(p)
 587         except OSError, exp:
 588             # If directory already exist, it's not a problem
 589             if not exp.errno != '17':
 590                 print "Cannot create the directory '%s' : '%s'" % (p, exp)
 591                 return
 592         cfg_p = os.path.join(p, host+'.cfg')
 593         if os.path.exists(cfg_p) and not overright:
 594             print "The file '%s' already exists" % cfg_p
 595             return
 596
 597         buf = self.get_cfg_bufer(d, 'host')
 598
 599         # Ok, we create it so (or overright)
 600         try:
 601             fd = open(cfg_p, 'w')
 602             fd.write(buf)
 603             fd.close()
 604         except OSError, exp:
 605             print "Cannot create the file '%s' : '%s'" % (cfg_p, exp)
 606             return
 607
 608
 609     # Generate all service for a host
 610     def write_service_config(self, host):
 611         srv_rules = {}
 612         for (name, rules) in self.disco_matches.items():
 613             if name != host:
 614                 continue
 615             rs = [r for r in rules if r.creation_type == 'service']
 616             print "RS", rs
 617             for r in rs:
 618                 if 'service_description' in r.writing_properties:
 619                     desc = r.writing_properties['service_description']
 620                     if not desc in srv_rules:
 621                         srv_rules[desc] = []
 622                     srv_rules[desc].append(r)
 623
 624         #print "Generate services for", host
 625         #print srv_rules
 626         for (desc, rules) in srv_rules.items():
 627             d = {'service_description' : desc, 'host_name' : host}
 628             for r in rules:
 629                 d.update(r.writing_properties)
 630             print "Generating", desc, d
 631             self.write_service_config_to_file(host, desc, d)
 632
 633
 634     # Will wrote all properties/values of d for the host
 635     # in the file
 636     def write_service_config_to_file(self, host, desc, d):
 637         p = os.path.join(self.output_dir, host)
 638
 639         # The host conf should already exist
 640         cfg_host_p = os.path.join(p, host+'.cfg')
 641         if not os.path.exists(cfg_host_p):
 642             print "No host configuration available, I bail out"
 643             return
 644
 645         cfg_p = os.path.join(p, desc+'.cfg')
 646         if os.path.exists(cfg_p) and not overright:
 647             print "The file '%s' already exists" % cfg_p
 648             return
 649
 650         buf = self.get_cfg_bufer(d, 'service')
 651
 652         # Ok, we create it so (or overright)
 653         try:
 654             fd = open(cfg_p, 'w')
 655             fd.write(buf)
 656             fd.close()
 657         except OSError, exp:
 658             print "Cannot create the file '%s' : '%s'" % (cfg_p, exp)
 659             return
 660
 661
 662     # Create a define t { } with data in d
 663     def get_cfg_bufer(self, d, t):
 664         tab = ['define %s {' % t]
 665         for (key, value) in d.items():
 666             tab.append('  %s   %s' % (key, value))
 667         tab.append('}\n')
 668         return '\n'.join(tab)
 669
 670
 671
 672
 673
 674 cfg_input = opts.cfg_input
 675 output_dir = opts.output_dir
 676 overright = opts.overright
 677 d = DiscoveryMerger(cfg_input, output_dir, macros, overright, runners)
 678
 679
 680
 681
 682 d.launch_runners()
 683 d.wait_for_runners_ends()
 684 d.get_runners_outputs()
 685
 686 d.read_disco_buf()
 687
 688 # Now look for rules
 689 d.match_rules()
 690 #print d.disco_matches
 691
 692 d.write_config()