import.c

   1 #define _GNU_SOURCE 1
   2 #include <sys/types.h>
   3 #include <signal.h>
   4
   5 #include <nagios/broker.h>
   6 #include <nagios/nebcallbacks.h>
   7 #include "sql.h"
   8 #include "hooks.h"
   9 #include "logging.h"
  10 #include "hash.h"
  11 #include "lparse.h"
  12 #include "logutils.h"
  13
  14 #define IGNORE_LINE 0
  15
  16 #define CONCERNS_HOST 50
  17 #define CONCERNS_SERVICE 60
  18
  19 #define MAX_NVECS 16
  20 #define HASH_TABLE_SIZE 128
  21
  22 /* for some reason these aren't defined inside Nagios' headers */
  23 #define SERVICE_OK 0
  24 #define SERVICE_WARNING 1
  25 #define SERVICE_CRITICAL 2
  26 #define SERVICE_UNKNOWN 3
  27
  28 #define PROGRESS_INTERVAL 500 /* lines to parse between progress updates */
  29
  30
  31 static uint imported, totsize, totlines;
  32 static int lines_since_progress, do_progress;
  33 static struct timeval import_start;
  34 static time_t daemon_start, daemon_stop, incremental;
  35 static int daemon_is_running;
  36 static uint max_dt_depth;
  37
  38 static time_t next_dt_purge; /* when next to purge expired downtime */
  39 #define DT_PURGE_GRACETIME 300 /* seconds to add to next_dt_purge */
  40
  41 static time_t ltime; /* the timestamp from the current log-line */
  42
  43 static int dt_start, dt_stop;
  44 #define dt_depth (dt_start - dt_stop)
  45 static hash_table *host_downtime;
  46 static hash_table *service_downtime;
  47 static int downtime_id;
  48 static time_t probably_ignore_downtime;
  49
  50 struct downtime_entry {
  51         int id;
  52         int code;
  53         char *host;
  54         char *service;
  55         time_t start;
  56         time_t stop;
  57         int fixed;
  58         time_t duration;
  59         time_t started;
  60         time_t ended;
  61         int purged;
  62         int trigger;
  63         int slot;
  64         struct downtime_entry *next;
  65 };
  66
  67 #define NUM_DENTRIES 1024
  68 static struct downtime_entry **dentry;
  69 static time_t last_downtime_start;
  70
  71 static struct string_code event_codes[] = {
  72         add_ignored("Error"),
  73         add_ignored("Warning"),
  74         add_ignored("LOG ROTATION"),
  75         add_ignored("HOST NOTIFICATION"),
  76         add_ignored("HOST FLAPPING ALERT"),
  77         add_ignored("SERVICE NOTIFICATION"),
  78         add_ignored("SERVICE FLAPPING ALERT"),
  79         add_ignored("SERVICE EVENT HANDLER"),
  80         add_ignored("HOST EVENT HANDLER"),
  81         add_ignored("LOG VERSION"),
  82
  83         add_code(0, "EXTERNAL COMMAND", NEBTYPE_EXTERNALCOMMAND_END),
  84         add_code(5, "HOST ALERT", NEBTYPE_HOSTCHECK_PROCESSED),
  85         add_code(5, "INITIAL HOST STATE", NEBTYPE_HOSTCHECK_PROCESSED),
  86         add_code(5, "CURRENT HOST STATE", NEBTYPE_HOSTCHECK_PROCESSED),
  87         add_code(6, "SERVICE ALERT", NEBTYPE_SERVICECHECK_PROCESSED),
  88         add_code(6, "INITIAL SERVICE STATE", NEBTYPE_SERVICECHECK_PROCESSED),
  89         add_code(6, "CURRENT SERVICE STATE", NEBTYPE_SERVICECHECK_PROCESSED),
  90         add_code(3, "HOST DOWNTIME ALERT", NEBTYPE_DOWNTIME_LOAD + CONCERNS_HOST),
  91         add_code(4, "SERVICE DOWNTIME ALERT", NEBTYPE_DOWNTIME_LOAD + CONCERNS_SERVICE),
  92         { 0, NULL, 0, 0 },
  93 };
  94
  95 static struct string_code command_codes[] = {
  96         add_cdef(1, DEL_HOST_DOWNTIME),
  97         add_cdef(1, DEL_SVC_DOWNTIME),
  98         add_cdef(8, SCHEDULE_AND_PROPAGATE_HOST_DOWNTIME),
  99         add_cdef(8, SCHEDULE_AND_PROPAGATE_TRIGGERED_HOST_DOWNTIME),
 100         add_cdef(8, SCHEDULE_HOSTGROUP_HOST_DOWNTIME),
 101         add_cdef(8, SCHEDULE_HOSTGROUP_SVC_DOWNTIME),
 102         add_cdef(8, SCHEDULE_HOST_DOWNTIME),
 103         add_cdef(8, SCHEDULE_HOST_SVC_DOWNTIME),
 104         add_cdef(8, SCHEDULE_SERVICEGROUP_HOST_DOWNTIME),
 105         add_cdef(8, SCHEDULE_SERVICEGROUP_SVC_DOWNTIME),
 106         add_cdef(8, SCHEDULE_SVC_DOWNTIME),
 107
 108         /*
 109          * These really have one more field than listed here. We omit one
 110          * to make author and comment concatenated with a semi-colon by default.
 111          */
 112         add_cdef(6, ACKNOWLEDGE_SVC_PROBLEM),
 113         add_cdef(5, ACKNOWLEDGE_HOST_PROBLEM),
 114         { 0, NULL, 0, 0 },
 115 };
 116
 117
 118 static inline void print_strvec(char **v, int n)
 119 {
 120         int i;
 121
 122         for (i = 0; i < n; i++)
 123                 printf("v[%2d]: %s\n", i, v[i]);
 124 }
 125
 126
 127 static const char *tobytes(uint n)
 128 {
 129         const char *suffix = "KMGT";
 130         static char tbuf[2][30];
 131         static int t = 0;
 132         int shift = 1;
 133
 134         t ^= 1;
 135         if (n < 1024) {
 136                 sprintf(tbuf[t], "%d bytes", n);
 137                 return tbuf[t];
 138         }
 139
 140         while (n >> (shift * 10) > 1024)
 141                 shift++;
 142
 143         sprintf(tbuf[t], "%0.2f %ciB",
 144                         (float)n / (float)(1 << (shift * 10)), suffix[shift - 1]);
 145
 146         return tbuf[t];
 147 }
 148
 149 static void show_progress(void)
 150 {
 151         time_t eta, elapsed;
 152         float pct_done;
 153
 154         totlines += lines_since_progress;
 155         lines_since_progress = 0;
 156
 157         if (!do_progress)
 158                 return;
 159
 160         elapsed = time(NULL) - import_start.tv_sec;
 161         if (!elapsed)
 162                 elapsed = 1;
 163
 164         pct_done = ((float)imported / (float)totsize) * 100;
 165         eta = (elapsed / pct_done) * (100.0 - pct_done);
 166
 167         printf("\rImporting data: %.2f%% (%s) done ",
 168                    pct_done, tobytes(imported));
 169         if (elapsed > 10) {
 170                 printf("ETA: ");
 171                 if (eta > 60)
 172                         printf("%lum%lus", eta / 60, eta % 60);
 173                 else
 174                         printf("%lus", eta);
 175         }
 176         printf("        ");
 177 }
 178
 179 static void end_progress(void)
 180 {
 181         struct timeval tv;
 182         int mins;
 183         float secs;
 184
 185         gettimeofday(&tv, NULL);
 186
 187         /*
 188          * If any of the logfiles doesn't have a newline
 189          * at end of file, imported will be slightly off.
 190          * We set it hard here so as to make sure that
 191          * the final progress output stops at exactly 100%
 192          */
 193         imported = totsize;
 194
 195         show_progress();
 196         putchar('\n');
 197         secs = (tv.tv_sec - import_start.tv_sec) * 1000000;
 198         secs += tv.tv_usec - import_start.tv_usec;
 199         mins = (tv.tv_sec - import_start.tv_sec) / 60;
 200         secs /= 1000000;
 201         secs -= (mins * 60);
 202         printf("%s in %u lines imported in ", tobytes(totsize), totlines);
 203         if (mins)
 204                 printf("%dm ", mins);
 205         printf("%.3fs\n", secs);
 206 }
 207
 208 static int use_sql = 1;
 209 static int insert_downtime_event(int type, char *host, char *service, int id)
 210 {
 211         nebstruct_downtime_data ds;
 212         int result;
 213
 214         if (!is_interesting_service(host, service))
 215                 return 0;
 216
 217         dt_start += type == NEBTYPE_DOWNTIME_START;
 218         dt_stop += type == NEBTYPE_DOWNTIME_STOP;
 219         if (dt_depth > max_dt_depth)
 220                 max_dt_depth = dt_depth;
 221
 222         if (!use_sql)
 223                 return 0;
 224
 225         memset(&ds, 0, sizeof(ds));
 226
 227         ds.type = type;
 228         ds.timestamp.tv_sec = ltime;
 229         ds.host_name = host;
 230         ds.service_description = service;
 231         ds.downtime_id = id;
 232
 233         result = hook_downtime(NEBCALLBACK_DOWNTIME_DATA, (void *)&ds);
 234         if (result < 0)
 235                 crash("Failed to insert downtime:\n  type=%d, host=%s, service=%s, id=%d",
 236                           type, host, service, id);
 237
 238         return result;
 239 }
 240
 241 static int insert_service_check(struct string_code *sc)
 242 {
 243         nebstruct_service_check_data ds;
 244
 245         if (!use_sql)
 246                 return 0;
 247
 248         if (!is_interesting_service(strv[0], strv[1]))
 249                 return 0;
 250
 251         memset(&ds, 0, sizeof(ds));
 252
 253         ds.timestamp.tv_sec = ltime;
 254         ds.type = sc->code;
 255         ds.host_name = strv[0];
 256         ds.service_description = strv[1];
 257         ds.state = parse_service_state(strv[2]);
 258         ds.state_type = soft_hard(strv[3]);
 259         ds.current_attempt = atoi(strv[4]);
 260         ds.output = strv[5];
 261
 262         return hook_service_result(NEBCALLBACK_SERVICE_CHECK_DATA, (void *)&ds);
 263 }
 264
 265 static int insert_host_check(struct string_code *sc)
 266 {
 267         nebstruct_host_check_data ds;
 268
 269         if (!use_sql)
 270                 return 0;
 271
 272         if (!is_interesting_host(strv[0]))
 273                 return 0;
 274
 275         memset(&ds, 0, sizeof(ds));
 276
 277         ds.timestamp.tv_sec = ltime;
 278         ds.type = sc->code;
 279         ds.host_name = strv[0];
 280         ds.state = parse_host_state(strv[1]);
 281         ds.state_type = soft_hard(strv[2]);
 282         ds.current_attempt = atoi(strv[3]);
 283         ds.output = strv[4];
 284
 285         return hook_host_result(NEBCALLBACK_HOST_CHECK_DATA, (void *)&ds);
 286 }
 287
 288 static int insert_process_event(int type)
 289 {
 290         nebstruct_process_data ds;
 291
 292         if (!use_sql)
 293                 return 0;
 294
 295         memset(&ds, 0, sizeof(ds));
 296         ds.timestamp.tv_sec = ltime;
 297         ds.type = type;
 298         return hook_process_data(NEBCALLBACK_PROCESS_DATA, (void *)&ds);
 299 }
 300
 301 static int insert_acknowledgement(struct string_code *sc)
 302 {
 303         return 0;
 304 }
 305
 306 static void dt_print(char *tpc, time_t when, struct downtime_entry *dt)
 307 {
 308         if (!debug_level)
 309                 return;
 310
 311         printf("%s: time=%lu started=%lu start=%lu stop=%lu duration=%lu id=%d ",
 312                    tpc, when, dt->started, dt->start, dt->stop, dt->duration, dt->id);
 313         printf("%s", dt->host);
 314         if (dt->service)
 315                 printf(";%s", dt->service);
 316         putchar('\n');
 317 }
 318
 319 static struct downtime_entry *last_dte;
 320 static struct downtime_entry *del_dte;
 321
 322 static void remove_downtime(struct downtime_entry *dt);
 323 static int del_matching_dt(void *data)
 324 {
 325         struct downtime_entry *dt = data;
 326
 327         if (del_dte->id == dt->id) {
 328                 dt_print("ALSO", 0, dt);
 329                 remove_downtime(dt);
 330         }
 331
 332         return 0;
 333 }
 334
 335 static void stash_downtime_command(struct downtime_entry *dt)
 336 {
 337         dt->slot = dt->start % NUM_DENTRIES;
 338         dt->next = dentry[dt->slot];
 339         dentry[dt->slot] = dt;
 340 }
 341
 342 static void remove_downtime(struct downtime_entry *dt)
 343 {
 344         struct downtime_entry *old;
 345
 346         if (!is_interesting_service(dt->host, dt->service))
 347                 return;
 348
 349         insert_downtime_event(NEBTYPE_DOWNTIME_STOP, dt->host, dt->service, dt->id);
 350
 351         if (!dt->service)
 352                 old = hash_remove(host_downtime, dt->host);
 353         else
 354                 old = hash_remove2(service_downtime, dt->host, dt->service);
 355
 356         dt_print("RM_DT", ltime, dt);
 357         dt->purged = 1;
 358 }
 359
 360 static struct downtime_entry *
 361 dt_matches_command(struct downtime_entry *dt, char *host, char *service)
 362 {
 363         for (; dt; dt = dt->next) {
 364                 time_t diff;
 365
 366                 if (ltime > dt->stop || ltime < dt->start) {
 367                         continue;
 368                 }
 369
 370                 switch (dt->code) {
 371                 case SCHEDULE_SVC_DOWNTIME:
 372                         if (service && strcmp(service, dt->service))
 373                                 continue;
 374
 375                         /* fallthrough */
 376                 case SCHEDULE_HOST_DOWNTIME:
 377                 case SCHEDULE_HOST_SVC_DOWNTIME:
 378                         if (strcmp(host, dt->host)) {
 379                                 continue;
 380                         }
 381
 382                 case SCHEDULE_AND_PROPAGATE_HOST_DOWNTIME:
 383                 case SCHEDULE_AND_PROPAGATE_TRIGGERED_HOST_DOWNTIME:
 384                         /* these two have host set in dt, but
 385                          * it will not match all the possible hosts */
 386
 387                         /* fallthrough */
 388                 case SCHEDULE_HOSTGROUP_HOST_DOWNTIME:
 389                 case SCHEDULE_HOSTGROUP_SVC_DOWNTIME:
 390                 case SCHEDULE_SERVICEGROUP_HOST_DOWNTIME:
 391                 case SCHEDULE_SERVICEGROUP_SVC_DOWNTIME:
 392                         break;
 393                 default:
 394                         crash("dt->code not set properly\n");
 395                 }
 396
 397                 /*
 398                  * Once we get here all the various other criteria have
 399                  * been matched, so we need to check if the daemon was
 400                  * running when this downtime was supposed to have
 401                  * started, and otherwise use the daemon start time
 402                  * as the value to diff against
 403                  */
 404                 if (daemon_stop < dt->start && daemon_start > dt->start) {
 405                         debug("Adjusting dt->start (%lu) to (%lu)\n",
 406                                   dt->start, daemon_start);
 407                         dt->start = daemon_start;
 408                         if (dt->trigger && dt->duration)
 409                                 dt->stop = dt->start + dt->duration;
 410                 }
 411
 412                 diff = ltime - dt->start;
 413                 if (diff < 3 || dt->trigger || !dt->fixed)
 414                         return dt;
 415         }
 416
 417         return NULL;
 418 }
 419
 420 static struct downtime_entry *
 421 find_downtime_command(char *host, char *service)
 422 {
 423         int i;
 424         struct downtime_entry *shortcut = NULL;
 425
 426         if (last_dte && last_dte->start == ltime) {
 427                 shortcut = last_dte;
 428 //              return last_dte;
 429         }
 430         for (i = 0; i < NUM_DENTRIES; i++) {
 431                 struct downtime_entry *dt;
 432                 dt = dt_matches_command(dentry[i], host, service);
 433                 if (dt) {
 434                         if (shortcut && dt != shortcut)
 435                                 if (debug_level)
 436                                         printf("FIND shortcut no good\n");
 437                         last_dte = dt;
 438                         return dt;
 439                 }
 440         }
 441
 442         debug("FIND not\n");
 443         return NULL;
 444 }
 445
 446 static int print_downtime(void *data)
 447 {
 448         struct downtime_entry *dt = (struct downtime_entry *)data;
 449
 450         dt_print("UNCLOSED", ltime, dt);
 451
 452         return 0;
 453 }
 454
 455 static inline void set_next_dt_purge(time_t base, time_t add)
 456 {
 457         if (!next_dt_purge || next_dt_purge > base + add)
 458                 next_dt_purge = base + add;
 459
 460         if (next_dt_purge <= ltime)
 461                 next_dt_purge = ltime + 1;
 462 }
 463
 464 static inline void add_downtime(char *host, char *service, int id)
 465 {
 466         struct downtime_entry *dt, *cmd, *old;
 467
 468         if (!is_interesting_service(host, service))
 469                 return;
 470
 471         dt = malloc(sizeof(*dt));
 472         cmd = find_downtime_command(host, service);
 473         if (!cmd) {
 474                 warn("DT with no ext cmd? %lu %s;%s", ltime, host, service);
 475                 memset(dt, 0, sizeof(*dt));
 476                 dt->duration = 7200; /* the default downtime duration in nagios */
 477                 dt->start = ltime;
 478                 dt->stop = dt->start + dt->duration;
 479         }
 480         else
 481                 memcpy(dt, cmd, sizeof(*dt));
 482
 483         dt->host = strdup(host);
 484         dt->id = id;
 485         dt->started = ltime;
 486
 487         set_next_dt_purge(ltime, dt->duration);
 488
 489         if (!service) {
 490                 dt->service = NULL;
 491                 old = hash_update(host_downtime, dt->host, dt);
 492         }
 493         else {
 494                 dt->service = strdup(service);
 495                 old = hash_update2(service_downtime, dt->host, dt->service, dt);
 496         }
 497
 498         if (old && old != dt) {
 499                 free(old->host);
 500                 if (old->service)
 501                         free(old->service);
 502                 free(old);
 503         }
 504
 505         dt_print("IN_DT", ltime, dt);
 506         insert_downtime_event(NEBTYPE_DOWNTIME_START, dt->host, dt->service, dt->id);
 507 }
 508
 509 static time_t last_host_dt_del, last_svc_dt_del;
 510 static int register_downtime_command(struct string_code *sc)
 511 {
 512         struct downtime_entry *dt;
 513         char *start_time, *end_time, *duration = NULL;
 514         char *host = NULL, *service = NULL, *fixed, *triggered_by = NULL;
 515         time_t foo;
 516
 517         switch (sc->code) {
 518         case DEL_HOST_DOWNTIME:
 519                 last_host_dt_del = ltime;
 520                 return 0;
 521         case DEL_SVC_DOWNTIME:
 522                 last_svc_dt_del = ltime;
 523                 return 0;
 524
 525         case SCHEDULE_HOST_DOWNTIME:
 526                 if (strtotimet(strv[5], &foo))
 527                         duration = strv[4];
 528                 /* fallthrough */
 529         case SCHEDULE_AND_PROPAGATE_HOST_DOWNTIME:
 530         case SCHEDULE_AND_PROPAGATE_TRIGGERED_HOST_DOWNTIME:
 531         case SCHEDULE_HOST_SVC_DOWNTIME:
 532                 host = strv[0];
 533                 /* fallthrough */
 534         case SCHEDULE_HOSTGROUP_HOST_DOWNTIME:
 535         case SCHEDULE_HOSTGROUP_SVC_DOWNTIME:
 536         case SCHEDULE_SERVICEGROUP_HOST_DOWNTIME:
 537         case SCHEDULE_SERVICEGROUP_SVC_DOWNTIME:
 538                 start_time = strv[1];
 539                 end_time = strv[2];
 540                 fixed = strv[3];
 541                 if (strtotimet(strv[5], &foo))
 542                         triggered_by = strv[4];
 543                 if (!duration)
 544                         duration = strv[5];
 545
 546                 break;
 547
 548         case SCHEDULE_SVC_DOWNTIME:
 549                 host = strv[0];
 550                 service = strv[1];
 551                 start_time = strv[2];
 552                 end_time = strv[3];
 553                 fixed = strv[4];
 554                 if (strtotimet(strv[6], &foo)) {
 555                         triggered_by = strv[5];
 556                         duration = strv[6];
 557                 }
 558                 else {
 559                         duration = strv[5];
 560                 }
 561                 break;
 562
 563         default:
 564                 crash("Unknown downtime type: %d", sc->code);
 565         }
 566
 567         if (!(dt = calloc(sizeof(*dt), 1)))
 568                 crash("calloc(%u, 1) failed: %s", (uint)sizeof(*dt), strerror(errno));
 569
 570         dt->code = sc->code;
 571         if (host)
 572                 dt->host = strdup(host);
 573         if (service)
 574                 dt->service = strdup(service);
 575
 576         dt->trigger = triggered_by ? !!(*triggered_by - '0') : 0;
 577         if (strtotimet(start_time, &dt->start) || strtotimet(end_time, &dt->stop))
 578         {
 579                 print_strvec(strv, sc->nvecs);
 580                 crash("strtotime(): type: %s; start_time='%s'; end_time='%s'; duration='%s';",
 581                           command_codes[sc->code - 1].str, start_time, end_time, duration);
 582         }
 583
 584         /*
 585          * sometimes downtime commands can be logged according to
 586          * log version 1, while the log still claims to be version 2.
 587          * Apparently, this happens when using a daemon supporting
 588          * version 2 logging but a downtime command is added that
 589          * follows the version 1 standard.
 590          * As such, we simply ignore the result of the "duration"
 591          * field conversion and just accept that it might not work
 592          */
 593         (void)strtotimet(duration, &dt->duration);
 594         dt->fixed = *fixed - '0';
 595
 596         /*
 597          * ignore downtime scheduled to take place in the future.
 598          * It will be picked up by the module anyways
 599          */
 600         if (dt->start > time(NULL)) {
 601                 free(dt);
 602                 return 0;
 603         }
 604
 605         if (dt->duration > time(NULL)) {
 606                 warn("Bizarrely large duration (%lu)", dt->duration);
 607         }
 608         if (dt->start < ltime) {
 609                 if (dt->duration && dt->duration > ltime - dt->start)
 610                         dt->duration -= ltime - dt->start;
 611
 612                 dt->start = ltime;
 613         }
 614         if (dt->stop < ltime || dt->stop < dt->start)  {
 615                 /* retroactively scheduled downtime, or just plain wrong */
 616                 dt->stop = dt->start;
 617                 dt->duration = 0;
 618         }
 619
 620         if (dt->fixed && dt->duration != dt->stop - dt->start) {
 621 //              warn("duration doesn't match stop - start: (%lu : %lu)",
 622 //                       dt->duration, dt->stop - dt->start);
 623
 624                 dt->duration = dt->stop - dt->start;
 625         }
 626         else if (dt->duration > 86400 * 14) {
 627                 warn("Oddly long duration: %lu", dt->duration);
 628         }
 629
 630         debug("start=%lu; stop=%lu; duration=%lu; fixed=%d; trigger=%d; host=%s service=%s\n",
 631                   dt->start, dt->stop, dt->duration, dt->fixed, dt->trigger, dt->host, dt->service);
 632
 633         stash_downtime_command(dt);
 634         return 0;
 635 }
 636
 637 static int insert_downtime(struct string_code *sc)
 638 {
 639         int type;
 640         struct downtime_entry *dt = NULL;
 641         int id = 0;
 642         time_t dt_del_cmd;
 643         char *host, *service = NULL;
 644
 645         host = strv[0];
 646         if (sc->nvecs == 4) {
 647                 service = strv[1];
 648                 dt = hash_find2(service_downtime, host, service);
 649         }
 650         else
 651                 dt = hash_find(host_downtime, host);
 652
 653         /*
 654          * to stop a downtime we can either get STOPPED or
 655          * CANCELLED. So far, I've only ever seen STARTED
 656          * for when it actually starts though, and since
 657          * the Nagios daemon is reponsible for launching
 658          * it, it's unlikely there are more variants of
 659          * that string
 660          */
 661         type = NEBTYPE_DOWNTIME_STOP;
 662         if (!strcmp(strv[sc->nvecs - 2], "STARTED"))
 663                 type = NEBTYPE_DOWNTIME_START;
 664
 665         switch (type) {
 666         case NEBTYPE_DOWNTIME_START:
 667                 if (dt) {
 668                         if (!probably_ignore_downtime)
 669                                 dt_print("ALRDY", ltime, dt);
 670                         return 0;
 671                 }
 672
 673                 if (probably_ignore_downtime)
 674                         debug("Should probably ignore this downtime: %lu : %lu %s;%s\n",
 675                                   probably_ignore_downtime, ltime, host, service);
 676
 677                 if (ltime - last_downtime_start > 1)
 678                         downtime_id++;
 679
 680                 id = downtime_id;
 681                 add_downtime(host, service, id);
 682                 last_downtime_start = ltime;
 683                 break;
 684
 685         case NEBTYPE_DOWNTIME_STOP:
 686                 if (!dt) {
 687                         /*
 688                          * this can happen when overlapping downtime entries
 689                          * occur, and the start event for the second (or nth)
 690                          * downtime starts before the first downtime has had
 691                          * a stop event. It basically means we've almost
 692                          * certainly done something wrong.
 693                          */
 694                         //printf("no dt. ds.host_name == '%s'\n", ds.host_name);
 695                         //fprintf(stderr, "CRASHING: %s;%s\n", ds.host_name, ds.service_description);
 696                         //crash("DOWNTIME_STOP without matching DOWNTIME_START");
 697                         return 0;
 698                 }
 699
 700                 dt_del_cmd = !dt->service ? last_host_dt_del : last_svc_dt_del;
 701
 702                 if ((ltime - dt_del_cmd) > 1 && dt->duration - (ltime - dt->started) > 60) {
 703                         debug("Short dt duration (%lu) for %s;%s (dt->duration=%lu)\n",
 704                                    ltime - dt->started, dt->host, dt->service, dt->duration);
 705                 }
 706                 if (ltime - dt->started > dt->duration + DT_PURGE_GRACETIME)
 707                         dt_print("Long", ltime, dt);
 708
 709                 remove_downtime(dt);
 710                 /*
 711                  * Now delete whatever matching downtimes we can find.
 712                  * this must be here, or we'll recurse like crazy into
 713                  * remove_downtime(), possibly exhausting the stack
 714                  * frame buffer
 715                  */
 716                 del_dte = dt;
 717                 if (!dt->service)
 718                         hash_walk_data(host_downtime, del_matching_dt);
 719                 else
 720                         hash_walk_data(service_downtime, del_matching_dt);
 721                 break;
 722
 723         default:
 724                 return -1;
 725         }
 726
 727         return 0;
 728 }
 729
 730 static int dt_purged;
 731 static int purge_expired_dt(void *data)
 732 {
 733         struct downtime_entry *dt = data;
 734
 735         if (dt->purged) {
 736                 return 0;
 737         }
 738
 739         if (ltime + DT_PURGE_GRACETIME > dt->stop) {
 740                 dt_purged++;
 741                 debug("PURGE %lu: purging expired dt %d (start=%lu; started=%lu; stop=%lu; duration=%lu; host=%s; service=%s",
 742                           ltime, dt->id, dt->start, dt->started, dt->stop, dt->duration, dt->host, dt->service);
 743                 remove_downtime(dt);
 744         }
 745         else {
 746                 dt_print("PURGED_NOT_TIME", ltime, dt);
 747         }
 748
 749         set_next_dt_purge(dt->started, dt->duration);
 750
 751         return 0;
 752 }
 753
 754 static int purged_downtimes;
 755 static void purge_expired_downtime(void)
 756 {
 757         int tot_purged = 0;
 758
 759         next_dt_purge = 0;
 760         dt_purged = 0;
 761         hash_walk_data(host_downtime, purge_expired_dt);
 762         if (dt_purged)
 763                 debug("PURGE %d host downtimes purged", dt_purged);
 764         tot_purged += dt_purged;
 765         dt_purged = 0;
 766         hash_walk_data(service_downtime, purge_expired_dt);
 767         if (dt_purged)
 768                 debug("PURGE %d service downtimes purged", dt_purged);
 769         tot_purged += dt_purged;
 770         if (tot_purged)
 771                 debug("PURGE total %d entries purged", tot_purged);
 772
 773         if (next_dt_purge)
 774                 debug("PURGE next downtime purge supposed to run @ %lu, in %lu seconds",
 775                           next_dt_purge, next_dt_purge - ltime);
 776
 777         purged_downtimes += tot_purged;
 778 }
 779
 780 static inline void handle_start_event(void)
 781 {
 782         if (!daemon_is_running)
 783                 insert_process_event(NEBTYPE_PROCESS_START);
 784
 785         probably_ignore_downtime = daemon_start = ltime;
 786         daemon_is_running = 1;
 787 }
 788
 789 static inline void handle_stop_event(void)
 790 {
 791         if (daemon_is_running) {
 792                 insert_process_event(NEBTYPE_PROCESS_SHUTDOWN);
 793                 daemon_is_running = 0;
 794         }
 795         daemon_stop = ltime;
 796 }
 797
 798 static int parse_line(char *line, uint len)
 799 {
 800         char *ptr, *colon;
 801         int nvecs = 0;
 802         struct string_code *sc;
 803         static time_t last_ltime = 0;
 804
 805         imported += len + 1; /* make up for 1 lost byte per newline */
 806
 807         /* ignore empty lines */
 808         if (!len)
 809                 return 0;
 810
 811         if (++lines_since_progress >= PROGRESS_INTERVAL)
 812                 show_progress();
 813
 814         /* skip obviously bogus lines */
 815         if (len < 12 || *line != '[') {
 816                 warn("line %d; len too short, or line doesn't start with '[' (%s)", line_no, line);
 817                 return -1;
 818         }
 819
 820         ltime = strtoul(line + 1, &ptr, 10);
 821         if (line + 1 == ptr) {
 822                 crash("Failed to parse log timestamp from '%s'. I can't handle malformed logdata", line);
 823                 return -1;
 824         }
 825
 826         if (ltime < last_ltime) {
 827 //              warn("ltime < last_ltime (%lu < %lu) by %lu. Compensating...",
 828 //                       ltime, last_ltime, last_ltime - ltime);
 829                 ltime = last_ltime;
 830         }
 831         else
 832                 last_ltime = ltime;
 833
 834         /*
 835          * Incremental will be 0 if not set, or 1 if set but
 836          * the database is currently empty.
 837          * Note that this will not always do the correct thing,
 838          * as downtime entries that might have been scheduled for
 839          * purging may never show up as "stopped" in the database
 840          * with this scheme. As such, incremental imports absolutely
 841          * require that nothing is in scheduled downtime when the
 842          * import is running (well, started really, but it amounts
 843          * to the same thing).
 844          */
 845         if (ltime < incremental)
 846                 return 0;
 847
 848         if (next_dt_purge && ltime >= next_dt_purge)
 849                 purge_expired_downtime();
 850
 851         if (probably_ignore_downtime && ltime - probably_ignore_downtime > 1)
 852                 probably_ignore_downtime = 0;
 853
 854         while (*ptr == ']' || *ptr == ' ')
 855                 ptr++;
 856
 857         if (!is_interesting(ptr))
 858                 return 0;
 859
 860         if (!(colon = strchr(ptr, ':'))) {
 861                 /* stupid heuristic, but might be good for something,
 862                  * somewhere, sometime. if nothing else, it should suppress
 863                  * annoying output */
 864                 if (is_start_event(ptr)) {
 865                         handle_start_event();
 866                         return 0;
 867                 }
 868                 if (is_stop_event(ptr)) {
 869                         handle_stop_event();
 870                         return 0;
 871                 }
 872
 873                 /*
 874                  * An unhandled event. We should probably crash here
 875                  */
 876                 handle_unknown_event(line);
 877                 return -1;
 878         }
 879
 880         /* an event happened without us having gotten a start-event */
 881         if (!daemon_is_running) {
 882                 insert_process_event(NEBTYPE_PROCESS_START);
 883                 daemon_start = ltime;
 884                 daemon_is_running = 1;
 885         }
 886
 887         if (!(sc = get_event_type(ptr, colon - ptr))) {
 888                 handle_unknown_event(line);
 889                 return -1;
 890         }
 891
 892         if (sc->code == IGNORE_LINE)
 893                 return 0;
 894
 895         *colon = 0;
 896         ptr = colon + 1;
 897         while (*ptr == ' ')
 898                 ptr++;
 899
 900         if (sc->nvecs) {
 901                 int i;
 902
 903                 nvecs = vectorize_string(ptr, sc->nvecs);
 904
 905                 if (nvecs != sc->nvecs) {
 906                         /* broken line */
 907                         warn("Line %d in %s seems to not have all the fields it should",
 908                                  line_no, cur_file->path);
 909                         return -1;
 910                 }
 911
 912                 for (i = 0; i < sc->nvecs; i++) {
 913                         if (!strv[i]) {
 914                                 /* this should never happen */
 915                                 warn("Line %d in %s seems to be broken, or we failed to parse it into a vector",
 916                                          line_no, cur_file->path);
 917                                 return -1;
 918                         }
 919                 }
 920         }
 921
 922         switch (sc->code) {
 923                 char *semi_colon;
 924
 925         case NEBTYPE_EXTERNALCOMMAND_END:
 926                 semi_colon = strchr(ptr, ';');
 927                 if (!semi_colon)
 928                         return 0;
 929                 if (!(sc = get_command_type(ptr, semi_colon - ptr))) {
 930                         return 0;
 931                 }
 932                 if (sc->code == RESTART_PROGRAM) {
 933                         handle_stop_event();
 934                         return 0;
 935                 }
 936
 937                 nvecs = vectorize_string(semi_colon + 1, sc->nvecs);
 938                 if (nvecs != sc->nvecs) {
 939                         warn("nvecs discrepancy: %d vs %d (%s)\n", nvecs, sc->nvecs, ptr);
 940                 }
 941                 if (sc->code != ACKNOWLEDGE_HOST_PROBLEM &&
 942                         sc->code != ACKNOWLEDGE_SVC_PROBLEM)
 943                 {
 944                         register_downtime_command(sc);
 945                 } else {
 946                         insert_acknowledgement(sc);
 947                 }
 948                 break;
 949
 950         case NEBTYPE_HOSTCHECK_PROCESSED:
 951                 return insert_host_check(sc);
 952
 953         case NEBTYPE_SERVICECHECK_PROCESSED:
 954                 return insert_service_check(sc);
 955
 956         case NEBTYPE_DOWNTIME_LOAD + CONCERNS_HOST:
 957         case NEBTYPE_DOWNTIME_LOAD + CONCERNS_SERVICE:
 958                 return insert_downtime(sc);
 959
 960         case IGNORE_LINE:
 961                 return 0;
 962         }
 963
 964         return 0;
 965 }
 966
 967 static int parse_one_line(char *str, uint len)
 968 {
 969         if (parse_line(str, len) && use_sql && sql_errno())
 970                 crash("sql error: %s", sql_error());
 971
 972         return 0;
 973 }
 974
 975 static int hash_one_line(char *line, uint len)
 976 {
 977         return add_interesting_object(line);
 978 }
 979
 980 static int hash_interesting(const char *path)
 981 {
 982         struct stat st;
 983
 984         if (stat(path, &st) < 0)
 985                 crash("failed to stat %s: %s", path, strerror(errno));
 986
 987         lparse_path(path, st.st_size, hash_one_line);
 988
 989         return 0;
 990 }
 991
 992 extern const char *__progname;
 993 int main(int argc, char **argv)
 994 {
 995         int i, truncate_db = 0;
 996         struct naglog_file *nfile;
 997         char *db_name = "monitor_reports";
 998         char *db_user = "monitor";
 999         char *db_pass = "monitor";
1000         char *db_table = "report_data";
1001
1002         do_progress = isatty(fileno(stdout));
1003
1004         strv = calloc(sizeof(char *), MAX_NVECS);
1005         nfile = calloc(sizeof(*nfile), argc - 1);
1006         dentry = calloc(sizeof(*dentry), NUM_DENTRIES);
1007         if (!strv || !nfile || !dentry)
1008                 crash("Failed to alloc initial structs");
1009
1010
1011         for (num_nfile = 0,i = 1; i < argc; i++) {
1012                 char *opt, *arg = argv[i];
1013                 struct naglog_file *nf;
1014                 int eq_opt = 0;
1015
1016                 if ((opt = strchr(arg, '='))) {
1017                         *opt++ = '\0';
1018                         eq_opt = 1;
1019                 }
1020                 else if (i < argc - 1) {
1021                         opt = argv[i + 1];
1022                 }
1023
1024                 if (!prefixcmp(arg, "--incremental")) {
1025                         incremental = 1;
1026                         continue;
1027                 }
1028                 if (!prefixcmp(arg, "--no-sql")) {
1029                         use_sql = 0;
1030                         continue;
1031                 }
1032                 if (!prefixcmp(arg, "--no-progress")) {
1033                         do_progress = 0;
1034                         continue;
1035                 }
1036                 if (!prefixcmp(arg, "--debug") || !prefixcmp(arg, "-d")) {
1037                         do_progress = 0;
1038                         debug_level++;
1039                         continue;
1040                 }
1041                 if (!prefixcmp(arg, "--truncate-db")) {
1042                         truncate_db = 1;
1043                         continue;
1044                 }
1045                 if (!prefixcmp(arg, "--db-name")) {
1046                         if (!opt || !*opt)
1047                                 crash("%s requires a database name as an argument", arg);
1048                         db_name = opt;
1049                         if (opt && !eq_opt)
1050                                 i++;
1051                         continue;
1052                 }
1053                 if (!prefixcmp(arg, "--db-user")) {
1054                         if (!opt || !*opt)
1055                                 crash("%s requires a database username as argument", arg);
1056                         db_user = opt;
1057                         if (opt && !eq_opt)
1058                                 i++;
1059                         continue;
1060                 }
1061                 if (!prefixcmp(arg, "--db-pass")) {
1062                         if (!opt || !*opt)
1063                                 crash("%s requires a database username as argument", arg);
1064                         db_pass = opt;
1065                         if (opt && !eq_opt)
1066                                 i++;
1067                         continue;
1068                 }
1069                 if (!prefixcmp(arg, "--db-table")) {
1070                         if (!opt || !*opt)
1071                                 crash("%s requires a database table name as argument", arg);
1072                         db_table = opt;
1073                         if (opt && !eq_opt)
1074                                 i++;
1075                         continue;
1076                 }
1077                 if (!prefixcmp(arg, "--interesting") || !prefixcmp(arg, "-i")) {
1078                         if (!opt || !*opt)
1079                                 crash("%s requires a filename as argument", arg);
1080                         hash_interesting(opt);
1081                         if (opt && !eq_opt)
1082                                 i++;
1083                         continue;
1084                 }
1085
1086                 /* non-argument, so treat as file */
1087                 nf = &nfile[num_nfile++];
1088                 nf->path = arg;
1089                 first_log_time(nf);
1090                 totsize += nf->size;
1091         }
1092
1093         if (use_sql) {
1094                 sql_config("db_database", db_name);
1095                 sql_config("db_user", db_user);
1096                 sql_config("db_pass", db_pass);
1097                 sql_config("db_table", db_table);
1098
1099                 if (sql_init() < 0)
1100                         crash("sql_init() failed");
1101                 if (truncate_db)
1102                         sql_query("TRUNCATE %s", sql_table_name());
1103
1104                 if (incremental) {
1105                         MYSQL_RES *result;
1106                         MYSQL_ROW row;
1107                         sql_query("SELECT timestamp FROM %s.%s ORDER BY timestamp DESC LIMIT 1",
1108                                           db_name, db_table);
1109
1110                         if (!(result = sql_get_result()))
1111                                 crash("Failed to get last timestamp: %s\n", sql_error());
1112
1113                         /* someone might use --incremental with an empty
1114                          * database. We shouldn't crash in that case */
1115                         if ((row = sql_fetch_row(result)))
1116                                 incremental = strtoul(row[0], NULL, 0);
1117
1118                         sql_free_result(result);
1119                 }
1120                 /*
1121                  * We lock the table we'll be working with and disable
1122                  * indexes on it. Otherwise doing the actual inserts
1123                  * will take just about forever, as MySQL has to update
1124                  * and flush the index cache between each operation.
1125                  */
1126                 if (sql_query("ALTER TABLE %s DISABLE KEYS", sql_table_name()))
1127                         crash("Failed to disable keys: %s", sql_error());
1128                 if (sql_query("LOCK TABLES %s WRITE", sql_table_name()))
1129                         crash("Failed to lock report_data table: %s", sql_error());
1130         }
1131
1132         log_grok_var("logfile", "/dev/null");
1133         log_grok_var("log_levels", "warn");
1134
1135         if (!num_nfile)
1136                 crash("Usage: %s [--incremental] [--interesting <file>] [--truncate-db] logfiles\n",
1137                           __progname);
1138
1139         if (log_init() < 0)
1140                 crash("log_init() failed");
1141
1142         qsort(nfile, num_nfile, sizeof(*nfile), nfile_cmp);
1143
1144         host_downtime = hash_init(HASH_TABLE_SIZE);
1145         service_downtime = hash_init(HASH_TABLE_SIZE);
1146
1147         if (hook_init() < 0)
1148                 crash("Failed to initialize hooks");
1149
1150         gettimeofday(&import_start, NULL);
1151         printf("Importing %s of data from %d files\n",
1152                    tobytes(totsize), num_nfile);
1153
1154         for (i = 0; i < num_nfile; i++) {
1155                 struct naglog_file *nf = &nfile[i];
1156                 cur_file = nf;
1157                 show_progress();
1158                 debug("importing from %s (%lu : %u)\n", nf->path, nf->first, nf->cmp);
1159                 line_no = 0;
1160                 lparse_path(nf->path, nf->size, parse_one_line);
1161                 imported++; /* make up for one lost byte per file */
1162         }
1163
1164         end_progress();
1165
1166         if (debug_level) {
1167                 if (dt_depth) {
1168                         printf("Unclosed host downtimes:\n");
1169                         puts("------------------------");
1170                         hash_walk_data(host_downtime, print_downtime);
1171                         printf("Unclosed service downtimes:\n");
1172                         puts("---------------------------");
1173                         hash_walk_data(service_downtime, print_downtime);
1174
1175                         printf("dt_depth: %d\n", dt_depth);
1176                 }
1177                 printf("purged downtimes: %d\n", purged_downtimes);
1178                 printf("max simultaneous host downtime hashes: %u\n",
1179                        hash_get_max_entries(host_downtime));
1180                 printf("max simultaneous service downtime hashes: %u\n",
1181                        hash_get_max_entries(service_downtime));
1182                 printf("max downtime depth: %u\n", max_dt_depth);
1183         }
1184
1185         if (use_sql) {
1186                 SQL_RESULT *res;
1187                 SQL_ROW row;
1188                 time_t start;
1189                 unsigned long entries;
1190
1191                 sql_query("SELECT id FROM report_data ORDER BY id DESC LIMIT 1");
1192                 if (!(res = sql_get_result()))
1193                         entries = 0;
1194                 else {
1195                         row = sql_fetch_row(res);
1196                         entries = strtoul(row[0], NULL, 0);
1197                         sql_free_result(res);
1198                 }
1199
1200                 signal(SIGINT, SIG_IGN);
1201                 sql_query("UNLOCK TABLES");
1202                 start = time(NULL);
1203                 printf("Creating sql table indexes. This will likely take ~%lu seconds\n",
1204                            (entries / 50000) + 1);
1205                 sql_query("ALTER TABLE %s ENABLE KEYS", sql_table_name());
1206                 printf("%lu database entries indexed in %lu seconds\n",
1207                            entries, time(NULL) - start);
1208                 sql_close();
1209         }
1210
1211         if (warnings && debug_level)
1212                 fprintf(stderr, "Total warnings: %d\n", warnings);
1213
1214         if (debug_level || dt_start != dt_stop)
1215                 fprintf(stderr, "Downtime data %s\n  started: %d\n  stopped: %d\n",
1216                         dt_depth ? "mismatch!" : "consistent", dt_start, dt_stop);
1217         if (hash_check_table(host_downtime))
1218                 fprintf(stderr, "Hash table inconsistencies for host_downtime\n");
1219         if (hash_check_table(service_downtime))
1220                 fprintf(stderr, "Hash table inconsistencies for service_downtime\n");
1221
1222         print_unhandled_events();
1223
1224         return 0;
1225 }