geoip.c

   1 /*
   2  * netsniff-ng - the packet sniffing beast
   3  * Copyright 2013 Daniel Borkmann.
   4  * Subject to the GPL, version 2.
   5  */
   6
   7 #include <stdio.h>
   8 #include <stdlib.h>
   9 #include <string.h>
  10 #include <GeoIP.h>
  11 #include <GeoIPCity.h>
  12 #include <netinet/in.h>
  13 #include <netdb.h>
  14 #include <sys/types.h>
  15 #include <sys/stat.h>
  16 #include <fcntl.h>
  17 #include <unistd.h>
  18
  19 #include "built_in.h"
  20 #include "die.h"
  21 #include "ioops.h"
  22 #include "str.h"
  23 #include "xmalloc.h"
  24 #include "zlib.h"
  25 #include "geoip.h"
  26
  27 struct file {
  28         const char *desc, *local;
  29         const char *remote, *possible_prefix;
  30 };
  31
  32 #define PRE     "/download/geoip/database"
  33 static const struct file files[] = {
  34         [GEOIP_CITY_EDITION_REV1] = {
  35                         .desc = "City IPv4",
  36                         .local = DATDIR_STRING "/city4.dat",
  37                         .remote = "/GeoLiteCity.dat.gz",
  38                         .possible_prefix = PRE,
  39                 },
  40         [GEOIP_CITY_EDITION_REV1_V6] = {
  41                         .desc = "City IPv6",
  42                         .local = DATDIR_STRING "/city6.dat",
  43                         .remote = "/GeoLiteCityv6.dat.gz",
  44                         .possible_prefix = PRE "/GeoLiteCityv6-beta",
  45                 },
  46         [GEOIP_COUNTRY_EDITION] = {
  47                         .desc = "Country IPv4",
  48                         .local = DATDIR_STRING "/country4.dat",
  49                         .remote = "/GeoIP.dat.gz",
  50                         .possible_prefix = PRE "/GeoLiteCountry",
  51                 },
  52         [GEOIP_COUNTRY_EDITION_V6] = {
  53                         .desc = "Country IPv6",
  54                         .local = DATDIR_STRING "/country6.dat",
  55                         .remote = "/GeoIPv6.dat.gz",
  56                         .possible_prefix = PRE,
  57                 },
  58         [GEOIP_ASNUM_EDITION] = {
  59                         .desc = "AS Numbers IPv4",
  60                         .local = DATDIR_STRING "/asname4.dat",
  61                         .remote = "/GeoIPASNum.dat.gz",
  62                         .possible_prefix = PRE "/asnum",
  63                 },
  64         [GEOIP_ASNUM_EDITION_V6] = {
  65                         .desc = "AS Numbers IPv6",
  66                         .local = DATDIR_STRING "/asname6.dat",
  67                         .remote = "/GeoIPASNumv6.dat.gz",
  68                         .possible_prefix = PRE "/asnum",
  69                 },
  70 };
  71
  72 static GeoIP *gi4_asname = NULL, *gi6_asname = NULL;
  73 static GeoIP *gi4_country = NULL, *gi6_country = NULL;
  74 static GeoIP *gi4_city = NULL, *gi6_city = NULL;
  75
  76 static char *servers[16] = { NULL };
  77
  78 #define CITYV4          (1 << 0)
  79 #define CITYV6          (1 << 1)
  80 #define COUNTRYV4       (1 << 2)
  81 #define COUNTRYV6       (1 << 3)
  82 #define ASNAMV4         (1 << 4)
  83 #define ASNAMV6         (1 << 5)
  84
  85 #define HAVEALL         (CITYV4 | CITYV6 | COUNTRYV4 | COUNTRYV6 | ASNAMV4 | ASNAMV6)
  86
  87 static int geoip_db_present = 0;
  88
  89 int geoip_working(void)
  90 {
  91         return geoip_db_present == HAVEALL;
  92 }
  93
  94 static int geoip_get_remote_fd(const char *server, const char *port)
  95 {
  96         int ret, fd = -1;
  97         struct addrinfo hints, *ahead, *ai;
  98
  99         bug_on(!server || !port);
 100
 101         memset(&hints, 0, sizeof(hints));
 102
 103         hints.ai_family = PF_UNSPEC;
 104         hints.ai_socktype = SOCK_STREAM;
 105         hints.ai_protocol = IPPROTO_TCP;
 106         hints.ai_flags = AI_NUMERICSERV;
 107
 108         ret = getaddrinfo(server, port, &hints, &ahead);
 109         if (ret != 0)
 110                 return -EIO;
 111
 112         for (ai = ahead; ai != NULL && fd < 0; ai = ai->ai_next) {
 113                 fd = socket(ai->ai_family, ai->ai_socktype, ai->ai_protocol);
 114                 if (fd < 0)
 115                         continue;
 116
 117                 ret = connect(fd, ai->ai_addr, ai->ai_addrlen);
 118                 if (ret < 0) {
 119                         close(fd);
 120                         fd = -1;
 121                         continue;
 122                 }
 123
 124                 break;
 125         }
 126
 127         freeaddrinfo(ahead);
 128
 129         return fd;
 130 }
 131
 132 static void geoip_inflate(int which)
 133 {
 134         int ret, ret2 = 1;
 135         gzFile fpi;
 136         FILE *fpo;
 137         char zfile[128], raw[4096];
 138
 139         slprintf(zfile, sizeof(zfile), "%s.gz", files[which].local);
 140         fpi = gzopen(zfile, "rb");
 141         if (fpi == NULL)
 142                 panic("No %s file!\n", zfile);
 143
 144         fpo = fopen(files[which].local, "wb");
 145         if (fpo == NULL)
 146                 panic("Cannot create %s!\n", files[which].local);
 147
 148         while ((ret = gzread(fpi, raw, sizeof(raw))) && ret2)
 149                 ret2 = fwrite(raw, ret, 1, fpo);
 150
 151         if (!gzeof(fpi))
 152                 panic("Error in gzread: %s\n", gzerror(fpi, &ret));
 153
 154         gzclose(fpi);
 155         fclose(fpo);
 156 }
 157
 158 static int geoip_get_database(const char *host, int which)
 159 {
 160         int found, sock, fd, i, good, retry = 0;
 161         ssize_t ret, len, rtotlen = 0, totlen = 0;
 162         char raw[4096], *ptr, zfile[128];
 163         size_t lenl = strlen("Content-Length: ");
 164         size_t lent = strlen("HTTP/1.1 200 OK");
 165         size_t lenc = strlen("\r\n\r\n");
 166         const char *http_req_fmt = "GET %s%s HTTP/1.1\r\n"
 167                                    "Connection: close\r\n"
 168                                    "Host: %s\r\n\r\n";
 169 again:
 170         found = good = 0;
 171         ptr = NULL;
 172         len = 0;
 173
 174         sock = geoip_get_remote_fd(host, "80");
 175         if (sock < 0)
 176                 return -EIO;
 177
 178         slprintf(raw, sizeof(raw), http_req_fmt,
 179                  retry ? files[which].possible_prefix : "",
 180                  files[which].remote, host);
 181
 182         ret = write(sock, raw, strlen(raw));
 183         if (ret <= 0) {
 184                 close(sock);
 185                 return -EIO;
 186         }
 187
 188         slprintf(zfile, sizeof(zfile), "%s.gz", files[which].local);
 189         fd = open_or_die_m(zfile, O_WRONLY | O_CREAT | O_TRUNC, DEFFILEMODE);
 190
 191         memset(raw, 0, sizeof(raw));
 192         ret = read(sock, raw, sizeof(raw));
 193         if (ret <= 0) {
 194                 close(fd);
 195                 close(sock);
 196                 return -EIO;
 197         }
 198
 199         for (i = 0; i < ret; i++) {
 200                 if (!strncmp(raw + i, "Content-Length: ", min_t(size_t, ret - i, lenl))) {
 201                         ptr = raw + i + lenl;
 202                         rtotlen = strtoul(ptr, NULL, 10);
 203                 }
 204
 205                 if (!strncmp(raw + i, "HTTP/1.1 200 OK", min_t(size_t, ret - i, lent)))
 206                         good = 1;
 207
 208                 if (!strncmp(raw + i, "\r\n\r\n", min_t(size_t, ret - i, lenc))) {
 209                         ptr = raw + i + lenc;
 210                         len = ret - i - lenc;
 211                         found = 1;
 212                         break;
 213                 }
 214         }
 215
 216         if (!found || ptr >= raw + ret || len < 0 || rtotlen == 0 || good == 0) {
 217                 close(fd);
 218                 close(sock);
 219
 220                 if (retry == 0) {
 221                         retry = 1;
 222                         goto again;
 223                 }
 224
 225                 return -ENOENT;
 226         }
 227
 228         do {
 229                 write_or_die(fd, ptr, len);
 230                 totlen += len;
 231                 printf("\r%s [%.2f%%, %zd/%zd, %s]", files[which].desc,
 232                        100.f * totlen / rtotlen, totlen, rtotlen, host);
 233                 fflush(stdout);
 234
 235                 memset(raw, 0, sizeof(raw));
 236                 ret = read(sock, raw, sizeof(raw));
 237
 238                 ptr = raw;
 239                 len = ret;
 240         } while (ret > 0);
 241
 242         printf("\n");
 243
 244         close(fd);
 245         close(sock);
 246
 247         if (totlen != rtotlen) {
 248                 unlink(files[which].local);
 249                 return -EIO;
 250         }
 251
 252         geoip_inflate(which);
 253         unlink(zfile);
 254
 255         return 0;
 256 }
 257
 258 static GeoIPRecord *geoip4_get_record(struct sockaddr_in *sa)
 259 {
 260         bug_on(gi4_city == NULL);
 261
 262         return GeoIP_record_by_ipnum(gi4_city, ntohl(sa->sin_addr.s_addr));
 263 }
 264
 265 static GeoIPRecord *geoip6_get_record(struct sockaddr_in6 *sa)
 266 {
 267         bug_on(gi6_city == NULL);
 268
 269         return GeoIP_record_by_ipnum_v6(gi6_city, sa->sin6_addr);
 270 }
 271
 272 const char *geoip4_as_name(struct sockaddr_in *sa)
 273 {
 274         bug_on(gi4_asname == NULL);
 275
 276         return GeoIP_name_by_ipnum(gi4_asname, ntohl(sa->sin_addr.s_addr));
 277 }
 278
 279 const char *geoip6_as_name(struct sockaddr_in6 *sa)
 280 {
 281         bug_on(gi6_asname == NULL);
 282
 283         return GeoIP_name_by_ipnum_v6(gi6_asname, sa->sin6_addr);
 284 }
 285
 286 float geoip4_longitude(struct sockaddr_in *sa)
 287 {
 288         GeoIPRecord *record;
 289         float longitude;
 290
 291         record = geoip4_get_record(sa);
 292         if (!record)
 293                 return 0;
 294
 295         longitude = record->longitude;
 296
 297         GeoIPRecord_delete(record);
 298         return longitude;
 299 }
 300
 301 float geoip4_latitude(struct sockaddr_in *sa)
 302 {
 303         GeoIPRecord *record;
 304         float latitude;
 305
 306         record = geoip4_get_record(sa);
 307         if (!record)
 308                 return 0;
 309
 310         latitude = record->latitude;
 311
 312         GeoIPRecord_delete(record);
 313         return latitude;
 314 }
 315
 316 float geoip6_longitude(struct sockaddr_in6 *sa)
 317 {
 318         GeoIPRecord *record;
 319         float longitude;
 320
 321         record = geoip6_get_record(sa);
 322         if (!record)
 323                 return 0;
 324
 325         longitude = record->longitude;
 326
 327         GeoIPRecord_delete(record);
 328         return longitude;
 329 }
 330
 331 float geoip6_latitude(struct sockaddr_in6 *sa)
 332 {
 333         GeoIPRecord *record;
 334         float latitude;
 335
 336         record = geoip6_get_record(sa);
 337         if (!record)
 338                 return 0;
 339
 340         latitude = record->latitude;
 341
 342         GeoIPRecord_delete(record);
 343         return latitude;
 344 }
 345
 346 char *geoip4_city_name(struct sockaddr_in *sa)
 347 {
 348         GeoIPRecord *record;
 349         char *city = NULL;
 350
 351         record = geoip4_get_record(sa);
 352         if (!record)
 353                 return NULL;
 354
 355         if (record->city)
 356                 city = xstrdup(record->city);
 357
 358         GeoIPRecord_delete(record);
 359         return city;
 360 }
 361
 362 char *geoip6_city_name(struct sockaddr_in6 *sa)
 363 {
 364         GeoIPRecord *record;
 365         char *city = NULL;
 366
 367         record = geoip6_get_record(sa);
 368         if (!record)
 369                 return NULL;
 370
 371         if (record->city)
 372                 city = xstrdup(record->city);
 373
 374         GeoIPRecord_delete(record);
 375         return city;
 376 }
 377
 378 char *geoip4_region_name(struct sockaddr_in *sa)
 379 {
 380         GeoIPRecord *record;
 381         char *region = NULL;
 382
 383         record = geoip4_get_record(sa);
 384         if (!record)
 385                 return NULL;
 386
 387         if (record->region)
 388                 region = xstrdup(record->region);
 389
 390         GeoIPRecord_delete(record);
 391         return region;
 392 }
 393
 394 char *geoip6_region_name(struct sockaddr_in6 *sa)
 395 {
 396         GeoIPRecord *record;
 397         char *region = NULL;
 398
 399         record = geoip6_get_record(sa);
 400         if (!record)
 401                 return NULL;
 402
 403         if (record->region)
 404                 region = xstrdup(record->region);
 405
 406         GeoIPRecord_delete(record);
 407         return region;
 408 }
 409
 410 const char *geoip4_country_name(struct sockaddr_in *sa)
 411 {
 412         bug_on(gi4_country == NULL);
 413
 414         return GeoIP_country_name_by_ipnum(gi4_country, ntohl(sa->sin_addr.s_addr));
 415 }
 416
 417 const char *geoip6_country_name(struct sockaddr_in6 *sa)
 418 {
 419         bug_on(gi6_country == NULL);
 420
 421         return GeoIP_country_name_by_ipnum_v6(gi6_country, sa->sin6_addr);
 422 }
 423
 424 const char *geoip4_country_code3_name(struct sockaddr_in *sa)
 425 {
 426         bug_on(gi4_country == NULL);
 427
 428         return GeoIP_country_code3_by_ipnum(gi4_country, ntohl(sa->sin_addr.s_addr));
 429 }
 430
 431 const char *geoip6_country_code3_name(struct sockaddr_in6 *sa)
 432 {
 433         bug_on(gi6_country == NULL);
 434
 435         return GeoIP_country_code3_by_ipnum_v6(gi6_country, sa->sin6_addr);
 436 }
 437
 438 static int fdout, fderr;
 439
 440 /* GeoIP people were too stupid to come to the idea that you could set
 441  * errno appropriately and return NULL instead of printing stuff from
 442  * the library directly that no one can turn off.
 443  */
 444
 445 static void geoip_open_prepare(void)
 446 {
 447         fflush(stdout);
 448         fdout = dup_or_die(1);
 449
 450         fflush(stderr);
 451         fderr = dup_or_die(2);
 452
 453         close(1);
 454         close(2);
 455 }
 456
 457 static void geoip_open_restore(void)
 458 {
 459         dup2_or_die(fdout, 1);
 460         dup2_or_die(fderr, 2);
 461
 462         close(fdout);
 463         close(fderr);
 464 }
 465
 466 static GeoIP *geoip_open_type(int type, int flags)
 467 {
 468         GeoIP *ret;
 469
 470         geoip_open_prepare();
 471         ret = GeoIP_open_type(type, flags);
 472         geoip_open_restore();
 473
 474         return ret;
 475 }
 476
 477 static GeoIP *geoip_open(const char *filename, int flags)
 478 {
 479         GeoIP *ret;
 480
 481         geoip_open_prepare();
 482         ret = GeoIP_open(filename, flags);
 483         geoip_open_restore();
 484
 485         return ret;
 486 }
 487
 488 static void init_geoip_city_open4(int enforce)
 489 {
 490         gi4_city = geoip_open(files[GEOIP_CITY_EDITION_REV1].local, GEOIP_MMAP_CACHE);
 491         if (gi4_city == NULL) {
 492                 gi4_city = geoip_open_type(GEOIP_CITY_EDITION_REV1, GEOIP_MMAP_CACHE);
 493                 if (gi4_city == NULL)
 494                         if (enforce)
 495                                 panic("Cannot open GeoIP4 city database, try --update!\n");
 496         }
 497
 498         if (gi4_city) {
 499                 GeoIP_set_charset(gi4_city, GEOIP_CHARSET_UTF8);
 500                 geoip_db_present |= CITYV4;
 501         }
 502 }
 503
 504 static void init_geoip_city_open6(int enforce)
 505 {
 506         gi6_city = geoip_open(files[GEOIP_CITY_EDITION_REV1_V6].local, GEOIP_MMAP_CACHE);
 507         if (gi6_city == NULL) {
 508                 gi6_city = geoip_open_type(GEOIP_CITY_EDITION_REV1_V6, GEOIP_MMAP_CACHE);
 509                 if (gi6_city == NULL)
 510                         if (enforce)
 511                                 panic("Cannot open GeoIP6 city database, try --update!\n");
 512         }
 513
 514         if (gi6_city) {
 515                 GeoIP_set_charset(gi6_city, GEOIP_CHARSET_UTF8);
 516                 geoip_db_present |= CITYV6;
 517         }
 518 }
 519
 520 static void init_geoip_city(int enforce)
 521 {
 522         init_geoip_city_open4(enforce);
 523         init_geoip_city_open6(enforce);
 524 }
 525
 526 static void destroy_geoip_city(void)
 527 {
 528         GeoIP_delete(gi4_city);
 529         GeoIP_delete(gi6_city);
 530 }
 531
 532 static void init_geoip_country_open4(int enforce)
 533 {
 534         gi4_country = geoip_open(files[GEOIP_COUNTRY_EDITION].local, GEOIP_MMAP_CACHE);
 535         if (gi4_country == NULL) {
 536                 gi4_country = geoip_open_type(GEOIP_COUNTRY_EDITION, GEOIP_MMAP_CACHE);
 537                 if (gi4_country == NULL)
 538                         if (enforce)
 539                                 panic("Cannot open GeoIP4 country database, try --update!\n");
 540         }
 541
 542         if (gi4_country) {
 543                 GeoIP_set_charset(gi4_country, GEOIP_CHARSET_UTF8);
 544                 geoip_db_present |= COUNTRYV4;
 545         }
 546 }
 547
 548 static void init_geoip_country_open6(int enforce)
 549 {
 550         gi6_country = geoip_open(files[GEOIP_COUNTRY_EDITION_V6].local, GEOIP_MMAP_CACHE);
 551         if (gi6_country == NULL) {
 552                 gi6_country = geoip_open_type(GEOIP_COUNTRY_EDITION_V6, GEOIP_MMAP_CACHE);
 553                 if (gi6_country == NULL)
 554                         if (enforce)
 555                                 panic("Cannot open GeoIP6 country database, try --update!\n");
 556         }
 557
 558         if (gi6_country) {
 559                 GeoIP_set_charset(gi6_country, GEOIP_CHARSET_UTF8);
 560                 geoip_db_present |= COUNTRYV6;
 561         }
 562 }
 563
 564 static void init_geoip_country(int enforce)
 565 {
 566         init_geoip_country_open4(enforce);
 567         init_geoip_country_open6(enforce);
 568 }
 569
 570 static void destroy_geoip_country(void)
 571 {
 572         GeoIP_delete(gi4_country);
 573         GeoIP_delete(gi6_country);
 574 }
 575
 576 static void init_geoip_asname_open4(int enforce)
 577 {
 578         gi4_asname = geoip_open(files[GEOIP_ASNUM_EDITION].local, GEOIP_MMAP_CACHE);
 579         if (gi4_asname == NULL) {
 580                 gi4_asname = geoip_open_type(GEOIP_ASNUM_EDITION, GEOIP_MMAP_CACHE);
 581                 if (gi4_asname == NULL)
 582                         if (enforce)
 583                                 panic("Cannot open GeoIP4 AS database, try --update!\n");
 584         }
 585
 586         if (gi4_asname) {
 587                 GeoIP_set_charset(gi4_asname, GEOIP_CHARSET_UTF8);
 588                 geoip_db_present |= ASNAMV4;
 589         }
 590 }
 591
 592 static void init_geoip_asname_open6(int enforce)
 593 {
 594         gi6_asname = geoip_open(files[GEOIP_ASNUM_EDITION_V6].local, GEOIP_MMAP_CACHE);
 595         if (gi6_asname == NULL) {
 596                 gi6_asname = geoip_open_type(GEOIP_ASNUM_EDITION_V6, GEOIP_MMAP_CACHE);
 597                 if (gi6_asname == NULL)
 598                         if (enforce)
 599                                 panic("Cannot open GeoIP6 AS database, try --update!\n");
 600         }
 601
 602         if (gi6_asname) {
 603                 GeoIP_set_charset(gi6_asname, GEOIP_CHARSET_UTF8);
 604                 geoip_db_present |= ASNAMV6;
 605         }
 606 }
 607
 608 static void init_geoip_asname(int enforce)
 609 {
 610         init_geoip_asname_open4(enforce);
 611         init_geoip_asname_open6(enforce);
 612 }
 613
 614 static void destroy_geoip_asname(void)
 615 {
 616         GeoIP_delete(gi4_asname);
 617         GeoIP_delete(gi6_asname);
 618 }
 619
 620 static void init_mirrors(void)
 621 {
 622         size_t i = 0;
 623         FILE *fp;
 624         char buff[256];
 625
 626         fp = fopen(ETCDIRE_STRING "/geoip.conf", "r");
 627         if (!fp)
 628                 panic("Cannot open geoip.conf!\n");
 629
 630         memset(buff, 0, sizeof(buff));
 631         while (fgets(buff, sizeof(buff), fp) != NULL &&
 632                i < array_size(servers)) {
 633                 buff[strcspn(buff, "\r\n")] = 0;
 634                 buff[strcspn(buff, "\r\n")] = 0;
 635
 636                 if (buff[0] == '#') {
 637                         memset(buff, 0, sizeof(buff));
 638                         continue;
 639                 }
 640
 641                 servers[i++] = xstrdup(buff);
 642                 memset(buff, 0, sizeof(buff));
 643         }
 644
 645         fclose(fp);
 646 }
 647
 648 static void destroy_mirrors(void)
 649 {
 650         size_t i;
 651
 652         for (i = 0; i < array_size(servers); ++i)
 653                 free(servers[i]);
 654 }
 655
 656 void init_geoip(int enforce)
 657 {
 658         init_geoip_city(enforce);
 659         init_geoip_country(enforce);
 660         init_geoip_asname(enforce);
 661 }
 662
 663 void update_geoip(void)
 664 {
 665         size_t i, j;
 666         int ret, good = 0;
 667
 668         init_mirrors();
 669
 670         for (i = 0; i < array_size(files); ++i) {
 671                 if (files[i].local && files[i].remote) {
 672                         good = 0;
 673
 674                         for (j = 0; j < array_size(servers); ++j) {
 675                                 if (servers[j] == NULL)
 676                                         continue;
 677                                 ret = geoip_get_database(servers[j], i);
 678                                 if (!ret) {
 679                                         good = 1;
 680                                         break;
 681                                 }
 682                         }
 683
 684                         if (good == 0)
 685                                 panic("Cannot get %s from mirrors!\n",
 686                                       files[i].remote);
 687                 }
 688         }
 689
 690         destroy_mirrors();
 691 }
 692
 693 void destroy_geoip(void)
 694 {
 695         destroy_geoip_city();
 696         destroy_geoip_country();
 697         destroy_geoip_asname();
 698
 699         geoip_db_present = 0;
 700 }