kismet-log-viewer/klc_dutch_08-2006-R2.pl

   1 #!/usr/bin/perl
   2 # Original script copyright 2005 tud.thaunderdog
   3 #
   4 #Changes 17 sep 2006 by Dutch:
   5 #Disable external DTD validation for the parser, so that the instanced LibXML
   6 #parsers don't go out on the net in order to validate each and every gps-point
   7 #network line the parsers read and write. By disabling this check, the script
   8 #runs faster as well as can be run without an active internet connection.
   9 #Default for newer versions of LibXML is to validate against any external DTD's
  10 #referenced in DOCTYPE statements.
  11 #
  12 #Changes 16 feb 2006 by Dutch:
  13 #Skipped the foreach loop for comparing new networks to previous networks, and
  14 #changed it to use a hash and lookup from the hash for the existing networks
  15 #Speeded the script up by a factor of 10+. Previously 3000 existing networks
  16 #merged/updated with 1200 networks took over 24 hours on an 850 Mhz P3 machine.
  17 #With the hashing it takes about 1 hour.
  18 #
  19 #Changes 10 feb 2006 by Dutch:
  20 #Minor bugfix and change to last iso next in foreach loops, to speed up a bit
  21 #also added checks for first log read in, then no need to compare with array
  22 #and no need to update the outputfile untill all data from first file read in.
  23 #
  24 #
  25
  26 use XML::LibXML;
  27
  28 unless ( @ARGV > 2 ) {
  29     print "Usage: $0 <input-1st> <input-Nth> <output>\n";
  30     exit;
  31 }
  32
  33 $start_time = localtime(time);
  34
  35 $out_file_name = pop @ARGV;
  36 @log_files = @ARGV;
  37 $net_num = 0;
  38 $firstlog = 0;
  39 %seen = ();
  40 $outcounter = 1;
  41 foreach $this_log (@log_files) {
  42         $firstlog++;
  43         print "KLC: Reading Logfile $this_log....\n";
  44
  45
  46         $doc = XML::LibXML::Document->new();
  47         $root = $doc->createElement('html');
  48         $doc->setDocumentElement($root);
  49         $body = $doc->createElement('body');
  50         $root->appendChild($body);
  51
  52
  53
  54         if (!$xml_body) {
  55                 open(LOG_FILE,"<$this_log");
  56                 @log_file = <LOG_FILE>;
  57                 close(LOG_FILE);
  58
  59                 foreach my $line (@log_file) {
  60                         if ($line=~/<?xml/) { $xml_head .= $line; }
  61                         elsif ($line=~/<!DOCTYPE/) {$xml_head .= $line; }
  62                         elsif ($line=~/<detection-run/) { $xml_head .= $line; }
  63                         elsif ($line=~/<\/detection-run/) { $xml_foot .= $line; }
  64                 }
  65
  66                 open(OUT_FILE,">$out_file_name");
  67                 print OUT_FILE ("$xml_head");
  68                 print OUT_FILE ("$xml_foot");
  69                 close(OUT_FILE);
  70                 $xml_body = 1;
  71         }
  72
  73
  74         $parser = XML::LibXML->new();
  75         $parser->expand_entities(0);
  76         $parser->load_ext_dtd(0);
  77         $tree = $parser->parse_file($this_log);
  78         $root = $tree->getDocumentElement;
  79
  80         @networks = $root->getElementsByTagName('wireless-network');
  81         foreach $this_network (@networks) {
  82
  83                 $net_bssid = $this_network->getElementsByTagName('BSSID')->[0]->getFirstChild->getData;
  84                 if ("$firstlog" eq "1") {
  85                     $seen{$net_bssid} = $outcounter;
  86                     $outcounter++;
  87                 }; #if reading the first log, fill the hash with each BSSID
  88                 $out_parser = XML::LibXML->new();
  89                 $out_parser->expand_entities(0);
  90                 $out_parser->load_ext_dtd(0);
  91                 $out_tree = $out_parser->parse_file($out_file_name);
  92                 $out_root = $out_tree->getDocumentElement;
  93
  94                 if ( $temp = $out_root->getElementsByTagName('wireless-network') ) { @out_networks = $out_root->getElementsByTagName('wireless-network'); }
  95
  96                 $unique = 1;
  97
  98                 if ("$firstlog" > "1") {
  99                 if (@out_networks) {
 100                                 if ( $seen{$net_bssid} ) {
 101                                 #if the new read BSSID is seen previously then clear unique flag, output the network data for later
 102                                 #processing. We use the hash value as the index into the array, to get the data. As the array
 103                                 #is 0 based and the counter we store in the hash is 1 based, we subtract 1.
 104                                 #the counter can't be zerobased as zero is the false flagvalue in perl, so the first entry in the hash
 105                                 #would be evaluated as false, in the above if test.
 106                                         undef $unique;
 107                                         $out_net = $out_networks[($seen{$net_bssid}-1)];
 108                                 } else {
 109                                         $seen{$net_bssid} = $outcounter;
 110                                         $outcounter++;
 111                                 #if not seen we add it to the hash and increment the counter
 112                                 }
 113                  } # end if there are networks in out file
 114                 } # end if firstlog different from the initial read log
 115                 if ($unique) {
 116                         print "KLC: Merging $net_bssid  -  $net_num - $firstlog\n";
 117                          $net_num++;
 118                          $this_network_xml = $this_network->toString;
 119                          $this_network_xml=~ s/<wireless-network number="\d+"/  <wireless-network number="$net_num"/;
 120                          $networks .= "\n$this_network_xml";
 121                 } else {
 122                         print "KLC: Merging $net_bssid  -  Updated\n";
 123                         undef $networks;
 124                         undef $gps;
 125                         $net_packets_LLC = $this_network->getElementsByTagName('LLC')->[0]->getFirstChild->getData;
 126                         $out_net_packets_LLC = $out_net->getElementsByTagName('LLC')->[0]->getFirstChild->getData;
 127                         $net_packets_LLC += $out_net_packets_LLC;
 128
 129                         $net_packets_data = $this_network->getElementsByTagName('data')->[0]->getFirstChild->getData;
 130                         $out_net_packets_data = $out_net->getElementsByTagName('data')->[0]->getFirstChild->getData;
 131                         $net_packets_data += $out_net_packets_data;
 132
 133                         $net_packets_crypt = $this_network->getElementsByTagName('crypt')->[0]->getFirstChild->getData;
 134                         $out_net_packets_crypt = $out_net->getElementsByTagName('crypt')->[0]->getFirstChild->getData;
 135                         $net_packets_crypt += $out_net_packets_crypt;
 136
 137                         $net_packets_weak = $this_network->getElementsByTagName('weak')->[0]->getFirstChild->getData;
 138                         $out_net_packets_weak = $out_net->getElementsByTagName('weak')->[0]->getFirstChild->getData;
 139                         $net_packets_weak += $out_net_packets_weak;
 140
 141                         $net_packets_dupeiv = $this_network->getElementsByTagName('dupeiv')->[0]->getFirstChild->getData;
 142                         $out_net_packets_dupeiv = $out_net->getElementsByTagName('dupeiv')->[0]->getFirstChild->getData;
 143                         $net_packets_dupeiv += $out_net_packets_dupeiv;
 144
 145                         $net_packets_total = $this_network->getElementsByTagName('total')->[0]->getFirstChild->getData;
 146                         $out_net_packets_total = $out_net->getElementsByTagName('total')->[0]->getFirstChild->getData;
 147                         $net_packets_total += $out_net_packets_total;
 148
 149                         $net_packets_datasize = $this_network->getElementsByTagName('datasize')->[0]->getFirstChild->getData;
 150                         $out_net_packets_datasize = $out_net->getElementsByTagName('datasize')->[0]->getFirstChild->getData;
 151                         $net_packets_datasize += $out_net_packets_datasize;
 152
 153                         if ( ( $temp = $this_network->getElementsByTagName('min-lat')->[0] ) && ( $temp = $out_net->getElementsByTagName('min-lat')->[0] ) ) {
 154                                 $gps = 1;
 155
 156                                 $net_gps_min_lat = $this_network->getElementsByTagName('min-lat')->[0]->getFirstChild->getData;
 157                                 $out_net_gps_min_lat = $out_net->getElementsByTagName('min-lat')->[0]->getFirstChild->getData;
 158                                 if ( $net_gps_min_lat > $out_net_gps_min_lat ) { $net_gps_min_lat = $out_net_gps_min_lat; }
 159
 160                                 $net_gps_min_lon = $this_network->getElementsByTagName('min-lon')->[0]->getFirstChild->getData;
 161                                 $out_net_gps_min_lon = $out_net->getElementsByTagName('min-lon')->[0]->getFirstChild->getData;
 162                                 if ( $net_gps_min_lon > $out_net_gps_min_lon ) { $net_gps_min_lon = $out_net_gps_min_lon; }
 163
 164                                 $net_gps_min_alt = $this_network->getElementsByTagName('min-alt')->[0]->getFirstChild->getData;
 165                                 $out_net_gps_min_alt = $out_net->getElementsByTagName('min-alt')->[0]->getFirstChild->getData;
 166                                 if ( $net_gps_min_alt > $out_net_gps_min_alt ) { $net_gps_min_alt = $out_net_gps_min_alt; }
 167
 168                                 $net_gps_min_spd = $this_network->getElementsByTagName('min-spd')->[0]->getFirstChild->getData;
 169                                 $out_net_gps_min_spd = $out_net->getElementsByTagName('min-spd')->[0]->getFirstChild->getData;
 170                                 if ( $net_gps_min_spd > $out_net_gps_min_spd ) { $net_gps_min_spd = $out_net_gps_min_spd; }
 171
 172                                 $net_gps_max_lat = $this_network->getElementsByTagName('max-lat')->[0]->getFirstChild->getData;
 173                                 $out_net_gps_max_lat = $out_net->getElementsByTagName('max-lat')->[0]->getFirstChild->getData;
 174                                 if ( $net_gps_max_lat < $out_net_gps_max_lat ) { $net_gps_max_lat = $out_net_gps_max_lat; }
 175
 176                                 $net_gps_max_lon = $this_network->getElementsByTagName('max-lon')->[0]->getFirstChild->getData;
 177                                 $out_net_gps_max_lon = $out_net->getElementsByTagName('max-lon')->[0]->getFirstChild->getData;
 178                                 if ( $net_gps_max_lon < $out_net_gps_max_lon ) { $net_gps_max_lon = $out_net_gps_max_lon; }
 179
 180                                 $net_gps_max_alt = $this_network->getElementsByTagName('max-alt')->[0]->getFirstChild->getData;
 181                                 $out_net_gps_max_alt = $out_net->getElementsByTagName('max-alt')->[0]->getFirstChild->getData;
 182                                 if ( $net_gps_max_alt < $out_net_gps_max_alt ) { $net_gps_max_alt = $out_net_gps_max_alt; }
 183
 184                                 $net_gps_max_spd = $this_network->getElementsByTagName('max-spd')->[0]->getFirstChild->getData;
 185                                 $out_net_gps_max_spd = $out_net->getElementsByTagName('max-spd')->[0]->getFirstChild->getData;
 186                                 if ( $net_gps_max_spd < $out_net_gps_max_spd ) { $net_gps_max_spd = $out_net_gps_max_spd; }
 187
 188                         }
 189
 190                         foreach  $this_out_network (@out_networks) {
 191                                 $this_network_xml = $this_out_network->toString;
 192                                 if ($this_out_network eq $out_net) {
 193                                         $this_network_xml=~ s/<LLC>\d*<\/LLC>/<LLC>$net_packets_LLC<\/LLC>/;
 194                                         $this_network_xml=~ s/<data>\d*<\/data>/<data>$net_packets_data<\/data>/;
 195                                         $this_network_xml=~ s/<crypt>\d*<\/crypt>/<crypt>$net_packets_crypt<\/crypt>/;
 196                                         $this_network_xml=~ s/<weak>\d*<\/weak>/<weak>$net_packets_weak<\/weak>/;
 197                                         $this_network_xml=~ s/<dupeiv>\d*<\/dupeiv>/<dupeiv>$net_packets_dupeiv<\/dupeiv>/;
 198                                         $this_network_xml=~ s/<total>\d*<\/total>/<total>$net_packets_total<\/total>/;
 199                                         $this_network_xml=~ s/<datasize>\d*<\/datasize>/<datasize>$net_packets_datasize<\/datasize>/;
 200
 201                                         if ($gps) {
 202                                                 $this_network_xml=~ s/<min-lat>\d*\.\d*<\/min-lat>/<min-lat>$net_gps_min_lat<\/min-lat>/;
 203                                                 $this_network_xml=~ s/<min-lon>\d*\.\d*<\/min-lon>/<min-lon>$net_gps_min_lon<\/min-lon>/;
 204                                                 $this_network_xml=~ s/<min-alt>\d*\.\d*<\/min-alt>/<min-alt>$net_gps_min_alt<\/min-alt>/;
 205                                                 $this_network_xml=~ s/<min-spd>\d*\.\d*<\/min-spd>/<min-spd>$net_gps_min_spd<\/min-spd>/;
 206                                                 $this_network_xml=~ s/<max-lat>\d*\.\d*<\/max-lat>/<max-lat>$net_gps_max_lat<\/max-lat>/;
 207                                                 $this_network_xml=~ s/<max-lon>\d*\.\d*<\/max-lon>/<max-lon>$net_gps_max_lon<\/max-lon>/;
 208                                                 $this_network_xml=~ s/<max-alt>\d*\.\d*<\/max-alt>/<max-alt>$net_gps_max_alt<\/max-alt>/;
 209                                                 $this_network_xml=~ s/<max-spd>\d*\.\d*<\/max-spd>/<max-spd>$net_gps_max_spd<\/max-spd>/;
 210                                         }
 211
 212
 213                                         $client_num = @net_clients = $this_network->getElementsByTagName('wireless-client');
 214                                         undef $client_num;
 215                                         if (@net_clients) {
 216
 217                                                 while ($this_network_xml=~ /.*client.*\n/) { $this_network_xml=~ s/.*client.*\n//; }
 218                                                 $this_network_xml=~ s/.*<\/wireless-network>.*//;
 219
 220
 221                                                 foreach  $this_client (@net_clients) {
 222                                                         $unique = 1;
 223                                                         $client_mac = $this_client->getElementsByTagName('client-mac')->[0]->getFirstChild->getData;
 224                                                         $out_net_clients_total = @out_net_clients = $out_net->getElementsByTagName('wireless-client');
 225
 226                                                         foreach  $out_this_client (@out_net_clients) {
 227                                                                  $out_client_mac = $out_this_client->getElementsByTagName('client-mac')->[0]->getFirstChild->getData;
 228
 229                                                                 if ( "$client_mac" eq "$out_client_mac" ) {
 230                                                                         undef $unique;
 231                                                                         $out_client = $this_out_client;
 232                                                                         last;
 233                                                                 }
 234
 235                                                         }
 236
 237                                                         if ($unique) {
 238                                                                 print "     Client  $client_mac\n";
 239                                                                 $this_client_xml = $this_client->toString;
 240                                                                 $client_num++;
 241                                                                 $this_client_xml=~ s/<wireless-client number="\d+"/<wireless-client number="$client_num"/;
 242
 243                                                         } else {
 244                                                                 print "     Client  $client_mac  -  Updated\n";
 245                                                                 undef $gps;
 246                                                                 $client_packets_data = $this_client->getElementsByTagName('client-data')->[0]->getFirstChild->getData;
 247                                                                 $out_client_net_packets_data = $out_net->getElementsByTagName('client-data')->[0]->getFirstChild->getData;
 248                                                                 $client_packets_data += $out_client_net_packets_data;
 249
 250                                                                 $client_packets_crypt = $this_client->getElementsByTagName('client-crypt')->[0]->getFirstChild->getData;
 251                                                                 $out_client_net_packets_crypt = $out_net->getElementsByTagName('client-crypt')->[0]->getFirstChild->getData;
 252                                                                 $client_packets_crypt += $out_client_net_packets_crypt;
 253
 254                                                                 $client_packets_weak = $this_client->getElementsByTagName('client-weak')->[0]->getFirstChild->getData;
 255                                                                 $out_client_net_packets_weak = $out_net->getElementsByTagName('client-weak')->[0]->getFirstChild->getData;
 256                                                                 $client_packets_weak += $out_client_net_packets_weak;
 257
 258                                                                 $client_packets_datasize = $this_client->getElementsByTagName('client-datasize')->[0]->getFirstChild->getData;
 259                                                                 $out_client_packets_datasize = $out_net->getElementsByTagName('client-datasize')->[0]->getFirstChild->getData;
 260                                                                 $client_packets_datasize += $out_client_packets_datasize;
 261
 262                                                                 if ( ( $temp = $this_client->getElementsByTagName('client-min-lat')->[0] ) && ( $temp = $out_net->getElementsByTagName('client-min-lat')->[0] ) ) {
 263                                                                         $gps = 1;
 264
 265                                                                         $client_gps_min_lat = $this_client->getElementsByTagName('client-min-lat')->[0]->getFirstChild->getData;
 266                                                                         $out_net_gps_min_lat = $out_net->getElementsByTagName('client-min-lat')->[0]->getFirstChild->getData;
 267                                                                         if ( $client_gps_min_lat > $out_net_gps_min_lat ) { $client_gps_min_lat = $out_net_gps_min_lat; }
 268
 269                                                                         $client_gps_min_lon = $this_client->getElementsByTagName('client-min-lon')->[0]->getFirstChild->getData;
 270                                                                         $out_net_gps_min_lon = $out_net->getElementsByTagName('client-min-lon')->[0]->getFirstChild->getData;
 271                                                                         if ( $client_gps_min_lon > $out_net_gps_min_lon ) { $client_gps_min_lon = $out_net_gps_min_lon; }
 272
 273                                                                         $client_gps_min_alt = $this_client->getElementsByTagName('client-min-alt')->[0]->getFirstChild->getData;
 274                                                                         $out_net_gps_min_alt = $out_net->getElementsByTagName('client-min-alt')->[0]->getFirstChild->getData;
 275                                                                         if ( $client_gps_min_alt > $out_net_gps_min_alt ) { $client_gps_min_alt = $out_net_gps_min_alt; }
 276
 277                                                                         $client_gps_min_spd = $this_client->getElementsByTagName('client-min-spd')->[0]->getFirstChild->getData;
 278                                                                         $out_net_gps_min_spd = $out_net->getElementsByTagName('client-min-spd')->[0]->getFirstChild->getData;
 279                                                                         if ( $client_gps_min_spd > $out_net_gps_min_spd ) { $client_gps_min_spd = $out_net_gps_min_spd; }
 280
 281                                                                         $client_gps_max_lat = $this_client->getElementsByTagName('client-max-lat')->[0]->getFirstChild->getData;
 282                                                                         $out_net_gps_max_lat = $out_net->getElementsByTagName('client-max-lat')->[0]->getFirstChild->getData;
 283                                                                         if ( $client_gps_max_lat < $out_net_gps_max_lat ) { $client_gps_max_lat = $out_net_gps_max_lat; }
 284
 285                                                                         $client_gps_max_lon = $this_client->getElementsByTagName('client-max-lon')->[0]->getFirstChild->getData;
 286                                                                         $out_net_gps_max_lon = $out_net->getElementsByTagName('client-max-lon')->[0]->getFirstChild->getData;
 287                                                                         if ( $client_gps_max_lon < $out_net_gps_max_lon ) { $client_gps_max_lon = $out_net_gps_max_lon; }
 288
 289                                                                         $client_gps_max_alt = $this_client->getElementsByTagName('client-max-alt')->[0]->getFirstChild->getData;
 290                                                                         $out_net_gps_max_alt = $out_net->getElementsByTagName('client-max-alt')->[0]->getFirstChild->getData;
 291                                                                         if ( $client_gps_max_alt < $out_net_gps_max_alt ) { $client_gps_max_alt = $out_net_gps_max_alt; }
 292
 293                                                                         $client_gps_max_spd = $this_client->getElementsByTagName('client-max-spd')->[0]->getFirstChild->getData;
 294                                                                         $out_net_gps_max_spd = $out_net->getElementsByTagName('client-max-spd')->[0]->getFirstChild->getData;
 295                                                                         if ( $client_gps_max_spd < $out_net_gps_max_spd ) { $client_gps_max_spd = $out_net_gps_max_spd; }
 296                                                                 } # endif client if gps
 297
 298                                                                 $this_client_xml = $this_client->toString;
 299                                                                 $this_client_xml=~ s/<client-data>\d*<\/client-data>/<client-data>$client_packets_data<\/client-data>/;
 300                                                                 $this_client_xml=~ s/<client-crypt>\d*<\/client-crypt>/<client-crypt>$client_packets_crypt<\/client-crypt>/;
 301                                                                 $this_client_xml=~ s/<client-weak>\d*<\/client-weak>/<client-weak>$client_packets_weak<\/client-weak>/;
 302                                                                 $this_client_xml=~ s/<client-datasize>\d*<\/client-datasize>/<client-datasize>$client_packets_datasize<\/client-datasize>/;
 303                                                                 if ($gps) {
 304                                                                         $this_client_xml=~ s/<client-min-lat>\d*\.\d*<\/client-min-lat>/<client-min-lat>$client_gps_min_lat<\/client-min-lat>/;
 305                                                                         $this_client_xml=~ s/<client-min-lon>\d*\.\d*<\/client-min-lon>/<client-min-lon>$client_gps_min_lon<\/client-min-lon>/;
 306                                                                         $this_client_xml=~ s/<client-min-alt>\d*\.\d*<\/client-min-alt>/<client-min-alt>$client_gps_min_alt<\/client-min-alt>/;
 307                                                                         $this_client_xml=~ s/<client-min-spd>\d*\.\d*<\/client-min-spd>/<client-min-spd>$client_gps_min_spd<\/client-min-spd>/;
 308                                                                         $this_client_xml=~ s/<client-max-lat>\d*\.\d*<\/client-max-lat>/<client-max-lat>$client_gps_max_lat<\/client-max-lat>/;
 309                                                                         $this_client_xml=~ s/<client-max-lon>\d*\.\d*<\/client-max-lon>/<client-max-lon>$client_gps_max_lon<\/client-max-lon>/;
 310                                                                         $this_client_xml=~ s/<client-max-alt>\d*\.\d*<\/client-max-alt>/<client-max-alt>$client_gps_max_alt<\/client-max-alt>/;
 311                                                                         $this_client_xml=~ s/<client-max-spd>\d*\.\d*<\/client-max-spd>/<client-max-spd>$client_gps_max_spd<\/client-max-spd>/;
 312                                                                 } # end gps
 313                                                                 $client_num++;
 314                                                                 $this_client_xml=~ s/<wireless-client number="\d+"/<wireless-client number="$client_num"/;
 315
 316
 317
 318                                                         } # end if client unique
 319
 320                                                         $this_network_xml .= "$this_client_xml\n";
 321
 322                                                 } # end foreach client this net
 323                                                 $this_network_xml.= "</wireless-network>\n";
 324                                         } # end if net has clients
 325
 326                                         $networks .= "\n$this_network_xml";
 327
 328                                 } else {
 329                                         $networks .= "\n$this_network_xml";
 330                                 } # end if found unique net
 331
 332                         } # end foreach out network
 333
 334                 } # end if net unique
 335             if ("$firstlog" > "1") {
 336                 open(OUT_FILE,">$out_file_name");
 337                 print OUT_FILE ("$xml_head");
 338                 print OUT_FILE ("$networks\n");
 339                 print OUT_FILE ("$xml_foot\n");
 340                 close(OUT_FILE);
 341             }
 342         } # end foreach $this_network
 343             if ("$firstlog" eq "1") {
 344                 open(OUT_FILE,">$out_file_name");
 345                 print OUT_FILE ("$xml_head");
 346                 print OUT_FILE ("$networks\n");
 347                 print OUT_FILE ("$xml_foot\n");
 348                 close(OUT_FILE);
 349             }
 350
 351
 352                 print "saving data sofar\n";
 353
 354
 355 } # end foreach $this_log
 356
 357 $end_time = localtime(time);
 358
 359 print "KLC: Combined Logs to $out_file_name\n";
 360 print "KLC: Operation Started: $start_time\n";
 361 print "KLC: Operation Ended:   $end_time\n";
 362 print "KLC: Done\n";
 363 # EOF