wiretap/etl.c

   1 /* etl.c
   2  *
   3  * Basic support for Microsoft ETL traces
   4  * Copyright (c) 2019 by Aurelien Aptel <aurelien.aptel@gmail.com>
   5  *
   6  * Wiretap Library
   7  * Copyright (c) 1998 by Gilbert Ramirez <gram@alumni.rice.edu>
   8  *
   9  * SPDX-License-Identifier: GPL-2.0-or-later
  10  */
  11
  12 #include "config.h"
  13
  14 #include <stdlib.h>
  15 #include <errno.h>
  16 #include <string.h>
  17 #include "wtap-int.h"
  18 #include "file_wrappers.h"
  19
  20 /*
  21  * ETL files are Windows native traces. They can be generated using
  22  * netsh:
  23  *
  24  *     netsh trace start tracefile=c:\mytrace.etl capture=yes
  25  *     netsh trace stop
  26  *
  27  * They are quite versatile: they store all sorts of system
  28  * information (TCP/IP stack state, processes running, ...) and system
  29  * events (syscalls, kernel stacks, ...), including network
  30  * traffic. It's pretty much the equivalent of strace, ftrace, /proc/
  31  * and tcpdump all bundled into one file.
  32  *
  33  * The API to consume and produce those events on a Window system is
  34  * called ETW and it uses a myriad of different structs, some
  35  * undocumented.
  36  *
  37  * https://docs.microsoft.com/en-us/windows/win32/api/evntcons/
  38  *
  39  * ETL files are made of those structs which are simply dumped from
  40  * memory. The file format remains undocumented but can be figured out
  41  * by looking at the API struct definitions, hexdumps and some guess
  42  * work. Microsoft also has its own tool to explore those traces
  43  * (MessageAnalyzer) which was very useful to double-check some of the
  44  * findings.
  45  *
  46  * Each event producer is called a Provider and they all have a
  47  * GUID. It seems the Provider responsible for generating the network
  48  * traffic events is "Microsoft-Windows-NDIS-PacketCapture".
  49  *
  50  * Here is a pseudo-grammar of an ETL file
  51  *
  52  * ETL        := ETW_BUFFER+
  53  * ETW_BUFFER := WMI_BUFFER_HEADER EVENT+
  54  * EVENT      := SYSTEM_TRACE_HEADER TRACE_LOGFILE_HEADER
  55  *            |  PERFINFO_TRACE_HEADER
  56  *            |  EVENT_HEADER <-- packets are here
  57  *            |  EVENT_INSTANCE_HEADER
  58  *            |  (other ignored types)
  59  *
  60  * NETMON files use EVENT_HEADER stuctures similar enough that we can
  61  * reuse its dissector. But the structure varies enough that we have
  62  * to introduce a new WTAP_ENCAP_ETL value.
  63  *
  64  * This ETL reader reuses existing Wireshark support:
  65  * - the netmon dissector will dissect the EVENT_HEADER itself.
  66  * - the messageanalyzer dissector knows about the NDIS Provider and
  67  *   will handle the ETW NDIS sub-payload.
  68  */
  69
  70 /*
  71  * The packet in the trace are out of order unfortunately. This define
  72  * controls whether we should expose them to wiretap sorted, so that
  73  * frame number increases correspond to timestamp increases. MS tool
  74  * MessageAnalyzer does the same thing.
  75  */
  76 #define SORTED_PACKETS
  77
  78 static int etl_file_type_subtype = -1;
  79
  80 void register_etl(void);
  81
  82 #define WMI_BUFFER_HEADER_SIZE 0x48
  83 #define EVENT_HEADER_SIZE 0x50
  84 #define NDIS_HEADER_SIZE  0x0C
  85 #define GUID_SIZE 0x10
  86 #define WMI_BUFFER_MAX_SIZE 0x800000 /* 8MB */
  87
  88 /* #define DEBUG_ETL */
  89
  90 #ifdef DEBUG_ETL
  91 /* workaround for stupid checkAPIs checks *sigh*... */
  92 #define DBG(...) g_ ## debug(__VA_ARGS__)
  93 #define WARN(...) g_ ## warning(__VA_ARGS__)
  94 #else
  95 #define DBG(...) do {} while (0)
  96 #define WARN(...) do {} while (0)
  97 #endif
  98
  99 struct wmi {
 100         uint32_t size;
 101         uint32_t useful_size;
 102         uint16_t buffer_flag;
 103         int64_t time_start;
 104 };
 105
 106 struct etl_index {
 107         int64_t etime;
 108 #ifdef SORTED_PACKETS
 109         int64_t sorted_off;
 110 #endif
 111         int64_t off;
 112         uint32_t size;
 113 };
 114
 115 struct etl {
 116         int64_t start_time;
 117         GArray *pkts;
 118         bool is_indexed;
 119 };
 120
 121 wtap_open_return_val etl_open(wtap *wth, int *err, char **err_info);
 122 static int read_wmi(struct wmi *wmi, wtap *wth, bool next, int *err);
 123 static int etl_index_all_packets(struct etl *etl, wtap *wth, int *err);
 124 static void etl_free(struct etl *etl);
 125 static struct etl_index *etl_find_index(struct etl *etl, int64_t off);
 126 static bool etl_seek_read(wtap *wth, int64_t seek_off, wtap_rec *rec, Buffer *buf, int *err, char **err_info);
 127 static bool etl_read_rec(struct etl *etl, struct etl_index *e, FILE_T fh, wtap_rec *rec, Buffer *buf, int *err, char **err_info);
 128 static bool etl_read(wtap *wth, wtap_rec *rec, Buffer *buf, int *err, char **err_info, int64_t *data_offset);
 129
 130 enum {
 131       ETL_OK,  /* seems ok */
 132       ETL_EOF, /* reached EOF */
 133       ETL_BAD, /* bad input */
 134       ETL_ERR, /* io failure */
 135 };
 136
 137 static int
 138 read_wmi(struct wmi *wmi, wtap *wth, bool next, int *err)
 139 {
 140         int64_t rc;
 141         int64_t start, off;
 142         uint8_t buf[WMI_BUFFER_HEADER_SIZE];
 143
 144         /*
 145           4 uint32 BufferSize
 146           4 uint32 SavedOffset
 147           4 uint32 CurrentOffset
 148           4  int32 ReferenceCount
 149           8  int64 TimeStamp
 150           8  int64 SequenceNumber
 151           8 uint64 ClockType/Frequency
 152           1  uint8 ProcessorNumber
 153           1  uint8 Alignment
 154           2 uint16 LoggerId
 155           4 uint32 BufferState
 156           4 uint32 Offset <--- useful size
 157           2 uint16 BufferFlag
 158           2 uint16 BufferType
 159           8  int64 StartTime
 160           8  int64 StartPerfClock
 161          */
 162
 163         start = file_tell(wth->fh);
 164         rc = file_read(buf, sizeof(buf), wth->fh);
 165         if (rc != sizeof(buf)) {
 166                 if (rc == 0 && file_eof(wth->fh))
 167                         return ETL_EOF;
 168                 return ETL_BAD;
 169         }
 170
 171         wmi->size = pletoh32(buf + 0);
 172         wmi->time_start = (int64_t)pletoh64(buf + 4+4+4+4);
 173         wmi->useful_size = pletoh32(buf + 4+4+4+4+8+8+8+1+1+2+4);
 174         wmi->buffer_flag = pletoh16(buf + 4+4+4+4+8+8+8+1+1+2+4+4);
 175
 176         /* check min/max sizes */
 177         if (wmi->size < WMI_BUFFER_HEADER_SIZE || wmi->size > WMI_BUFFER_MAX_SIZE)
 178                 return ETL_BAD;
 179
 180         /* alignement and useful size */
 181         if (wmi->size % 8 != 0 || wmi->useful_size > wmi->size)
 182                 return ETL_BAD;
 183
 184         /* overflow */
 185         if (start > INT64_MAX - (int64_t)wmi->size)
 186                 return ETL_BAD;
 187
 188         off = start + (int64_t)wmi->size;
 189         if (next) {
 190                 rc = file_seek(wth->fh, off, SEEK_SET, err);
 191                 if (rc < 0 || err)
 192                         return ETL_ERR;
 193         }
 194         return ETL_OK;
 195 }
 196
 197 enum event_header_type {
 198     TRACE_HEADER_TYPE_SYSTEM32       = 1,
 199     TRACE_HEADER_TYPE_SYSTEM64       = 2,
 200     TRACE_HEADER_TYPE_COMPACT32      = 3,
 201     TRACE_HEADER_TYPE_COMPACT64      = 4,
 202     TRACE_HEADER_TYPE_FULL_HEADER32  = 10,
 203     TRACE_HEADER_TYPE_INSTANCE32     = 11,
 204     TRACE_HEADER_TYPE_TIMED          = 12,  /* not used */
 205     TRACE_HEADER_TYPE_ERROR          = 13,  /* error while logging event */
 206     TRACE_HEADER_TYPE_WNODE_HEADER   = 14,  /* not used */
 207     TRACE_HEADER_TYPE_MESSAGE        = 15,
 208     TRACE_HEADER_TYPE_PERFINFO32     = 16,
 209     TRACE_HEADER_TYPE_PERFINFO64     = 17,
 210     TRACE_HEADER_TYPE_EVENT_HEADER32 = 18,
 211     TRACE_HEADER_TYPE_EVENT_HEADER64 = 19,
 212     TRACE_HEADER_TYPE_FULL_HEADER64  = 20,
 213     TRACE_HEADER_TYPE_INSTANCE64     = 21
 214 };
 215
 216 static void
 217 etl_add_index(struct etl *etl, int64_t off, int64_t etime, uint32_t size)
 218 {
 219         struct etl_index v = {
 220               .off = off,
 221               .etime = etime,
 222               .size = size,
 223         };
 224
 225         g_array_append_val(etl->pkts, v);
 226 }
 227
 228 #ifdef SORTED_PACKETS
 229 static int
 230 pkt_cmp(const void *a, const void *b)
 231 {
 232         int64_t ta = ((const struct etl_index *)a)->etime, tb = ((const struct etl_index *)b)->etime;
 233         return ta > tb ? 1 : (ta < tb ? -1 : 0);
 234 }
 235
 236 static void
 237 etl_sort(struct etl *etl)
 238 {
 239         unsigned i;
 240         int64_t sorted_off = 0;
 241
 242         g_array_sort(etl->pkts, pkt_cmp);
 243
 244         for (i = 0; i < etl->pkts->len; i++) {
 245                 struct etl_index *e = &g_array_index(etl->pkts, struct etl_index, i);
 246                 e->sorted_off = sorted_off;
 247                 sorted_off += e->size;
 248         }
 249 }
 250 #endif
 251
 252 static int
 253 etl_index_all_packets(struct etl *etl, wtap *wth, int *err)
 254 {
 255         int64_t rc;
 256         struct wmi wmi;
 257         uint8_t ev_type;
 258         int64_t wmi_start, ev_start, ev_size, off, pad, etime;
 259         uint8_t buf[EVENT_HEADER_SIZE];
 260
 261         wmi_start = off = 0;
 262
 263         while (!(rc = read_wmi(&wmi, wth, false, err))) {
 264                 DBG("ETL: WMI 0x%" PRIx64, wmi_start);
 265                 DBG("ETL: WMI size 0x%x", wmi.size);
 266
 267                 if (wmi.buffer_flag & 0x40) {
 268                         WARN("ETL: WMI is compressed");
 269                         goto next_wmi;
 270                 }
 271
 272                 while (off < wmi_start + wmi.useful_size) {
 273                         bool ok = false;
 274                         ev_start = off = file_tell(wth->fh);
 275                         DBG("ETL: EV 0x%" PRIx64, ev_start);
 276
 277                         rc = file_read(buf, 6, wth->fh);
 278                         if (rc != 6)
 279                                 return ETL_BAD;
 280
 281                         ev_type = buf[2];
 282
 283                         switch (ev_type) {
 284                         case TRACE_HEADER_TYPE_SYSTEM32:
 285                         case TRACE_HEADER_TYPE_SYSTEM64:
 286                         case TRACE_HEADER_TYPE_COMPACT32:
 287                         case TRACE_HEADER_TYPE_COMPACT64:
 288                         case TRACE_HEADER_TYPE_PERFINFO32:
 289                         case TRACE_HEADER_TYPE_PERFINFO64:
 290                                 ev_size = pletoh16(buf + 4);
 291                                 goto next_event;
 292                         case TRACE_HEADER_TYPE_EVENT_HEADER32:
 293                         case TRACE_HEADER_TYPE_EVENT_HEADER64:
 294                                 ev_size = pletoh16(buf + 0);
 295                                 break;
 296                         default:
 297                                 ev_size = pletoh16(buf + 0);
 298                                 goto next_event;
 299                         }
 300
 301                         /* EVENT_HEADER
 302                            2 uint16 Size
 303                            1  uint8 Type
 304                            1  uint8 MarkerFlags
 305                            2 uint16 Flags
 306                            2 uint16 EventProperty
 307                            4 uint32 ThreadId
 308                            4 uint32 ProcessId
 309                            8  int64 TimeStamp
 310                            16  GUID ProviderId
 311                            struct EVENT_DESCRIPTOR {
 312                            2            uint16 Id
 313                            1             uint8 Version
 314                            1             uint8 Channel
 315                            1             uint8 Level
 316                            1             uint8 Opcode
 317                            2            uint16 Task
 318                            8            uint64 Keyword
 319                            } EventDescriptor
 320                            4 uint32  KernelTime
 321                            4 uint32  UserTime
 322                            16  GUID  ActivityId
 323                         */
 324
 325                         rc = file_seek(wth->fh, ev_start, SEEK_SET, err);
 326                         if (rc != ev_start)
 327                                 return ETL_ERR;
 328
 329                         rc = file_read(buf, EVENT_HEADER_SIZE, wth->fh);
 330                         if (rc != EVENT_HEADER_SIZE)
 331                                 return ETL_BAD;
 332
 333                         etime = (int64_t)pletoh64(buf+2+1+1+2+2+4+4);
 334                         etime += wmi.time_start;
 335
 336                         ok = true;
 337
 338                 next_event:
 339                         DBG("ETL: EV size 0x%" PRIx64, ev_size);
 340                         if (ev_start > INT64_MAX - ev_size)
 341                                 return ETL_BAD;
 342                         if (ev_size <= 6 || ev_size >= UINT32_MAX)
 343                                 return ETL_BAD;
 344                         if (ev_start+ev_size > wmi_start + wmi.useful_size)
 345                                 return ETL_BAD;
 346
 347                         if (ok)
 348                                 etl_add_index(etl, ev_start, etime, (uint32_t)ev_size);
 349
 350                         off = ev_start + ev_size;
 351                         pad = 8 - (off % 8);
 352                         if (pad < 8)
 353                                 off += pad;
 354
 355                         rc = file_seek(wth->fh, off, SEEK_SET, err);
 356                         if (rc != off)
 357                                 return ETL_ERR;
 358                 }
 359
 360         next_wmi:
 361                 off = wmi_start + wmi.size;
 362                 rc = file_seek(wth->fh, off, SEEK_SET, err);
 363                 if (rc != off)
 364                         return ETL_ERR;
 365                 wmi_start = off;
 366         }
 367
 368         if (rc == 0 || rc == ETL_EOF) {
 369 #ifdef SORTED_PACKETS
 370                 etl_sort(etl);
 371 #endif
 372                 etl->is_indexed = true;
 373                 rc = 0;
 374         }
 375
 376         return (int)rc;
 377 }
 378
 379 static void
 380 etl_close(wtap *wth)
 381 {
 382         struct etl* etl = (struct etl *)wth->priv;
 383         etl_free(etl);
 384         wth->priv = NULL;
 385 }
 386
 387 static void
 388 etl_free(struct etl *etl)
 389 {
 390         if (!etl)
 391                 return;
 392         g_array_free(etl->pkts, true);
 393         g_free(etl);
 394 }
 395
 396 wtap_open_return_val
 397 etl_open(wtap *wth, int *err, char **err_info _U_)
 398 {
 399         /*
 400          * ETL files have no magic header. To check for ETL we try to
 401          * read a couple of WMI headers and check if some the fields
 402          * make sense.
 403          */
 404
 405         struct etl *etl;
 406         int rc;
 407
 408         etl = g_new0(struct etl, 1);
 409         etl->pkts = g_array_sized_new(false, true, sizeof(struct etl_index), 256);
 410
 411         rc = etl_index_all_packets(etl, wth, err);
 412
 413         switch (rc) {
 414         case ETL_OK:
 415                 break;
 416         case ETL_EOF:
 417                 if (!etl->pkts || etl->pkts->len <= 0) {
 418                         etl_free(etl);
 419                         return WTAP_OPEN_NOT_MINE;
 420                 }
 421                 break;
 422         case ETL_BAD:
 423                 etl_free(etl);
 424                 return WTAP_OPEN_NOT_MINE;
 425         default:
 426                 etl_free(etl);
 427                 return WTAP_OPEN_ERROR;
 428         }
 429
 430         if (file_seek(wth->fh, 0, SEEK_SET, err) < 0) {
 431                 etl_free(etl);
 432                 return WTAP_OPEN_ERROR;
 433         }
 434
 435         wth->file_type_subtype = etl_file_type_subtype;
 436         wth->priv = (void *)etl;
 437         wth->subtype_read = etl_read;
 438         wth->subtype_seek_read = etl_seek_read;
 439         wth->subtype_close = etl_close;
 440         wth->snapshot_length = 0; /* not known */
 441
 442         wth->file_encap = WTAP_ENCAP_ETL;
 443         wth->file_tsprec = WTAP_TSPREC_USEC;
 444
 445         return WTAP_OPEN_MINE;
 446 }
 447
 448 static struct etl_index *
 449 etl_find_index(struct etl *etl, int64_t off)
 450 {
 451         int64_t i;
 452
 453         if (off == 0)
 454                 return &g_array_index(etl->pkts, struct etl_index, 0);
 455
 456         for (i = 0; i < etl->pkts->len; i++) {
 457                 struct etl_index *e = &g_array_index(etl->pkts, struct etl_index, i);
 458 #ifdef SORTED_PACKETS
 459                 if (e->sorted_off == off)
 460                         return e;
 461 #else
 462                 if (e->off >= off)
 463                         return e;
 464 #endif
 465         }
 466
 467         return NULL;
 468 }
 469
 470 static bool etl_seek_read_handle(struct etl *etl, FILE_T fh, int64_t off,
 471                             int64_t *data_offset, wtap_rec *rec, Buffer *buf,
 472                             int *err, char **err_info)
 473 {
 474         struct etl_index *e = etl_find_index(etl, off);
 475         bool r;
 476
 477         if (!e)
 478                 return false;
 479
 480 #ifdef SORTED_PACKETS
 481         *data_offset = e->sorted_off;
 482
 483         if (file_seek(fh, e->off, SEEK_SET, err) == -1)
 484                 return false;
 485
 486         r = etl_read_rec(etl, e, fh, rec, buf, err, err_info);
 487         if (!r)
 488                 return false;
 489
 490         if (file_seek(fh, e->sorted_off + e->size, SEEK_SET, err) == -1)
 491                 return false;
 492 #else
 493         *data_offset = e->off;
 494
 495         if (file_seek(fh, e->off, SEEK_SET, err) == -1)
 496                 return false;
 497
 498         r = etl_read_rec(etl, e, fh, rec, buf, err, err_info);
 499 #endif
 500         return r;
 501 }
 502
 503 /* Read the next packet */
 504 static bool
 505 etl_read(wtap *wth, wtap_rec *rec, Buffer *buf,  int *err,
 506          char **err_info, int64_t *data_offset)
 507 {
 508         struct etl *etl = (struct etl *)wth->priv;
 509         int64_t off = file_tell(wth->fh);
 510
 511         return etl_seek_read_handle(etl, wth->fh, off, data_offset, rec, buf, err, err_info);
 512 }
 513
 514 static bool
 515 etl_seek_read(wtap *wth, int64_t seek_off, wtap_rec *rec,
 516               Buffer *buf, int *err, char **err_info)
 517 {
 518         struct etl *etl = (struct etl *)wth->priv;
 519         int64_t data_offset;
 520
 521         return etl_seek_read_handle(etl, wth->random_fh, seek_off, &data_offset, rec, buf, err, err_info);
 522 }
 523
 524 static bool
 525 etl_read_rec(struct etl *etl _U_, struct etl_index *e, FILE_T fh, wtap_rec *rec, Buffer *buf,
 526              int *err, char **err_info)
 527 {
 528         uint32_t size = e->size;
 529
 530         if (e->size < NDIS_HEADER_SIZE || e->size > WTAP_MAX_PACKET_SIZE_STANDARD) {
 531                 *err = WTAP_ERR_BAD_FILE;
 532                 return false;
 533         }
 534
 535         rec->rec_type = REC_TYPE_PACKET;
 536         rec->presence_flags = WTAP_HAS_TS;
 537         filetime_to_nstime(&rec->ts, e->etime);
 538         rec->rec_header.packet_header.len = size;
 539         rec->rec_header.packet_header.caplen = size;
 540         rec->rec_header.packet_header.pkt_encap = WTAP_ENCAP_ETL;
 541
 542         return wtap_read_packet_bytes(fh, buf, size, err, err_info);
 543 }
 544
 545 static const struct supported_block_type etl_blocks_supported[] = {
 546         /*
 547          * We support packet blocks, with no comments or other options.
 548          */
 549         { WTAP_BLOCK_PACKET, MULTIPLE_BLOCKS_SUPPORTED, NO_OPTIONS_SUPPORTED }
 550 };
 551
 552 static const struct file_type_subtype_info etl_info = {
 553         "Windows ETL", "etl", "etl", NULL,
 554         true, BLOCKS_SUPPORTED(etl_blocks_supported),
 555         NULL, NULL, NULL
 556 };
 557
 558 void register_etl(void)
 559 {
 560         etl_file_type_subtype = wtap_register_file_type_subtype(&etl_info);
 561 }
 562
 563 /*
 564  * Editor modelines  -  https://www.wireshark.org/tools/modelines.html
 565  *
 566  * Local variables:
 567  * c-basic-offset: 8
 568  * tab-width: 8
 569  * indent-tabs-mode: t
 570  * End:
 571  *
 572  * vi: set shiftwidth=8 tabstop=8 noexpandtab:
 573  * :indentSize=8:tabSize=8:noTabs=false:
 574  */