tools/asterix/update-specs.py

   1 #!/usr/bin/env python3
   2 # -*- coding: utf-8 -*-
   3 #
   4 # By Zoran Bošnjak <zoran.bosnjak@sloveniacontrol.si>
   5 #
   6 # Use asterix specifications in JSON format,
   7 # to generate C/C++ structures, suitable for wireshark.
   8 #
   9 # SPDX-License-Identifier: GPL-2.0-or-later
  10 #
  11
  12 import argparse
  13
  14 import urllib.request
  15 import json
  16 from copy import copy, deepcopy
  17 from itertools import chain, repeat, takewhile
  18 from functools import reduce
  19 import os
  20 import sys
  21 import re
  22
  23 import convertspec as convert
  24
  25 # Path to default upstream repository
  26 upstream_repo = 'https://zoranbosnjak.github.io/asterix-specs'
  27 dissector_file = 'epan/dissectors/packet-asterix.c'
  28
  29 class Offset(object):
  30     """Keep track of number of added bits.
  31     It's like integer, except when offsets are added together,
  32     a 'modulo 8' is applied, such that offset is always between [0,7].
  33     """
  34
  35     def __init__(self):
  36         self.current = 0
  37
  38     def __add__(self, other):
  39         self.current = (self.current + other) % 8
  40         return self
  41
  42     @property
  43     def get(self):
  44         return self.current
  45
  46 class Context(object):
  47     """Support class to be used as a context manager.
  48     The 'tell' method is used to output (print) some data.
  49     All output is first collected to a buffer, then rendered
  50     using a template file.
  51     """
  52     def __init__(self):
  53         self.buffer = {}
  54         self.offset = Offset()
  55         self.inside_repetitive = False
  56
  57     def __enter__(self):
  58         return self
  59
  60     def __exit__(self, exc_type, exc_value, exc_traceback):
  61         pass
  62
  63     def tell(self, channel, s):
  64         """Append string 's' to an output channel."""
  65         lines = self.buffer.get(channel, [])
  66         lines.append(s)
  67         self.buffer[channel] = lines
  68
  69     def reset_offset(self):
  70         self.offset = Offset()
  71
  72 def get_number(value):
  73     t = value['type']
  74     if t == 'Integer':
  75         return float(value['value'])
  76     if t == 'Div':
  77         a = get_number(value['numerator'])
  78         b = get_number(value['denominator'])
  79         return a/b
  80     if t == 'Pow':
  81         return float(pow(value['base'], value['exponent']))
  82     raise Exception('unexpected value type {}'.format(t))
  83
  84 def replace_string(s, mapping):
  85     """Helper function to replace each entry from the mapping."""
  86     for (key,val) in mapping.items():
  87         s = s.replace(key, val)
  88     return s
  89
  90 def safe_string(s):
  91     """String replacement table."""
  92     return replace_string(s, {
  93         # from C reference manual
  94         chr(92): r"\\", # Backslash character.
  95         '?':    r"\?",  # Question mark character.
  96         "'":    r"\'",  # Single quotation mark.
  97         '"':    r'\"',  # Double quotation mark.
  98         "\a":   "",     # Audible alert.
  99         "\b":   "",     # Backspace character.
 100         chr(27): "",    # <ESC> character.
 101         "\f":   "",     # Form feed.
 102         "\n":   "",     # Newline character.
 103         "\r":   "",     # Carriage return.
 104         "\t":   " ",    # Horizontal tab.
 105         "\v":   "",     # Vertical tab.
 106     })
 107
 108 def get_scaling(content):
 109     """Get scaling factor from the content."""
 110     lsb = content.get('lsb')
 111     if lsb is None:
 112         return None
 113     return '{}'.format(get_number(lsb))
 114
 115 def get_fieldpart(content):
 116     """Get FIELD_PART* from the content."""
 117     t = content['type']
 118     if t == 'Raw': return 'FIELD_PART_HEX'
 119     elif t == 'Table': return 'FIELD_PART_UINT'
 120     elif t == 'String':
 121         var = content['variation']
 122         if var == 'StringAscii': return 'FIELD_PART_ASCII'
 123         elif var == 'StringICAO': return 'FIELD_PART_CALLSIGN'
 124         elif var == 'StringOctal': return 'FIELD_PART_SQUAWK'
 125         else:
 126             raise Exception('unexpected string variation: {}'.format(var))
 127     elif t == 'Integer':
 128         if content['signed']:
 129             return 'FIELD_PART_INT'
 130         else:
 131             return 'FIELD_PART_UINT'
 132     elif t == 'Quantity':
 133         if content['signed']:
 134             return 'FIELD_PART_FLOAT'
 135         else:
 136             return 'FIELD_PART_UFLOAT'
 137     elif t == 'Bds':
 138         return 'FIELD_PART_HEX'
 139     else:
 140         raise Exception('unexpected content type: {}'.format(t))
 141
 142 def download_url(path):
 143     """Download url and return content as a string."""
 144     with urllib.request.urlopen(upstream_repo + path) as url:
 145         return url.read()
 146
 147 def read_file(path):
 148     """Read file content, return string."""
 149     with open(path) as f:
 150         return f.read()
 151
 152 def load_jsons(paths):
 153     """Load json files from either URL or from local disk."""
 154
 155     # load from url
 156     if paths == []:
 157         manifest = download_url('/manifest.json').decode()
 158         listing = []
 159         for spec in json.loads(manifest):
 160             cat = spec['category']
 161             for edition in spec['cats']:
 162                 listing.append('/specs/cat{}/cats/cat{}/definition.json'.format(cat, edition))
 163             for edition in spec['refs']:
 164                 listing.append('/specs/cat{}/refs/ref{}/definition.json'.format(cat, edition))
 165         return [download_url(i).decode() for i in listing]
 166
 167     # load from disk
 168     else:
 169         listing = []
 170         for path in paths:
 171             if os.path.isdir(path):
 172                 for root, dirs, files in os.walk(path):
 173                     for i in files:
 174                         (a,b) = os.path.splitext(i)
 175                         if (a,b) != ('definition', '.json'):
 176                             continue
 177                         listing.append(os.path.join(root, i))
 178             elif os.path.isfile(path):
 179                 listing.append(path)
 180             else:
 181                 raise Exception('unexpected path type: {}'.path)
 182         return [read_file(f) for f in listing]
 183
 184 def load_gitrev(paths):
 185     """Read git revision reference."""
 186
 187     # load from url
 188     if paths == []:
 189         gitrev = download_url('/gitrev.txt').decode().strip()
 190         return [upstream_repo, 'git revision: {}'.format(gitrev)]
 191
 192     # load from disk
 193     else:
 194         return ['(local disk)']
 195
 196 def get_ft(ref, n, content, offset):
 197     """Get FT... from the content."""
 198     a = offset.get
 199
 200     # bruto bit size (next multiple of 8)
 201     (m, b) = divmod(a+n, 8)
 202     m = m if b == 0 else m + 1
 203     m *= 8
 204
 205     mask = '0x00'
 206     if a != 0 or b != 0:
 207         bits = chain(repeat(0, a), repeat(1, n), repeat(0, m-n-a))
 208         mask = 0
 209         for (a,b) in zip(bits, reversed(range(m))):
 210             mask += a*pow(2,b)
 211         mask = hex(mask)
 212         # prefix mask with zeros '0x000...', to adjust mask size
 213         assert mask[0:2] == '0x'
 214         mask = mask[2:]
 215         required_mask_size = (m//8)*2
 216         add_some = required_mask_size - len(mask)
 217         mask = '0x' + '0'*add_some + mask
 218
 219     t = content['type']
 220
 221     if t == 'Raw':
 222         if n > 64:          # very long items
 223             assert (n % 8) == 0, "very long items require byte alignment"
 224             return 'FT_NONE, BASE_NONE, NULL, 0x00'
 225
 226         if (n % 8):         # not byte aligned
 227             base = 'DEC'
 228         else:               # byte aligned
 229             if n >= 32:             # long items
 230                 base = 'HEX'
 231             else:                   # short items
 232                 base = 'HEX_DEC'
 233         return 'FT_UINT{}, BASE_{}, NULL, {}'.format(m, base, mask)
 234     elif t == 'Table':
 235         return 'FT_UINT{}, BASE_DEC, VALS (valstr_{}), {}'.format(m, ref, mask)
 236     elif t == 'String':
 237         var = content['variation']
 238         if var == 'StringAscii':
 239             return 'FT_STRING, BASE_NONE, NULL, {}'.format(mask)
 240         elif var == 'StringICAO':
 241             return 'FT_STRING, BASE_NONE, NULL, {}'.format(mask)
 242         elif var == 'StringOctal':
 243             return 'FT_UINT{}, BASE_OCT, NULL, {}'.format(m, mask)
 244         else:
 245             raise Exception('unexpected string variation: {}'.format(var))
 246     elif t == 'Integer':
 247         signed = content['signed']
 248         if signed:
 249             return 'FT_INT{}, BASE_DEC, NULL, {}'.format(m, mask)
 250         else:
 251             return 'FT_UINT{}, BASE_DEC, NULL, {}'.format(m, mask)
 252     elif t == 'Quantity':
 253         return 'FT_DOUBLE, BASE_NONE, NULL, 0x00'
 254     elif t == 'Bds':
 255         return 'FT_UINT{}, BASE_DEC, NULL, {}'.format(m, mask)
 256     else:
 257         raise Exception('unexpected content type: {}'.format(t))
 258
 259 def reference(cat, edition, path):
 260     """Create reference string."""
 261     name = '_'.join(path)
 262     if edition is None:
 263         return('{:03d}_{}'.format(cat, name))
 264     return('{:03d}_V{}_{}_{}'.format(cat, edition['major'], edition['minor'], name))
 265
 266 def get_rule(rule):
 267     t = rule['type']
 268     if t == 'ContextFree':
 269         return rule['value']
 270     elif t == 'Dependent':
 271         return rule['default']
 272     else:
 273         raise Exception('unexpected type: {}'.format(t))
 274
 275 def get_bit_size(item):
 276     """Return bit size of a (spare) item."""
 277     if item['spare']:
 278         return item['length']
 279     else:
 280         return get_rule(item['rule'])['size']
 281
 282 def get_description(item, content=None):
 283     """Return item description."""
 284     name = item['name'] if not is_generated(item) else None
 285     title = item.get('title')
 286     if content is not None and content.get('unit'):
 287         unit = '[{}]'.format(safe_string(content['unit']))
 288     else:
 289         unit = None
 290
 291     parts = filter(lambda x: bool(x), [name, title, unit])
 292     if not parts:
 293         return ''
 294     return reduce(lambda a,b: a + ', ' + b, parts)
 295
 296 def generate_group(item, variation=None):
 297     """Generate group-item from element-item."""
 298     level2 = copy(item)
 299     level2['name'] = 'VALUE'
 300     level2['is_generated'] = True
 301     if variation is None:
 302         level1 = copy(item)
 303         level1['rule'] = {
 304             'type': 'ContextFree',
 305             'value': {
 306                 'type': 'Group',
 307                 'items': [level2],
 308             },
 309         }
 310     else:
 311         level2['rule'] = {
 312             'type': 'ContextFree',
 313             'value': variation,
 314         }
 315         level1 = {
 316             'type': "Group",
 317             'items': [level2],
 318         }
 319     return level1
 320
 321 def is_generated(item):
 322     return item.get('is_generated') is not None
 323
 324 def ungroup(item):
 325     """Convert group of items of known size to element"""
 326     n = sum([get_bit_size(i) for i in get_rule(item['rule'])['items']])
 327     result = copy(item)
 328     result['rule'] = {
 329         'type': 'ContextFree',
 330         'value': {
 331             'type': 'Element',
 332             'size': n,
 333             'rule': {
 334                 'type': 'ContextFree',
 335                 'value': {'type': 'Raw'},
 336             },
 337         },
 338     }
 339     return result
 340
 341 def part1(ctx, get_ref, catalogue):
 342     """Generate components in order
 343     - static int hf_...
 344     - FiledPart
 345     - FieldPart[]
 346     - AsterixField
 347     """
 348
 349     tell = lambda s: ctx.tell('insert1', s)
 350     tell_pr = lambda s: ctx.tell('insert2', s)
 351
 352     ctx.reset_offset()
 353
 354     def handle_item(path, item):
 355         """Handle 'spare' or regular 'item'.
 356         This function is used recursively, depending on the item structure.
 357         """
 358
 359         def handle_variation(path, variation):
 360             """Handle 'Element, Group...' variations.
 361             This function is used recursively, depending on the item structure."""
 362
 363             t = variation['type']
 364
 365             ref = get_ref(path)
 366
 367             def part_of(item):
 368                 if item['spare']:
 369                     return '&IXXX_{}bit_spare'.format(item['length'])
 370                 return '&I{}_{}'.format(ref, item['name'])
 371
 372             if t == 'Element':
 373                 tell('static int hf_{};'.format(ref))
 374                 n = variation['size']
 375                 content = get_rule(variation['rule'])
 376                 scaling = get_scaling(content)
 377                 scaling = scaling if scaling is not None else 1.0
 378                 fp = get_fieldpart(content)
 379
 380                 if content['type'] == 'Table':
 381                     tell('static const value_string valstr_{}[] = {}'.format(ref, '{'))
 382                     for (a,b) in content['values']:
 383                         tell('    {} {}, "{}" {},'.format('{', a, safe_string(b), '}'))
 384                     tell('    {} 0, NULL {}'.format('{', '}'))
 385                     tell('};')
 386
 387                 tell('static const FieldPart I{} = {} {}, {}, {}, &hf_{}, NULL {};'.format(ref, '{', n, scaling, fp, ref, '}'))
 388                 description = get_description(item, content)
 389
 390                 ft = get_ft(ref, n, content, ctx.offset)
 391                 tell_pr('        {} &hf_{}, {} "{}", "asterix.{}", {}, NULL, HFILL {} {},'.format('{', ref, '{', description, ref, ft, '}', '}'))
 392
 393                 ctx.offset += n
 394
 395             elif t == 'Group':
 396                 ctx.reset_offset()
 397
 398                 description = get_description(item)
 399                 tell_pr('        {} &hf_{}, {} "{}", "asterix.{}", FT_NONE, BASE_NONE, NULL, 0x00, NULL, HFILL {} {},'.format('{', ref, '{', description, ref, '}', '}'))
 400
 401                 tell('static int hf_{};'.format(ref))
 402                 for i in variation['items']:
 403                     handle_item(path, i)
 404
 405                 # FieldPart[]
 406                 tell('static const FieldPart * const I{}_PARTS[] = {}'.format(ref,'{'))
 407                 for i in variation['items']:
 408                     tell('    {},'.format(part_of(i)))
 409                 tell('    NULL')
 410                 tell('};')
 411
 412                 # AsterixField
 413                 bit_size = sum([get_bit_size(i) for i in variation['items']])
 414                 byte_size = bit_size // 8
 415                 parts = 'I{}_PARTS'.format(ref)
 416                 comp = '{ NULL }'
 417                 if not ctx.inside_repetitive:
 418                     tell('static const AsterixField I{} = {} FIXED, {}, 0, 0, &hf_{}, {}, {} {};'.format
 419                         (ref, '{', byte_size, ref, parts, comp, '}'))
 420
 421             elif t == 'Extended':
 422                 ctx.reset_offset()
 423
 424                 description = get_description(item)
 425                 tell_pr('        {} &hf_{}, {} "{}", "asterix.{}", FT_NONE, BASE_NONE, NULL, 0x00, NULL, HFILL {} {},'.format('{', ref, '{', description, ref, '}', '}'))
 426                 tell('static int hf_{};'.format(ref))
 427
 428                 items = []
 429                 for i in variation['items']:
 430                     if i is None:
 431                         items.append(i)
 432                         continue
 433                     if i.get('rule') is not None:
 434                         if get_rule(i['rule'])['type'] == 'Group':
 435                             i = ungroup(i)
 436                     items.append(i)
 437
 438                 for i in items:
 439                     if i is None:
 440                         ctx.offset += 1
 441                     else:
 442                         handle_item(path, i)
 443
 444                 tell('static const FieldPart * const I{}_PARTS[] = {}'.format(ref,'{'))
 445                 for i in items:
 446                     if i is None:
 447                         tell('    &IXXX_FX,')
 448                     else:
 449                         tell('    {},'.format(part_of(i)))
 450
 451                 tell('    NULL')
 452                 tell('};')
 453
 454                 # AsterixField
 455                 parts = 'I{}_PARTS'.format(ref)
 456                 comp = '{ NULL }'
 457                 tell('static const AsterixField I{} = {} FX, 0, 0, 0, &hf_{}, {}, {} {};'.format
 458                     (ref, '{', ref, parts, comp, '}'))
 459
 460             elif t == 'Repetitive':
 461                 ctx.reset_offset()
 462                 ctx.inside_repetitive = True
 463
 464                 # Group is required below this item.
 465                 if variation['variation']['type'] == 'Element':
 466                     subvar = generate_group(item, variation['variation'])
 467                 else:
 468                     subvar = variation['variation']
 469                 handle_variation(path, subvar)
 470
 471                 # AsterixField
 472                 bit_size = sum([get_bit_size(i) for i in subvar['items']])
 473                 byte_size = bit_size // 8
 474                 rep = variation['rep']['size'] // 8
 475                 parts = 'I{}_PARTS'.format(ref)
 476                 comp = '{ NULL }'
 477                 tell('static const AsterixField I{} = {} REPETITIVE, {}, {}, 0, &hf_{}, {}, {} {};'.format
 478                     (ref, '{', byte_size, rep, ref, parts, comp, '}'))
 479                 ctx.inside_repetitive = False
 480
 481             elif t == 'Explicit':
 482                 ctx.reset_offset()
 483                 tell('static int hf_{};'.format(ref))
 484                 description = get_description(item)
 485                 tell_pr('        {} &hf_{}, {} "{}", "asterix.{}", FT_NONE, BASE_NONE, NULL, 0x00, NULL, HFILL {} {},'.format('{', ref, '{', description, ref, '}', '}'))
 486                 tell('static const AsterixField I{} = {} EXP, 0, 0, 1, &hf_{}, NULL, {} NULL {} {};'.format(ref, '{', ref, '{', '}', '}'))
 487
 488             elif t == 'Compound':
 489                 ctx.reset_offset()
 490                 tell('static int hf_{};'.format(ref))
 491                 description = get_description(item)
 492                 tell_pr('        {} &hf_{}, {} "{}", "asterix.{}", FT_NONE, BASE_NONE, NULL, 0x00, NULL, HFILL {} {},'.format('{', ref, '{', description, ref, '}', '}'))
 493                 comp = '{'
 494                 for i in variation['items']:
 495                     if i is None:
 496                         comp += ' &IX_SPARE,'
 497                         continue
 498                     # Group is required below this item.
 499                     if get_rule(i['rule'])['type'] == 'Element':
 500                         subitem = generate_group(i)
 501                     else:
 502                         subitem = i
 503                     comp += ' &I{}_{},'.format(ref, subitem['name'])
 504                     handle_item(path, subitem)
 505                 comp += ' NULL }'
 506
 507                 # AsterixField
 508                 tell('static const AsterixField I{} = {} COMPOUND, 0, 0, 0, &hf_{}, NULL, {} {};'.format
 509                     (ref, '{', ref, comp, '}'))
 510
 511             else:
 512                 raise Exception('unexpected variation type: {}'.format(t))
 513
 514         if item['spare']:
 515             ctx.offset += item['length']
 516             return
 517
 518         # Group is required on the first level.
 519         if path == [] and get_rule(item['rule'])['type'] == 'Element':
 520             variation = get_rule(generate_group(item)['rule'])
 521         else:
 522             variation = get_rule(item['rule'])
 523         handle_variation(path + [item['name']], variation)
 524
 525     for item in catalogue:
 526         # adjust 'repetitive fx' item
 527         if get_rule(item['rule'])['type'] == 'Repetitive' and \
 528            get_rule(item['rule'])['rep']['type'] == 'Fx':
 529             var = get_rule(item['rule'])['variation'].copy()
 530             vt = var['type']
 531             item = item.copy()
 532             if vt == 'Element':
 533                 items = [{
 534                     'definition': None,
 535                     'description': None,
 536                     'name': 'Subitem',
 537                     'remark': None,
 538                     'spare': False,
 539                     'title': 'Subitem',
 540                     'rule': {
 541                         'type': 'ContextFree',
 542                         'value': var,
 543                     },}]
 544             elif vt == 'Group':
 545                 items = var['items']
 546             else:
 547                 raise Exception("Unexpected type", vt)
 548             item['rule'] = {
 549                 'type': 'ContextFree',
 550                 'value': {
 551                     'type': 'Extended',
 552                     'items': items + [None],
 553                 }
 554             }
 555         handle_item([], item)
 556     tell('')
 557
 558 def part2(ctx, ref, uap):
 559     """Generate UAPs"""
 560
 561     tell = lambda s: ctx.tell('insert1', s)
 562
 563     ut = uap['type']
 564     if ut == 'uap':
 565         variations = [{'name': 'uap', 'items': uap['items']}]
 566     elif ut == 'uaps':
 567         variations = uap['variations']
 568     else:
 569         raise Exception('unexpected uap type {}'.format(ut))
 570
 571     for var in variations:
 572         tell('static const AsterixField * const I{}_{}[] = {}'.format(ref, var['name'], '{'))
 573         for i in var['items']:
 574             if i is None:
 575                 tell('    &IX_SPARE,')
 576             else:
 577                 tell('    &I{}_{},'.format(ref, i))
 578         tell('    NULL')
 579         tell('};')
 580
 581     tell('static const AsterixField * const * const I{}[] = {}'.format(ref, '{'))
 582     for var in variations:
 583         tell('    I{}_{},'.format(ref, var['name']))
 584     tell('    NULL')
 585     tell('};')
 586     tell('')
 587
 588 def part3(ctx, specs):
 589     """Generate
 590         - static const AsterixField ***...
 591         - static const enum_val_t ..._versions[]...
 592     """
 593     tell = lambda s: ctx.tell('insert1', s)
 594     def fmt_edition(cat, edition):
 595         return 'I{:03d}_V{}_{}'.format(cat, edition['major'], edition['minor'])
 596
 597     cats = set([spec['number'] for spec in specs])
 598     for cat in sorted(cats):
 599         lst = [spec for spec in specs if spec['number'] == cat]
 600         editions = sorted([val['edition'] for val in lst], key = lambda x: (x['major'], x['minor']), reverse=True)
 601         editions_fmt = [fmt_edition(cat, edition) for edition in editions]
 602         editions_str = ', '.join(['I{:03d}'.format(cat)] + editions_fmt)
 603         tell('static const AsterixField * const * const * const I{:03d}all[] = {} {} {};'.format(cat, '{', editions_str, '}'))
 604         tell('')
 605
 606         tell('static const enum_val_t I{:03d}_versions[] = {}'.format(cat, '{'))
 607         edition = editions[0]
 608         a = edition['major']
 609         b = edition['minor']
 610         tell('    {} "I{:03d}", "Version {}.{} (latest)", 0 {},'.format('{', cat, a, b, '}'))
 611         for ix, edition in enumerate(editions, start=1):
 612             a = edition['major']
 613             b = edition['minor']
 614             tell('    {} "I{:03d}_v{}_{}", "Version {}.{}", {} {},'.format('{', cat, a, b, a, b, ix, '}'))
 615         tell('    { NULL, NULL, 0 }')
 616         tell('};')
 617         tell('')
 618
 619 def part4(ctx, cats):
 620     """Generate
 621         - static const AsterixField ****categories[]...
 622         - prefs_register_enum_preference ...
 623     """
 624     tell = lambda s: ctx.tell('insert1', s)
 625     tell_pr = lambda s: ctx.tell('insert3', s)
 626
 627     tell('static const AsterixField * const * const * const * const categories[] = {')
 628     for i in range(0, 256):
 629         val = 'I{:03d}all'.format(i) if i in cats else 'NULL'
 630         tell('    {}, /* {:03d} */'.format(val, i))
 631     tell('    NULL')
 632     tell('};')
 633
 634     for cat in sorted(cats):
 635         tell_pr('    prefs_register_enum_preference (asterix_prefs_module, "i{:03d}_version", "I{:03d} version", "Select the CAT{:03d} version", &global_categories_version[{}], I{:03d}_versions, false);'.format(cat, cat, cat, cat, cat))
 636
 637 class Output(object):
 638     """Output context manager. Write either to stdout or to a dissector
 639     file directly, depending on 'update' argument"""
 640     def __init__(self, update):
 641         self.update = update
 642         self.f = None
 643
 644     def __enter__(self):
 645         if self.update:
 646             self.f = open(dissector_file, 'w')
 647         return self
 648
 649     def __exit__(self, exc_type, exc_value, exc_traceback):
 650         if self.f is not None:
 651             self.f.close()
 652
 653     def dump(self, line):
 654         if self.f is None:
 655             print(line)
 656         else:
 657             self.f.write(line+'\n')
 658
 659 def remove_rfs(spec):
 660     """Remove RFS item. It's present in specs, but not used."""
 661     catalogue = []  # create new catalogue without RFS
 662     rfs_items = []
 663     for i in spec['catalogue']:
 664         if get_rule(i['rule'])['type'] == 'Rfs':
 665             rfs_items.append(i['name'])
 666         else:
 667             catalogue.append(i)
 668     if not rfs_items:
 669         return spec
 670     spec2 = copy(spec)
 671     spec2['catalogue'] = catalogue
 672     # remove RFS from UAP(s)
 673     uap = deepcopy(spec['uap'])
 674     ut = uap['type']
 675     if ut == 'uap':
 676         items = [None if i in rfs_items else i for i in uap['items']]
 677         if items[-1] is None: items = items[:-1]
 678         uap['items'] = items
 679     elif ut == 'uaps':
 680         variations = []
 681         for var in uap['variations']:
 682             items = [None if i in rfs_items else i for i in var['items']]
 683             if items[-1] is None: items = items[:-1]
 684             var['items'] = items
 685             variations.append(var)
 686         uap['variations'] = variations
 687     else:
 688         raise Exception('unexpected uap type {}'.format(ut))
 689     spec2['uap'] = uap
 690     return spec2
 691
 692 def is_valid(spec):
 693     """Check spec"""
 694     def check_item(item):
 695         if item['spare']:
 696             return True
 697         return check_variation(get_rule(item['rule']))
 698     def check_variation(variation):
 699         t = variation['type']
 700         if t == 'Element':
 701             return True
 702         elif t == 'Group':
 703             # do not allow nested items
 704             for i in variation['items']:
 705                 if not i['spare']:
 706                     if get_rule(i['rule'])['type'] != 'Element':
 707                         return False
 708             return all([check_item(i) for i in variation['items']])
 709         elif t == 'Extended':
 710             trailing_fx = variation['items'][-1] == None
 711             if not trailing_fx:
 712                 return False
 713             return all([check_item(i) for i in variation['items'] if i is not None])
 714         elif t == 'Repetitive':
 715             return check_variation(variation['variation'])
 716         elif t == 'Explicit':
 717             return True
 718         elif t == 'Compound':
 719             items = [i for i in variation['items'] if i is not None]
 720             return all([check_item(i) for i in items])
 721         else:
 722             raise Exception('unexpected variation type {}'.format(t))
 723     return all([check_item(i) for i in spec['catalogue']])
 724
 725 def main():
 726     parser = argparse.ArgumentParser(description='Process asterix specs files.')
 727     parser.add_argument('paths', metavar='PATH', nargs='*',
 728         help='json spec file(s), use upstream repository in no input is given')
 729     parser.add_argument('--reference', action='store_true',
 730         help='print upstream reference and exit')
 731     parser.add_argument("--update", action="store_true",
 732         help="Update %s as needed instead of writing to stdout" % dissector_file)
 733     args = parser.parse_args()
 734
 735     if args.reference:
 736         gitrev_short = download_url('/gitrev.txt').decode().strip()[0:10]
 737         print(gitrev_short)
 738         sys.exit(0)
 739
 740     # read and json-decode input files
 741     jsons = load_jsons(args.paths)
 742     jsons = [json.loads(i) for i in jsons]
 743     jsons = [convert.handle_asterix(i) for i in jsons]
 744     jsons = sorted(jsons, key = lambda x: (x['number'], x['edition']['major'], x['edition']['minor']))
 745     jsons = [spec for spec in jsons if spec['type'] == 'Basic']
 746     jsons = [remove_rfs(spec) for spec in jsons]
 747     jsons = [spec for spec in jsons if is_valid(spec)]
 748
 749     cats = list(set([x['number'] for x in jsons]))
 750     latest_editions = {cat: sorted(
 751         filter(lambda x: x['number'] == cat, jsons),
 752         key = lambda x: (x['edition']['major'], x['edition']['minor']), reverse=True)[0]['edition']
 753         for cat in cats}
 754
 755     # regular expression for template rendering
 756     ins = re.compile(r'---\{([A-Za-z0-9_]*)\}---')
 757
 758     gitrev = load_gitrev(args.paths)
 759     with Context() as ctx:
 760         for i in gitrev:
 761             ctx.tell('gitrev', i)
 762
 763         # generate parts into the context buffer
 764         for spec in jsons:
 765             is_latest = spec['edition'] == latest_editions[spec['number']]
 766
 767             ctx.tell('insert1', '/* Category {:03d}, edition {}.{} */'.format(
 768                 spec['number'], spec['edition']['major'], spec['edition']['minor']))
 769
 770             # handle part1
 771             get_ref = lambda path: reference(spec['number'], spec['edition'], path)
 772             part1(ctx, get_ref, spec['catalogue'])
 773             if is_latest:
 774                 ctx.tell('insert1', '/* Category {:03d}, edition {}.{} (latest) */'.format(
 775                     spec['number'], spec['edition']['major'], spec['edition']['minor']))
 776                 get_ref = lambda path: reference(spec['number'], None, path)
 777                 part1(ctx, get_ref, spec['catalogue'])
 778
 779             # handle part2
 780             cat = spec['number']
 781             edition = spec['edition']
 782             ref = '{:03d}_V{}_{}'.format(cat, edition['major'], edition['minor'])
 783             part2(ctx, ref, spec['uap'])
 784             if is_latest:
 785                 ref = '{:03d}'.format(cat)
 786                 part2(ctx, ref, spec['uap'])
 787
 788         part3(ctx, jsons)
 789         part4(ctx, set([spec['number'] for spec in jsons]))
 790
 791         # use context buffer to render template
 792         script_path = os.path.dirname(os.path.realpath(__file__))
 793         with open(os.path.join(script_path, 'packet-asterix-template.c')) as f:
 794             template_lines = f.readlines()
 795
 796         # All input is collected and rendered.
 797         # It's safe to update the dissector.
 798
 799         # copy each line of the template to required output,
 800         # if the 'insertion' is found in the template,
 801         # replace it with the buffer content
 802         with Output(args.update) as out:
 803             for line in template_lines:
 804                 line = line.rstrip()
 805
 806                 insertion = ins.match(line)
 807                 if insertion is None:
 808                     out.dump(line)
 809                 else:
 810                     segment = insertion.group(1)
 811                     [out.dump(i) for i in ctx.buffer[segment]]
 812
 813 if __name__ == '__main__':
 814     main()