client/pyscripts/pm3_help2json.py

   1 #!/usr/bin/env python3
   2 """
   3 PM3 Help 2 JSON
   4
   5 This script takes the full text help output from the PM3 client and converts it to JSON.
   6
   7 Authors / Maintainers:
   8  - Samuel Windall
   9
  10 Note:
  11     This file is used during the pm3 client build
  12     any changes to the call script parameters should be reflected in the makefile
  13 """
  14
  15 import re
  16 import json
  17 import datetime
  18 import argparse
  19 import logging
  20
  21 ##############################################################################
  22 # Script version data: (Please increment when making updates)
  23
  24 APP_NAME = 'PM3Help2JSON'
  25
  26 VERSION_MAJOR = 1
  27 VERSION_MINOR = 0
  28
  29 ##############################################################################
  30 # Main Application Code:
  31
  32
  33 def main():
  34     """The main function for the script"""
  35     args = build_arg_parser().parse_args()
  36     logging_format = '%(message)s'
  37     if args.debug:
  38         logging.basicConfig(level=logging.DEBUG, format=logging_format)
  39     else:
  40         logging.basicConfig(level=logging.WARN, format=logging_format)
  41     logging.info(f'{get_version()} starting...')
  42     help_text = args.input_file.read()
  43     command_data = parse_all_command_data(help_text)
  44     meta_data = build_metadata(args.meta, command_data)
  45     output_data = {
  46         'metadata': meta_data,
  47         'commands': command_data,
  48     }
  49     json.dump(output_data, args.output_file, indent=4, sort_keys=True)
  50     logging.info(f'{get_version()} completed!')
  51
  52
  53 def build_arg_parser():
  54     """Build the argument parser for reading the program arguments"""
  55     parser = argparse.ArgumentParser()
  56     parser.add_argument('input_file', type=argparse.FileType('r'), help='Source of full text help from the PM3 client.')
  57     parser.add_argument('output_file', type=argparse.FileType('w'), help='Destination for JSON output.')
  58     parser.add_argument('--meta', action='append', help='Additional metadata to be included.', metavar='key:value')
  59     parser.add_argument('--version', '-v', action='version', version=get_version(), help='Version data about this app.')
  60     parser.add_argument('--debug', '-d', action='store_true', help='Log debug messages.')
  61     return parser
  62
  63
  64 def build_help_regex():
  65     """The regex uses to parse the full text output of help data from the pm3 client."""
  66     # Reads the divider followed by the command itself
  67     re_command = r'-{87}\n(?P<command>.+)\n'
  68     # Reads if the command is available offline
  69     re_offline = r'available offline: (?P<offline>yes|no)\n+'
  70     # Reads the description lines
  71     re_description = r'(?P<description>(.|\n)+?)\n+'
  72     # Reads the usage string
  73     re_usage = r'usage:\n(?P<usage>(?:.+\n)+)\n+'
  74     # Reads the options and there individual descriptions
  75     re_options = r'options:\n(?P<options>(?:.+\n)+)\n'
  76     # Reads the notes and examples
  77     re_notes = r'examples\/notes:\n(?P<notes>(?:.+\n)+)'
  78     # Combine them into a single regex object
  79     re_full = re.compile(re_command+re_offline+re_description+re_usage+re_options+re_notes, re.MULTILINE);
  80     return re_full
  81
  82
  83 def parse_all_command_data(help_text):
  84     """Turns the full text output of help data from the pm3 client into a list of dictionaries"""
  85     command_dicts = {}
  86     # Strip out ANSI escape sequences
  87     help_text = remove_ansi_escape_codes(help_text)
  88     # Find all commands in the full text help output
  89     matches = build_help_regex().finditer(help_text)
  90     for match in matches:
  91         # Turn a match into a dictionary with keys for the extracted fields
  92         command_object = parse_command_data(match)
  93         # Store this command against its name for easy lookup
  94         command_dicts[command_object['command']] = command_object
  95     return command_dicts
  96
  97
  98 def parse_command_data(match):
  99     """Turns a regex match of a command in the help text and converts it into a dictionary"""
 100     logging.info('Parsing new command...')
 101     # Get and clean the command string
 102     command = remove_extra_whitespace(match.group('command'))
 103     logging.info(f'    Command: {command}')
 104     # Get the online status as a boolean. Note: the regex only picks up 'yes' or 'no' so this check is safe.
 105     offline = (match.group('offline') == 'yes')
 106     logging.debug(f'    Offline: {offline}')
 107     # Get and clean the description paragraph
 108     description = text_to_oneliner(match.group('description'))
 109     logging.debug(f'    Description: {description}')
 110     # Get and clen the usage string
 111     usage = text_to_oneliner(match.group('usage'))
 112     logging.debug(f'    Usage: {usage}')
 113     # Get and clean the list of options
 114     options = text_to_list(match.group('options'))
 115     logging.debug(f'    Options: {options}')
 116     # Get and clean the list of examples and notes
 117     notes = text_to_list(match.group('notes'))
 118     logging.debug(f'    Notes: {notes}')
 119     # Construct the command dictionary
 120     command_data = {
 121         'command': command,
 122         'offline': offline,
 123         'description': description,
 124         'usage': usage,
 125         'options': options,
 126         'notes': notes
 127     }
 128     logging.info('Completed parsing command!')
 129     return command_data
 130
 131
 132 def build_metadata(extra_data, command_data):
 133     """Turns the full text output of help data from the pm3 client into a list of dictionaries."""
 134     logging.info('Building metadata...')
 135     metadata = {
 136         'extracted_by': get_version(),
 137         'extracted_on': datetime.datetime.utcnow().replace(microsecond=0).isoformat(),
 138         'commands_extracted': len(command_data)
 139     }
 140     for key, value in metadata.items():
 141         logging.debug(f'    {key} - {value}')
 142     if extra_data:
 143         for extra in extra_data:
 144             parts = extra.split(':')
 145             if len(parts) == 2:
 146                 metadata[parts[0]] = parts[1]
 147                 logging.debug(f'    {parts[0]} - {parts[1]}')
 148             else:
 149                 logging.warning(f'Error building metadata. '
 150                                 f'Skipped "{extra}". '
 151                                 f'Extra metadata must be in the format "key:value".')
 152     logging.info('Completed building metadata!')
 153     return metadata
 154
 155
 156 ##############################################################################
 157 # Helper Functions:
 158
 159
 160 def get_version():
 161     """Get the version string for this script"""
 162     return f'{APP_NAME} v{VERSION_MAJOR}.{VERSION_MINOR:02}'
 163
 164
 165 def remove_ansi_escape_codes(text):
 166     """Remove ANSI escape sequences that may be left in the text."""
 167     re_ansi_escape = re.compile(r'(\x9B|\x1B\[)[0-?]*[ -/]*[@-~]')
 168     return re_ansi_escape.sub('', str(text)).lower()
 169
 170
 171 def remove_extra_whitespace(text):
 172     """Removes extra whitespace that may be in the text."""
 173     # Ensure input is a string
 174     text = str(text)
 175     # Remove whitespace from the start and end of the text
 176     text = text.strip()
 177     # Deduplicate spaces in the string
 178     text = re.sub(r' +', ' ', text)
 179     return text
 180
 181
 182 def text_to_oneliner(text):
 183     """Converts a multi line string into a single line string and removes extra whitespace"""
 184     # Ensure input is a string
 185     text = str(text)
 186     # Replace newlines with spaces
 187     text = re.sub(r'\n+', ' ', text)
 188     # Remove the extra whitespace
 189     text = remove_extra_whitespace(text)
 190     return text
 191
 192
 193 def text_to_list(text):
 194     """Converts a multi line string into a list of lines and removes extra whitespace"""
 195     # Ensure input is a string
 196     text = str(text)
 197     # Get all the lines
 198     lines = text.strip().split('\n')
 199     # For each line clean up any extra whitespace
 200     return [remove_extra_whitespace(line) for line in lines]
 201
 202
 203 ##############################################################################
 204 # Application entrypoint:
 205
 206 if __name__ == '__main__':
 207     main()