client/pyscripts/pm3_help2json.py

   1 #!/usr/bin/env python3
   2 """
   3 PM3 Help 2 JSON
   4
   5 This script takes the full text help output from the PM3 client and converts it to JSON.
   6
   7 Authors / Maintainers:
   8  - Samuel Windall
   9
  10 Note:
  11     This file is used during the pm3 client build
  12     any changes to the call script parameters should be reflected in the makefile
  13 """
  14
  15 import re
  16 import json
  17 import datetime
  18 import argparse
  19 import logging
  20
  21 ##############################################################################
  22 # Script version data: (Please increment when making updates)
  23
  24 APP_NAME = 'PM3Help2JSON'
  25
  26 VERSION_MAJOR = 1
  27 VERSION_MINOR = 0
  28
  29 ##############################################################################
  30 # Main Application Code:
  31
  32
  33 def main():
  34     """The main function for the script"""
  35     args = build_arg_parser().parse_args()
  36     logging_format = '%(message)s'
  37     if args.debug:
  38         logging.basicConfig(level=logging.DEBUG, format=logging_format)
  39     else:
  40         logging.basicConfig(level=logging.WARN, format=logging_format)
  41     logging.info(f'{get_version()} starting...')
  42     help_text = args.input_file.read()
  43     command_data = parse_all_command_data(help_text)
  44     meta_data = build_metadata(args.meta, command_data)
  45     output_data = {
  46         'metadata': meta_data,
  47         'commands': command_data,
  48     }
  49     json.dump(output_data, args.output_file, indent=4, sort_keys=True)
  50     args.output_file.write("\n") # add trailing newline as json.dump does not
  51     logging.info(f'{get_version()} completed!')
  52
  53
  54 def build_arg_parser():
  55     """Build the argument parser for reading the program arguments"""
  56     parser = argparse.ArgumentParser()
  57     parser.add_argument('input_file', type=argparse.FileType('r'), help='Source of full text help from the PM3 client.')
  58     parser.add_argument('output_file', type=argparse.FileType('w'), help='Destination for JSON output.')
  59     parser.add_argument('--meta', action='append', help='Additional metadata to be included.', metavar='key:value')
  60     parser.add_argument('--version', '-v', action='version', version=get_version(), help='Version data about this app.')
  61     parser.add_argument('--debug', '-d', action='store_true', help='Log debug messages.')
  62     return parser
  63
  64
  65 def build_help_regex():
  66     re_command = r'-{87}\n(?P<command>.+)\n'
  67     # Reads if the command is available offline
  68     re_offline = r'available offline: (?P<offline>yes|no)\n+'
  69     # Reads the description lines
  70     re_description = r'(?P<description>\n[\s\S]*?(?=usage:))'
  71     # Reads the usage string
  72     re_usage = r'(?:usage:\n(?P<usage>(?:.+\n)+)\n+)?'
  73     # Reads the options and there individual descriptions
  74     re_options = r'(?:options:\n(?P<options>(?:.+\n)+)\n+)?'
  75     # Reads the notes and examples
  76     re_notes = r'(?:examples\/notes:\n(?P<notes>[\s\S]*?(?=(===|---|\n\n))))'
  77     # Combine them into a single regex object
  78     re_full = re.compile(re_command+re_offline+re_description+re_usage+re_options+re_notes, re.MULTILINE)
  79     return re_full
  80
  81
  82 def parse_all_command_data(help_text):
  83     """Turns the full text output of help data from the pm3 client into a list of dictionaries"""
  84     command_dicts = {}
  85     # Strip out ANSI escape sequences
  86     help_text = remove_ansi_escape_codes(help_text)
  87     # Find all commands in the full text help output
  88     matches = build_help_regex().finditer(help_text)
  89     for match in matches:
  90         # Turn a match into a dictionary with keys for the extracted fields
  91         command_object = parse_command_data(match)
  92         # Store this command against its name for easy lookup
  93         command_dicts[command_object['command']] = command_object
  94     return command_dicts
  95
  96
  97 def parse_command_data(match):
  98     """Turns a regex match of a command in the help text and converts it into a dictionary"""
  99     logging.info('Parsing new command...')
 100     # Get and clean the command string
 101     command = remove_extra_whitespace(match.group('command'))
 102     logging.info(f'    Command: {command}')
 103     # Get the online status as a boolean. Note: the regex only picks up 'yes' or 'no' so this check is safe.
 104     offline = (match.group('offline') == 'yes')
 105     logging.debug(f'    Offline: {offline}')
 106     # Get and clean the description paragraph
 107     description = text_to_oneliner(match.group('description'))
 108     logging.debug(f'    Description: {description}')
 109     # Get and clen the usage string
 110     usage = text_to_oneliner(match.group('usage'))
 111     logging.debug(f'    Usage: {usage}')
 112     # Get and clean the list of options
 113     options = text_to_list(match.group('options'))
 114     logging.debug(f'    Options: {options}')
 115     # Get and clean the list of examples and notes
 116     notes = text_to_list(match.group('notes'))
 117     logging.debug(f'    Notes: {notes}')
 118     # Construct the command dictionary
 119     command_data = {
 120         'command': command,
 121         'offline': offline,
 122         'description': description,
 123         'usage': usage,
 124         'options': options,
 125         'notes': notes
 126     }
 127     logging.info('Completed parsing command!')
 128     return command_data
 129
 130
 131 def build_metadata(extra_data, command_data):
 132     """Turns the full text output of help data from the pm3 client into a list of dictionaries."""
 133     logging.info('Building metadata...')
 134     metadata = {
 135         'extracted_by': get_version(),
 136         'extracted_on': datetime.datetime.utcnow().replace(microsecond=0).isoformat(),
 137         'commands_extracted': len(command_data)
 138     }
 139     for key, value in metadata.items():
 140         logging.debug(f'    {key} - {value}')
 141     if extra_data:
 142         for extra in extra_data:
 143             parts = extra.split(':')
 144             if len(parts) == 2:
 145                 metadata[parts[0]] = parts[1]
 146                 logging.debug(f'    {parts[0]} - {parts[1]}')
 147             else:
 148                 logging.warning(f'Error building metadata. '
 149                                 f'Skipped "{extra}". '
 150                                 f'Extra metadata must be in the format "key:value".')
 151     logging.info('Completed building metadata!')
 152     return metadata
 153
 154
 155 ##############################################################################
 156 # Helper Functions:
 157
 158
 159 def get_version():
 160     """Get the version string for this script"""
 161     return f'{APP_NAME} v{VERSION_MAJOR}.{VERSION_MINOR:02}'
 162
 163
 164 def remove_ansi_escape_codes(text):
 165     """Remove ANSI escape sequences that may be left in the text."""
 166     re_ansi_escape = re.compile(r'(\x9B|\x1B\[)[0-?]*[ -/]*[@-~]')
 167     return re_ansi_escape.sub('', str(text))
 168
 169
 170 def remove_extra_whitespace(text):
 171     """Removes extra whitespace that may be in the text."""
 172     # Ensure input is a string
 173     text = str(text)
 174     # Remove whitespace from the start and end of the text
 175     text = text.strip()
 176     # Deduplicate spaces in the string
 177     text = re.sub(r' +', ' ', text)
 178     return text
 179
 180
 181 def text_to_oneliner(text):
 182     """Converts a multi line string into a single line string and removes extra whitespace"""
 183     if text is None:
 184         return ""
 185     # Ensure input is a string
 186     text = str(text)
 187     # Replace newlines with spaces
 188     text = re.sub(r'\n+', ' ', text)
 189     # Remove the extra whitespace
 190     text = remove_extra_whitespace(text)
 191     return text
 192
 193
 194 def text_to_list(text):
 195     """Converts a multi line string into a list of lines and removes extra whitespace"""
 196     if text is None:
 197         return []
 198     # Ensure input is a string
 199     text = str(text)
 200     # Get all the lines
 201     lines = text.strip().split('\n')
 202     # For each line clean up any extra whitespace
 203     return [remove_extra_whitespace(line) for line in lines]
 204
 205
 206 ##############################################################################
 207 # Application entrypoint:
 208
 209 if __name__ == '__main__':
 210     main()