openmp/runtime/tools/message-converter.py

   1 #!/usr/bin/env python3
   2
   3 #
   4 # //===----------------------------------------------------------------------===//
   5 # //
   6 # // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   7 # // See https://llvm.org/LICENSE.txt for license information.
   8 # // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   9 # //
  10 # //===----------------------------------------------------------------------===//
  11 #
  12
  13 import argparse
  14 import datetime
  15 import os
  16 import platform
  17 import re
  18 import sys
  19 from libomputils import ScriptError, error
  20
  21
  22 class TargetPlatform:
  23     """Convenience class for handling the target platform for configuration/compilation"""
  24
  25     system_override = None
  26     """
  27     Target system name override by the user.
  28     It follows the conventions from https://docs.python.org/3/library/platform.html#platform.system
  29     """
  30
  31     def set_system_override(override_system):
  32         """
  33         Set a system override for the target.
  34         Please follow the style from https://docs.python.org/3/library/platform.html#platform.system
  35         """
  36         TargetPlatform.system_override = override_system
  37
  38     def system():
  39         """
  40         Target System name.
  41         It follows the conventions from https://docs.python.org/3/library/platform.html#platform.system
  42         """
  43         if TargetPlatform.system_override is None:
  44             return platform.system()
  45         return TargetPlatform.system_override
  46
  47
  48 class ParseMessageDataError(ScriptError):
  49     """Convenience class for parsing message data file errors"""
  50
  51     def __init__(self, filename, line, msg):
  52         super(ParseMessageDataError, self).__init__(msg)
  53         self.filename = filename
  54         self.line = line
  55
  56
  57 def parse_error(filename, line, msg):
  58     raise ParseMessageDataError(filename, line, msg)
  59
  60
  61 def display_language_id(inputFile):
  62     """Quickly parse file for LangId and print it"""
  63     regex = re.compile(r'^LangId\s+"([0-9]+)"')
  64     with open(inputFile, encoding="utf-8") as f:
  65         for line in f:
  66             m = regex.search(line)
  67             if not m:
  68                 continue
  69             print(m.group(1))
  70
  71
  72 class Message(object):
  73     special = {
  74         "n": "\n",
  75         "t": "\t",
  76     }
  77
  78     def __init__(self, lineNumber, name, text):
  79         self.lineNumber = lineNumber
  80         self.name = name
  81         self.text = text
  82
  83     def toSrc(self):
  84         if TargetPlatform.system().casefold() == "Windows".casefold():
  85             return re.sub(r"%([0-9])\$(s|l?[du])", r"%\1!\2!", self.text)
  86         return str(self.text)
  87
  88     def toMC(self):
  89         retval = self.toSrc()
  90         for special, substitute in Message.special.items():
  91             retval = re.sub(r"\\{}".format(special), substitute, retval)
  92         return retval
  93
  94
  95 class MessageData(object):
  96     """
  97     Convenience class representing message data parsed from i18n/* files
  98
  99     Generate these objects using static create() factory method
 100     """
 101
 102     sectionInfo = {
 103         "meta": {"short": "prp", "long": "meta", "set": 1, "base": 1 << 16},
 104         "strings": {"short": "str", "long": "strings", "set": 2, "base": 2 << 16},
 105         "formats": {"short": "fmt", "long": "formats", "set": 3, "base": 3 << 16},
 106         "messages": {"short": "msg", "long": "messages", "set": 4, "base": 4 << 16},
 107         "hints": {"short": "hnt", "long": "hints", "set": 5, "base": 5 << 16},
 108     }
 109     orderedSections = ["meta", "strings", "formats", "messages", "hints"]
 110
 111     def __init__(self):
 112         self.filename = None
 113         self.sections = {}
 114
 115     def getMeta(self, name):
 116         metaList = self.sections["meta"]
 117         for meta in metaList:
 118             if meta.name == name:
 119                 return meta.text
 120         error(
 121             'No "{}" detected in meta data' " for file {}".format(name, self.filename)
 122         )
 123
 124     @staticmethod
 125     def create(inputFile):
 126         """Creates MessageData object from inputFile"""
 127         data = MessageData()
 128         data.filename = os.path.abspath(inputFile)
 129         obsolete = 1
 130         sectionRegex = re.compile(r"-\*- ([a-zA-Z0-9_]+) -\*-")
 131         keyValueRegex = re.compile(r'([a-zA-Z_][a-zA-Z0-9_]*)\s+"(.*)"')
 132         moreValueRegex = re.compile(r'"(.*)"')
 133
 134         with open(inputFile, "r", encoding="utf-8") as f:
 135             currentSection = None
 136             currentKey = None
 137             for lineNumber, line in enumerate(f, 1):
 138                 line = line.strip()
 139                 # Skip empty lines
 140                 if not line:
 141                     continue
 142                 # Skip comment lines
 143                 if line.startswith("#"):
 144                     continue
 145                 # Matched a section header
 146                 match = sectionRegex.search(line)
 147                 if match:
 148                     currentSection = match.group(1).lower()
 149                     if currentSection in data.sections:
 150                         parse_error(
 151                             inputFile,
 152                             lineNumber,
 153                             "section: {} already defined".format(currentSection),
 154                         )
 155                     data.sections[currentSection] = []
 156                     continue
 157                 # Matched a Key "Value" line (most lines)
 158                 match = keyValueRegex.search(line)
 159                 if match:
 160                     if not currentSection:
 161                         parse_error(inputFile, lineNumber, "no section defined yet.")
 162                     key = match.group(1)
 163                     if key == "OBSOLETE":
 164                         key = "OBSOLETE{}".format(obsolete)
 165                         obsolete += 1
 166                     value = match.group(2)
 167                     currentKey = key
 168                     data.sections[currentSection].append(
 169                         Message(lineNumber, key, value)
 170                     )
 171                     continue
 172                 # Matched a Continuation of string line
 173                 match = moreValueRegex.search(line)
 174                 if match:
 175                     value = match.group(1)
 176                     if not currentSection:
 177                         parse_error(inputFile, lineNumber, "no section defined yet.")
 178                     if not currentKey:
 179                         parse_error(inputFile, lineNumber, "no key defined yet.")
 180                     data.sections[currentSection][-1].text += value
 181                     continue
 182                 # Unknown line syntax
 183                 parse_error(inputFile, lineNumber, "bad line:\n{}".format(line))
 184         return data
 185
 186
 187 def insert_header(f, data, commentChar="//"):
 188     f.write(
 189         "{0} Do not edit this file! {0}\n"
 190         "{0} The file was generated from"
 191         " {1} by {2} on {3}. {0}\n\n".format(
 192             commentChar,
 193             os.path.basename(data.filename),
 194             os.path.basename(__file__),
 195             datetime.datetime.now().ctime(),
 196         )
 197     )
 198
 199
 200 def generate_enum_file(enumFile, prefix, data):
 201     """Create the include file with message enums"""
 202     global g_sections
 203     with open(enumFile, "w") as f:
 204         insert_header(f, data)
 205         f.write(
 206             "enum {0}_id {1}\n"
 207             "\n"
 208             "    // A special id for absence of message.\n"
 209             "    {0}_null = 0,\n"
 210             "\n".format(prefix, "{")
 211         )
 212         for section in MessageData.orderedSections:
 213             messages = data.sections[section]
 214             info = MessageData.sectionInfo[section]
 215             shortName = info["short"]
 216             longName = info["long"]
 217             base = info["base"]
 218             setIdx = info["set"]
 219             f.write(
 220                 "    // Set #{}, {}.\n"
 221                 "    {}_{}_first = {},\n".format(
 222                     setIdx, longName, prefix, shortName, base
 223                 )
 224             )
 225             for message in messages:
 226                 f.write("    {}_{}_{},\n".format(prefix, shortName, message.name))
 227             f.write("    {}_{}_last,\n\n".format(prefix, shortName))
 228         f.write(
 229             "    {0}_xxx_lastest\n\n"
 230             "{1}; // enum {0}_id\n\n"
 231             "typedef enum {0}_id  {0}_id_t;\n\n\n"
 232             "// end of file //\n".format(prefix, "}")
 233         )
 234
 235
 236 def generate_signature_file(signatureFile, data):
 237     """Create the signature file"""
 238     sigRegex = re.compile(r"(%[0-9]\$(s|l?[du]))")
 239     with open(signatureFile, "w") as f:
 240         f.write("// message catalog signature file //\n\n")
 241         for section in MessageData.orderedSections:
 242             messages = data.sections[section]
 243             longName = MessageData.sectionInfo[section]["long"]
 244             f.write("-*- {}-*-\n\n".format(longName.upper()))
 245             for message in messages:
 246                 sigs = sorted(list(set([a for a, b in sigRegex.findall(message.text)])))
 247                 i = 0
 248                 # Insert empty placeholders if necessary
 249                 while i != len(sigs):
 250                     num = i + 1
 251                     if not sigs[i].startswith("%{}".format(num)):
 252                         sigs.insert(i, "%{}$-".format(num))
 253                     else:
 254                         i += 1
 255                 f.write("{:<40} {}\n".format(message.name, " ".join(sigs)))
 256             f.write("\n")
 257         f.write("// end of file //\n")
 258
 259
 260 def generate_default_messages_file(defaultFile, prefix, data):
 261     """Create the include file with message strings organized"""
 262     with open(defaultFile, "w", encoding="utf-8") as f:
 263         insert_header(f, data)
 264         for section in MessageData.orderedSections:
 265             f.write(
 266                 "static char const *\n"
 267                 "__{}_default_{}[] =\n"
 268                 "    {}\n"
 269                 "        NULL,\n".format(prefix, section, "{")
 270             )
 271             messages = data.sections[section]
 272             for message in messages:
 273                 f.write('        "{}",\n'.format(message.toSrc()))
 274             f.write("        NULL\n" "    {};\n\n".format("}"))
 275         f.write(
 276             "struct kmp_i18n_section {0}\n"
 277             "    int           size;\n"
 278             "    char const ** str;\n"
 279             "{1}; // struct kmp_i18n_section\n"
 280             "typedef struct kmp_i18n_section  kmp_i18n_section_t;\n\n"
 281             "static kmp_i18n_section_t\n"
 282             "__{2}_sections[] =\n"
 283             "    {0}\n"
 284             "        {0} 0, NULL {1},\n".format("{", "}", prefix)
 285         )
 286
 287         for section in MessageData.orderedSections:
 288             messages = data.sections[section]
 289             f.write(
 290                 "        {} {}, __{}_default_{} {},\n".format(
 291                     "{", len(messages), prefix, section, "}"
 292                 )
 293             )
 294         numSections = len(MessageData.orderedSections)
 295         f.write(
 296             "        {0} 0, NULL {1}\n"
 297             "    {1};\n\n"
 298             "struct kmp_i18n_table {0}\n"
 299             "    int                   size;\n"
 300             "    kmp_i18n_section_t *  sect;\n"
 301             "{1}; // struct kmp_i18n_table\n"
 302             "typedef struct kmp_i18n_table  kmp_i18n_table_t;\n\n"
 303             "static kmp_i18n_table_t __kmp_i18n_default_table =\n"
 304             "    {0}\n"
 305             "        {3},\n"
 306             "        __{2}_sections\n"
 307             "    {1};\n\n"
 308             "// end of file //\n".format("{", "}", prefix, numSections)
 309         )
 310
 311
 312 def generate_message_file_unix(messageFile, data):
 313     """
 314     Create the message file for Unix OSes
 315
 316     Encoding is in UTF-8
 317     """
 318     with open(messageFile, "w", encoding="utf-8") as f:
 319         insert_header(f, data, commentChar="$")
 320         f.write('$quote "\n\n')
 321         for section in MessageData.orderedSections:
 322             setIdx = MessageData.sectionInfo[section]["set"]
 323             f.write(
 324                 "$ ------------------------------------------------------------------------------\n"
 325                 "$ {}\n"
 326                 "$ ------------------------------------------------------------------------------\n\n"
 327                 "$set {}\n\n".format(section, setIdx)
 328             )
 329             messages = data.sections[section]
 330             for num, message in enumerate(messages, 1):
 331                 f.write('{} "{}"\n'.format(num, message.toSrc()))
 332             f.write("\n")
 333         f.write("\n$ end of file $")
 334
 335
 336 def generate_message_file_windows(messageFile, data):
 337     """
 338     Create the message file for Windows OS
 339
 340     Encoding is in UTF-16LE
 341     """
 342     language = data.getMeta("Language")
 343     langId = data.getMeta("LangId")
 344     with open(messageFile, "w", encoding="utf-16-le") as f:
 345         insert_header(f, data, commentChar=";")
 346         f.write("\nLanguageNames = ({0}={1}:msg_{1})\n\n".format(language, langId))
 347         f.write("FacilityNames=(\n")
 348         for section in MessageData.orderedSections:
 349             setIdx = MessageData.sectionInfo[section]["set"]
 350             shortName = MessageData.sectionInfo[section]["short"]
 351             f.write(" {}={}\n".format(shortName, setIdx))
 352         f.write(")\n\n")
 353
 354         for section in MessageData.orderedSections:
 355             shortName = MessageData.sectionInfo[section]["short"]
 356             n = 0
 357             messages = data.sections[section]
 358             for message in messages:
 359                 n += 1
 360                 f.write(
 361                     "MessageId={}\n"
 362                     "Facility={}\n"
 363                     "Language={}\n"
 364                     "{}\n.\n\n".format(n, shortName, language, message.toMC())
 365                 )
 366         f.write("\n; end of file ;")
 367
 368
 369 def main():
 370     parser = argparse.ArgumentParser(description="Generate message data files")
 371     parser.add_argument(
 372         "--lang-id",
 373         action="store_true",
 374         help="Print language identifier of the message catalog source file",
 375     )
 376     parser.add_argument(
 377         "--prefix",
 378         default="kmp_i18n",
 379         help="Prefix to be used for all C identifiers (type and variable names)"
 380         " in enum and default message files.",
 381     )
 382     parser.add_argument("--enum", metavar="FILE", help="Generate enum file named FILE")
 383     parser.add_argument(
 384         "--default", metavar="FILE", help="Generate default messages file named FILE"
 385     )
 386     parser.add_argument(
 387         "--signature", metavar="FILE", help="Generate signature file named FILE"
 388     )
 389     parser.add_argument(
 390         "--message", metavar="FILE", help="Generate message file named FILE"
 391     )
 392     parser.add_argument(
 393         "--target-system-override",
 394         metavar="TARGET_SYSTEM_NAME",
 395         help="Target System override.\n"
 396         "By default the target system is the host system\n"
 397         "See possible values at https://docs.python.org/3/library/platform.html#platform.system",
 398     )
 399     parser.add_argument("inputfile")
 400     commandArgs = parser.parse_args()
 401
 402     if commandArgs.lang_id:
 403         display_language_id(commandArgs.inputfile)
 404         return
 405     data = MessageData.create(commandArgs.inputfile)
 406     prefix = commandArgs.prefix
 407     if commandArgs.target_system_override:
 408         TargetPlatform.set_system_override(commandArgs.target_system_override)
 409     if commandArgs.enum:
 410         generate_enum_file(commandArgs.enum, prefix, data)
 411     if commandArgs.default:
 412         generate_default_messages_file(commandArgs.default, prefix, data)
 413     if commandArgs.signature:
 414         generate_signature_file(commandArgs.signature, data)
 415     if commandArgs.message:
 416         if TargetPlatform.system().casefold() == "Windows".casefold():
 417             generate_message_file_windows(commandArgs.message, data)
 418         else:
 419             generate_message_file_unix(commandArgs.message, data)
 420
 421
 422 if __name__ == "__main__":
 423     try:
 424         main()
 425     except ScriptError as e:
 426         print("error: {}".format(e))
 427         sys.exit(1)
 428
 429 # end of file