NexTest/parse_message_file.py

   1 import json
   2 import gzip
   3 import bz2
   4 import lzma
   5 from io import StringIO
   6
   7 def open_compressed_file(filename):
   8     """ Open a file, trying various compression methods if available. """
   9     if filename.endswith('.gz'):
  10         return gzip.open(filename, 'rt', encoding='utf-8')
  11     elif filename.endswith('.bz2'):
  12         return bz2.open(filename, 'rt', encoding='utf-8')
  13     elif filename.endswith('.xz') or filename.endswith('.lzma'):
  14         return lzma.open(filename, 'rt', encoding='utf-8')
  15     else:
  16         return open(filename, 'r', encoding='utf-8')
  17
  18 def parse_line(line):
  19     """ Parse a line in the format 'var: value' and return the key and value. """
  20     parts = line.split(":", 1)
  21     if len(parts) == 2:
  22         key = parts[0].strip()
  23         value = parts[1].strip()
  24         return key, value
  25     return None, None
  26
  27 def validate_integer(value, variable_name, line_number):
  28     """ Validate and convert a value to an integer. """
  29     try:
  30         return int(value)
  31     except ValueError:
  32         raise ValueError(f"{variable_name} on line {line_number} should be an integer, but got '{value}'.")
  33
  34 def parse_file(filename, validate_only=False, verbose=False):
  35     with open_compressed_file(filename) as file:
  36         lines = file.readlines()
  37     return parse_lines(lines, validate_only, verbose)
  38
  39 def parse_string(data, validate_only=False, verbose=False):
  40     lines = StringIO(data).readlines()
  41     return parse_lines(lines, validate_only, verbose)
  42
  43 def parse_lines(lines, validate_only=False, verbose=False):
  44     services = []
  45     current_service = None
  46     in_user_list = False
  47     in_message_list = False
  48     in_message_thread = False
  49     in_user_info = False
  50     in_message_post = False
  51     in_bio_body = False
  52     in_message_body = False
  53     in_comment_section = False
  54     in_include_service = False
  55     in_include_users = False
  56     in_include_messages = False
  57     include_files = []
  58     user_id = None
  59     current_bio = None
  60     current_message = None
  61     current_thread = None
  62     post_id = 1
  63
  64     def parse_include_files(file_list):
  65         included_services = []
  66         for include_file in file_list:
  67             included_services.extend(parse_file(include_file, validate_only, verbose))
  68         return included_services
  69
  70     def parse_include_users(file_list):
  71         users = {}
  72         for include_file in file_list:
  73             included_users = parse_file(include_file, validate_only, verbose)
  74             for service in included_users:
  75                 users.update(service['Users'])
  76         return users
  77
  78     def parse_include_messages(file_list):
  79         messages = []
  80         for include_file in file_list:
  81             included_messages = parse_file(include_file, validate_only, verbose)
  82             for service in included_messages:
  83                 messages.extend(service['MessageThreads'])
  84         return messages
  85
  86     try:
  87         for line_number, line in enumerate(lines, 1):
  88             line = line.strip()
  89             if line == "--- Include Service Start ---":
  90                 in_include_service = True
  91                 include_files = []
  92                 if verbose:
  93                     print(f"Line {line_number}: {line} (Starting include service section)")
  94                 continue
  95             elif line == "--- Include Service End ---":
  96                 in_include_service = False
  97                 if verbose:
  98                     print(f"Line {line_number}: {line} (Ending include service section)")
  99                 services.extend(parse_include_files(include_files))
 100                 continue
 101             elif in_include_service:
 102                 include_files.append(line)
 103                 if verbose:
 104                     print(f"Line {line_number}: {line} (Including file for service)")
 105                 continue
 106             elif line == "--- Include Users Start ---":
 107                 in_include_users = True
 108                 include_files = []
 109                 if verbose:
 110                     print(f"Line {line_number}: {line} (Starting include users section)")
 111                 continue
 112             elif line == "--- Include Users End ---":
 113                 in_include_users = False
 114                 if verbose:
 115                     print(f"Line {line_number}: {line} (Ending include users section)")
 116                 if current_service:
 117                     current_service['Users'].update(parse_include_users(include_files))
 118                 continue
 119             elif in_include_users:
 120                 include_files.append(line)
 121                 if verbose:
 122                     print(f"Line {line_number}: {line} (Including file for users)")
 123                 continue
 124             elif line == "--- Include Messages Start ---":
 125                 in_include_messages = True
 126                 include_files = []
 127                 if verbose:
 128                     print(f"Line {line_number}: {line} (Starting include messages section)")
 129                 continue
 130             elif line == "--- Include Messages End ---":
 131                 in_include_messages = False
 132                 if verbose:
 133                     print(f"Line {line_number}: {line} (Ending include messages section)")
 134                 if current_service:
 135                     current_service['MessageThreads'].extend(parse_include_messages(include_files))
 136                 continue
 137             elif in_include_messages:
 138                 include_files.append(line)
 139                 if verbose:
 140                     print(f"Line {line_number}: {line} (Including file for messages)")
 141                 continue
 142             elif line == "--- Start Archive Service ---":
 143                 current_service = {'Users': {}, 'MessageThreads': [], 'Interactions': []}
 144                 if verbose:
 145                     print(f"Line {line_number}: {line} (Starting new archive service)")
 146                 continue
 147             elif line == "--- End Archive Service ---":
 148                 services.append(current_service)
 149                 current_service = None
 150                 if verbose:
 151                     print(f"Line {line_number}: {line} (Ending archive service)")
 152                 continue
 153             elif line == "--- Start Comment Section ---":
 154                 in_comment_section = True
 155                 if verbose:
 156                     print(f"Line {line_number}: {line} (Starting comment section)")
 157                 continue
 158             elif line == "--- End Comment Section ---":
 159                 in_comment_section = False
 160                 if verbose:
 161                     print(f"Line {line_number}: {line} (Ending comment section)")
 162                 continue
 163             elif in_comment_section:
 164                 if verbose:
 165                     print(f"Line {line_number}: {line} (Comment)")
 166                 continue
 167             elif current_service is not None:
 168                 key, value = parse_line(line)
 169                 if key == "Entry":
 170                     current_service['Entry'] = validate_integer(value, "Entry", line_number)
 171                 elif key == "Service":
 172                     current_service['Service'] = value
 173                 elif line == "--- Start User List ---":
 174                     in_user_list = True
 175                     if verbose:
 176                         print(f"Line {line_number}: {line} (Starting user list)")
 177                     continue
 178                 elif line == "--- End User List ---":
 179                     in_user_list = False
 180                     if verbose:
 181                         print(f"Line {line_number}: {line} (Ending user list)")
 182                     continue
 183                 elif line == "--- Start User Info ---":
 184                     in_user_info = True
 185                     if verbose:
 186                         print(f"Line {line_number}: {line} (Starting user info)")
 187                     continue
 188                 elif line == "--- End User Info ---":
 189                     in_user_info = False
 190                     user_id = None
 191                     if verbose:
 192                         print(f"Line {line_number}: {line} (Ending user info)")
 193                     continue
 194                 elif line == "--- Start Message List ---":
 195                     in_message_list = True
 196                     if verbose:
 197                         print(f"Line {line_number}: {line} (Starting message list)")
 198                     continue
 199                 elif line == "--- End Message List ---":
 200                     in_message_list = False
 201                     if verbose:
 202                         print(f"Line {line_number}: {line} (Ending message list)")
 203                     continue
 204                 elif line == "--- Start Message Thread ---":
 205                     in_message_thread = True
 206                     current_thread = {'Title': '', 'Messages': []}
 207                     post_id = 1
 208                     if verbose:
 209                         print(f"Line {line_number}: {line} (Starting message thread)")
 210                     continue
 211                 elif line == "--- End Message Thread ---":
 212                     in_message_thread = False
 213                     current_service['MessageThreads'].append(current_thread)
 214                     current_thread = None
 215                     if verbose:
 216                         print(f"Line {line_number}: {line} (Ending message thread)")
 217                     continue
 218                 elif line == "--- Start Message Post ---":
 219                     in_message_post = True
 220                     current_message = {}
 221                     if verbose:
 222                         print(f"Line {line_number}: {line} (Starting message post)")
 223                     continue
 224                 elif line == "--- End Message Post ---":
 225                     in_message_post = False
 226                     if current_message:
 227                         current_thread['Messages'].append(current_message)
 228                     current_message = None
 229                     if verbose:
 230                         print(f"Line {line_number}: {line} (Ending message post)")
 231                     continue
 232                 elif in_message_list and key == "Interactions":
 233                     current_service['Interactions'] = [interaction.strip() for interaction in value.split(",")]
 234                     if verbose:
 235                         print(f"Line {line_number}: Interactions set to {current_service['Interactions']}")
 236
 237                 if in_user_list and in_user_info:
 238                     if key == "User":
 239                         user_id = validate_integer(value, "User", line_number)
 240                         current_service['Users'][user_id] = {'Bio': ""}
 241                         if verbose:
 242                             print(f"Line {line_number}: User ID set to {user_id}")
 243                     elif key == "Name":
 244                         if user_id is not None:
 245                             current_service['Users'][user_id]['Name'] = value
 246                             if verbose:
 247                                 print(f"Line {line_number}: Name set to {value}")
 248                     elif key == "Handle":
 249                         if user_id is not None:
 250                             current_service['Users'][user_id]['Handle'] = value
 251                             if verbose:
 252                                 print(f"Line {line_number}: Handle set to {value}")
 253                     elif key == "Location":
 254                         if user_id is not None:
 255                             current_service['Users'][user_id]['Location'] = value
 256                             if verbose:
 257                                 print(f"Line {line_number}: Location set to {value}")
 258                     elif key == "Joined":
 259                         if user_id is not None:
 260                             current_service['Users'][user_id]['Joined'] = value
 261                             if verbose:
 262                                 print(f"Line {line_number}: Joined date set to {value}")
 263                     elif key == "Birthday":
 264                         if user_id is not None:
 265                             current_service['Users'][user_id]['Birthday'] = value
 266                             if verbose:
 267                                 print(f"Line {line_number}: Birthday set to {value}")
 268                     elif line == "--- Start Bio Body ---":
 269                         if user_id is not None:
 270                             current_bio = []
 271                             in_bio_body = True
 272                             if verbose:
 273                                 print(f"Line {line_number}: Starting bio body")
 274                     elif line == "--- End Bio Body ---":
 275                         if user_id is not None and current_bio is not None:
 276                             current_service['Users'][user_id]['Bio'] = "\n".join(current_bio)
 277                             current_bio = None
 278                             in_bio_body = False
 279                             if verbose:
 280                                 print(f"Line {line_number}: Ending bio body")
 281                     elif in_bio_body and current_bio is not None:
 282                         current_bio.append(line)
 283                         if verbose:
 284                             print(f"Line {line_number}: Adding to bio body: {line}")
 285                 elif in_message_list and in_message_thread:
 286                     if key == "Thread":
 287                         current_thread['Thread'] = validate_integer(value, "Thread", line_number)
 288                         if verbose:
 289                             print(f"Line {line_number}: Thread ID set to {value}")
 290                     elif key == "Title":
 291                         current_thread['Title'] = value
 292                         if verbose:
 293                             print(f"Line {line_number}: Title set to {value}")
 294                     elif key == "Author":
 295                         current_message['Author'] = value
 296                         if verbose:
 297                             print(f"Line {line_number}: Author set to {value}")
 298                     elif key == "Time":
 299                         current_message['Time'] = value
 300                         if verbose:
 301                             print(f"Line {line_number}: Time set to {value}")
 302                     elif key == "Type":
 303                         message_type = value
 304                         if message_type not in current_service['Interactions']:
 305                             raise ValueError(f"Unexpected message type '{message_type}' found on line {line_number}. Expected one of {current_service['Interactions']}")
 306                         current_message['Type'] = message_type
 307                         if verbose:
 308                             print(f"Line {line_number}: Type set to {message_type}")
 309                     elif key == "Post":
 310                         post_value = validate_integer(value, "Post", line_number)
 311                         current_message['Post'] = post_value
 312                         if 'post_ids' not in current_thread:
 313                             current_thread['post_ids'] = set()
 314                         current_thread['post_ids'].add(post_value)
 315                         if verbose:
 316                             print(f"Line {line_number}: Post ID set to {post_value}")
 317                     elif key == "Nested":
 318                         nested_value = validate_integer(value, "Nested", line_number)
 319                         if nested_value != 0 and nested_value not in current_thread.get('post_ids', set()):
 320                             raise ValueError(
 321                                 f"Nested value '{nested_value}' on line {line_number} does not match any existing Post values in the current thread. Existing Post IDs: {list(current_thread.get('post_ids', set()))}"
 322                             )
 323                         current_message['Nested'] = nested_value
 324                         if verbose:
 325                             print(f"Line {line_number}: Nested set to {nested_value}")
 326                     elif line == "--- Start Message Body ---":
 327                         if current_message is not None:
 328                             current_message['Message'] = []
 329                             in_message_body = True
 330                             if verbose:
 331                                 print(f"Line {line_number}: Starting message body")
 332                     elif line == "--- End Message Body ---":
 333                         if current_message is not None and 'Message' in current_message:
 334                             current_message['Message'] = "\n".join(current_message['Message'])
 335                             in_message_body = False
 336                             if verbose:
 337                                 print(f"Line {line_number}: Ending message body")
 338                     elif in_message_body and current_message is not None and 'Message' in current_message:
 339                         current_message['Message'].append(line)
 340                         if verbose:
 341                             print(f"Line {line_number}: Adding to message body: {line}")
 342     except Exception as e:
 343         if validate_only:
 344             return False, f"Error: {str(e)}", lines[line_number - 1]
 345         else:
 346             raise
 347
 348     if validate_only:
 349         return True, "", ""
 350
 351     return services
 352
 353 def display_services(services):
 354     for service in services:
 355         print(f"Service Entry: {service['Entry']}")
 356         print(f"Service: {service['Service']}")
 357         print(f"Interactions: {', '.join(service['Interactions'])}")
 358         print("User List:")
 359         for user_id, user_info in service['Users'].items():
 360             print(f"  User ID: {user_id}")
 361             print(f"    Name: {user_info['Name']}")
 362             print(f"    Handle: {user_info['Handle']}")
 363             print(f"    Location: {user_info.get('Location', '')}")
 364             print(f"    Joined: {user_info.get('Joined', '')}")
 365             print(f"    Birthday: {user_info.get('Birthday', '')}")
 366             print(f"    Bio: {user_info.get('Bio', '').strip()}")
 367             print("")
 368         print("Message Threads:")
 369         for idx, thread in enumerate(service['MessageThreads']):
 370             print(f"  --- Message Thread {idx+1} ---")
 371             if thread['Title']:
 372                 print(f"    Title: {thread['Title']}")
 373             for message in thread['Messages']:
 374                 print(f"    {message['Author']} ({message['Time']}): [{message['Type']}] Post ID: {message['Post']} Nested: {message['Nested']}")
 375                 print(f"    {message['Message'].strip()}")
 376             print("")
 377
 378 def to_json(services):
 379     """ Convert the services data structure to JSON """
 380     return json.dumps(services, indent=2)
 381
 382 def from_json(json_str):
 383     """ Convert a JSON string back to the services data structure """
 384     return json.loads(json_str)