parse_message_file.py

   1 from __future__ import absolute_import, division, print_function, unicode_literals
   2 import json
   3 import gzip
   4 import bz2
   5 import sys
   6 import io
   7
   8 try:
   9     import lzma
  10 except ImportError:
  11     try:
  12         from backports import lzma
  13     except ImportError:
  14         lzma = None
  15
  16 try:
  17     from io import StringIO
  18 except ImportError:
  19     try:
  20         from cStringIO import StringIO
  21     except ImportError:
  22         from StringIO import StringIO
  23
  24 PY2 = sys.version_info[0] == 2
  25
  26 def open_compressed_file(filename):
  27     """ Open a file, trying various compression methods if available. """
  28     if filename.endswith('.gz'):
  29         return gzip.open(filename, 'rt', encoding='utf-8')
  30     elif filename.endswith('.bz2'):
  31         return bz2.open(filename, 'rt', encoding='utf-8')
  32     elif filename.endswith('.xz') or filename.endswith('.lzma'):
  33         if lzma:
  34             return lzma.open(filename, 'rt', encoding='utf-8')
  35         else:
  36             raise ImportError("lzma module is not available")
  37     else:
  38         return io.open(filename, 'r', encoding='utf-8')
  39
  40 def save_compressed_file(data, filename):
  41     """ Save data to a file, using various compression methods if specified. """
  42     if filename.endswith('.gz'):
  43         with gzip.open(filename, 'wt', encoding='utf-8') as file:
  44             file.write(data)
  45     elif filename.endswith('.bz2'):
  46         with bz2.open(filename, 'wt', encoding='utf-8') as file:
  47             file.write(data)
  48     elif filename.endswith('.xz') or filename.endswith('.lzma'):
  49         if lzma:
  50             with lzma.open(filename, 'wt', encoding='utf-8') as file:
  51                 file.write(data)
  52         else:
  53             raise ImportError("lzma module is not available")
  54     else:
  55         with io.open(filename, 'w', encoding='utf-8') as file:
  56             file.write(data)
  57
  58 def parse_line(line):
  59     """ Parse a line in the format 'var: value' and return the key and value. """
  60     parts = line.split(":", 1)
  61     if len(parts) == 2:
  62         key = parts[0].strip()
  63         value = parts[1].strip()
  64         return key, value
  65     return None, None
  66
  67 def validate_non_negative_integer(value, variable_name, line_number):
  68     """ Validate and convert a value to a non-negative integer. """
  69     try:
  70         int_value = int(value)
  71         if int_value < 0:
  72             raise ValueError
  73         return int_value
  74     except ValueError:
  75         raise ValueError("{0} on line {1} should be a non-negative integer, but got '{2}'.".format(variable_name, line_number, value))
  76
  77 def parse_file(filename, validate_only=False, verbose=False):
  78     with open_compressed_file(filename) as file:
  79         lines = file.readlines()
  80     return parse_lines(lines, validate_only, verbose)
  81
  82 def parse_string(data, validate_only=False, verbose=False):
  83     lines = StringIO(data).readlines()
  84     return parse_lines(lines, validate_only, verbose)
  85
  86 def parse_lines(lines, validate_only=False, verbose=False):
  87     services = []
  88     current_service = None
  89     in_user_list = False
  90     in_message_list = False
  91     in_message_thread = False
  92     in_user_info = False
  93     in_message_post = False
  94     in_bio_body = False
  95     in_message_body = False
  96     in_comment_section = False
  97     in_include_service = False
  98     in_include_users = False
  99     in_include_messages = False
 100     in_category_list = False
 101     in_description_body = False
 102     in_include_categories = False
 103     in_categorization_list = False
 104     include_files = []
 105     user_id = None
 106     current_bio = None
 107     current_message = None
 108     current_thread = None
 109     current_category = None
 110     categorization_values = {'Categories': [], 'Forums': []}
 111     category_ids = {'Categories': set(), 'Forums': set()}
 112     post_id = 1
 113
 114     def parse_include_files(file_list):
 115         included_services = []
 116         for include_file in file_list:
 117             included_services.extend(parse_file(include_file, validate_only, verbose))
 118         return included_services
 119
 120     def parse_include_users(file_list):
 121         users = {}
 122         for include_file in file_list:
 123             included_users = parse_file(include_file, validate_only, verbose)
 124             for service in included_users:
 125                 users.update(service['Users'])
 126         return users
 127
 128     def parse_include_messages(file_list):
 129         messages = []
 130         for include_file in file_list:
 131             included_messages = parse_file(include_file, validate_only, verbose)
 132             for service in included_messages:
 133                 messages.extend(service['MessageThreads'])
 134         return messages
 135
 136     def parse_include_categories(file_list):
 137         categories = []
 138         for include_file in file_list:
 139             included_categories = parse_file(include_file, validate_only, verbose)
 140             for service in included_categories:
 141                 categories.extend(service['Categories'])
 142         return categories
 143
 144     try:
 145         for line_number, line in enumerate(lines, 1):
 146             line = line.strip()
 147             if line == "--- Include Service Start ---":
 148                 in_include_service = True
 149                 include_files = []
 150                 if verbose:
 151                     print("Line {0}: {1} (Starting include service section)".format(line_number, line))
 152                 continue
 153             elif line == "--- Include Service End ---":
 154                 in_include_service = False
 155                 if verbose:
 156                     print("Line {0}: {1} (Ending include service section)".format(line_number, line))
 157                 services.extend(parse_include_files(include_files))
 158                 continue
 159             elif in_include_service:
 160                 include_files.append(line)
 161                 if verbose:
 162                     print("Line {0}: {1} (Including file for service)".format(line_number, line))
 163                 continue
 164             elif line == "--- Include Users Start ---":
 165                 in_include_users = True
 166                 include_files = []
 167                 if verbose:
 168                     print("Line {0}: {1} (Starting include users section)".format(line_number, line))
 169                 continue
 170             elif line == "--- Include Users End ---":
 171                 in_include_users = False
 172                 if verbose:
 173                     print("Line {0}: {1} (Ending include users section)".format(line_number, line))
 174                 if current_service:
 175                     current_service['Users'].update(parse_include_users(include_files))
 176                 continue
 177             elif in_include_users:
 178                 include_files.append(line)
 179                 if verbose:
 180                     print("Line {0}: {1} (Including file for users)".format(line_number, line))
 181                 continue
 182             elif line == "--- Include Messages Start ---":
 183                 in_include_messages = True
 184                 include_files = []
 185                 if verbose:
 186                     print("Line {0}: {1} (Starting include messages section)".format(line_number, line))
 187                 continue
 188             elif line == "--- Include Messages End ---":
 189                 in_include_messages = False
 190                 if verbose:
 191                     print("Line {0}: {1} (Ending include messages section)".format(line_number, line))
 192                 if current_service:
 193                     current_service['MessageThreads'].extend(parse_include_messages(include_files))
 194                 continue
 195             elif in_include_messages:
 196                 include_files.append(line)
 197                 if verbose:
 198                     print("Line {0}: {1} (Including file for messages)".format(line_number, line))
 199                 continue
 200             elif line == "--- Include Categories Start ---":
 201                 in_include_categories = True
 202                 include_files = []
 203                 if verbose:
 204                     print("Line {0}: {1} (Starting include categories section)".format(line_number, line))
 205                 continue
 206             elif line == "--- Include Categories End ---":
 207                 in_include_categories = False
 208                 if verbose:
 209                     print("Line {0}: {1} (Ending include categories section)".format(line_number, line))
 210                 if current_service:
 211                     current_service['Categories'].extend(parse_include_categories(include_files))
 212                     for category in current_service['Categories']:
 213                         kind_split = category.get('Kind', '').split(",")
 214                         category['Type'] = kind_split[0].strip() if len(kind_split) > 0 else ""
 215                         category['Level'] = kind_split[1].strip() if len(kind_split) > 1 else ""
 216                         category_ids[category['Type']].add(category['ID'])
 217                 continue
 218             elif in_include_categories:
 219                 include_files.append(line)
 220                 if verbose:
 221                     print("Line {0}: {1} (Including file for categories)".format(line_number, line))
 222                 continue
 223             elif line == "--- Start Archive Service ---":
 224                 current_service = {'Users': {}, 'MessageThreads': [], 'Categories': [], 'Interactions': [], 'Categorization': {}}
 225                 if verbose:
 226                     print("Line {0}: {1} (Starting new archive service)".format(line_number, line))
 227                 continue
 228             elif line == "--- End Archive Service ---":
 229                 services.append(current_service)
 230                 current_service = None
 231                 if verbose:
 232                     print("Line {0}: {1} (Ending archive service)".format(line_number, line))
 233                 continue
 234             elif line == "--- Start Comment Section ---":
 235                 in_comment_section = True
 236                 if verbose:
 237                     print("Line {0}: {1} (Starting comment section)".format(line_number, line))
 238                 continue
 239             elif line == "--- End Comment Section ---":
 240                 in_comment_section = False
 241                 if verbose:
 242                     print("Line {0}: {1} (Ending comment section)".format(line_number, line))
 243                 continue
 244             elif in_comment_section:
 245                 if verbose:
 246                     print("Line {0}: {1} (Comment)".format(line_number, line))
 247                 continue
 248             elif line == "--- Start Category List ---":
 249                 in_category_list = True
 250                 current_category = {}
 251                 if verbose:
 252                     print("Line {0}: {1} (Starting category list)".format(line_number, line))
 253                 continue
 254             elif line == "--- End Category List ---":
 255                 in_category_list = False
 256                 if current_category:
 257                     kind_split = current_category.get('Kind', '').split(",")
 258                     current_category['Type'] = kind_split[0].strip() if len(kind_split) > 0 else ""
 259                     current_category['Level'] = kind_split[1].strip() if len(kind_split) > 1 else ""
 260                     if current_category['Type'] not in categorization_values:
 261                         raise ValueError("Invalid 'Type' value '{0}' on line {1}. Expected one of {2}.".format(current_category['Type'], line_number, categorization_values.keys()))
 262                     if current_category['InSub'] != 0 and current_category['InSub'] not in category_ids[current_category['Type']]:
 263                         raise ValueError("InSub value '{0}' on line {1} does not match any existing ID values.".format(current_category['InSub'], line_number))
 264                     current_service['Categories'].append(current_category)
 265                     category_ids[current_category['Type']].add(current_category['ID'])
 266                 current_category = None
 267                 if verbose:
 268                     print("Line {0}: {1} (Ending category list)".format(line_number, line))
 269                 continue
 270             elif line == "--- Start Categorization List ---":
 271                 in_categorization_list = True
 272                 current_service['Categorization'] = {}
 273                 if verbose:
 274                     print("Line {0}: {1} (Starting categorization list)".format(line_number, line))
 275                 continue
 276             elif line == "--- End Categorization List ---":
 277                 in_categorization_list = False
 278                 if verbose:
 279                     print("Line {0}: {1} (Ending categorization list)".format(line_number, line))
 280                 categorization_values = current_service['Categorization']
 281                 continue
 282             elif current_service is not None:
 283                 key, value = parse_line(line)
 284                 if key == "Entry":
 285                     current_service['Entry'] = validate_non_negative_integer(value, "Entry", line_number)
 286                 elif key == "Service":
 287                     current_service['Service'] = value
 288                 elif key == "Categories":
 289                     current_service['Categorization']['Categories'] = [category.strip() for category in value.split(",")]
 290                     if verbose:
 291                         print("Line {0}: Categories set to {1}".format(line_number, current_service['Categorization']['Categories']))
 292                 elif key == "Forums":
 293                     current_service['Categorization']['Forums'] = [forum.strip() for forum in value.split(",")]
 294                     if verbose:
 295                         print("Line {0}: Forums set to {1}".format(line_number, current_service['Categorization']['Forums']))
 296                 elif in_category_list:
 297                     if key == "Kind":
 298                         current_category['Kind'] = value
 299                     elif key == "ID":
 300                         current_category['ID'] = validate_non_negative_integer(value, "ID", line_number)
 301                     elif key == "InSub":
 302                         current_category['InSub'] = validate_non_negative_integer(value, "InSub", line_number)
 303                     elif key == "Headline":
 304                         current_category['Headline'] = value
 305                     elif key == "Description":
 306                         current_category['Description'] = value
 307                 elif line == "--- Start User List ---":
 308                     in_user_list = True
 309                     if verbose:
 310                         print("Line {0}: {1} (Starting user list)".format(line_number, line))
 311                     continue
 312                 elif line == "--- End User List ---":
 313                     in_user_list = False
 314                     if verbose:
 315                         print("Line {0}: {1} (Ending user list)".format(line_number, line))
 316                     continue
 317                 elif line == "--- Start User Info ---":
 318                     in_user_info = True
 319                     if verbose:
 320                         print("Line {0}: {1} (Starting user info)".format(line_number, line))
 321                     continue
 322                 elif line == "--- End User Info ---":
 323                     in_user_info = False
 324                     user_id = None
 325                     if verbose:
 326                         print("Line {0}: {1} (Ending user info)".format(line_number, line))
 327                     continue
 328                 elif line == "--- Start Message List ---":
 329                     in_message_list = True
 330                     if verbose:
 331                         print("Line {0}: {1} (Starting message list)".format(line_number, line))
 332                     continue
 333                 elif line == "--- End Message List ---":
 334                     in_message_list = False
 335                     if verbose:
 336                         print("Line {0}: {1} (Ending message list)".format(line_number, line))
 337                     continue
 338                 elif line == "--- Start Message Thread ---":
 339                     in_message_thread = True
 340                     current_thread = {'Title': '', 'Messages': []}
 341                     post_id = 1
 342                     if verbose:
 343                         print("Line {0}: {1} (Starting message thread)".format(line_number, line))
 344                     continue
 345                 elif line == "--- End Message Thread ---":
 346                     in_message_thread = False
 347                     current_service['MessageThreads'].append(current_thread)
 348                     current_thread = None
 349                     if verbose:
 350                         print("Line {0}: {1} (Ending message thread)".format(line_number, line))
 351                     continue
 352                 elif line == "--- Start Message Post ---":
 353                     in_message_post = True
 354                     current_message = {}
 355                     if verbose:
 356                         print("Line {0}: {1} (Starting message post)".format(line_number, line))
 357                     continue
 358                 elif line == "--- End Message Post ---":
 359                     in_message_post = False
 360                     if current_message:
 361                         current_thread['Messages'].append(current_message)
 362                     current_message = None
 363                     if verbose:
 364                         print("Line {0}: {1} (Ending message post)".format(line_number, line))
 365                     continue
 366                 elif in_message_list and key == "Interactions":
 367                     current_service['Interactions'] = [interaction.strip() for interaction in value.split(",")]
 368                     if verbose:
 369                         print("Line {0}: Interactions set to {1}".format(line_number, current_service['Interactions']))
 370
 371                 if in_user_list and in_user_info:
 372                     if key == "User":
 373                         user_id = validate_non_negative_integer(value, "User", line_number)
 374                         current_service['Users'][user_id] = {'Bio': ""}
 375                         if verbose:
 376                             print("Line {0}: User ID set to {1}".format(line_number, user_id))
 377                     elif key == "Name":
 378                         if user_id is not None:
 379                             current_service['Users'][user_id]['Name'] = value
 380                             if verbose:
 381                                 print("Line {0}: Name set to {1}".format(line_number, value))
 382                     elif key == "Handle":
 383                         if user_id is not None:
 384                             current_service['Users'][user_id]['Handle'] = value
 385                             if verbose:
 386                                 print("Line {0}: Handle set to {1}".format(line_number, value))
 387                     elif key == "Location":
 388                         if user_id is not None:
 389                             current_service['Users'][user_id]['Location'] = value
 390                             if verbose:
 391                                 print("Line {0}: Location set to {1}".format(line_number, value))
 392                     elif key == "Joined":
 393                         if user_id is not None:
 394                             current_service['Users'][user_id]['Joined'] = value
 395                             if verbose:
 396                                 print("Line {0}: Joined date set to {1}".format(line_number, value))
 397                     elif key == "Birthday":
 398                         if user_id is not None:
 399                             current_service['Users'][user_id]['Birthday'] = value
 400                             if verbose:
 401                                 print("Line {0}: Birthday set to {1}".format(line_number, value))
 402                     elif line == "--- Start Bio Body ---":
 403                         if user_id is not None:
 404                             current_bio = []
 405                             in_bio_body = True
 406                             if verbose:
 407                                 print("Line {0}: Starting bio body".format(line_number))
 408                     elif line == "--- End Bio Body ---":
 409                         if user_id is not None and current_bio is not None:
 410                             current_service['Users'][user_id]['Bio'] = "\n".join(current_bio)
 411                             current_bio = None
 412                             in_bio_body = False
 413                             if verbose:
 414                                 print("Line {0}: Ending bio body".format(line_number))
 415                     elif in_bio_body and current_bio is not None:
 416                         current_bio.append(line)
 417                         if verbose:
 418                             print("Line {0}: Adding to bio body: {1}".format(line_number, line))
 419                 elif in_message_list and in_message_thread:
 420                     if key == "Thread":
 421                         current_thread['Thread'] = validate_non_negative_integer(value, "Thread", line_number)
 422                         if verbose:
 423                             print("Line {0}: Thread ID set to {1}".format(line_number, value))
 424                     elif key == "Category":
 425                         current_thread['Category'] = [category.strip() for category in value.split(",")]
 426                         if verbose:
 427                             print("Line {0}: Category set to {1}".format(line_number, current_thread['Category']))
 428                     elif key == "Forum":
 429                         current_thread['Forum'] = [forum.strip() for forum in value.split(",")]
 430                         if verbose:
 431                             print("Line {0}: Forum set to {1}".format(line_number, current_thread['Forum']))
 432                     elif key == "Title":
 433                         current_thread['Title'] = value
 434                         if verbose:
 435                             print("Line {0}: Title set to {1}".format(line_number, value))
 436                     elif key == "Author":
 437                         current_message['Author'] = value
 438                         if verbose:
 439                             print("Line {0}: Author set to {1}".format(line_number, value))
 440                     elif key == "Time":
 441                         current_message['Time'] = value
 442                         if verbose:
 443                             print("Line {0}: Time set to {1}".format(line_number, value))
 444                     elif key == "Date":
 445                         current_message['Date'] = value
 446                         if verbose:
 447                             print("Line {0}: Date set to {1}".format(line_number, value))
 448                     elif key == "Type":
 449                         message_type = value
 450                         if message_type not in current_service['Interactions']:
 451                             raise ValueError("Unexpected message type '{0}' found on line {1}. Expected one of {2}".format(message_type, line_number, current_service['Interactions']))
 452                         current_message['Type'] = message_type
 453                         if verbose:
 454                             print("Line {0}: Type set to {1}".format(line_number, message_type))
 455                     elif key == "Post":
 456                         post_value = validate_non_negative_integer(value, "Post", line_number)
 457                         current_message['Post'] = post_value
 458                         if 'post_ids' not in current_thread:
 459                             current_thread['post_ids'] = set()
 460                         current_thread['post_ids'].add(post_value)
 461                         if verbose:
 462                             print("Line {0}: Post ID set to {1}".format(line_number, post_value))
 463                     elif key == "Nested":
 464                         nested_value = validate_non_negative_integer(value, "Nested", line_number)
 465                         if nested_value != 0 and nested_value not in current_thread.get('post_ids', set()):
 466                             raise ValueError(
 467                                 "Nested value '{0}' on line {1} does not match any existing Post values in the current thread. Existing Post IDs: {2}".format(
 468                                     nested_value, line_number, list(current_thread.get('post_ids', set())))
 469                             )
 470                         current_message['Nested'] = nested_value
 471                         if verbose:
 472                             print("Line {0}: Nested set to {1}".format(line_number, nested_value))
 473                     elif line == "--- Start Message Body ---":
 474                         if current_message is not None:
 475                             current_message['Message'] = []
 476                             in_message_body = True
 477                             if verbose:
 478                                 print("Line {0}: Starting message body".format(line_number))
 479                     elif line == "--- End Message Body ---":
 480                         if current_message is not None and 'Message' in current_message:
 481                             current_message['Message'] = "\n".join(current_message['Message'])
 482                             in_message_body = False
 483                             if verbose:
 484                                 print("Line {0}: Ending message body".format(line_number))
 485                     elif in_message_body and current_message is not None and 'Message' in current_message:
 486                         current_message['Message'].append(line)
 487                         if verbose:
 488                             print("Line {0}: Adding to message body: {1}".format(line_number, line))
 489     except Exception as e:
 490         if validate_only:
 491             return False, "Error: {0}".format(str(e)), lines[line_number - 1]
 492         else:
 493             raise
 494
 495     if validate_only:
 496         return True, "", ""
 497
 498     return services
 499
 500 def display_services(services):
 501     for service in services:
 502         print("Service Entry: {0}".format(service['Entry']))
 503         print("Service: {0}".format(service['Service']))
 504         print("Interactions: {0}".format(', '.join(service['Interactions'])))
 505         if 'Categorization' in service and service['Categorization']:
 506             for category_type, category_levels in service['Categorization'].items():
 507                 print("{0}: {1}".format(category_type, ', '.join(category_levels)))
 508         print("Category List:")
 509         for category in service['Categories']:
 510             kind_split = category.get('Kind', '').split(",")
 511             category['Type'] = kind_split[0].strip() if len(kind_split) > 0 else ""
 512             category['Level'] = kind_split[1].strip() if len(kind_split) > 1 else ""
 513             print("  Type: {0}, Level: {1}".format(category['Type'], category['Level']))
 514             print("  ID: {0}".format(category['ID']))
 515             print("  InSub: {0}".format(category['InSub']))
 516             print("  Headline: {0}".format(category['Headline']))
 517             print("  Description: {0}".format(category['Description'].strip()))
 518             print("")
 519         print("User List:")
 520         for user_id, user_info in service['Users'].items():
 521             print("  User ID: {0}".format(user_id))
 522             print("    Name: {0}".format(user_info['Name']))
 523             print("    Handle: {0}".format(user_info['Handle']))
 524             print("    Location: {0}".format(user_info.get('Location', '')))
 525             print("    Joined: {0}".format(user_info.get('Joined', '')))
 526             print("    Birthday: {0}".format(user_info.get('Birthday', '')))
 527             print("    Bio: {0}".format(user_info.get('Bio', '').strip()))
 528             print("")
 529         print("Message Threads:")
 530         for idx, thread in enumerate(service['MessageThreads']):
 531             print("  --- Message Thread {0} ---".format(idx+1))
 532             if thread['Title']:
 533                 print("    Title: {0}".format(thread['Title']))
 534             if 'Category' in thread:
 535                 print("    Category: {0}".format(', '.join(thread['Category'])))
 536             if 'Forum' in thread:
 537                 print("    Forum: {0}".format(', '.join(thread['Forum'])))
 538             for message in thread['Messages']:
 539                 print("    {0} ({1} on {2}): [{3}] Post ID: {4} Nested: {5}".format(
 540                     message['Author'], message['Time'], message['Date'], message['Type'], message['Post'], message['Nested']))
 541                 print("    {0}".format(message['Message'].strip()))
 542             print("")
 543
 544 def to_json(services):
 545     """ Convert the services data structure to JSON """
 546     return json.dumps(services, indent=2)
 547
 548 def from_json(json_str):
 549     """ Convert a JSON string back to the services data structure """
 550     return json.loads(json_str)
 551
 552 def load_from_json_file(json_filename):
 553     """ Load the services data structure from a JSON file """
 554     with open_compressed_file(json_filename) as file:
 555         return json.load(file)
 556
 557 def save_to_json_file(services, json_filename):
 558     """ Save the services data structure to a JSON file """
 559     json_data = json.dumps(services, indent=2)
 560     save_compressed_file(json_data, json_filename)
 561
 562 def services_to_string(services, line_ending="lf"):
 563     """ Convert the services data structure back to the original text format """
 564     lines = []
 565     for service in services:
 566         lines.append("--- Start Archive Service ---")
 567         lines.append("Entry: {0}".format(service['Entry']))
 568         lines.append("Service: {0}".format(service['Service']))
 569
 570         lines.append("--- Start User List ---")
 571         for user_id, user_info in service['Users'].items():
 572             lines.append("--- Start User Info ---")
 573             lines.append("User: {0}".format(user_id))
 574             lines.append("Name: {0}".format(user_info['Name']))
 575             lines.append("Handle: {0}".format(user_info['Handle']))
 576             if 'Location' in user_info:
 577                 lines.append("Location: {0}".format(user_info['Location']))
 578             if 'Joined' in user_info:
 579                 lines.append("Joined: {0}".format(user_info['Joined']))
 580             if 'Birthday' in user_info:
 581                 lines.append("Birthday: {0}".format(user_info['Birthday']))
 582             if 'Bio' in user_info:
 583                 lines.append("Bio:")
 584                 lines.append("--- Start Bio Body ---")
 585                 lines.extend(user_info['Bio'].split("\n"))
 586                 lines.append("--- End Bio Body ---")
 587             lines.append("--- End User Info ---")
 588         lines.append("--- End User List ---")
 589
 590         if 'Categorization' in service and service['Categorization']:
 591             lines.append("--- Start Categorization List ---")
 592             for category_type, category_levels in service['Categorization'].items():
 593                 lines.append("{0}: {1}".format(category_type, ', '.join(category_levels)))
 594             lines.append("--- End Categorization List ---")
 595
 596         if 'Categories' in service and service['Categories']:
 597             for category in service['Categories']:
 598                 lines.append("--- Start Category List ---")
 599                 lines.append("Kind: {0}, {1}".format(category['Type'], category['Level']))
 600                 lines.append("ID: {0}".format(category['ID']))
 601                 lines.append("InSub: {0}".format(category['InSub']))
 602                 lines.append("Headline: {0}".format(category['Headline']))
 603                 lines.append("Description: {0}".format(category['Description']))
 604                 lines.append("--- End Category List ---")
 605
 606         lines.append("--- Start Message List ---")
 607         lines.append("Interactions: {0}".format(', '.join(service['Interactions'])))
 608         for thread in service['MessageThreads']:
 609             lines.append("--- Start Message Thread ---")
 610             lines.append("Thread: {0}".format(thread['Thread']))
 611             if 'Category' in thread:
 612                 lines.append("Category: {0}".format(', '.join(thread['Category'])))
 613             if 'Forum' in thread:
 614                 lines.append("Forum: {0}".format(', '.join(thread['Forum'])))
 615             if 'Title' in thread:
 616                 lines.append("Title: {0}".format(thread['Title']))
 617             for message in thread['Messages']:
 618                 lines.append("--- Start Message Post ---")
 619                 lines.append("Author: {0}".format(message['Author']))
 620                 lines.append("Time: {0}".format(message['Time']))
 621                 lines.append("Date: {0}".format(message['Date']))
 622                 lines.append("Type: {0}".format(message['Type']))
 623                 lines.append("Post: {0}".format(message['Post']))
 624                 lines.append("Nested: {0}".format(message['Nested']))
 625                 lines.append("Message:")
 626                 lines.append("--- Start Message Body ---")
 627                 lines.extend(message['Message'].split("\n"))
 628                 lines.append("--- End Message Body ---")
 629                 lines.append("--- End Message Post ---")
 630             lines.append("--- End Message Thread ---")
 631         lines.append("--- End Message List ---")
 632
 633         lines.append("--- End Archive Service ---")
 634
 635     line_sep = {"lf": "\n", "cr": "\r", "crlf": "\r\n"}
 636     return line_sep.get(line_ending, "\n").join(lines)
 637
 638 def save_services_to_file(services, filename, line_ending="lf"):
 639     """ Save the services data structure to a file in the original text format """
 640     data = services_to_string(services, line_ending)
 641     save_compressed_file(data, filename)
 642
 643 def init_empty_service(entry, service_name):
 644     """ Initialize an empty service structure """
 645     return {
 646         'Entry': entry,
 647         'Service': service_name,
 648         'Users': {},
 649         'MessageThreads': [],
 650         'Categories': [],
 651         'Interactions': [],
 652         'Categorization': {}
 653     }
 654
 655 def add_user(service, user_id, name, handle, location='', joined='', birthday='', bio=''):
 656     """ Add a user to the service """
 657     service['Users'][user_id] = {
 658         'Name': name,
 659         'Handle': handle,
 660         'Location': location,
 661         'Joined': joined,
 662         'Birthday': birthday,
 663         'Bio': bio
 664     }
 665
 666 def remove_user(service, user_id):
 667     """ Remove a user from the service """
 668     if user_id in service['Users']:
 669         del service['Users'][user_id]
 670
 671 def add_category(service, kind, category_type, category_level, category_id, insub, headline, description):
 672     """ Add a category to the service """
 673     category = {
 674         'Kind': f"{kind}, {category_level}",
 675         'ID': category_id,
 676         'InSub': insub,
 677         'Headline': headline,
 678         'Description': description
 679     }
 680     service['Categories'].append(category)
 681     if category_type not in service['Categorization']:
 682         service['Categorization'][category_type] = []
 683     if category_level not in service['Categorization'][category_type]:
 684         service['Categorization'][category_type].append(category_level)
 685
 686 def remove_category(service, category_id):
 687     """ Remove a category from the service """
 688     service['Categories'] = [category for category in service['Categories'] if category['ID'] != category_id]
 689
 690 def add_message_thread(service, thread_id, title='', category='', forum=''):
 691     """ Add a message thread to the service """
 692     thread = {
 693         'Thread': thread_id,
 694         'Title': title,
 695         'Category': category.split(',') if category else [],
 696         'Forum': forum.split(',') if forum else [],
 697         'Messages': []
 698     }
 699     service['MessageThreads'].append(thread)
 700
 701 def remove_message_thread(service, thread_id):
 702     """ Remove a message thread from the service """
 703     service['MessageThreads'] = [thread for thread in service['MessageThreads'] if thread['Thread'] != thread_id]
 704
 705 def add_message_post(service, thread_id, author, time, date, msg_type, post_id, nested, message):
 706     """ Add a message post to a thread in the service """
 707     for thread in service['MessageThreads']:
 708         if thread['Thread'] == thread_id:
 709             post = {
 710                 'Author': author,
 711                 'Time': time,
 712                 'Date': date,
 713                 'Type': msg_type,
 714                 'Post': post_id,
 715                 'Nested': nested,
 716                 'Message': message
 717             }
 718             thread['Messages'].append(post)
 719             return
 720     raise ValueError("Thread ID {0} not found in service".format(thread_id))
 721
 722 def remove_message_post(service, thread_id, post_id):
 723     """ Remove a message post from a thread in the service """
 724     for thread in service['MessageThreads']:
 725         if thread['Thread'] == thread_id:
 726             thread['Messages'] = [post for post in thread['Messages'] if post['Post'] != post_id]
 727             return
 728     raise ValueError("Thread ID {0} not found in service".format(thread_id))
 729
 730 def add_service(services, entry, service_name):
 731     """ Add a new service to the list of services """
 732     new_service = init_empty_service(entry, service_name)
 733     services.append(new_service)
 734     return new_service
 735
 736 def remove_service(services, entry):
 737     """ Remove an existing service from the list of services """
 738     services[:] = [service for service in services if service['Entry'] != entry]