solenv/bin/pack_images.py

   1 # -*- coding: utf-8 -*-
   2 # -*- tab-width: 4; indent-tabs-mode: nil; py-indent-offset: 4 -*-
   3 #
   4 # This file is part of the LibreOffice project.
   5 #
   6 # This Source Code Form is subject to the terms of the Mozilla Public
   7 # License, v. 2.0. If a copy of the MPL was not distributed with this
   8 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   9 #
  10 """ Pack images into archives. """
  11
  12 from __future__ import with_statement
  13
  14 import argparse
  15 from collections import OrderedDict
  16 import contextlib
  17 import logging
  18 import os
  19 import shutil
  20 import sys
  21 import tempfile
  22 import zipfile
  23
  24
  25 logging.basicConfig(format='%(message)s')
  26 LOGGER = logging.getLogger()
  27
  28
  29 def main(args):
  30     """ Main function. """
  31
  32     tmp_output_file = "%s.%d.tmp" % (args.output_file, os.getpid())
  33
  34     # Sanity checks
  35     if not os.path.exists(args.imagelist_file):
  36         LOGGER.error("imagelist_file '%s' doesn't exists", args.imagelist_file)
  37         sys.exit(2)
  38
  39     if args.links_file is not None and not os.path.exists(args.links_file):
  40         LOGGER.error("link_file '%s' doesn't exists", args.links_file)
  41         sys.exit(2)
  42
  43     out_path = os.path.dirname(args.output_file)
  44     for path in (out_path, args.global_path, args.module_path):
  45         if not os.path.exists(path):
  46             LOGGER.error("Path '%s' doesn't exists", path)
  47             sys.exit(2)
  48         if not os.access(path, os.X_OK):
  49             LOGGER.error("Unable to search path %s", path)
  50             sys.exit(2)
  51
  52     if not os.access(out_path, os.W_OK):
  53         LOGGER.error("Unable to write into path %s", out_path)
  54
  55     custom_paths = []
  56     for path in args.custom_path:
  57         if not os.path.exists(path):
  58             LOGGER.warning("Skipping non-existing path: %s", path)
  59             continue
  60         elif not os.access(path, os.X_OK):
  61             LOGGER.error("Unable to search path %s", path)
  62             sys.exit(2)
  63
  64         custom_paths.append(path)
  65
  66     imagelist_filenames = get_imagelist_filenames(args.imagelist_file)
  67     global_image_list, module_image_list = parse_image_list(imagelist_filenames)
  68     custom_image_list = find_custom(custom_paths)
  69
  70     links = {}
  71     if args.links_file is not None:
  72         read_links(links, args.links_file)
  73     else:
  74         read_links(links, os.path.join(ARGS.global_path, "links.txt"))
  75         for path in custom_paths:
  76             read_links(links, os.path.join(path, "links.txt"))
  77     check_links(links)
  78
  79     zip_list = create_zip_list(global_image_list, module_image_list, custom_image_list,
  80                                args.global_path, args.module_path)
  81     remove_links_from_zip_list(zip_list, links)
  82
  83     if check_rebuild(args.output_file, imagelist_filenames, custom_paths, zip_list, args.links_file):
  84         tmp_dir = copy_images(zip_list)
  85         create_zip_archive(zip_list, links, tmp_dir, tmp_output_file, args.sort_file)
  86
  87         replace_zip_file(tmp_output_file, args.output_file)
  88         try:
  89             LOGGER.info("Remove temporary directory %s", tmp_dir)
  90             shutil.rmtree(tmp_dir)
  91         except Exception as e:
  92             LOGGER.error("Unable to remove temporary directory %s", tmp_dir)
  93             sys.exit(2)
  94     else:
  95         LOGGER.info("No rebuild needed. %s is up to date.", args.output_file)
  96
  97
  98 def check_rebuild(zip_file, imagelist_filenames, custom_paths, zip_list, links_file):
  99     """ Check if a rebuild is needed.
 100
 101     :type zip_file: str
 102     :param zip_file: File to check against (use st_mtime).
 103
 104     :type imagelist_filenames: dict
 105     :param imagelist_filenames: List of imagelist filename.
 106
 107     :type custom_paths: list
 108     :param custom_paths: List of paths to use with links.txt files.
 109
 110     :type zip_list: dict
 111     :param zip_list: List of filenames to create the zip archive.
 112
 113     :type links_file: str
 114     :param links_file: filename to read the links from
 115
 116     :rtype: bool
 117     :return: True if rebuild is needed and False if not.
 118     """
 119
 120     if not os.path.exists(zip_file):
 121         return True
 122
 123     zip_file_stat = os.stat(zip_file)
 124
 125     def compare_modtime(filenames):
 126         """ Check if modification time of zip archive is older the provided
 127         list of files.
 128
 129         :type filenames: dict
 130         :param filenames: List of filenames to check against.
 131
 132         :rtype: bool
 133         :return: True if zip archive is older and False if not.
 134         """
 135         for filename, path in filenames.items():
 136             filename = os.path.join(path, filename) if filename else path
 137             if os.path.exists(filename) and zip_file_stat.st_mtime < os.stat(filename).st_mtime:
 138                 return True
 139         return False
 140
 141     if compare_modtime(imagelist_filenames):
 142         return True
 143     if compare_modtime(zip_list):
 144         return True
 145     if links_file is not None:
 146         if zip_file_stat.st_mtime < os.stat(links_file).st_mtime:
 147             return True
 148     else:
 149         for path in custom_paths:
 150             link_file = os.path.join(path, 'links.txt')
 151             if os.path.exists(link_file):
 152                 if zip_file_stat.st_mtime < os.stat(link_file).st_mtime:
 153                     return True
 154
 155     return False
 156
 157
 158 def replace_zip_file(src, dst):
 159     """ Replace the old archive file with the newly created one.
 160
 161     :type src: str
 162     :param src: Source file name.
 163
 164     :type dst: str
 165     :param dst: Destination file name.
 166     """
 167     LOGGER.info("Replace old zip archive with new archive")
 168     if os.path.exists(dst):
 169         try:
 170             os.unlink(dst)
 171         except OSError as e:
 172             if os.path.exists(src):
 173                 os.unlink(src)
 174
 175             LOGGER.error("Unable to unlink old archive '%s'", dst)
 176             LOGGER.error(str(e))
 177             sys.exit(1)
 178
 179     try:
 180         LOGGER.info("Copy archive '%s' to '%s'", src, dst)
 181         shutil.move(src, dst)
 182     except (shutil.SameFileError, OSError) as e:
 183         os.unlink(src)
 184         LOGGER.error("Cannot copy file '%s' to %s: %s", src, dst, str(e))
 185         sys.exit(1)
 186
 187
 188 def optimize_zip_layout(zip_list, sort_file=None):
 189     """ Optimize the zip layout by ordering the list of filename alphabetically
 190     or with provided sort file (can be partly).
 191
 192     :type zip_list: dict
 193     :param zip_list: List of files to include in the zip archive.
 194
 195     :type sort_file: str
 196     :param sort_file: Path/Filename to file with sort order.
 197
 198     :rtype: OrderedDict
 199     :return: Dict with right sort order.
 200     """
 201     if sort_file is None:
 202         LOGGER.info("No sort file provided")
 203         LOGGER.info("Sorting entries alphabetically")
 204
 205         return OrderedDict(sorted(zip_list.items(), key=lambda t: t[0]))
 206
 207     LOGGER.info("Sort entries from file '%s'", sort_file)
 208     sorted_zip_list = OrderedDict()
 209     try:
 210         fh = open(sort_file)
 211     except IOError:
 212         LOGGER.error("Cannot open file %s", sort_file)
 213         sys.exit(1)
 214     else:
 215         with fh:
 216             for line in fh:
 217                 line = line.strip()
 218                 if line == '' or line.startswith('#'):
 219                     continue
 220
 221                 if line in zip_list:
 222                     sorted_zip_list[line] = ''
 223                 else:
 224                     LOGGER.warning("Unknown file '%s'", line)
 225
 226     for key in sorted(zip_list.keys()):
 227         if key not in sorted_zip_list:
 228             sorted_zip_list[key] = ''
 229
 230     return sorted_zip_list
 231
 232
 233 def create_zip_archive(zip_list, links, tmp_dir, tmp_zip_file, sort_file=None):
 234     """ Create the zip archive.
 235
 236     :type zip_list: dict
 237     :param zip_list: All filenames to be included in the archive.
 238
 239     :type links: dict
 240     :param links: All filenames to create links.txt file.
 241
 242     :type tmp_dir: str
 243     :param tmp_dir: Path to temporary saved files.
 244
 245     :type tmp_zip_file: str
 246     :param tmp_zip_file: Filename/Path of temporary zip archive.
 247
 248     :type sort_file: str|None
 249     :param sort_file: Optional filename with sort order to apply.
 250     """
 251     LOGGER.info("Creating image archive")
 252
 253     old_pwd = os.getcwd()
 254     os.chdir(tmp_dir)
 255
 256     ordered_zip_list = optimize_zip_layout(zip_list, sort_file)
 257
 258     with contextlib.closing(zipfile.ZipFile(tmp_zip_file, 'w')) as tmp_zip:
 259         if links.keys():
 260             LOGGER.info("Add file 'links.txt' to zip archive")
 261             create_links_file(tmp_dir, links)
 262             tmp_zip.write('links.txt', compress_type=zipfile.ZIP_DEFLATED)
 263
 264         for link in ordered_zip_list:
 265             LOGGER.info("Add file '%s' from path '%s' to zip archive", link, tmp_dir)
 266             try:
 267                 if (link.endswith(".svg")):
 268                     tmp_zip.write(link, compress_type=zipfile.ZIP_DEFLATED)
 269                 else:
 270                     tmp_zip.write(link)
 271             except OSError:
 272                 LOGGER.warning("Unable to add file '%s' to zip archive", link)
 273
 274     os.chdir(old_pwd)
 275
 276
 277 def create_links_file(path, links):
 278     """ Create file links.txt. Contains all links.
 279
 280     :type path: str
 281     :param path: Path where to create the file.
 282
 283     :type links: dict
 284     :param links: Dict with links (source -> target).
 285     """
 286     LOGGER.info("Create file links.txt")
 287
 288     try:
 289         filename = os.path.join(path, 'links.txt')
 290         fh = open(filename, 'w')
 291     except IOError:
 292         LOGGER.error("Cannot open file %s", filename)
 293         sys.exit(1)
 294     else:
 295         with fh:
 296             for key in sorted(links.keys()):
 297                 fh.write("%s %s\n" % (key, links[key]))
 298
 299
 300 def copy_images(zip_list):
 301     """ Create a temporary directory and copy images to that directory.
 302
 303     :type zip_list: dict
 304     :param zip_list: Dict with all files.
 305
 306     :rtype: str
 307     :return: Path of the temporary directory.
 308     """
 309     LOGGER.info("Copy image files to temporary directory")
 310
 311     tmp_dir = tempfile.mkdtemp()
 312     for key, value in zip_list.items():
 313         path = os.path.join(value, key)
 314         out_path = os.path.join(tmp_dir, key)
 315
 316         LOGGER.debug("Copying '%s' to '%s'", path, out_path)
 317         if os.path.exists(path):
 318             dirname = os.path.dirname(out_path)
 319             if not os.path.exists(dirname):
 320                 os.makedirs(dirname)
 321
 322             try:
 323                 shutil.copyfile(path, out_path)
 324             except (shutil.SameFileError, OSError) as e:
 325                 LOGGER.error("Cannot add file '%s' to image dir: %s", path, str(e))
 326                 sys.exit(1)
 327
 328     LOGGER.debug("Temporary directory '%s'", tmp_dir)
 329     return tmp_dir
 330
 331
 332 def remove_links_from_zip_list(zip_list, links):
 333     """ Remove any files from zip list that are linked.
 334
 335     :type zip_list: dict
 336     :param zip_list: Files to include in the zip archive.
 337
 338     :type links: dict
 339     :param links: Images which are linked.
 340     """
 341     LOGGER.info("Remove linked files from zip list")
 342
 343     for link in links.keys():
 344         if link in zip_list:
 345             del zip_list[link]
 346
 347     LOGGER.debug("Cleaned zip list:\n%s", "\n".join(zip_list))
 348
 349
 350 def create_zip_list(global_image_list, module_image_list, custom_image_list, global_path, module_path):
 351     """ Create list of images for the zip archive.
 352
 353     :type global_image_list: dict
 354     :param global_image_list: Global images.
 355
 356     :type module_image_list: dict
 357     :param module_image_list: Module images.
 358
 359     :type custom_image_list: dict
 360     :param custom_image_list: Custom images.
 361
 362     :type global_path: str
 363     :param global_path: Global path (use when no custom path is available).
 364
 365     :type module_path: str
 366     :param module_path: Module path (use when no custom path is available).
 367
 368     :rtype: dict
 369     :return: List of images to include in zip archive.
 370     """
 371     LOGGER.info("Assemble image list")
 372
 373     zip_list = {}
 374     duplicates = []
 375
 376     for key in global_image_list.keys():
 377         if key in module_image_list:
 378             duplicates.append(key)
 379             continue
 380
 381         if key in custom_image_list:
 382             zip_list[key] = custom_image_list[key]
 383             continue
 384
 385         zip_list[key] = global_path
 386
 387     for key in module_image_list.keys():
 388         if key in custom_image_list:
 389             zip_list[key] = custom_image_list[key]
 390             continue
 391
 392         zip_list[key] = module_path
 393
 394     if duplicates:
 395         LOGGER.warning("Found duplicate entries in 'global' and 'module' list")
 396         LOGGER.warning("\n".join(duplicates))
 397
 398     LOGGER.debug("Assembled image list:\n%s", "\n".join(zip_list))
 399     return zip_list
 400
 401
 402 def check_links(links):
 403     """ Check that no link points to another link.
 404
 405     :type links: dict
 406     :param links: Links to icons
 407     """
 408
 409     stop = False
 410
 411     for link, target in links.items():
 412         if target in links.keys():
 413             LOGGER.error("Link %s -> %s -> %s", link, target, links[target])
 414             stop = True
 415
 416     if stop:
 417         LOGGER.error("Some icons in links.txt were found to link to other linked icons")
 418         sys.exit(1)
 419
 420
 421 def read_links(links, filename):
 422     """ Read links from file.
 423
 424     :type links: dict
 425     :param links: Hash to store all links
 426
 427     :type filename: str
 428     :param filename: filename to read the links from
 429     """
 430
 431     LOGGER.info("Read links from file '%s'", filename)
 432     if not os.path.isfile(filename):
 433         LOGGER.info("No file to read")
 434         return
 435
 436     try:
 437         fh = open(filename)
 438     except IOError:
 439         LOGGER.error("Cannot open file %s", filename)
 440         sys.exit(1)
 441     else:
 442         with fh:
 443             for line in fh:
 444                 line = line.strip()
 445                 if line == '' or line.startswith('#'):
 446                     continue
 447
 448                 tmp = line.split(' ')
 449                 if len(tmp) == 2:
 450                     links[tmp[0]] = tmp[1]
 451                 else:
 452                     LOGGER.error("Malformed links line '%s' in file %s", line, filename)
 453                     sys.exit(1)
 454
 455     LOGGER.debug("Read links:\n%s", "\n".join(links))
 456
 457
 458 def find_custom(custom_paths):
 459     """ Find all custom images.
 460
 461     :type custom_paths: list
 462     :param custom_paths: List of custom paths to search within.
 463
 464     :rtype: dict
 465     :return: List of all custom images.
 466     """
 467     LOGGER.info("Find all images in custom paths")
 468
 469     custom_image_list = {}
 470     for path in custom_paths:
 471         # find all png files under the path including subdirs
 472         custom_files = [val for sublist in [
 473             [os.path.join(i[0], j) for j in i[2]
 474              if j.endswith('.png') and os.path.isfile(os.path.join(i[0], j))]
 475             for i in os.walk(path)] for val in sublist]
 476
 477         for filename in custom_files:
 478             if filename.startswith(path):
 479                 key = filename.replace(os.path.join(path, ''), '')
 480                 if key not in custom_image_list:
 481                     custom_image_list[key] = path
 482
 483     LOGGER.debug("Custom image list:\n%s", "\n".join(custom_image_list.keys()))
 484     return custom_image_list
 485
 486
 487 def parse_image_list(imagelist_filenames):
 488     """ Parse and extract filename from the imagelist files.
 489
 490     :type imagelist_filenames: list
 491     :param imagelist_filenames: List of imagelist files.
 492
 493     :rtype: tuple
 494     :return: Tuple with dicts containing the global or module image list.
 495     """
 496
 497     global_image_list = {}
 498     module_image_list = {}
 499     for imagelist_filename in imagelist_filenames.keys():
 500         LOGGER.info("Parsing '%s'", imagelist_filename)
 501
 502         try:
 503             fh = open(imagelist_filename)
 504         except IOError as e:
 505             LOGGER.error("Cannot open imagelist file %s", imagelist_filename)
 506             sys.exit(2)
 507         else:
 508             line_count = 0
 509             with fh:
 510                 for line in fh:
 511                     line_count += 1
 512                     line = line.strip()
 513                     if line == '' or line.startswith('#'):
 514                         continue
 515                     # clean up backslashes and double slashes
 516                     line = line.replace('\\', '/')
 517                     line = line.replace('//', '/')
 518
 519                     if line.startswith('%GLOBALRES%'):
 520                         key = "res/%s" % line.replace('%GLOBALRES%/', '')
 521                         global_image_list[key] = True
 522                         if key.endswith('.png'):
 523                             global_image_list[key[:-4] + '.svg'] = True
 524                         continue
 525
 526                     if line.startswith('%MODULE%'):
 527                         key = line.replace('%MODULE%/', '')
 528                         module_image_list[key] = True
 529                         if key.endswith('.png'):
 530                             module_image_list[key[:-4] + '.svg'] = True
 531                         continue
 532
 533                     LOGGER.error("Cannot parse line %s:%d", imagelist_filename, line_count)
 534                     sys.exit(1)
 535
 536     LOGGER.debug("Global image list:\n%s", "\n".join(global_image_list))
 537     LOGGER.debug("Module image list:\n%s", "\n".join(module_image_list))
 538     return global_image_list, module_image_list
 539
 540
 541 def get_imagelist_filenames(filename):
 542     """ Extract a list of imagelist filenames.
 543
 544     :type filename: str
 545     :param filename: Name of file from extracting the list.
 546
 547     :rtype: dict
 548     :return: List of imagelist filenames.
 549     """
 550     LOGGER.info("Extract the imagelist filenames")
 551
 552     imagelist_filenames = {}
 553     try:
 554         fh = open(filename)
 555     except IOError:
 556         LOGGER.error("Cannot open imagelist file %s", filename)
 557         sys.exit(1)
 558     else:
 559         with fh:
 560             for line in fh:
 561                 line = line.strip()
 562                 if not line or line == '':
 563                     continue
 564
 565                 for line_split in line.split(' '):
 566                     line_split.strip()
 567                     imagelist_filenames[line_split] = ''
 568
 569     LOGGER.debug("Extracted imagelist filenames:\n%s", "\n".join(imagelist_filenames.keys()))
 570     return imagelist_filenames
 571
 572
 573 if __name__ == '__main__':
 574     parser = argparse.ArgumentParser("Pack images into archives")
 575     parser.add_argument('-o', '--output-file', dest='output_file',
 576                         action='store', required=True,
 577                         help='path to output archive')
 578     parser.add_argument('-l', '--imagelist-file', dest='imagelist_file',
 579                         action='store', required=True,
 580                         help='file containing list of image list file')
 581     parser.add_argument('-L', '--links-file', dest='links_file',
 582                         action='store', required=False,
 583                         help='file containing linked images')
 584     parser.add_argument('-s', '--sort-file', dest='sort_file',
 585                         action='store', required=True, default=None,
 586                         help='image sort order file')
 587     parser.add_argument('-c', '--custom-path', dest='custom_path',
 588                         action='append', required=True,
 589                         help='path to custom path directory')
 590     parser.add_argument('-g', '--global-path', dest='global_path',
 591                         action='store', required=True,
 592                         help='path to global images directory')
 593     parser.add_argument('-m', '--module-path', dest='module_path',
 594                         action='store', required=True,
 595                         help='path to module images directory')
 596     parser.add_argument('-v', '--verbose', dest='verbose',
 597                         action='count', default=0,
 598                         help='set the verbosity (can be used multiple times)')
 599
 600     ARGS = parser.parse_args()
 601     LOGGER.setLevel(logging.ERROR - (10 * ARGS.verbose if ARGS.verbose <= 3 else 3))
 602
 603     main(ARGS)
 604     sys.exit(0)
 605
 606 # vim: set shiftwidth=4 softtabstop=4 expandtab: