solenv/bin/pack_images.py

   1 # -*- coding: utf-8 -*-
   2 # -*- tab-width: 4; indent-tabs-mode: nil; py-indent-offset: 4 -*-
   3 #
   4 # This file is part of the LibreOffice project.
   5 #
   6 # This Source Code Form is subject to the terms of the Mozilla Public
   7 # License, v. 2.0. If a copy of the MPL was not distributed with this
   8 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   9 #
  10 """ Pack images into archives. """
  11
  12 from __future__ import with_statement
  13
  14 import argparse
  15 from collections import OrderedDict
  16 import contextlib
  17 import logging
  18 import os
  19 import shutil
  20 import sys
  21 import tempfile
  22 import zipfile
  23
  24
  25 logging.basicConfig(format='%(message)s')
  26 LOGGER = logging.getLogger()
  27
  28
  29 def main(args):
  30     """ Main function. """
  31
  32     tmp_output_file = "%s.%d.tmp" % (args.output_file, os.getpid())
  33
  34     # Sanity checks
  35     if not os.path.exists(args.imagelist_file):
  36         LOGGER.error("imagelist_file '%s' doesn't exists", args.imagelist_file)
  37         sys.exit(2)
  38
  39     if args.links_file is not None and not os.path.exists(args.links_file):
  40         LOGGER.error("link_file '%s' doesn't exists", args.links_file)
  41         sys.exit(2)
  42
  43     out_path = os.path.dirname(args.output_file)
  44     for path in (out_path, args.global_path, args.module_path):
  45         if not os.path.exists(path):
  46             LOGGER.error("Path '%s' doesn't exists", path)
  47             sys.exit(2)
  48         if not os.access(path, os.X_OK):
  49             LOGGER.error("Unable to search path %s", path)
  50             sys.exit(2)
  51
  52     if not os.access(out_path, os.W_OK):
  53         LOGGER.error("Unable to write into path %s", out_path)
  54
  55     custom_paths = []
  56     for path in args.custom_path:
  57         if not os.path.exists(path):
  58             LOGGER.warning("Skipping non-existing path: %s", path)
  59             continue
  60         elif not os.access(path, os.X_OK):
  61             LOGGER.error("Unable to search path %s", path)
  62             sys.exit(2)
  63
  64         custom_paths.append(path)
  65
  66     imagelist_filenames = get_imagelist_filenames(args.imagelist_file)
  67     global_image_list, module_image_list = parse_image_list(imagelist_filenames)
  68     custom_image_list = find_custom(custom_paths)
  69
  70     links = {}
  71     if args.links_file is not None:
  72         read_links(links, args.links_file)
  73     else:
  74         read_links(links, os.path.join(ARGS.global_path, "links.txt"))
  75         for path in custom_paths:
  76             read_links(links, os.path.join(path, "links.txt"))
  77     check_links(links)
  78
  79     zip_list = create_zip_list(global_image_list, module_image_list, custom_image_list,
  80                                args.global_path, args.module_path)
  81     remove_links_from_zip_list(zip_list, links)
  82
  83     if check_rebuild(args.output_file, imagelist_filenames, custom_paths, zip_list, args.links_file):
  84         tmp_dir = copy_images(zip_list)
  85         create_zip_archive(zip_list, links, tmp_dir, tmp_output_file, args.sort_file)
  86
  87         replace_zip_file(tmp_output_file, args.output_file)
  88         try:
  89             LOGGER.info("Remove temporary directory %s", tmp_dir)
  90             shutil.rmtree(tmp_dir)
  91         except Exception as e:
  92             LOGGER.error("Unable to remove temporary directory %s", tmp_dir)
  93             sys.exit(2)
  94     else:
  95         LOGGER.info("No rebuild needed. %s is up to date.", args.output_file)
  96
  97
  98 def check_rebuild(zip_file, imagelist_filenames, custom_paths, zip_list, links_file):
  99     """ Check if a rebuild is needed.
 100
 101     :type zip_file: str
 102     :param zip_file: File to check against (use st_mtime).
 103
 104     :type imagelist_filenames: dict
 105     :param imagelist_filenames: List of imagelist filename.
 106
 107     :type custom_paths: list
 108     :param custom_paths: List of paths to use with links.txt files.
 109
 110     :type zip_list: dict
 111     :param zip_list: List of filenames to create the zip archive.
 112
 113     :type links_file: str
 114     :param links_file: filename to read the links from
 115
 116     :rtype: bool
 117     :return: True if rebuild is needed and False if not.
 118     """
 119
 120     if not os.path.exists(zip_file):
 121         return True
 122
 123     zip_file_stat = os.stat(zip_file)
 124
 125     def compare_modtime(filenames):
 126         """ Check if modification time of zip archive is older the provided
 127         list of files.
 128
 129         :type filenames: dict
 130         :param filenames: List of filenames to check against.
 131
 132         :rtype: bool
 133         :return: True if zip archive is older and False if not.
 134         """
 135         for filename, path in filenames.items():
 136             filename = os.path.join(path, filename) if filename else path
 137             if os.path.exists(filename) and zip_file_stat.st_mtime < os.stat(filename).st_mtime:
 138                 return True
 139         return False
 140
 141     if compare_modtime(imagelist_filenames):
 142         return True
 143     if compare_modtime(zip_list):
 144         return True
 145     if links_file is not None:
 146         if zip_file_stat.st_mtime < os.stat(links_file).st_mtime:
 147             return True
 148     else:
 149         for path in custom_paths:
 150             link_file = os.path.join(path, 'links.txt')
 151             if os.path.exists(link_file):
 152                 if zip_file_stat.st_mtime < os.stat(link_file).st_mtime:
 153                     return True
 154
 155     return False
 156
 157
 158 def replace_zip_file(src, dst):
 159     """ Replace the old archive file with the newly created one.
 160
 161     :type src: str
 162     :param src: Source file name.
 163
 164     :type dst: str
 165     :param dst: Destination file name.
 166     """
 167     LOGGER.info("Replace old zip archive with new archive")
 168     if os.path.exists(dst):
 169         try:
 170             os.unlink(dst)
 171         except OSError as e:
 172             if os.path.exists(src):
 173                 os.unlink(src)
 174
 175             LOGGER.error("Unable to unlink old archive '%s'", dst)
 176             LOGGER.error(str(e))
 177             sys.exit(1)
 178
 179     try:
 180         LOGGER.info("Copy archive '%s' to '%s'", src, dst)
 181         shutil.move(src, dst)
 182     except (shutil.SameFileError, OSError) as e:
 183         os.unlink(src)
 184         LOGGER.error("Cannot copy file '%s' to %s: %s", src, dst, str(e))
 185         sys.exit(1)
 186
 187
 188 def optimize_zip_layout(zip_list, sort_file=None):
 189     """ Optimize the zip layout by ordering the list of filename alphabetically
 190     or with provided sort file (can be partly).
 191
 192     :type zip_list: dict
 193     :param zip_list: List of files to include in the zip archive.
 194
 195     :type sort_file: str
 196     :param sort_file: Path/Filename to file with sort order.
 197
 198     :rtype: OrderedDict
 199     :return: Dict with right sort order.
 200     """
 201     if sort_file is None:
 202         LOGGER.info("No sort file provided")
 203         LOGGER.info("Sorting entries alphabetically")
 204
 205         return OrderedDict(sorted(zip_list.items(), key=lambda t: t[0]))
 206
 207     LOGGER.info("Sort entries from file '%s'", sort_file)
 208     sorted_zip_list = OrderedDict()
 209     try:
 210         fh = open(sort_file)
 211     except IOError:
 212         LOGGER.error("Cannot open file %s", sort_file)
 213         sys.exit(1)
 214     else:
 215         with fh:
 216             for line in fh:
 217                 line = line.strip()
 218                 if line == '' or line.startswith('#'):
 219                     continue
 220
 221                 if line in zip_list:
 222                     sorted_zip_list[line] = ''
 223                 else:
 224                     LOGGER.warning("Unknown file '%s'", line)
 225
 226     for key in sorted(zip_list.keys()):
 227         if key not in sorted_zip_list:
 228             sorted_zip_list[key] = ''
 229
 230     return sorted_zip_list
 231
 232
 233 def create_zip_archive(zip_list, links, tmp_dir, tmp_zip_file, sort_file=None):
 234     """ Create the zip archive.
 235
 236     :type zip_list: dict
 237     :param zip_list: All filenames to be included in the archive.
 238
 239     :type links: dict
 240     :param links: All filenames to create links.txt file.
 241
 242     :type tmp_dir: str
 243     :param tmp_dir: Path to temporary saved files.
 244
 245     :type tmp_zip_file: str
 246     :param tmp_zip_file: Filename/Path of temporary zip archive.
 247
 248     :type sort_file: str|None
 249     :param sort_file: Optional filename with sort order to apply.
 250     """
 251     LOGGER.info("Creating image archive")
 252
 253     old_pwd = os.getcwd()
 254     os.chdir(tmp_dir)
 255
 256     ordered_zip_list = optimize_zip_layout(zip_list, sort_file)
 257
 258     with contextlib.closing(zipfile.ZipFile(tmp_zip_file, 'w')) as tmp_zip:
 259         if links.keys():
 260             LOGGER.info("Add file 'links.txt' to zip archive")
 261             create_links_file(tmp_dir, links)
 262             tmp_zip.write('links.txt', compress_type=zipfile.ZIP_DEFLATED)
 263
 264         for link in ordered_zip_list:
 265             LOGGER.info("Add file '%s' from path '%s' to zip archive", link, tmp_dir)
 266             try:
 267                 if (link.endswith(".svg")):
 268                     tmp_zip.write(link, compress_type=zipfile.ZIP_DEFLATED)
 269                 else:
 270                     tmp_zip.write(link)
 271             except OSError:
 272                 LOGGER.warning("Unable to add file '%s' to zip archive", link)
 273
 274     os.chdir(old_pwd)
 275
 276
 277 def create_links_file(path, links):
 278     """ Create file links.txt. Contains all links.
 279
 280     :type path: str
 281     :param path: Path where to create the file.
 282
 283     :type links: dict
 284     :param links: Dict with links (source -> target).
 285     """
 286     LOGGER.info("Create file links.txt")
 287
 288     try:
 289         filename = os.path.join(path, 'links.txt')
 290         fh = open(filename, 'w')
 291     except IOError:
 292         LOGGER.error("Cannot open file %s", filename)
 293         sys.exit(1)
 294     else:
 295         with fh:
 296             for key in sorted(links.keys()):
 297                 fh.write("%s %s\n" % (key, links[key]))
 298
 299
 300 def copy_images(zip_list):
 301     """ Create a temporary directory and copy images to that directory.
 302
 303     :type zip_list: dict
 304     :param zip_list: Dict with all files.
 305
 306     :rtype: str
 307     :return: Path of the temporary directory.
 308     """
 309     LOGGER.info("Copy image files to temporary directory")
 310
 311     tmp_dir = tempfile.mkdtemp()
 312     for key, value in zip_list.items():
 313         path = os.path.join(value, key)
 314         out_path = os.path.join(tmp_dir, key)
 315
 316         LOGGER.debug("Copying '%s' to '%s'", path, out_path)
 317         if os.path.exists(path):
 318             dirname = os.path.dirname(out_path)
 319             if not os.path.exists(dirname):
 320                 os.makedirs(dirname)
 321
 322             try:
 323                 shutil.copyfile(path, out_path)
 324             except (shutil.SameFileError, OSError) as e:
 325                 LOGGER.error("Cannot add file '%s' to image dir: %s", path, str(e))
 326                 sys.exit(1)
 327
 328     LOGGER.debug("Temporary directory '%s'", tmp_dir)
 329     return tmp_dir
 330
 331
 332 def remove_links_from_zip_list(zip_list, links):
 333     """ Remove any files from zip list that are linked.
 334
 335     :type zip_list: dict
 336     :param zip_list: Files to include in the zip archive.
 337
 338     :type links: dict
 339     :param links: Images which are linked.
 340     """
 341     LOGGER.info("Remove linked files from zip list")
 342
 343     for link in links.keys():
 344         if link in zip_list:
 345             module = zip_list[link]
 346             del zip_list[link]
 347             # tdf#135369 if we removed something that is a link to a real file
 348             # from our list of images to pack, but that real image is not in
 349             # the list of images to pack, add it in instead now
 350             if links[link] not in zip_list:
 351                 zip_list[links[link]] = module
 352
 353     LOGGER.debug("Cleaned zip list:\n%s", "\n".join(zip_list))
 354
 355
 356 def create_zip_list(global_image_list, module_image_list, custom_image_list, global_path, module_path):
 357     """ Create list of images for the zip archive.
 358
 359     :type global_image_list: dict
 360     :param global_image_list: Global images.
 361
 362     :type module_image_list: dict
 363     :param module_image_list: Module images.
 364
 365     :type custom_image_list: dict
 366     :param custom_image_list: Custom images.
 367
 368     :type global_path: str
 369     :param global_path: Global path (use when no custom path is available).
 370
 371     :type module_path: str
 372     :param module_path: Module path (use when no custom path is available).
 373
 374     :rtype: dict
 375     :return: List of images to include in zip archive.
 376     """
 377     LOGGER.info("Assemble image list")
 378
 379     zip_list = {}
 380     duplicates = []
 381
 382     for key in global_image_list.keys():
 383         if key in module_image_list:
 384             duplicates.append(key)
 385             continue
 386
 387         if key in custom_image_list:
 388             zip_list[key] = custom_image_list[key]
 389             continue
 390
 391         zip_list[key] = global_path
 392
 393     for key in module_image_list.keys():
 394         if key in custom_image_list:
 395             zip_list[key] = custom_image_list[key]
 396             continue
 397
 398         zip_list[key] = module_path
 399
 400     if duplicates:
 401         LOGGER.warning("Found duplicate entries in 'global' and 'module' list")
 402         LOGGER.warning("\n".join(duplicates))
 403
 404     LOGGER.debug("Assembled image list:\n%s", "\n".join(zip_list))
 405     return zip_list
 406
 407
 408 def check_links(links):
 409     """ Check that no link points to another link.
 410
 411     :type links: dict
 412     :param links: Links to icons
 413     """
 414
 415     stop = False
 416
 417     for link, target in links.items():
 418         if target in links.keys():
 419             LOGGER.error("Link %s -> %s -> %s", link, target, links[target])
 420             stop = True
 421
 422     if stop:
 423         LOGGER.error("Some icons in links.txt were found to link to other linked icons")
 424         sys.exit(1)
 425
 426
 427 def read_links(links, filename):
 428     """ Read links from file.
 429
 430     :type links: dict
 431     :param links: Hash to store all links
 432
 433     :type filename: str
 434     :param filename: filename to read the links from
 435     """
 436
 437     LOGGER.info("Read links from file '%s'", filename)
 438     if not os.path.isfile(filename):
 439         LOGGER.info("No file to read")
 440         return
 441
 442     try:
 443         fh = open(filename)
 444     except IOError:
 445         LOGGER.error("Cannot open file %s", filename)
 446         sys.exit(1)
 447     else:
 448         with fh:
 449             for line in fh:
 450                 line = line.strip()
 451                 if line == '' or line.startswith('#'):
 452                     continue
 453
 454                 tmp = line.split(' ')
 455                 if len(tmp) == 2:
 456                     links[tmp[0]] = tmp[1]
 457                 else:
 458                     LOGGER.error("Malformed links line '%s' in file %s", line, filename)
 459                     sys.exit(1)
 460
 461     LOGGER.debug("Read links:\n%s", "\n".join(links))
 462
 463
 464 def find_custom(custom_paths):
 465     """ Find all custom images.
 466
 467     :type custom_paths: list
 468     :param custom_paths: List of custom paths to search within.
 469
 470     :rtype: dict
 471     :return: List of all custom images.
 472     """
 473     LOGGER.info("Find all images in custom paths")
 474
 475     custom_image_list = {}
 476     for path in custom_paths:
 477         # find all png files under the path including subdirs
 478         custom_files = [val for sublist in [
 479             [os.path.join(i[0], j) for j in i[2]
 480              if j.endswith('.png') and os.path.isfile(os.path.join(i[0], j))]
 481             for i in os.walk(path)] for val in sublist]
 482
 483         for filename in custom_files:
 484             if filename.startswith(path):
 485                 key = filename.replace(os.path.join(path, ''), '')
 486                 if key not in custom_image_list:
 487                     custom_image_list[key] = path
 488
 489     LOGGER.debug("Custom image list:\n%s", "\n".join(custom_image_list.keys()))
 490     return custom_image_list
 491
 492
 493 def parse_image_list(imagelist_filenames):
 494     """ Parse and extract filename from the imagelist files.
 495
 496     :type imagelist_filenames: list
 497     :param imagelist_filenames: List of imagelist files.
 498
 499     :rtype: tuple
 500     :return: Tuple with dicts containing the global or module image list.
 501     """
 502
 503     global_image_list = {}
 504     module_image_list = {}
 505     for imagelist_filename in imagelist_filenames.keys():
 506         LOGGER.info("Parsing '%s'", imagelist_filename)
 507
 508         try:
 509             fh = open(imagelist_filename)
 510         except IOError as e:
 511             LOGGER.error("Cannot open imagelist file %s", imagelist_filename)
 512             sys.exit(2)
 513         else:
 514             line_count = 0
 515             with fh:
 516                 for line in fh:
 517                     line_count += 1
 518                     line = line.strip()
 519                     if line == '' or line.startswith('#'):
 520                         continue
 521                     # clean up backslashes and double slashes
 522                     line = line.replace('\\', '/')
 523                     line = line.replace('//', '/')
 524
 525                     if line.startswith('%GLOBALRES%'):
 526                         key = "res/%s" % line.replace('%GLOBALRES%/', '')
 527                         global_image_list[key] = True
 528                         if key.endswith('.png'):
 529                             global_image_list[key[:-4] + '.svg'] = True
 530                         continue
 531
 532                     if line.startswith('%MODULE%'):
 533                         key = line.replace('%MODULE%/', '')
 534                         module_image_list[key] = True
 535                         if key.endswith('.png'):
 536                             module_image_list[key[:-4] + '.svg'] = True
 537                         continue
 538
 539                     LOGGER.error("Cannot parse line %s:%d", imagelist_filename, line_count)
 540                     sys.exit(1)
 541
 542     LOGGER.debug("Global image list:\n%s", "\n".join(global_image_list))
 543     LOGGER.debug("Module image list:\n%s", "\n".join(module_image_list))
 544     return global_image_list, module_image_list
 545
 546
 547 def get_imagelist_filenames(filename):
 548     """ Extract a list of imagelist filenames.
 549
 550     :type filename: str
 551     :param filename: Name of file from extracting the list.
 552
 553     :rtype: dict
 554     :return: List of imagelist filenames.
 555     """
 556     LOGGER.info("Extract the imagelist filenames")
 557
 558     imagelist_filenames = {}
 559     try:
 560         fh = open(filename)
 561     except IOError:
 562         LOGGER.error("Cannot open imagelist file %s", filename)
 563         sys.exit(1)
 564     else:
 565         with fh:
 566             for line in fh:
 567                 line = line.strip()
 568                 if not line or line == '':
 569                     continue
 570
 571                 for line_split in line.split(' '):
 572                     line_split.strip()
 573                     imagelist_filenames[line_split] = ''
 574
 575     LOGGER.debug("Extracted imagelist filenames:\n%s", "\n".join(imagelist_filenames.keys()))
 576     return imagelist_filenames
 577
 578
 579 if __name__ == '__main__':
 580     parser = argparse.ArgumentParser("Pack images into archives")
 581     parser.add_argument('-o', '--output-file', dest='output_file',
 582                         action='store', required=True,
 583                         help='path to output archive')
 584     parser.add_argument('-l', '--imagelist-file', dest='imagelist_file',
 585                         action='store', required=True,
 586                         help='file containing list of image list file')
 587     parser.add_argument('-L', '--links-file', dest='links_file',
 588                         action='store', required=False,
 589                         help='file containing linked images')
 590     parser.add_argument('-s', '--sort-file', dest='sort_file',
 591                         action='store', required=True, default=None,
 592                         help='image sort order file')
 593     parser.add_argument('-c', '--custom-path', dest='custom_path',
 594                         action='append', required=True,
 595                         help='path to custom path directory')
 596     parser.add_argument('-g', '--global-path', dest='global_path',
 597                         action='store', required=True,
 598                         help='path to global images directory')
 599     parser.add_argument('-m', '--module-path', dest='module_path',
 600                         action='store', required=True,
 601                         help='path to module images directory')
 602     parser.add_argument('-v', '--verbose', dest='verbose',
 603                         action='count', default=0,
 604                         help='set the verbosity (can be used multiple times)')
 605
 606     ARGS = parser.parse_args()
 607     LOGGER.setLevel(logging.ERROR - (10 * ARGS.verbose if ARGS.verbose <= 3 else 3))
 608
 609     main(ARGS)
 610     sys.exit(0)
 611
 612 # vim: set shiftwidth=4 softtabstop=4 expandtab: