bin/update/create_partial_update.py

   1 #!/usr/bin/env python3
   2 import json
   3 import os
   4 import subprocess
   5 import sys
   6
   7 import requests
   8
   9 from config import parse_config
  10 from path import UpdaterPath, mkdir_p, convert_to_unix, convert_to_native
  11 from signing import sign_mar_file
  12 from tools import get_file_info, get_hash
  13 from uncompress_mar import extract_mar
  14
  15 BUF_SIZE = 1024
  16 current_dir_path = os.path.dirname(os.path.realpath(convert_to_unix(__file__)))
  17
  18
  19 class InvalidFileException(Exception):
  20
  21     def __init__(self, *args, **kwargs):
  22         super().__init__(self, *args, **kwargs)
  23
  24
  25 def download_file(filepath, url, hash_string):
  26     with open(filepath, "wb") as f:
  27         response = requests.get(url, stream=True)
  28
  29         if not response.ok:
  30             return
  31
  32         for block in response.iter_content(1024):
  33             f.write(block)
  34
  35     file_hash = get_hash(filepath)
  36
  37     if file_hash != hash_string:
  38         raise InvalidFileException(
  39             "file hash does not match for file %s: Expected %s, Got: %s" % (url, hash_string, file_hash))
  40
  41
  42 def handle_language(lang_entries, filedir):
  43     langs = {}
  44     for lang, data in lang_entries.items():
  45         lang_dir = os.path.join(filedir, lang)
  46         lang_file = os.path.join(lang_dir, "lang.mar")
  47         mkdir_p(lang_dir)
  48         download_file(lang_file, data["url"], data["hash"])
  49         dir_path = os.path.join(lang_dir, "lang")
  50         mkdir_p(dir_path)
  51         extract_mar(lang_file, dir_path)
  52         langs[lang] = dir_path
  53
  54     return langs
  55
  56
  57 def download_mar_for_update_channel_and_platform(config, platform, temp_dir):
  58     base_url = config.server_url + "update/partial-targets/1/"
  59     url = base_url + platform + "/" + config.channel
  60     r = requests.get(url)
  61     if r.status_code != 200:
  62         print(r.content)
  63         raise Exception("download failed")
  64
  65     update_info = json.loads(r.content.decode("utf-8"))
  66     update_files = update_info['updates']
  67     downloaded_updates = {}
  68     for update_file in update_files:
  69         build = update_file["build"]
  70         filedir = os.path.join(temp_dir, build)
  71
  72         mkdir_p(filedir)
  73
  74         filepath = filedir + "/complete.mar"
  75         url = update_file["update"]["url"]
  76         expected_hash = update_file["update"]["hash"]
  77         download_file(filepath, url, expected_hash)
  78
  79         dir_path = os.path.join(filedir, "complete")
  80         mkdir_p(dir_path)
  81         extract_mar(filepath, dir_path)
  82
  83         downloaded_updates[build] = {"complete": dir_path}
  84
  85         langs = handle_language(update_file["languages"], filedir)
  86         downloaded_updates[build]["languages"] = langs
  87
  88     return downloaded_updates
  89
  90
  91 def generate_file_name(old_build_id, mar_name_prefix):
  92     name = "%s_from_%s_partial.mar" % (mar_name_prefix, old_build_id)
  93     return name
  94
  95
  96 def generate_lang_file_name(old_build_id, mar_name_prefix, lang):
  97     name = "%s_%s_from_%s_partial.mar" % (mar_name_prefix, lang, old_build_id)
  98     return name
  99
 100
 101 def add_single_dir(path):
 102     dir_name = [os.path.join(path, name) for name in os.listdir(path) if os.path.isdir(os.path.join(path, name))]
 103     return dir_name[0]
 104
 105
 106 def main():
 107     workdir = sys.argv[1]
 108
 109     updater_path = UpdaterPath(workdir)
 110     updater_path.ensure_dir_exist()
 111
 112     mar_name_prefix = sys.argv[2]
 113     update_config = sys.argv[3]
 114     platform = sys.argv[4]
 115     build_id = sys.argv[5]
 116
 117     current_build_path = updater_path.get_current_build_dir()
 118     mar_dir = updater_path.get_mar_dir()
 119     temp_dir = updater_path.get_previous_build_dir()
 120     update_dir = updater_path.get_update_dir()
 121
 122     current_build_path = add_single_dir(current_build_path)
 123     if sys.platform == "cygwin":
 124         current_build_path = add_single_dir(current_build_path)
 125
 126     config = parse_config(update_config)
 127
 128     updates = download_mar_for_update_channel_and_platform(config, platform, temp_dir)
 129
 130     data = {"partials": []}
 131
 132     for build, update in updates.items():
 133         file_name = generate_file_name(build, mar_name_prefix)
 134         mar_file = os.path.join(update_dir, file_name)
 135         subprocess.call([os.path.join(current_dir_path, 'make_incremental_update.sh'), convert_to_native(mar_file),
 136                          convert_to_native(update["complete"]), convert_to_native(current_build_path)])
 137         sign_mar_file(update_dir, config, mar_file, mar_name_prefix)
 138
 139         partial_info = {"file": get_file_info(mar_file, config.base_url), "from": build, "to": build_id,
 140                         "languages": {}}
 141
 142         # on Windows we don't use language packs
 143         if sys.platform != "cygwin":
 144             for lang, lang_info in update["languages"].items():
 145                 lang_name = generate_lang_file_name(build, mar_name_prefix, lang)
 146
 147                 # write the file into the final directory
 148                 lang_mar_file = os.path.join(update_dir, lang_name)
 149
 150                 # the directory of the old language file is of the form
 151                 # workdir/mar/language/en-US/LibreOffice_<version>_<os>_archive_langpack_<lang>/
 152                 language_dir = add_single_dir(os.path.join(mar_dir, "language", lang))
 153                 subprocess.call(
 154                     [os.path.join(current_dir_path, 'make_incremental_update.sh'), convert_to_native(lang_mar_file),
 155                      convert_to_native(lang_info), convert_to_native(language_dir)])
 156                 sign_mar_file(update_dir, config, lang_mar_file, mar_name_prefix)
 157
 158                 # add the partial language info
 159                 partial_info["languages"][lang] = get_file_info(lang_mar_file, config.base_url)
 160
 161         data["partials"].append(partial_info)
 162
 163     with open(os.path.join(update_dir, "partial_update_info.json"), "w") as f:
 164         json.dump(data, f)
 165
 166
 167 if __name__ == '__main__':
 168     main()