bin/update/create_partial_update.py

   1 #!/usr/bin/env python3
   2 import requests
   3 import json
   4 import sys
   5 import hashlib
   6 import os
   7 import subprocess
   8 import errno
   9 import json
  10
  11 from config import parse_config
  12 from uncompress_mar import extract_mar
  13 from tools import get_file_info, get_hash
  14 from signing import sign_mar_file
  15
  16 from path import UpdaterPath, mkdir_p, convert_to_unix, convert_to_native
  17
  18 BUF_SIZE = 1024
  19 current_dir_path = os.path.dirname(os.path.realpath(convert_to_unix(__file__)))
  20
  21 class InvalidFileException(Exception):
  22
  23     def __init__(self, *args, **kwargs):
  24         super().__init__(self, *args, **kwargs)
  25
  26 def download_file(filepath, url, hash_string):
  27     with open(filepath, "wb") as f:
  28         response = requests.get(url, stream=True)
  29
  30         if not response.ok:
  31             return
  32
  33         for block in response.iter_content(1024):
  34             f.write(block)
  35
  36     file_hash = get_hash(filepath)
  37
  38     if file_hash != hash_string:
  39         raise InvalidFileException("file hash does not match for file %s: Expected %s, Got: %s" % (url, hash_string, file_hash))
  40
  41 def handle_language(lang_entries, filedir):
  42     mar = os.environ.get('MAR', 'mar')
  43     langs = {}
  44     for lang, data in lang_entries.items():
  45         lang_dir = os.path.join(filedir, lang)
  46         lang_file = os.path.join(lang_dir, "lang.mar")
  47         mkdir_p(lang_dir)
  48         download_file(lang_file , data["url"], data["hash"])
  49         dir_path = os.path.join(lang_dir, "lang")
  50         mkdir_p(dir_path)
  51         extract_mar(lang_file, dir_path)
  52         langs[lang] = dir_path
  53
  54     return langs
  55
  56 def download_mar_for_update_channel_and_platform(config, platform, temp_dir):
  57     mar = os.environ.get('MAR', 'mar')
  58     base_url = config.server_url + "update/partial-targets/1/"
  59     url = base_url + platform + "/" + config.channel
  60     r = requests.get(url)
  61     if r.status_code is not 200:
  62         print(r.content)
  63         raise Exception("download failed")
  64
  65     update_info = json.loads(r.content.decode("utf-8"))
  66     update_files = update_info['updates']
  67     downloaded_updates = {}
  68     for update_file in update_files:
  69         build = update_file["build"]
  70         filedir = os.path.join(temp_dir, build)
  71
  72         mkdir_p(filedir)
  73
  74         filepath = filedir + "/complete.mar"
  75         url = update_file["update"]["url"]
  76         expected_hash = update_file["update"]["hash"]
  77         download_file(filepath, url, expected_hash)
  78
  79         dir_path = os.path.join(filedir, "complete")
  80         mkdir_p(dir_path)
  81         extract_mar(filepath, dir_path)
  82
  83         downloaded_updates[build] = {"complete": dir_path}
  84
  85         langs = handle_language(update_file["languages"], filedir)
  86         downloaded_updates[build]["languages"] = langs
  87
  88     return downloaded_updates
  89
  90 def generate_file_name(current_build_id, old_build_id, mar_name_prefix):
  91     name = "%s_from_%s_partial.mar" %(mar_name_prefix, old_build_id)
  92     return name
  93
  94 def generate_lang_file_name(current_build_id, old_build_id, mar_name_prefix, lang):
  95     name = "%s_%s_from_%s_partial.mar" %(mar_name_prefix, lang, old_build_id)
  96     return name
  97
  98 def add_single_dir(path):
  99     dir_name =  [os.path.join(path, name) for name in os.listdir(path) if os.path.isdir(os.path.join(path, name))]
 100     return dir_name[0]
 101
 102 def main():
 103     workdir = sys.argv[1]
 104
 105     updater_path = UpdaterPath(workdir)
 106     updater_path.ensure_dir_exist()
 107
 108     mar_name_prefix = sys.argv[2]
 109     update_config = sys.argv[3]
 110     platform = sys.argv[4]
 111     build_id = sys.argv[5]
 112
 113     current_build_path = updater_path.get_current_build_dir()
 114     mar_dir = updater_path.get_mar_dir()
 115     temp_dir = updater_path.get_previous_build_dir()
 116     update_dir = updater_path.get_update_dir()
 117
 118     current_build_path = add_single_dir(current_build_path)
 119     if sys.platform == "cygwin":
 120         current_build_path = add_single_dir(current_build_path)
 121
 122     config = parse_config(update_config)
 123
 124     updates = download_mar_for_update_channel_and_platform(config, platform, temp_dir)
 125
 126     data = {"partials": []}
 127
 128     for build, update in updates.items():
 129         file_name = generate_file_name(build_id, build, mar_name_prefix)
 130         mar_file = os.path.join(update_dir, file_name)
 131         subprocess.call([os.path.join(current_dir_path, 'make_incremental_update.sh'), convert_to_native(mar_file), convert_to_native(update["complete"]), convert_to_native(current_build_path)])
 132         sign_mar_file(update_dir, config, mar_file, mar_name_prefix)
 133
 134         partial_info = {"file":get_file_info(mar_file, config.base_url), "from": build, "to": build_id, "languages": {}}
 135
 136         # on Windows we don't use language packs
 137         if sys.platform != "cygwin":
 138             for lang, lang_info in update["languages"].items():
 139                 lang_name = generate_lang_file_name(build_id, build, mar_name_prefix, lang)
 140
 141                 # write the file into the final directory
 142                 lang_mar_file = os.path.join(update_dir, lang_name)
 143
 144                 # the directory of the old language file is of the form
 145                 # workdir/mar/language/en-US/LibreOffice_<version>_<os>_archive_langpack_<lang>/
 146                 language_dir = add_single_dir(os.path.join(mar_dir, "language", lang))
 147                 subprocess.call([os.path.join(current_dir_path, 'make_incremental_update.sh'), convert_to_native(lang_mar_file), convert_to_native(lang_info), convert_to_native(language_dir)])
 148                 sign_mar_file(update_dir, config, lang_mar_file, mar_name_prefix)
 149
 150                 # add the partial language info
 151                 partial_info["languages"][lang] = get_file_info(lang_mar_file, config.base_url)
 152
 153         data["partials"].append(partial_info)
 154
 155     with open(os.path.join(update_dir, "partial_update_info.json"), "w") as f:
 156         json.dump(data, f)
 157
 158
 159 if __name__ == '__main__':
 160     main()