scripts/checktransupdate.py

   1 #!/usr/bin/env python3
   2 # SPDX-License-Identifier: GPL-2.0
   3
   4 """
   5 This script helps track the translation status of the documentation
   6 in different locales, e.g., zh_CN. More specially, it uses `git log`
   7 commit to find the latest english commit from the translation commit
   8 (order by author date) and the latest english commits from HEAD. If
   9 differences occur, report the file and commits that need to be updated.
  10
  11 The usage is as follows:
  12 - ./scripts/checktransupdate.py -l zh_CN
  13 This will print all the files that need to be updated or translated in the zh_CN locale.
  14 - ./scripts/checktransupdate.py Documentation/translations/zh_CN/dev-tools/testing-overview.rst
  15 This will only print the status of the specified file.
  16
  17 The output is something like:
  18 Documentation/dev-tools/kfence.rst
  19 No translation in the locale of zh_CN
  20
  21 Documentation/translations/zh_CN/dev-tools/testing-overview.rst
  22 commit 42fb9cfd5b18 ("Documentation: dev-tools: Add link to RV docs")
  23 1 commits needs resolving in total
  24 """
  25
  26 import os
  27 import time
  28 import logging
  29 from argparse import ArgumentParser, ArgumentTypeError, BooleanOptionalAction
  30 from datetime import datetime
  31
  32
  33 def get_origin_path(file_path):
  34     """Get the origin path from the translation path"""
  35     paths = file_path.split("/")
  36     tidx = paths.index("translations")
  37     opaths = paths[:tidx]
  38     opaths += paths[tidx + 2 :]
  39     return "/".join(opaths)
  40
  41
  42 def get_latest_commit_from(file_path, commit):
  43     """Get the latest commit from the specified commit for the specified file"""
  44     command = f"git log --pretty=format:%H%n%aD%n%cD%n%n%B {commit} -1 -- {file_path}"
  45     logging.debug(command)
  46     pipe = os.popen(command)
  47     result = pipe.read()
  48     result = result.split("\n")
  49     if len(result) <= 1:
  50         return None
  51
  52     logging.debug("Result: %s", result[0])
  53
  54     return {
  55         "hash": result[0],
  56         "author_date": datetime.strptime(result[1], "%a, %d %b %Y %H:%M:%S %z"),
  57         "commit_date": datetime.strptime(result[2], "%a, %d %b %Y %H:%M:%S %z"),
  58         "message": result[4:],
  59     }
  60
  61
  62 def get_origin_from_trans(origin_path, t_from_head):
  63     """Get the latest origin commit from the translation commit"""
  64     o_from_t = get_latest_commit_from(origin_path, t_from_head["hash"])
  65     while o_from_t is not None and o_from_t["author_date"] > t_from_head["author_date"]:
  66         o_from_t = get_latest_commit_from(origin_path, o_from_t["hash"] + "^")
  67     if o_from_t is not None:
  68         logging.debug("tracked origin commit id: %s", o_from_t["hash"])
  69     return o_from_t
  70
  71
  72 def get_commits_count_between(opath, commit1, commit2):
  73     """Get the commits count between two commits for the specified file"""
  74     command = f"git log --pretty=format:%H {commit1}...{commit2} -- {opath}"
  75     logging.debug(command)
  76     pipe = os.popen(command)
  77     result = pipe.read().split("\n")
  78     # filter out empty lines
  79     result = list(filter(lambda x: x != "", result))
  80     return result
  81
  82
  83 def pretty_output(commit):
  84     """Pretty print the commit message"""
  85     command = f"git log --pretty='format:%h (\"%s\")' -1 {commit}"
  86     logging.debug(command)
  87     pipe = os.popen(command)
  88     return pipe.read()
  89
  90
  91 def valid_commit(commit):
  92     """Check if the commit is valid or not"""
  93     msg = pretty_output(commit)
  94     return "Merge tag" not in msg
  95
  96 def check_per_file(file_path):
  97     """Check the translation status for the specified file"""
  98     opath = get_origin_path(file_path)
  99
 100     if not os.path.isfile(opath):
 101         logging.error("Cannot find the origin path for {file_path}")
 102         return
 103
 104     o_from_head = get_latest_commit_from(opath, "HEAD")
 105     t_from_head = get_latest_commit_from(file_path, "HEAD")
 106
 107     if o_from_head is None or t_from_head is None:
 108         logging.error("Cannot find the latest commit for %s", file_path)
 109         return
 110
 111     o_from_t = get_origin_from_trans(opath, t_from_head)
 112
 113     if o_from_t is None:
 114         logging.error("Error: Cannot find the latest origin commit for %s", file_path)
 115         return
 116
 117     if o_from_head["hash"] == o_from_t["hash"]:
 118         logging.debug("No update needed for %s", file_path)
 119     else:
 120         logging.info(file_path)
 121         commits = get_commits_count_between(
 122             opath, o_from_t["hash"], o_from_head["hash"]
 123         )
 124         count = 0
 125         for commit in commits:
 126             if valid_commit(commit):
 127                 logging.info("commit %s", pretty_output(commit))
 128                 count += 1
 129         logging.info("%d commits needs resolving in total\n", count)
 130
 131
 132 def valid_locales(locale):
 133     """Check if the locale is valid or not"""
 134     script_path = os.path.dirname(os.path.abspath(__file__))
 135     linux_path = os.path.join(script_path, "..")
 136     if not os.path.isdir(f"{linux_path}/Documentation/translations/{locale}"):
 137         raise ArgumentTypeError("Invalid locale: {locale}")
 138     return locale
 139
 140
 141 def list_files_with_excluding_folders(folder, exclude_folders, include_suffix):
 142     """List all files with the specified suffix in the folder and its subfolders"""
 143     files = []
 144     stack = [folder]
 145
 146     while stack:
 147         pwd = stack.pop()
 148         # filter out the exclude folders
 149         if os.path.basename(pwd) in exclude_folders:
 150             continue
 151         # list all files and folders
 152         for item in os.listdir(pwd):
 153             ab_item = os.path.join(pwd, item)
 154             if os.path.isdir(ab_item):
 155                 stack.append(ab_item)
 156             else:
 157                 if ab_item.endswith(include_suffix):
 158                     files.append(ab_item)
 159
 160     return files
 161
 162
 163 class DmesgFormatter(logging.Formatter):
 164     """Custom dmesg logging formatter"""
 165     def format(self, record):
 166         timestamp = time.time()
 167         formatted_time = f"[{timestamp:>10.6f}]"
 168         log_message = f"{formatted_time} {record.getMessage()}"
 169         return log_message
 170
 171
 172 def config_logging(log_level, log_file="checktransupdate.log"):
 173     """configure logging based on the log level"""
 174     # set up the root logger
 175     logger = logging.getLogger()
 176     logger.setLevel(log_level)
 177
 178     # Create console handler
 179     console_handler = logging.StreamHandler()
 180     console_handler.setLevel(log_level)
 181
 182     # Create file handler
 183     file_handler = logging.FileHandler(log_file)
 184     file_handler.setLevel(log_level)
 185
 186     # Create formatter and add it to the handlers
 187     formatter = DmesgFormatter()
 188     console_handler.setFormatter(formatter)
 189     file_handler.setFormatter(formatter)
 190
 191     # Add the handler to the logger
 192     logger.addHandler(console_handler)
 193     logger.addHandler(file_handler)
 194
 195
 196 def main():
 197     """Main function of the script"""
 198     script_path = os.path.dirname(os.path.abspath(__file__))
 199     linux_path = os.path.join(script_path, "..")
 200
 201     parser = ArgumentParser(description="Check the translation update")
 202     parser.add_argument(
 203         "-l",
 204         "--locale",
 205         default="zh_CN",
 206         type=valid_locales,
 207         help="Locale to check when files are not specified",
 208     )
 209
 210     parser.add_argument(
 211         "--print-missing-translations",
 212         action=BooleanOptionalAction,
 213         default=True,
 214         help="Print files that do not have translations",
 215     )
 216
 217     parser.add_argument(
 218         '--log',
 219         default='INFO',
 220         choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'],
 221         help='Set the logging level')
 222
 223     parser.add_argument(
 224         '--logfile',
 225         default='checktransupdate.log',
 226         help='Set the logging file (default: checktransupdate.log)')
 227
 228     parser.add_argument(
 229         "files", nargs="*", help="Files to check, if not specified, check all files"
 230     )
 231     args = parser.parse_args()
 232
 233     # Configure logging based on the --log argument
 234     log_level = getattr(logging, args.log.upper(), logging.INFO)
 235     config_logging(log_level)
 236
 237     # Get files related to linux path
 238     files = args.files
 239     if len(files) == 0:
 240         offical_files = list_files_with_excluding_folders(
 241             os.path.join(linux_path, "Documentation"), ["translations", "output"], "rst"
 242         )
 243
 244         for file in offical_files:
 245             # split the path into parts
 246             path_parts = file.split(os.sep)
 247             # find the index of the "Documentation" directory
 248             kindex = path_parts.index("Documentation")
 249             # insert the translations and locale after the Documentation directory
 250             new_path_parts = path_parts[:kindex + 1] + ["translations", args.locale] \
 251                            + path_parts[kindex + 1 :]
 252             # join the path parts back together
 253             new_file = os.sep.join(new_path_parts)
 254             if os.path.isfile(new_file):
 255                 files.append(new_file)
 256             else:
 257                 if args.print_missing_translations:
 258                     logging.info(os.path.relpath(os.path.abspath(file), linux_path))
 259                     logging.info("No translation in the locale of %s\n", args.locale)
 260
 261     files = list(map(lambda x: os.path.relpath(os.path.abspath(x), linux_path), files))
 262
 263     # cd to linux root directory
 264     os.chdir(linux_path)
 265
 266     for file in files:
 267         check_per_file(file)
 268
 269
 270 if __name__ == "__main__":
 271     main()