tools/lint/perfdocs/utils.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4 import difflib
   5 import filecmp
   6 import os
   7 import pathlib
   8
   9 import yaml
  10 from mozversioncontrol import get_repository_object
  11
  12 from perfdocs.logger import PerfDocLogger
  13
  14 logger = PerfDocLogger()
  15
  16 ON_TRY = "MOZ_AUTOMATION" in os.environ
  17
  18
  19 def save_file(file_content, path, extension="rst"):
  20     """
  21     Saves data into a file.
  22
  23     :param str path: Location and name of the file being saved
  24         (without an extension).
  25     :param str data: Content to write into the file.
  26     :param str extension: Extension to save the file as.
  27     """
  28     new_file = pathlib.Path("{}.{}".format(str(path), extension))
  29     with new_file.open("wb") as f:
  30         f.write(file_content.encode("utf-8"))
  31
  32
  33 def read_file(path, stringify=False):
  34     """
  35     Opens a file and returns its contents.
  36
  37     :param str path: Path to the file.
  38     :return list: List containing the lines in the file.
  39     """
  40     with path.open(encoding="utf-8") as f:
  41         return f.read() if stringify else f.readlines()
  42
  43
  44 def read_yaml(yaml_path):
  45     """
  46     Opens a YAML file and returns the contents.
  47
  48     :param str yaml_path: Path to the YAML to open.
  49     :return dict: Dictionary containing the YAML content.
  50     """
  51     contents = {}
  52     try:
  53         with yaml_path.open(encoding="utf-8") as f:
  54             contents = yaml.safe_load(f)
  55     except Exception as e:
  56         logger.warning(
  57             "Error opening file {}: {}".format(str(yaml_path), str(e)), str(yaml_path)
  58         )
  59
  60     return contents
  61
  62
  63 def are_dirs_equal(dir_1, dir_2):
  64     """
  65     Compare two directories to see if they are equal. Files in each
  66     directory are assumed to be equal if their names and contents
  67     are equal.
  68
  69     :param dir_1: First directory path
  70     :param dir_2: Second directory path
  71     :return: True if the directory trees are the same and False otherwise.
  72     """
  73
  74     dirs_cmp = filecmp.dircmp(str(dir_1.resolve()), str(dir_2.resolve()))
  75     if dirs_cmp.left_only or dirs_cmp.right_only or dirs_cmp.funny_files:
  76         logger.log("Some files are missing or are funny.")
  77         for file in dirs_cmp.left_only:
  78             logger.log(f"Missing in existing docs: {file}")
  79         for file in dirs_cmp.right_only:
  80             logger.log(f"Missing in new docs: {file}")
  81         for file in dirs_cmp.funny_files:
  82             logger.log(f"The following file is funny: {file}")
  83         return False
  84
  85     _, mismatch, errors = filecmp.cmpfiles(
  86         str(dir_1.resolve()), str(dir_2.resolve()), dirs_cmp.common_files, shallow=False
  87     )
  88
  89     if mismatch or errors:
  90         logger.log(f"Found mismatches: {mismatch}")
  91
  92         # The root for where to save the diff will be different based on
  93         # whether we are running in CI or not
  94         os_root = pathlib.Path.cwd().anchor
  95         diff_root = pathlib.Path(os_root, "builds", "worker")
  96         if not ON_TRY:
  97             diff_root = pathlib.Path(PerfDocLogger.TOP_DIR, "artifacts")
  98             diff_root.mkdir(parents=True, exist_ok=True)
  99
 100         diff_path = pathlib.Path(diff_root, "diff.txt")
 101         with diff_path.open("w", encoding="utf-8") as diff_file:
 102             for entry in mismatch:
 103                 logger.log(f"Mismatch found on {entry}")
 104
 105                 with pathlib.Path(dir_1, entry).open(encoding="utf-8") as f:
 106                     newlines = f.readlines()
 107                 with pathlib.Path(dir_2, entry).open(encoding="utf-8") as f:
 108                     baselines = f.readlines()
 109                 for line in difflib.unified_diff(
 110                     baselines, newlines, fromfile="base", tofile="new"
 111                 ):
 112                     logger.log(line)
 113
 114                 # Here we want to add to diff.txt in a patch format, we use
 115                 # the basedir to make the file names/paths relative and this is
 116                 # different in CI vs local runs.
 117                 basedir = pathlib.Path(
 118                     os_root, "builds", "worker", "checkouts", "gecko"
 119                 )
 120                 if not ON_TRY:
 121                     basedir = diff_root
 122
 123                 relative_path = str(pathlib.Path(dir_2, entry)).split(str(basedir))[-1]
 124                 patch = difflib.unified_diff(
 125                     baselines, newlines, fromfile=relative_path, tofile=relative_path
 126                 )
 127
 128                 write_header = True
 129                 for line in patch:
 130                     if write_header:
 131                         diff_file.write(
 132                             f"diff --git a/{relative_path} b/{relative_path}\n"
 133                         )
 134                         write_header = False
 135                     diff_file.write(line)
 136
 137                 logger.log(f"Completed diff on {entry}")
 138
 139         logger.log(f"Saved diff to {diff_path}")
 140
 141         return False
 142
 143     for common_dir in dirs_cmp.common_dirs:
 144         subdir_1 = pathlib.Path(dir_1, common_dir)
 145         subdir_2 = pathlib.Path(dir_2, common_dir)
 146         if not are_dirs_equal(subdir_1, subdir_2):
 147             return False
 148
 149     return True
 150
 151
 152 def get_changed_files(top_dir):
 153     """
 154     Returns the changed files found with duplicates removed.
 155     """
 156     repo = get_repository_object(top_dir)
 157     return list(set(repo.get_changed_files() + repo.get_outgoing_files()))