bin/test-hid-vs-ui.py

   1 #!/usr/bin/env python
   2 # -*- tab-width: 4; indent-tabs-mode: nil; py-indent-offset: 4 -*-
   3 #
   4 # This file is part of the LibreOffice project.
   5 #
   6 # This Source Code Form is subject to the terms of the Mozilla Public
   7 # License, v. 2.0. If a copy of the MPL was not distributed with this
   8 # file, you can obtain one at http://mozilla.org/MPL/2.0/.
   9 #
  10 # Parses all help files (.xhp) to check that hids referencing .ui are up-to-date
  11 # From fdo#67350
  12
  13
  14 import sys
  15 import argparse
  16 import os
  17 import subprocess
  18 import xml.etree.ElementTree as ET
  19 import collections
  20 import re
  21 import smtplib
  22 import email
  23 import email.mime.text
  24 import time
  25 import datetime
  26
  27 # retrieve all hids related to .ui files
  28 def init_hids():
  29     global args, local_repo
  30     if local_repo:
  31         repo_dir = os.path.join(core_repo_dir,'helpcontent2')
  32         os.chdir(repo_dir)
  33         return subprocess.check_output(['git','grep','hid="[^"]*/[^"]*">','.'])
  34     else:
  35         repo_dir = '/var/tmp/help.git'
  36         if not os.path.exists(repo_dir):
  37             os.makedirs(repo_dir)
  38         os.chdir(repo_dir)
  39
  40         if not os.path.exists(os.path.join(repo_dir,'config')):
  41             subprocess.call(['git','clone','--bare','git://gerrit.libreoffice.org/help',repo_dir])
  42         elif not args['git_static']:
  43             subprocess.call(['git','fetch','origin'])
  44         return subprocess.check_output(['git','grep','hid="[^"]*/[^"]*">','master','--'])
  45
  46 # retrieve .ui files list from the core
  47 def init_core_files():
  48     global core_repo_dir, local_repo
  49     core_repo_dir = args['core_repo_dir']
  50     if core_repo_dir is None:
  51         core_repo_dir = os.path.dirname(os.path.abspath(os.path.dirname(sys.argv[0])))
  52         local_repo = True
  53
  54     if not os.path.exists(core_repo_dir):
  55         os.makedirs(core_repo_dir)
  56     os.chdir(core_repo_dir)
  57
  58     if not os.path.exists(os.path.join(core_repo_dir,'.git')):
  59         subprocess.call(['git','clone','git://gerrit.libreoffice.org/core',core_repo_dir])
  60     elif not args['git_static']:
  61         subprocess.call(['git','fetch','origin'])
  62     allfiles = subprocess.check_output(['git','ls-tree','--name-only','--full-name','-r','master'])
  63     return re.findall(r'.*\.ui',allfiles)
  64
  65
  66 if __name__ == "__main__":
  67
  68     parser = argparse.ArgumentParser('hid for ui consistency parser')
  69     parser.add_argument('-s', '--send-to', action='append', help='email address to send the report to. Use one flag per address.', required=False)
  70     parser.add_argument('-g', '--git-static', action='store_true', help='to avoid contacting remote server to refresh repositories.', required=False)
  71     parser.add_argument('-r', '--core-repo-dir', help='enforce path to core repository when analyzing .ui files.', required=False)
  72     args=vars(parser.parse_args())
  73
  74     uifileslist = init_core_files()  # play it early to gain the local repo identification
  75
  76     rows = init_hids().splitlines()
  77     #<tree>:<relative_file>:<text>
  78     # handled as sets to remove duplicates (and we don't need an iterator)
  79     targets = collections.defaultdict(set)
  80     origin = collections.defaultdict(set)
  81
  82     # fill all matching hids and their parent file
  83     for row in rows:
  84         fname, rawtext = row.split(':',1)[0:]
  85         hid = rawtext.split('hid="')[1].split('"')[0]
  86         if hid.startswith('.uno'):
  87             continue
  88         uifileraw, compname = hid.rsplit('/',1)
  89         uifile = uifileraw + ".ui"
  90         # map modules/ etc, which exist only in install
  91         # back to their source location
  92         if uifile.startswith("modules/scalc"):
  93             uifile = "sc/scalc" + uifile[13:]
  94         elif uifile.startswith("modules/swriter"):
  95             uifile = "sw/swriter" + uifile[15:]
  96         elif uifile.startswith("modules/schart"):
  97             uifile = "chart2" + uifile[14:]
  98         elif uifile.startswith("modules/smath"):
  99             uifile = "starmath/smath" + uifile[13:]
 100         elif uifile.startswith("modules/sdraw"):
 101             uifile = "sd/sdraw" + uifile[13:]
 102         elif uifile.startswith("modules/simpress"):
 103             uifile = "sd/simpress" + uifile[16:]
 104         elif uifile.startswith("modules/BasicIDE"):
 105             uifile = "basctl/basicide" + uifile[16:]
 106         elif uifile.startswith("modules/sabpilot"):
 107             uifile = "extensions/sabpilot" + uifile[16:]
 108         elif uifile.startswith("modules/sbibliography"):
 109             uifile = "extensions/sbibliography" + uifile[21:]
 110         elif uifile.startswith("modules/scanner"):
 111             uifile = "extensions/scanner" + uifile[15:]
 112         elif uifile.startswith("modules/spropctrlr"):
 113             uifile = "extensions/spropctrlr" + uifile[18:]
 114         elif uifile.startswith("sfx"):
 115             uifile = "sfx2" + uifile[3:]
 116         elif uifile.startswith("svt"):
 117             uifile = "svtools" + uifile[3:]
 118         elif uifile.startswith("fps"):
 119             uifile = "fpicker" + uifile[3:]
 120         components = uifile.split('/',1)
 121         uifile = components[0] + '/uiconfig/' + components[1]
 122         targets[uifile].add(compname.split(':')[0])
 123         origin[uifile].add(fname)  # help file(s)
 124
 125     errors = ''
 126     # search in all .ui files referenced in help
 127     # 2 possible errors: file not found in repo, id not found in file
 128     for uikey in dict.keys(targets):
 129         if uikey not in uifileslist:
 130             if len(origin[uikey]) == 1:
 131                 errors += '\nFrom ' + origin[uikey].pop()
 132             else:
 133                 errors += '\nFrom one of ' + str(origin[uikey]).replace('set(','').replace(')','')
 134             errors += ', we did not find file '+ uikey+'.'
 135             continue
 136
 137         full_path = os.path.join(core_repo_dir,uikey)
 138         # print full_path
 139         root = ET.parse(full_path).getroot()
 140         ids = [element.attrib['id'].split(':')[0] for element in root.findall('.//object[@id]')]
 141         # print targets[uikey]
 142         missing_ids = [ element for element in targets[uikey] if element not in ids ]
 143         if missing_ids:
 144             if len(origin[uikey]) == 1:
 145                 errors += '\nFrom ' + origin[uikey].pop()
 146             else:
 147                 errors += '\nFrom one of ' + str(origin[uikey]).replace('set(','').replace(')','')
 148             errors += ', referenced items '+ str(missing_ids) + ' were not found inside '+ uikey+'.'
 149
 150     if not errors:
 151         errors = '\nall is clean\n'
 152
 153     if args['send_to']:
 154         msg_from = os.path.basename(sys.argv[0]) + '@libreoffice.org'
 155         if isinstance(args['send_to'], str):
 156             msg_to = [args['send_to']]
 157         else:
 158             msg_to = args['send_to']
 159             print("send to array " + msg_to[0])
 160
 161         server = smtplib.SMTP('localhost')
 162         body = '''
 163 Hello,
 164
 165 Here is the report for wrong hids from help related to .ui files
 166
 167 '''
 168         body += errors
 169         body += '''
 170
 171 Best,
 172
 173 Your friendly LibreOffice Help-ids Checker
 174
 175 Note: The bot generating this message can be found and improved here:
 176        https://gerrit.libreoffice.org/gitweb?p=dev-tools.git;a=blob;f=scripts/test-hid-vs-ui.py'''
 177         now = datetime.datetime.now()
 178         msg = email.mime.text.MIMEText(body, 'plain', 'UTF-8')
 179         msg['From'] = msg_from
 180         msg['To'] = msg_to[0]
 181         msg['Cc'] = ', '.join(msg_to[1:]) # Works only if at least 2 items in tuple
 182         msg['Date'] = email.utils.formatdate(time.mktime(now.timetuple()))
 183         msg['Subject'] = 'LibreOffice Gerrit News for python on %s' % (now.date().isoformat())
 184         msg['Reply-To'] = msg_to[0]
 185         msg['X-Mailer'] = 'LibreOfficeGerritDigestMailer 1.1'
 186
 187         server.sendmail(msg_from, msg_to, str(msg))
 188     else:
 189         print(errors)
 190
 191 # vim: set shiftwidth=4 softtabstop=4 expandtab: