bin/test-hid-vs-ui.py

   1 #!/usr/bin/env python
   2 # -*- Mode: makefile-gmake; tab-width: 4; indent-tabs-mode: t -*-
   3 #
   4 # This file is part of the LibreOffice project.
   5 #
   6 # This Source Code Form is subject to the terms of the Mozilla Public
   7 # License, v. 2.0. If a copy of the MPL was not distributed with this
   8 # file, you can obtain one at http://mozilla.org/MPL/2.0/.
   9 #
  10 # Parses all help files (.xhp) to check that hids referencing .ui are up-to-date
  11 # From fdo#67350
  12
  13
  14 import sys
  15 import argparse
  16 import os
  17 import subprocess
  18 import xml.etree.ElementTree as ET
  19 import collections
  20 import re
  21 import smtplib
  22 import email
  23 import email.mime.text
  24 import time
  25 import datetime
  26
  27 # retrieve all hids related to .ui files
  28 def init_hids():
  29     global args, local_repo
  30     if local_repo:
  31         repo_dir = os.path.join(core_repo_dir,'helpcontent2')
  32         os.chdir(repo_dir)
  33         return subprocess.check_output(['git','grep','hid="[^"]*/[^"]*">','.'])
  34     else:
  35         repo_dir = '/var/tmp/help.git'
  36         if not os.path.exists(repo_dir):os.makedirs(repo_dir)
  37         os.chdir(repo_dir)
  38
  39         if not os.path.exists(os.path.join(repo_dir,'config')):
  40             subprocess.call(['git','clone','--bare','git://gerrit.libreoffice.org/help',repo_dir])
  41         elif not args['git_static']:
  42             subprocess.call(['git','fetch','origin'])
  43         return subprocess.check_output(['git','grep','hid="[^"]*/[^"]*">','master','--'])
  44
  45 # retrieve .ui files list from the core
  46 def init_core_files():
  47     global core_repo_dir, local_repo
  48     core_repo_dir = args['core_repo_dir']
  49     if core_repo_dir is None:
  50         core_repo_dir = os.path.dirname(os.path.abspath(os.path.dirname(sys.argv[0])))
  51         local_repo = True
  52
  53     if not os.path.exists(core_repo_dir):os.makedirs(core_repo_dir)
  54     os.chdir(core_repo_dir)
  55
  56     if not os.path.exists(os.path.join(core_repo_dir,'.git')):
  57         subprocess.call(['git','clone','git://gerrit.libreoffice.org/core',core_repo_dir])
  58     elif not args['git_static']:
  59         subprocess.call(['git','fetch','origin'])
  60     allfiles = subprocess.check_output(['git','ls-tree','--name-only','--full-name','-r','master'])
  61     return re.findall('.*\.ui',allfiles)
  62
  63
  64 if __name__ == "__main__":
  65
  66     parser = argparse.ArgumentParser('hid for ui consistency parser')
  67     parser.add_argument('-s', '--send-to', action='append', help='email address to send the report to. Use one flag per address.', required=False)
  68     parser.add_argument('-g', '--git-static', action='store_true', help='to avoid contacting remote server to refresh repositories.', required=False)
  69     parser.add_argument('-r', '--core-repo-dir', help='enforce path to core repository when analyzing .ui files.', required=False)
  70     args=vars(parser.parse_args())
  71
  72     uifileslist = init_core_files()  # play it early to gain the local repo identification
  73
  74     rows = init_hids().splitlines()
  75     #<tree>:<relative_file>:<text>
  76     # handled as sets to remove duplicates (and we don't need an iterator)
  77     targets = collections.defaultdict(set)
  78     origin = collections.defaultdict(set)
  79
  80     # fill all matching hids and their parent file
  81     for row in rows:
  82         fname, rawtext = row.split(':',1)[0:]
  83         hid = rawtext.split('hid="')[1].split('"')[0]
  84         if hid.startswith('.uno'): continue
  85         uifileraw, compname = hid.rsplit('/',1)
  86         uifile = uifileraw + ".ui"
  87         # map modules/ etc, which exist only in install
  88         # back to their source location
  89         if uifile.startswith("modules/scalc"):
  90             uifile = "sc/scalc" + uifile[13:]
  91         elif uifile.startswith("modules/swriter"):
  92             uifile = "sw/swriter" + uifile[15:]
  93         elif uifile.startswith("modules/schart"):
  94             uifile = "chart2" + uifile[14:]
  95         elif uifile.startswith("modules/smath"):
  96             uifile = "starmath/smath" + uifile[13:]
  97         elif uifile.startswith("modules/sdraw"):
  98             uifile = "sd/sdraw" + uifile[13:]
  99         elif uifile.startswith("modules/simpress"):
 100             uifile = "sd/simpress" + uifile[16:]
 101         elif uifile.startswith("modules/BasicIDE"):
 102             uifile = "basctl/basicide" + uifile[16:]
 103         elif uifile.startswith("modules/sabpilot"):
 104             uifile = "extensions/sabpilot" + uifile[16:]
 105         elif uifile.startswith("modules/sbibliography"):
 106             uifile = "extensions/sbibliography" + uifile[21:]
 107         elif uifile.startswith("modules/scanner"):
 108             uifile = "extensions/scanner" + uifile[15:]
 109         elif uifile.startswith("modules/spropctrlr"):
 110             uifile = "extensions/spropctrlr" + uifile[18:]
 111         elif uifile.startswith("sfx"):
 112             uifile = "sfx2" + uifile[3:]
 113         elif uifile.startswith("svt"):
 114             uifile = "svtools" + uifile[3:]
 115         components = uifile.split('/',1);
 116         uifile = components[0] + '/uiconfig/' + components[1]
 117         targets[uifile].add(compname.split(':')[0])
 118         origin[uifile].add(fname)  # help file(s)
 119
 120     errors = ''
 121     # search in all .ui files referenced in help
 122     # 2 possible errors: file not found in repo, id not found in file
 123     for uikey in dict.keys(targets):
 124         if uikey not in uifileslist:
 125             if len(origin[uikey]) == 1:
 126                 errors += '\nFrom ' + origin[uikey].pop()
 127             else:
 128                 errors += '\nFrom one of ' + str(origin[uikey]).replace('set(','').replace(')','')
 129             errors += ', we did not find file '+ uikey+'.'
 130             continue
 131
 132         full_path = os.path.join(core_repo_dir,uikey)
 133         # print full_path
 134         root = ET.parse(full_path).getroot()
 135         ids = [element.attrib['id'].split(':')[0] for element in root.findall('.//object[@id]')]
 136         # print targets[uikey]
 137         missing_ids = [ element for element in targets[uikey] if element not in ids ]
 138         if missing_ids:
 139             if len(origin[uikey]) == 1:
 140                 errors += '\nFrom ' + origin[uikey].pop()
 141             else:
 142                 errors += '\nFrom one of ' + str(origin[uikey]).replace('set(','').replace(')','')
 143             errors += ', referenced items '+ str(missing_ids) + ' were not found inside '+ uikey+'.'
 144
 145     if not errors:
 146         errors = '\nall is clean\n'
 147
 148     if args['send_to']:
 149         msg_from = os.path.basename(sys.argv[0]) + '@libreoffice.org'
 150         if isinstance(args['send_to'], basestring):
 151             msg_to = [args['send_to']]
 152         else:
 153             msg_to = args['send_to']
 154             print "send to array " + msg_to[0]
 155
 156         server = smtplib.SMTP('localhost')
 157         body = '''
 158 Hello,
 159
 160 Here is the report for wrong hids from help related to .ui files
 161
 162 '''
 163         body += errors
 164         body += '''
 165
 166 Best,
 167
 168 Your friendly LibreOffice Help-ids Checker
 169
 170 Note: The bot generating this message can be found and improved here:
 171        https://gerrit.libreoffice.org/gitweb?p=dev-tools.git;a=blob;f=scripts/test-hid-vs-ui.py'''
 172         now = datetime.datetime.now()
 173         msg = email.mime.text.MIMEText(body, 'plain', 'UTF-8')
 174         msg['From'] = msg_from
 175         msg['To'] = msg_to[0]
 176         msg['Cc'] = ', '.join(msg_to[1:]) # Works only if at least 2 items in tuple
 177         msg['Date'] = email.utils.formatdate(time.mktime(now.timetuple()))
 178         msg['Subject'] = 'LibreOffice Gerrit News for python on %s' % (now.date().isoformat())
 179         msg['Reply-To'] = msg_to[0]
 180         msg['X-Mailer'] = 'LibreOfficeGerritDigestMailer 1.1'
 181
 182         server.sendmail(msg_from, msg_to, str(msg))
 183     else:
 184         print errors
 185
 186 # vim: set et sw=4 ts=4: