bin/test-hid-vs-ui.py

   1 #!/usr/bin/env python
   2 # -*- tab-width: 4; indent-tabs-mode: nil; py-indent-offset: 4 -*-
   3 #
   4 # This file is part of the LibreOffice project.
   5 #
   6 # This Source Code Form is subject to the terms of the Mozilla Public
   7 # License, v. 2.0. If a copy of the MPL was not distributed with this
   8 # file, you can obtain one at http://mozilla.org/MPL/2.0/.
   9 #
  10 # Parses all help files (.xhp) to check that hids referencing .ui are up-to-date
  11 # From fdo#67350
  12
  13
  14 import sys
  15 import argparse
  16 import os
  17 import subprocess
  18 import xml.etree.ElementTree as ET
  19 import collections
  20 import re
  21 import smtplib
  22 import email
  23 import email.mime.text
  24 import time
  25 import datetime
  26
  27 # retrieve all hids related to .ui files
  28 def init_hids():
  29     global args, local_repo
  30     if local_repo:
  31         repo_dir = os.path.join(core_repo_dir,'helpcontent2')
  32         os.chdir(repo_dir)
  33         return subprocess.check_output(['git','grep','hid="[^"]*/[^"]*">','.'])
  34     else:
  35         repo_dir = '/var/tmp/help.git'
  36         if not os.path.exists(repo_dir):os.makedirs(repo_dir)
  37         os.chdir(repo_dir)
  38
  39         if not os.path.exists(os.path.join(repo_dir,'config')):
  40             subprocess.call(['git','clone','--bare','git://gerrit.libreoffice.org/help',repo_dir])
  41         elif not args['git_static']:
  42             subprocess.call(['git','fetch','origin'])
  43         return subprocess.check_output(['git','grep','hid="[^"]*/[^"]*">','master','--'])
  44
  45 # retrieve .ui files list from the core
  46 def init_core_files():
  47     global core_repo_dir, local_repo
  48     core_repo_dir = args['core_repo_dir']
  49     if core_repo_dir is None:
  50         core_repo_dir = os.path.dirname(os.path.abspath(os.path.dirname(sys.argv[0])))
  51         local_repo = True
  52
  53     if not os.path.exists(core_repo_dir):os.makedirs(core_repo_dir)
  54     os.chdir(core_repo_dir)
  55
  56     if not os.path.exists(os.path.join(core_repo_dir,'.git')):
  57         subprocess.call(['git','clone','git://gerrit.libreoffice.org/core',core_repo_dir])
  58     elif not args['git_static']:
  59         subprocess.call(['git','fetch','origin'])
  60     allfiles = subprocess.check_output(['git','ls-tree','--name-only','--full-name','-r','master'])
  61     return re.findall('.*\.ui',allfiles)
  62
  63
  64 if __name__ == "__main__":
  65
  66     parser = argparse.ArgumentParser('hid for ui consistency parser')
  67     parser.add_argument('-s', '--send-to', action='append', help='email address to send the report to. Use one flag per address.', required=False)
  68     parser.add_argument('-g', '--git-static', action='store_true', help='to avoid contacting remote server to refresh repositories.', required=False)
  69     parser.add_argument('-r', '--core-repo-dir', help='enforce path to core repository when analyzing .ui files.', required=False)
  70     args=vars(parser.parse_args())
  71
  72     uifileslist = init_core_files()  # play it early to gain the local repo identification
  73
  74     rows = init_hids().splitlines()
  75     #<tree>:<relative_file>:<text>
  76     # handled as sets to remove duplicates (and we don't need an iterator)
  77     targets = collections.defaultdict(set)
  78     origin = collections.defaultdict(set)
  79
  80     # fill all matching hids and their parent file
  81     for row in rows:
  82         fname, rawtext = row.split(':',1)[0:]
  83         hid = rawtext.split('hid="')[1].split('"')[0]
  84         if hid.startswith('.uno'): continue
  85         uifileraw, compname = hid.rsplit('/',1)
  86         uifile = uifileraw + ".ui"
  87         # map modules/ etc, which exist only in install
  88         # back to their source location
  89         if uifile.startswith("modules/scalc"):
  90             uifile = "sc/scalc" + uifile[13:]
  91         elif uifile.startswith("modules/swriter"):
  92             uifile = "sw/swriter" + uifile[15:]
  93         elif uifile.startswith("modules/schart"):
  94             uifile = "chart2" + uifile[14:]
  95         elif uifile.startswith("modules/smath"):
  96             uifile = "starmath/smath" + uifile[13:]
  97         elif uifile.startswith("modules/sdraw"):
  98             uifile = "sd/sdraw" + uifile[13:]
  99         elif uifile.startswith("modules/simpress"):
 100             uifile = "sd/simpress" + uifile[16:]
 101         elif uifile.startswith("modules/BasicIDE"):
 102             uifile = "basctl/basicide" + uifile[16:]
 103         elif uifile.startswith("modules/sabpilot"):
 104             uifile = "extensions/sabpilot" + uifile[16:]
 105         elif uifile.startswith("modules/sbibliography"):
 106             uifile = "extensions/sbibliography" + uifile[21:]
 107         elif uifile.startswith("modules/scanner"):
 108             uifile = "extensions/scanner" + uifile[15:]
 109         elif uifile.startswith("modules/spropctrlr"):
 110             uifile = "extensions/spropctrlr" + uifile[18:]
 111         elif uifile.startswith("sfx"):
 112             uifile = "sfx2" + uifile[3:]
 113         elif uifile.startswith("svt"):
 114             uifile = "svtools" + uifile[3:]
 115         elif uifile.startswith("fps"):
 116             uifile = "fpicker" + uifile[3:]
 117         components = uifile.split('/',1);
 118         uifile = components[0] + '/uiconfig/' + components[1]
 119         targets[uifile].add(compname.split(':')[0])
 120         origin[uifile].add(fname)  # help file(s)
 121
 122     errors = ''
 123     # search in all .ui files referenced in help
 124     # 2 possible errors: file not found in repo, id not found in file
 125     for uikey in dict.keys(targets):
 126         if uikey not in uifileslist:
 127             if len(origin[uikey]) == 1:
 128                 errors += '\nFrom ' + origin[uikey].pop()
 129             else:
 130                 errors += '\nFrom one of ' + str(origin[uikey]).replace('set(','').replace(')','')
 131             errors += ', we did not find file '+ uikey+'.'
 132             continue
 133
 134         full_path = os.path.join(core_repo_dir,uikey)
 135         # print full_path
 136         root = ET.parse(full_path).getroot()
 137         ids = [element.attrib['id'].split(':')[0] for element in root.findall('.//object[@id]')]
 138         # print targets[uikey]
 139         missing_ids = [ element for element in targets[uikey] if element not in ids ]
 140         if missing_ids:
 141             if len(origin[uikey]) == 1:
 142                 errors += '\nFrom ' + origin[uikey].pop()
 143             else:
 144                 errors += '\nFrom one of ' + str(origin[uikey]).replace('set(','').replace(')','')
 145             errors += ', referenced items '+ str(missing_ids) + ' were not found inside '+ uikey+'.'
 146
 147     if not errors:
 148         errors = '\nall is clean\n'
 149
 150     if args['send_to']:
 151         msg_from = os.path.basename(sys.argv[0]) + '@libreoffice.org'
 152         if isinstance(args['send_to'], basestring):
 153             msg_to = [args['send_to']]
 154         else:
 155             msg_to = args['send_to']
 156             print "send to array " + msg_to[0]
 157
 158         server = smtplib.SMTP('localhost')
 159         body = '''
 160 Hello,
 161
 162 Here is the report for wrong hids from help related to .ui files
 163
 164 '''
 165         body += errors
 166         body += '''
 167
 168 Best,
 169
 170 Your friendly LibreOffice Help-ids Checker
 171
 172 Note: The bot generating this message can be found and improved here:
 173        https://gerrit.libreoffice.org/gitweb?p=dev-tools.git;a=blob;f=scripts/test-hid-vs-ui.py'''
 174         now = datetime.datetime.now()
 175         msg = email.mime.text.MIMEText(body, 'plain', 'UTF-8')
 176         msg['From'] = msg_from
 177         msg['To'] = msg_to[0]
 178         msg['Cc'] = ', '.join(msg_to[1:]) # Works only if at least 2 items in tuple
 179         msg['Date'] = email.utils.formatdate(time.mktime(now.timetuple()))
 180         msg['Subject'] = 'LibreOffice Gerrit News for python on %s' % (now.date().isoformat())
 181         msg['Reply-To'] = msg_to[0]
 182         msg['X-Mailer'] = 'LibreOfficeGerritDigestMailer 1.1'
 183
 184         server.sendmail(msg_from, msg_to, str(msg))
 185     else:
 186         print errors
 187
 188 # vim: set shiftwidth=4 softtabstop=4 expandtab: