bin/find-can-be-private-symbols.py

   1 #!/usr/bin/python3
   2 #
   3 # Find exported symbols that can be made non-exported.
   4 #
   5 # Noting that (a) parsing these commands is a pain, the output is quite irregular and (b) I'm fumbling in the
   6 # dark here, trying to guess what exactly constitutes an "import" vs an "export" of a symbol, linux linking
   7 # is rather complex.
   8 #
   9 # Takes about 5min to run on a decent machine.
  10 #
  11 # The standalone function analysis is reasonable reliable, but the class/method analysis is less so
  12 #   (something to do with destructor thunks not showing up in my results?)
  13 #
  14 # Also, the class/method analysis will not catch problems like
  15 #    'dynamic_cast from 'Foo' with hidden type visibility to 'Bar' with default type visibility'
  16 #    but loplugin:dyncastvisibility will do that for you
  17 #
  18
  19 import subprocess
  20 import sys
  21 import re
  22
  23 exported_symbols = set()
  24 imported_symbols = set()
  25 # standalone functions that are exported but not imported
  26 unused_function_exports = set()
  27 classes_with_exported_symbols = set()
  28 classes_with_imported_symbols = set()
  29 # all names that exist in the source code
  30 all_source_names = set()
  31
  32
  33 subprocess_find_all_source_names = subprocess.Popen("git grep -oh -P '\\b\\w\\w\\w+\\b' -- '*.h*' | sort -u", stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True)
  34 with subprocess_find_all_source_names.stdout as txt:
  35     for line in txt:
  36         line = line.strip()
  37         all_source_names.add(line)
  38 subprocess_find_all_source_names.terminate()
  39
  40 # find all our shared libs
  41 subprocess_find = subprocess.Popen("find ./instdir -name *.so && find ./workdir/LinkTarget/CppunitTest -name *.so", stdout=subprocess.PIPE, shell=True)
  42 with subprocess_find.stdout as txt:
  43     for line in txt:
  44         sharedlib = line.strip()
  45         # look for exported symbols
  46         subprocess_nm = subprocess.Popen(b"nm -D " + sharedlib, stdout=subprocess.PIPE, shell=True)
  47         with subprocess_nm.stdout as txt2:
  48             # We are looking for lines something like:
  49             # 0000000000036ed0 T flash_component_getFactory
  50             line_regex = re.compile(r'^[0-9a-fA-F]+ T ')
  51             for line2_bytes in txt2:
  52                 line2 = line2_bytes.strip().decode("utf-8")
  53                 if line_regex.match(line2):
  54                     sym = line2.split(" ")[2]
  55                     exported_symbols.add(sym)
  56         # look for imported symbols
  57         subprocess_objdump = subprocess.Popen(b"objdump -T " + sharedlib, stdout=subprocess.PIPE, shell=True)
  58         with subprocess_objdump.stdout as txt2:
  59             # ignore some header bumpf
  60             txt2.readline()
  61             txt2.readline()
  62             txt2.readline()
  63             txt2.readline()
  64             # We are looking for lines something like:
  65             # 0000000000000000      DF *UND*  0000000000000000     _ZN16FilterConfigItem10WriteInt32ERKN3rtl8OUStringEi
  66             for line2_bytes in txt2:
  67                 line2 = line2_bytes.strip().decode("utf-8")
  68                 tokens = line2.split(" ")
  69                 if len(tokens) < 7 or not(tokens[7].startswith("*UND*")): continue
  70                 sym = tokens[len(tokens)-1]
  71                 imported_symbols.add(sym)
  72 subprocess_find.terminate()
  73
  74 # look for imported symbols in executables
  75 subprocess_find = subprocess.Popen("find ./instdir -name *.bin", stdout=subprocess.PIPE, shell=True)
  76 with subprocess_find.stdout as txt:
  77     for line in txt:
  78         executable = line.strip()
  79         # look for exported symbols
  80         subprocess_nm = subprocess.Popen(b"nm -D " + executable + b" | grep -w U", stdout=subprocess.PIPE, shell=True)
  81         with subprocess_nm.stdout as txt2:
  82             # We are looking for lines something like:
  83             # U sal_detail_deinitialize
  84             for line2_bytes in txt2:
  85                 line2 = line2_bytes.strip().decode("utf-8")
  86                 sym = line2.split(" ")[1]
  87                 imported_symbols.add(sym)
  88 subprocess_find.terminate()
  89
  90 diff = exported_symbols - imported_symbols
  91 print("exported = " + str(len(exported_symbols)))
  92 print("imported = " + str(len(imported_symbols)))
  93 print("diff     = " + str(len(diff)))
  94
  95 for sym in exported_symbols:
  96     filtered_sym = subprocess.check_output(["c++filt", sym]).strip().decode("utf-8")
  97     if filtered_sym.startswith("non-virtual thunk to "): filtered_sym = filtered_sym[21:]
  98     elif filtered_sym.startswith("virtual thunk to "): filtered_sym = filtered_sym[17:]
  99     i = filtered_sym.find("(")
 100     i = filtered_sym.rfind("::", 0, i)
 101     if i != -1:
 102         classname = filtered_sym[:i]
 103         # find classes where all of the exported symbols are not imported
 104         classes_with_exported_symbols.add(classname)
 105     else:
 106         func = filtered_sym
 107         # find standalone functions which are exported but not imported
 108         if not(sym in imported_symbols): unused_function_exports.add(func)
 109
 110 for sym in imported_symbols:
 111     filtered_sym = subprocess.check_output(["c++filt", sym]).strip().decode("utf-8")
 112     if filtered_sym.startswith("non-virtual thunk to "): filtered_sym = filtered_sym[21:]
 113     elif filtered_sym.startswith("virtual thunk to "): filtered_sym = filtered_sym[17:]
 114     i = filtered_sym.find("(")
 115     i = filtered_sym.rfind("::", 0, i)
 116     if i != -1:
 117         classname = filtered_sym[:i]
 118         classes_with_imported_symbols.add(classname)
 119
 120 def extractFunctionNameFromSignature(sym):
 121     i = sym.find("(")
 122     if i == -1: return sym
 123     return sym[:i]
 124
 125 with open("bin/find-can-be-private-symbols.functions.results", "wt") as f:
 126     for sym in sorted(unused_function_exports):
 127         # Filter out most of the noise.
 128         # No idea where these are coming from, but not our code.
 129         if sym.startswith("CERT_"): continue
 130         elif sym.startswith("DER_"): continue
 131         elif sym.startswith("FORM_"): continue
 132         elif sym.startswith("FPDF"): continue
 133         elif sym.startswith("HASH_"): continue
 134         elif sym.startswith("Hunspell_"): continue
 135         elif sym.startswith("LL_"): continue
 136         elif sym.startswith("LP_"): continue
 137         elif sym.startswith("LU"): continue
 138         elif sym.startswith("MIP"): continue
 139         elif sym.startswith("MPS"): continue
 140         elif sym.startswith("NSS"): continue
 141         elif sym.startswith("NSC_"): continue
 142         elif sym.startswith("PK11"): continue
 143         elif sym.startswith("PL_"): continue
 144         elif sym.startswith("PQ"): continue
 145         elif sym.startswith("PBE_"): continue
 146         elif sym.startswith("PORT_"): continue
 147         elif sym.startswith("PRP_"): continue
 148         elif sym.startswith("PR_"): continue
 149         elif sym.startswith("PT_"): continue
 150         elif sym.startswith("QS_"): continue
 151         elif sym.startswith("REPORT_"): continue
 152         elif sym.startswith("RSA_"): continue
 153         elif sym.startswith("SEC"): continue
 154         elif sym.startswith("SGN"): continue
 155         elif sym.startswith("SOS"): continue
 156         elif sym.startswith("SSL_"): continue
 157         elif sym.startswith("VFY_"): continue
 158         elif sym.startswith("_PR_"): continue
 159         elif sym.startswith("_"): continue
 160         elif sym.startswith("ber_"): continue
 161         elif sym.startswith("bfp_"): continue
 162         elif sym.startswith("ldap_"): continue
 163         elif sym.startswith("ne_"): continue
 164         elif sym.startswith("opj_"): continue
 165         elif sym.startswith("pg_"): continue
 166         elif sym.startswith("pq"): continue
 167         elif sym.startswith("presolve_"): continue
 168         elif sym.startswith("sqlite3_"): continue
 169         # dynamically loaded
 170         elif sym.endswith("get_implementation"): continue
 171         elif sym.endswith("component_getFactory"): continue
 172         elif sym == "CreateDialogFactory": continue
 173         elif sym == "CreateUnoWrapper": continue
 174         elif sym == "CreateWindow": continue
 175         elif sym == "ExportDOC": continue
 176         elif sym == "ExportPPT": continue
 177         elif sym == "ExportRTF": continue
 178         elif sym == "GetSaveWarningOfMSVBAStorage_ww8": continue
 179         elif sym == "GetSpecialCharsForEdit": continue
 180         elif sym.startswith("Import"): continue
 181         elif sym.startswith("Java_com_sun_star_"): continue
 182         elif sym.startswith("TestImport"): continue
 183         elif sym.startswith("getAllCalendars_"): continue
 184         elif sym.startswith("getAllCurrencies_"): continue
 185         elif sym.startswith("getAllFormats"): continue
 186         elif sym.startswith("getBreakIteratorRules_"): continue
 187         elif sym.startswith("getCollationOptions_"): continue
 188         elif sym.startswith("getCollatorImplementation_"): continue
 189         elif sym.startswith("getContinuousNumberingLevels_"): continue
 190         elif sym.startswith("getDateAcceptancePatterns_"): continue
 191         elif sym.startswith("getForbiddenCharacters_"): continue
 192         elif sym.startswith("getIndexAlgorithm_"): continue
 193         elif sym.startswith("getLCInfo_"): continue
 194         elif sym.startswith("getLocaleItem_"): continue
 195         elif sym.startswith("getOutlineNumberingLevels_"): continue
 196         elif sym.startswith("getReservedWords_"): continue
 197         elif sym.startswith("getSTC_"): continue
 198         elif sym.startswith("getSearchOptions_"): continue
 199         elif sym.startswith("getTransliterations_"): continue
 200         elif sym.startswith("getUnicodeScripts_"): continue
 201         elif sym.startswith("lok_"): continue
 202         # UDK API
 203         elif sym.startswith("osl_"): continue
 204         elif sym.startswith("rtl_"): continue
 205         elif sym.startswith("typelib_"): continue
 206         elif sym.startswith("typereg_"): continue
 207         elif sym.startswith("uno_"): continue
 208         # remove things we found that do not exist in our source code, they're not ours
 209         if not(extractFunctionNameFromSignature(sym) in all_source_names): continue
 210         f.write(sym + "\n")
 211
 212 with open("bin/find-can-be-private-symbols.classes.results", "wt") as f:
 213     for sym in sorted(classes_with_exported_symbols - classes_with_imported_symbols):
 214         # externals
 215         if sym.startswith("libcdr"): continue
 216         elif sym.startswith("libabw"): continue
 217         elif sym.startswith("libebook"): continue
 218         elif sym.startswith("libepubgen"): continue
 219         elif sym.startswith("libfreehand"): continue
 220         elif sym.startswith("libmspub"): continue
 221         elif sym.startswith("libpagemaker"): continue
 222         elif sym.startswith("libqxp"): continue
 223         elif sym.startswith("libvisio"): continue
 224         elif sym.startswith("libzmf"): continue
 225         elif sym.startswith("lucene::"): continue
 226         elif sym.startswith("Sk"): continue
 227         f.write(sym + "\n")