compilerplugins/clang/pahole-all-classes.py

   1 #!/usr/bin/python3
   2 #
   3 # Find holes in structures, so that we can pack them and improve our memory density.
   4 #
   5 # In order to make this work, you need to
   6 # (1) Be operating in a workspace where you have a __NON-DEBUG__ build of LibreOffice, but __WITH SYMBOLS__.
   7 #     (A debug build has different sizes for some things in the standard library.)
   8 # (2) First run the unusedfields loplugin to generate a log file
   9 # (3) Install the pahole stuff into your gdb, I used this one:
  10 #     https://github.com/PhilArmstrong/pahole-gdb
  11 # (4) Run the script
  12 #     ./compilerplugins/clang/pahole-all-classes.py
  13 #
  14
  15 import _thread
  16 import io
  17 import os
  18 import subprocess
  19 import re
  20
  21 # search for all the class names in the file produced by the unusedfields loplugin
  22 #a = subprocess.Popen("grep 'definition:' workdir/loplugin.unusedfields.log | sort -u", stdout=subprocess.PIPE, shell=True)
  23 a = subprocess.Popen("cat n1", stdout=subprocess.PIPE, shell=True)
  24
  25 classSet = set()
  26 classSourceLocDict = dict()
  27 locToClassDict = dict()
  28 with a.stdout as txt:
  29     for line in txt:
  30         tokens = line.decode('utf8').strip().split("\t")
  31         className = tokens[2].strip()
  32         srcLoc = tokens[5].strip()
  33         # ignore things like unions
  34         if "anonymous" in className: continue
  35         # ignore duplicates
  36         if className in classSet: continue
  37         classSet.add(className)
  38         classSourceLocDict[className] = srcLoc
  39         locToClassDict[srcLoc] = className
  40 a.terminate()
  41
  42 # Some of the pahole commands are going to fail, and I cannot read the error stream and the input stream
  43 # together because python has no way of (easily) doing a non-blocking read.
  44 # So I have to write the commands out using a background thread, and then read the entire resulting
  45 # stream out below.
  46 def write_pahole_commands(classes):
  47     for className in classes:
  48         stdin.write("echo " + className + " " + classSourceLocDict[className] + "\n")
  49         stdin.write("pahole " + className + "\n")
  50         stdin.flush()
  51     stdin.write("echo all-done\n")
  52     stdin.flush()
  53     stdin.close() # only way to make it flush the last echo command
  54
  55 # Use generator because lines often end up merged together in gdb's output, and we need
  56 # to split them up, and that creates a mess in the parsing logic.
  57 def read_generator(gdbOutput):
  58     while True:
  59         line = gdbOutput.readline();
  60         if line == "": return # end of file
  61         line = line.decode('utf8').strip()
  62         print("gdb: " + line)
  63         for split in line.split("(gdb)"):
  64             split = split.strip()
  65             if len(split) == 0: continue
  66             if "all-done" in split: return
  67             yield split
  68
  69 # build list of classes sorted by source location to increase the chances of
  70 # processing stuff stored in the same DSO together
  71 sortedLocs = sorted(locToClassDict.keys())
  72 classList = list()
  73 for src in sortedLocs:
  74     if "/inc/" in src or "include/" in src:
  75         classList.append(locToClassDict[src])
  76
  77 with open("compilerplugins/clang/pahole.results", "wt") as f:
  78     # Process 400 classes at a time, otherwise gdb's memory usage blows up and kills the machine
  79     # This number is chosen to make gdb peak at around 8G.
  80     while len(classList) > 0:
  81
  82         currClassList = classList[0:500];
  83         classList = classList[500:]
  84
  85         gdbProc = subprocess.Popen("gdb", stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True)
  86
  87         stdin = io.TextIOWrapper(gdbProc.stdin, 'utf-8')
  88
  89         # make gdb load all the debugging info
  90         stdin.write("set confirm off\n")
  91         # make gdb not wrap output and mess up my parsing
  92         stdin.write("set width unlimited\n")
  93         for filename in sorted(os.listdir('instdir/program')):
  94             if filename.endswith(".so"):
  95                 stdin.write("add-symbol-file instdir/program/" + filename + "\n")
  96         stdin.flush()
  97
  98
  99         _thread.start_new_thread( write_pahole_commands, (currClassList,) )
 100
 101         firstLineRegex = re.compile(r"/\*\s+(\d+)\s+\*/ struct") # /* 16 */ struct Foo
 102         fieldLineRegex = re.compile(r"/\*\s+(\d+)\s+(\d+)\s+\*/ ") # /* 12 8 */ class rtl::OUString aName
 103         holeLineRegex = re.compile(r"/\* XXX (\d+) bit hole, try to pack \*/")
 104         # sometimes pahole can't determine the size of a sub-struct, and then it returns bad data
 105         bogusLineRegex = re.compile(r"/\*\s+\d+\s+0\s+\*/")
 106         structLines = list()
 107         foundHole = False
 108         cumulativeHoleBits = 0
 109         alignedStructSize = 0
 110         foundBogusLine = False
 111         # pahole doesn't report space at the end of the structure, so work it out myself
 112         sizeOfStructWithoutPadding = 0
 113         for line in read_generator(gdbProc.stdout):
 114             structLines.append(line)
 115             firstLineMatch = firstLineRegex.match(line)
 116             if firstLineMatch:
 117                 alignedStructSize = int(firstLineMatch.group(1))
 118                 structLines.clear()
 119                 structLines.append(line)
 120             holeLineMatch = holeLineRegex.match(line)
 121             if holeLineMatch:
 122                 foundHole = True
 123                 cumulativeHoleBits += int(holeLineMatch.group(1))
 124             fieldLineMatch = fieldLineRegex.match(line)
 125             if fieldLineMatch:
 126                 fieldPosInBytes = int(fieldLineMatch.group(1))
 127                 fieldSizeInBytes = int(fieldLineMatch.group(2))
 128                 sizeOfStructWithoutPadding = fieldPosInBytes + fieldSizeInBytes
 129             if bogusLineRegex.match(line):
 130                 foundBogusLine = True
 131             if line == "}":
 132                 # Ignore very large structs, packing those is not going to help much, and
 133                 # re-organising them can make them much less readable.
 134                 if foundHole and len(structLines) < 16 and alignedStructSize < 100 and not foundBogusLine:
 135                     # Verify that, after packing, and compiler alignment, the new structure will be actually smaller.
 136                     # Sometimes, we can save space, but the compiler will align the structure such that we don't
 137                     # actually save any space.
 138                     # TODO improve detection of the required alignment for a structure
 139                     holeAtEnd = alignedStructSize - sizeOfStructWithoutPadding
 140                     potentialSpace = (cumulativeHoleBits / 8) + holeAtEnd
 141                     if potentialSpace >= 8:
 142                         for line in structLines:
 143                             f.write(line + "\n")
 144                         if holeAtEnd > 0:
 145                             f.write("hole at end of struct: " + str(holeAtEnd) + "\n")
 146                         f.write("\n")
 147                 #  reset state
 148                 structLines.clear()
 149                 foundHole = False
 150                 cumulativeHoleBits = 0
 151                 structSize = 0
 152                 foundBogusLine = False
 153                 actualStructSize = 0
 154
 155         gdbProc.terminate()