compilerplugins/clang/pahole-all-classes.py

   1 #!/usr/bin/python3
   2 #
   3 # Find holes in structures, so that we can pack them and improve our memory density.
   4 #
   5 # In order to make this work, you need to
   6 # (1) Be operating in a workspace where you have a __NON-DEBUG__ build of LibreOffice, but __WITH SYMBOLS__.
   7 #     (A debug build has different sizes for some things in the standard library.)
   8 # (2) First run the unusedfields loplugin to generate a log file
   9 # (3) Install the pahole stuff into your gdb, I used this one:
  10 #     https://github.com/PhilArmstrong/pahole-gdb
  11 # (4) Edit the loop near the top of the script to only produce results for one of our modules.
  12 #     Note that this will make GDB soak up about 8G of RAM, which is why I don't do more than one module at a time
  13 # (5) Run the script
  14 #     ./compilerplugins/clang/pahole-all-classes.py > ./compilerplugins/clang/pahole.results
  15 #
  16
  17 import _thread
  18 import io
  19 import os
  20 import subprocess
  21 import time
  22 import re
  23
  24 # search for all the class names in the file produced by the unusedfields loplugin
  25 #a = subprocess.Popen("grep 'definition:' workdir/loplugin.unusedfields.log | sort -u", stdout=subprocess.PIPE, shell=True)
  26 a = subprocess.Popen("cat n1", stdout=subprocess.PIPE, shell=True)
  27
  28 classSet = set()
  29 classSourceLocDict = dict()
  30 with a.stdout as txt:
  31     for line in txt:
  32         tokens = line.decode('utf8').strip().split("\t")
  33         className = tokens[2].strip()
  34         srcLoc = tokens[5].strip()
  35         # ignore things like unions
  36         if "anonymous" in className: continue
  37         # ignore duplicates
  38         if className in classSet: continue
  39         classSet.add(className)
  40         classSourceLocDict[className] = srcLoc
  41 a.terminate()
  42
  43 # Some of the pahole commands are going to fail, and I cannot read the error stream and the input stream
  44 # together because python has no way of (easily) doing a non-blocking read.
  45 # So I have to write the commands out using a background thread, and then read the entire resulting
  46 # stream out below.
  47 def write_pahole_commands(classes):
  48     for className in classes:
  49         stdin.write("echo " + className + " " + classSourceLocDict[className] + "\n")
  50         stdin.write("pahole " + className + "\n")
  51         stdin.flush()
  52     stdin.write("echo all-done\n")
  53     stdin.flush()
  54     stdin.close() # only way to make it flush the last echo command
  55
  56 # Use generator because lines often end up merged together in gdb's output, and we need
  57 # to split them up, and that creates a mess in the parsing logic.
  58 def read_generator(gdbOutput):
  59     while True:
  60         line = gdbOutput.readline().decode('utf8').strip()
  61         for split in line.split("(gdb)"):
  62             split = split.strip()
  63             if len(split) == 0: continue
  64             if "all-done" in split: return
  65             yield split
  66
  67 classList = sorted(classSet)
  68
  69 # Process 200 classes at a time, otherwise gdb's memory usage blows up and kills the machine
  70 #
  71 while len(classList) > 0:
  72
  73     currClassList = classList[1:200];
  74     classList = classList[200:]
  75
  76     gdbProc = subprocess.Popen("gdb", stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True)
  77
  78     stdin = io.TextIOWrapper(gdbProc.stdin, 'utf-8')
  79
  80     # make gdb load all the debugging info
  81     stdin.write("set confirm off\n")
  82     for filename in sorted(os.listdir('instdir/program')):
  83         if filename.endswith(".so"):
  84             stdin.write("add-symbol-file instdir/program/" + filename + "\n")
  85     stdin.flush()
  86
  87
  88     _thread.start_new_thread( write_pahole_commands, (currClassList,) )
  89
  90     firstLineRegex = re.compile("/\*\s+(\d+)\s+\*/ struct")
  91     fieldLineRegex = re.compile("/\*\s+(\d+)\s+(\d+)\s+\*/ ")
  92     holeLineRegex = re.compile("/\* XXX (\d+) bit hole, try to pack \*/")
  93     # sometimes pahole can't determine the size of a sub-struct, and then it returns bad data
  94     bogusLineRegex = re.compile("/\*\s+\d+\s+0\s+\*/")
  95     structLines = list()
  96     foundHole = False
  97     cumulativeHoleBits = 0
  98     structSize = 0
  99     foundBogusLine = False
 100     # pahole doesn't report space at the end of the structure, so work it out myself
 101     sizeOfFields = 0
 102     for line in read_generator(gdbProc.stdout):
 103         structLines.append(line)
 104         firstLineMatch = firstLineRegex.match(line)
 105         if firstLineMatch:
 106             structSize = int(firstLineMatch.group(1))
 107         holeLineMatch = holeLineRegex.match(line)
 108         if holeLineMatch:
 109             foundHole = True
 110             cumulativeHoleBits += int(holeLineMatch.group(1))
 111         fieldLineMatch = fieldLineRegex.match(line)
 112         if fieldLineMatch:
 113             fieldSize = int(fieldLineMatch.group(2))
 114             sizeOfFields = int(fieldLineMatch.group(1)) + fieldSize
 115         if bogusLineRegex.match(line):
 116             foundBogusLine = True
 117         if line == "}":
 118             # Ignore very large structs, packing those is not going to help much, and
 119             # re-organising them can make them much less readable.
 120             if foundHole and len(structLines) < 12 and structSize < 100 and not foundBogusLine:
 121                 # Verify that we have enough hole-space that removing it will result in a structure
 122                 # that still satisfies alignment requirements, otherwise the compiler will just put empty
 123                 # space at the end of the struct.
 124                 # TODO improve detection of the required alignment for a structure
 125                 potentialSpace = (cumulativeHoleBits / 8) + (sizeOfFields - structSize)
 126                 if potentialSpace >= 8:
 127                     for line in structLines:
 128                         print(line)
 129                     if (sizeOfFields - structSize) > 0:
 130                         print("hole at end of struct: " + str(sizeOfFields - structSize))
 131             #  reset state
 132             structLines.clear()
 133             foundHole = False
 134             cumulativeHoleBits = 0
 135             structSize = 0
 136             foundBogusLine = False
 137             actualStructSize = 0
 138
 139     gdbProc.terminate()