maintenance/hiphop/make

   1 #!/usr/bin/hphpi -f
   2 <?php
   3
   4 require( dirname( __FILE__ ) . '/../Maintenance.php' );
   5
   6 class MakeHipHop extends Maintenance {
   7         function execute() {
   8                 global $wgHipHopBuildDirectory;
   9
  10                 $startTime = time();
  11
  12                 $thisDir = realpath( dirname( __FILE__ ) );
  13                 $IP = realpath( "$thisDir/../.." );
  14                 if ( strval( $wgHipHopBuildDirectory ) !== '' ) {
  15                         $buildDir = $wgHipHopBuildDirectory;
  16                 } else {
  17                         $buildDir = "$thisDir/build";
  18                 }
  19                 $extensionsDir = realpath( MWInit::getExtensionsDirectory() );
  20                 $outDir = "$buildDir/hiphop-output";
  21                 $persistentDir = "$buildDir/persistent";
  22
  23                 if ( !is_dir( $buildDir ) ) {
  24                         mkdir( $buildDir, 0777, true );
  25                 }
  26                 if ( !is_dir( $persistentDir ) ) {
  27                         mkdir( $persistentDir, 0777, true );
  28                 }
  29
  30                 if ( realpath( "$IP/../phase3" ) !== $IP
  31                         || realpath( "$IP/../extensions" ) !== $extensionsDir )
  32                 {
  33                         # Set up a fake source directory with the correct layout
  34                         $sourceBase = "$buildDir/source";
  35                         $this->setupFakeSourceBase( $IP, $extensionsDir, $sourceBase );
  36                 } else {
  37                         $sourceBase = realpath( "$IP/.." );
  38                         unlink( "$buildDir/source" );
  39                 }
  40
  41                 # With the CentOS RPMs, you just get g++44, no g++, so we have to
  42                 # use the environment
  43                 if ( isset( $_ENV['CXX'] ) ) {
  44                         $cxx = $_ENV['CXX'];
  45                 } else {
  46                         $cxx = 'g++';
  47                 }
  48
  49                 # Create a function that provides the HipHop compiler version, and
  50                 # doesn't exist when MediaWiki is invoked in interpreter mode.
  51                 $version = str_replace( PHP_EOL, ' ', trim( `hphp --version` ) );
  52                 file_put_contents(
  53                         "$buildDir/HipHopCompilerVersion.php",
  54                         "<" . "?php\n" .
  55                         "function wfHipHopCompilerVersion() {\n" .
  56                         "return " . var_export( $version, true ) . ";\n" .
  57                         "}\n"
  58                 );
  59
  60                 # Generate the file list
  61                 $files = $this->getFileList();
  62                 file_put_contents(
  63                         "$buildDir/file-list",
  64                         implode( "\n", $files ) . "\n" );
  65
  66                 # Generate the C++
  67                 passthru(
  68                         'hphp' .
  69                         ' --target=cpp' .
  70                         ' --format=file' .
  71                         ' --input-dir=' . wfEscapeShellArg( $sourceBase ) .
  72                         ' --input-list=' . wfEscapeShellArg( "$buildDir/file-list" ) .
  73                         ' --inputs=' . wfEscapeShellArg( "$buildDir/HipHopCompilerVersion.php" ) .
  74                         ' -c ' . wfEscapeShellArg( "$thisDir/compiler.conf" ) .
  75                         ' --parse-on-demand=false' .
  76                         ' --program=mediawiki-hphp' .
  77                         ' --output-dir=' . wfEscapeShellArg( $outDir ) .
  78                         ' --log=3', $ret );
  79
  80                 if ( $ret ) {
  81                         $this->error( "hphp hit an error. Stopping build.\n" );
  82                         exit( 1 );
  83                 }
  84
  85                 # Sanity check, quickly make sure we've got an output directory
  86                 if( !is_dir( $outDir ) ) {
  87                         $this->error( "No output directory", true );
  88                 }
  89
  90                 # Warn about volatile classes
  91                 $this->checkVolatileClasses( $outDir );
  92
  93                 # Copy the generated C++ files into the source directory for cmake
  94                 $iter = new RecursiveIteratorIterator(
  95                         new RecursiveDirectoryIterator( $outDir ),
  96                         RecursiveIteratorIterator::SELF_FIRST );
  97                 $sourceFiles = array();
  98                 $regenerateMakefile = false;
  99                 $numFiles = 0;
 100                 $numFilesChanged = 0;
 101                 foreach ( $iter as $sourcePath => $file ) {
 102                         $name = substr( $sourcePath, strlen( $outDir ) + 1 );
 103                         $sourceFiles[$name] = true;
 104                         $destPath = "$persistentDir/$name";
 105                         if ( $file->isDir() ) {
 106                                 if ( !is_dir( $destPath ) ) {
 107                                         mkdir( $destPath );
 108                                 }
 109                                 continue;
 110                         }
 111
 112                         $numFiles++;
 113                         # Remove any files that weren't touched, these may have been removed
 114                         # from file-list, we should not compile them
 115                         if ( $file->getMTime() < $startTime ) {
 116                                 if ( file_exists( $destPath ) ) {
 117                                         unlink( $destPath );
 118                                         # Files removed, regenerate the makefile
 119                                         $regenerateMakefile = true;
 120                                 }
 121                                 unlink( $sourcePath );
 122                                 $numFilesChanged++;
 123                                 continue;
 124                         }
 125
 126                         if ( file_exists( $destPath ) ) {
 127                                 $sourceHash = md5( file_get_contents( $sourcePath ) );
 128                                 $destHash = md5( file_get_contents( $destPath ) );
 129                                 if ( $sourceHash == $destHash ) {
 130                                         continue;
 131                                 }
 132                         } else {
 133                                 # New files added, regenerate the makefile
 134                                 $regenerateMakefile = true;
 135                         }
 136                         $numFilesChanged++;
 137                         copy( $sourcePath, $destPath );
 138                 }
 139
 140                 echo "MediaWiki: $numFilesChanged files changed out of $numFiles\n";
 141
 142                 if ( !file_exists( "$persistentDir/CMakeLists.txt" ) ) {
 143                         # Run cmake for the first time
 144                         $regenerateMakefile = true;
 145                 }
 146
 147                 # Do our own version of $HPHP_HOME/bin/run.sh, which isn't so broken.
 148                 # HipHop's RELEASE mode seems to be stuck always on, so symbols get
 149                 # stripped. Also we will try keeping the generated .o files instead of
 150                 # throwing away hours of CPU time every time you make a typo.
 151
 152                 chdir( $persistentDir );
 153
 154                 if ( $regenerateMakefile ) {
 155                         copy( $_ENV['HPHP_HOME'] . '/bin/CMakeLists.base.txt',
 156                                 "$persistentDir/CMakeLists.txt" );
 157
 158                         if ( file_exists( "$persistentDir/CMakeCache.txt" ) ) {
 159                                 unlink( "$persistentDir/CMakeCache.txt" );
 160                         }
 161
 162                         $cmd = 'cmake' .
 163                                 " -D CMAKE_BUILD_TYPE:string=" . wfEscapeShellArg( $GLOBALS['wgHipHopBuildType'] ) .
 164                                 ' -D PROGRAM_NAME:string=mediawiki-hphp';
 165
 166                         if ( file_exists( '/usr/bin/ccache' ) ) {
 167                                 $cmd .= ' -D CMAKE_CXX_COMPILER:string=ccache' .
 168                                         ' -D CMAKE_CXX_COMPILER_ARG1:string=' . wfEscapeShellArg( $cxx );
 169                         }
 170
 171                         $cmd .= ' .';
 172                         echo "$cmd\n";
 173                         passthru( $cmd );
 174                 }
 175
 176                 # Determine appropriate make concurrency
 177                 # Compilation can take a lot of memory, let's assume that that is limiting.
 178                 $procs = $this->getNumProcs();
 179
 180                 # Run make. This is the slow step.
 181                 passthru( 'make -j' . wfEscapeShellArg( $procs ) );
 182
 183                 $elapsed = time() - $startTime;
 184
 185                 echo "Completed in ";
 186                 if ( $elapsed >= 3600 ) {
 187                         $hours = floor( $elapsed / 3600 );
 188                         echo $hours . 'h ';
 189                         $elapsed -= $hours * 3600;
 190                 }
 191                 if ( $elapsed >= 60 ) {
 192                         $minutes = floor( $elapsed / 60 );
 193                         echo $minutes . 'm ';
 194                         $elapsed -= $minutes * 60;
 195                 }
 196                 echo $elapsed . "s\n";
 197                 echo "The MediaWiki executable is at $buildDir/persistent/mediawiki-hphp\n";
 198         }
 199
 200         function checkVolatileClasses( $dir ) {
 201                 $lines = file( "$dir/sys/dynamic_table_class.cpp" );
 202                 $classes = array();
 203                 foreach ( $lines as $line ) {
 204                         if ( preg_match( '/^\s+\(const char \*\)"([^"]*)", \(const char \*\)-1/', $line, $m ) ) {
 205                                 $classes[] = $m[1];
 206                         }
 207                 }
 208                 if ( !count( $classes ) ) {
 209                         print "No volatile classes found\n";
 210                         return;
 211                 }
 212                 sort( $classes );
 213                 $classes = array_unique( $classes );
 214                 print "WARNING: The following classes are volatile: " . implode( ', ', $classes ) . "\n";
 215         }
 216
 217         function getNumProcs() {
 218                 global $wgHipHopCompilerProcs;
 219                 if ( $wgHipHopCompilerProcs !== 'detect' ) {
 220                         return intval( $wgHipHopCompilerProcs );
 221                 }
 222
 223                 if ( !file_exists( '/proc/meminfo' ) ) {
 224                         return 1;
 225                 }
 226                 $mem = false;
 227                 foreach ( file( '/proc/meminfo' ) as $line ) {
 228                         if ( preg_match( '/^MemTotal:\s+(\d+)\s+kB/', $line, $m ) ) {
 229                                 $mem = intval( $m[1] );
 230                                 break;
 231                         }
 232                 }
 233                 if ( $mem ) {
 234                         // At least one process
 235                         return max( 1, floor( $mem / 1000000 ) );
 236                 } else {
 237                         return 1;
 238                 }
 239         }
 240
 241         function setupFakeSourceBase( $phase3, $extensions, $dest ) {
 242                 if ( !file_exists( $dest ) ) {
 243                         mkdir( $dest, 0777, true );
 244                 }
 245
 246                 $this->forceCreateLink( "$dest/phase3", $phase3 );
 247                 $this->forceCreateLink( "$dest/extensions", $extensions );
 248         }
 249
 250         function forceCreateLink( $target, $link ) {
 251                 if ( file_exists( $target ) ) {
 252                         if ( readlink( $target ) === $link ) {
 253                                 return;
 254                         }
 255                         unlink( $target );
 256                 }
 257                 symlink( $target, $link );
 258         }
 259
 260         function getFileList() {
 261                 global $wgAutoloadClasses, $wgAutoloadLocalClasses, $wgCompiledFiles;
 262                 $inputFiles = array_merge(
 263                         array_values( $wgAutoloadClasses ),
 264                         array_values( $wgAutoloadLocalClasses ),
 265                         $wgCompiledFiles
 266                 );
 267                 $processedFiles = array();
 268                 foreach ( $inputFiles as $file ) {
 269                         if ( substr( $file, 0, 1 ) === '/' ) {
 270                                 $processedFiles[] = $this->absoluteToRelative( $file );
 271                         } elseif ( preg_match( '/^extensions/', $file ) ) {
 272                                 $processedFiles[] = $file;
 273                         } else {
 274                                 $processedFiles[] = "phase3/$file";
 275                         }
 276                 }
 277
 278                 $extraCoreFiles = array_map( 'trim', file( dirname( __FILE__ ) . '/extra-files' ) );
 279                 foreach ( $extraCoreFiles as $file ) {
 280                         if ( $file === '' ) {
 281                                 continue;
 282                         }
 283                         $processedFiles[] = "phase3/$file";
 284                 }
 285                 return array_unique( $processedFiles );
 286         }
 287
 288         function absoluteToRelative( $file ) {
 289                 global $IP;
 290
 291                 $coreBase = realpath( $IP ) . '/';
 292                 $extBase = realpath( MWInit::getExtensionsDirectory() ) . '/';
 293                 $file = realpath( $file );
 294
 295                 if ( substr( $file, 0, strlen( $extBase ) ) === $extBase ) {
 296                         return 'extensions/' . substr( $file, strlen( $extBase ) );
 297                 } elseif ( substr( $file, 0, strlen( $coreBase ) ) === $coreBase ) {
 298                         return 'phase3/' . substr( $file, strlen( $coreBase ) );
 299                 } else {
 300                         $this->error( "The following file is registered for compilation but is not in \$IP or " .
 301                                 "\$wgExtensionsDirectory: $file \n" );
 302                         exit( 1 );
 303                 }
 304         }
 305 }
 306
 307 $maintClass = 'MakeHipHop';
 308 require_once( RUN_MAINTENANCE_IF_MAIN );