maintenance/dumpHTML.inc

   1 <?php
   2 /**
   3  * @package MediaWiki
   4  * @subpackage Maintenance
   5  */
   6
   7 define( 'REPORTING_INTERVAL', 10 );
   8
   9 require_once( 'includes/ImagePage.php' );
  10 require_once( 'includes/CategoryPage.php' );
  11
  12 class DumpHTML {
  13         var $dest, $interwiki, $depth, $sharedStaticPath;
  14
  15         function DumpHTML( $dest, $interwiki = true, $depth = 3 ) {
  16                 $this->dest = $dest;
  17                 $this->interwiki = $interwiki;
  18                 $this->depth = $depth;
  19         }
  20
  21         /**
  22          * Write a set of articles specified by start and end page_id
  23          * Skip categories and images, they will be done separately
  24          */
  25         function doArticles( $start, $end = false ) {
  26                 $fname = 'DumpHTML::doArticles';
  27
  28                 $this->setupGlobals();
  29
  30                 if ( $end === false ) {
  31                         $dbr =& wfGetDB( DB_SLAVE );
  32                         $end = $dbr->selectField( 'page', 'max(page_id)', false, $fname );
  33                 }
  34
  35
  36                 for ($id = $start; $id <= $end; $id++) {
  37                         if ( !($id % REPORTING_INTERVAL) ) {
  38                                 print "Processing ID: $id".chr(13);
  39                         }
  40                         $title = Title::newFromID( $id );
  41                         if ( $title ) {
  42                                 $ns = $title->getNamespace() ;
  43                                 if ( $ns != NS_CATEGORY && $ns != NS_IMAGE ) {
  44                                         $this->doArticle( $title );
  45                                 }
  46                         }
  47                 }
  48         }
  49
  50         function doSpecials() {
  51                 $this->doMainPage();
  52
  53                 $this->setupGlobals();
  54                 print "Special:Categories...";
  55                 $this->doArticle( Title::makeTitle( NS_SPECIAL, 'Categories' ) );
  56                 print "\n";
  57         }
  58
  59         /** Write the main page as index.html */
  60         function doMainPage() {
  61                 global $wgMakeDumpLinks;
  62
  63                 print "Making index.html  ";
  64
  65                 // Set up globals with no ../../.. in the link URLs
  66                 $this->setupGlobals( 0 );
  67
  68                 // But still use that directory style
  69                 $wgMakeDumpLinks = 3;
  70
  71                 $title = Title::newMainPage();
  72                 $text = $this->getArticleHTML( $title );
  73                 $file = fopen( "{$this->dest}/index.html", "w" );
  74                 if ( !$file ) {
  75                         print "\nCan't open index.html for writing\n";
  76                         return false;
  77                 }
  78                 fwrite( $file, $text );
  79                 fclose( $file );
  80                 print "\n";
  81         }
  82
  83         function doImageDescriptions() {
  84                 global $wgSharedUploadDirectory;
  85
  86                 $fname = 'DumpHTML::doImageDescriptions';
  87
  88                 $this->setupGlobals( 3 );
  89
  90                 /**
  91                  * Dump image description pages that don't have an associated article, but do
  92                  * have a local image
  93                  */
  94                 $dbr =& wfGetDB( DB_SLAVE );
  95                 extract( $dbr->tableNames( 'image', 'page' ) );
  96                 $res = $dbr->select( 'image', array( 'img_name' ), false, $fname );
  97
  98                 $i = 0;
  99                 print "Writing " . $dbr->numRows( $res ) . " image description pages for local images\n";
 100                 while ( $row = $dbr->fetchObject( $res ) ) {
 101                         if ( !( ++$i % REPORTING_INTERVAL ) ) {
 102                                 print "$i\t{$row->img_name}\n";
 103                         }
 104                         $title = Title::makeTitle( NS_IMAGE, $row->img_name );
 105                         if ( $title->getArticleID() ) {
 106                                 // Already done by dumpHTML
 107                                 continue;
 108                         }
 109                         $this->doArticle( $title );
 110                 }
 111                 /**
 112                  * Dump images which only have a real description page on commons
 113                  */
 114                 print "Writing description pages for commons images\n";
 115                 $i = 0;
 116                 for ( $hash = 0; $hash < 256; $hash++ ) {
 117                         $dir = sprintf( "%01x/%02x", intval( $hash / 16 ), $hash );
 118                         $paths = glob( "{$this->sharedStaticPath}/$dir/*" );
 119                         $paths += glob( "{$this->sharedStaticPath}/thumb/$dir/*" );
 120
 121                         foreach ( $paths as $path ) {
 122                                 $file = basename( $path );
 123                                 if ( !(++$i % REPORTING_INTERVAL ) ) {
 124                                         print "$i\t$file\n";
 125                                 }
 126
 127                                 $title = Title::makeTitle( NS_IMAGE, $file );
 128                                 $this->doArticle( $title );
 129                         }
 130                 }
 131         }
 132
 133         function doCategories() {
 134                 $fname = 'DumpHTML::doCategories';
 135                 $this->setupGlobals();
 136
 137                 $dbr =& wfGetDB( DB_SLAVE );
 138                 $categorylinks = $dbr->tableName( 'categorylinks' );
 139                 print "Selecting categories...";
 140                 $sql = 'SELECT DISTINCT cl_to FROM categorylinks';
 141                 $res = $dbr->query( $sql, $fname );
 142
 143                 print "\nWriting " . $dbr->numRows( $res ).  " category pages\n";
 144                 $i = 0;
 145                 while ( $row = $dbr->fetchObject( $res ) ) {
 146                         if ( !(++$i % REPORTING_INTERVAL ) ) {
 147                                 print "$i\t{$row->cl_to}\n";
 148                         }
 149                         $title = Title::makeTitle( NS_CATEGORY, $row->cl_to );
 150                         $this->doArticle( $title );
 151                 }
 152         }
 153
 154
 155         /** Write an article specified by title */
 156         function doArticle( $title ) {
 157                 global $wgTitle, $wgSharedUploadPath, $wgSharedUploadDirectory;
 158                 global $wgUploadDirectory;
 159
 160                 $text = $this->getArticleHTML( $title );
 161                 if ( $text === false ) {
 162                         return;
 163                 }
 164
 165                 # Parse the XHTML to find the images
 166                 $images = $this->findImages( $text );
 167                 $this->copyImages( $images );
 168
 169                 # Write to file
 170                 $this->writeArticle( $title, $text );
 171         }
 172
 173         /** Write the given text to the file identified by the given title object */
 174         function writeArticle( &$title, $text ) {
 175                 $filename = $title->getHashedFilename();
 176                 $fullName = "{$this->dest}/$filename";
 177                 $fullDir = dirname( $fullName );
 178
 179                 wfMkdirParents( $fullDir, 0755 );
 180
 181                 $file = fopen( $fullName, 'w' );
 182                 if ( !$file ) {
 183                         print("Can't open file $fullName for writing\n");
 184                         return;
 185                 }
 186
 187                 fwrite( $file, $text );
 188                 fclose( $file );
 189         }
 190
 191         /** Set up globals required for parsing */
 192         function setupGlobals( $depth = NULL ) {
 193                 global $wgUser, $wgTitle, $wgMakeDumpLinks, $wgStylePath, $wgArticlePath;
 194                 global $wgUploadPath, $wgLogo, $wgMaxCredits, $wgSharedUploadPath;
 195                 global $wgHideInterlanguageLinks, $wgUploadDirectory, $wgThumbnailScriptPath;
 196                 global $wgSharedThumbnailScriptPath, $wgEnableParserCache;
 197
 198                 if ( is_null( $depth ) ) {
 199                         $wgMakeDumpLinks = $this->depth;
 200                 } else {
 201                         $wgMakeDumpLinks = $depth;
 202                 }
 203
 204                 $wgScriptPath = '..' . str_repeat( '/..', $wgMakeDumpLinks );
 205                 $wgArticlePath = str_repeat( '../', $wgMakeDumpLinks ) . '$1';
 206                 $wgStylePath = "$wgScriptPath/skins";
 207                 $wgUploadPath = "$wgScriptPath/images";
 208                 $wgSharedUploadPath = "$wgUploadPath/shared";
 209                 $wgLogo = "$wgStylePath/common/images/wiki.png";
 210                 $wgMaxCredits = -1;
 211                 $wgHideInterlangageLinks = !$this->interwiki;
 212                 $wgThumbnailScriptPath = $wgSharedThumbnailScriptPath = false;
 213                 $wgEnableParserCache = false;
 214
 215                 $wgUser = new User;
 216                 $wgUser->setOption( 'skin', 'htmldump' );
 217                 $wgUser->setOption( 'editsection', 0 );
 218
 219                 $this->sharedStaticPath = "$wgUploadDirectory/shared";
 220
 221         }
 222
 223         /** Reads the content of a title object, executes the skin and captures the result */
 224         function getArticleHTML( &$title ) {
 225                 global $wgOut, $wgTitle, $wgArticle, $wgUser, $wgUseCategoryMagic;
 226
 227                 $wgOut = new OutputPage;
 228                 $wgOut->setParserOptions( new ParserOptions );
 229
 230                 $wgTitle =& $title;
 231                 if ( is_null( $wgTitle ) ) {
 232                         return false;
 233                 }
 234
 235                 $ns = $wgTitle->getNamespace();
 236                 if ( $ns == NS_SPECIAL ) {
 237                         SpecialPage::executePath( $wgTitle );
 238                 } else {
 239                         if ( $ns == NS_IMAGE ) {
 240                                 $wgArticle = new ImagePage( $wgTitle );
 241                         } elseif ( $wgUseCategoryMagic && $ns == NS_CATEGORY ) {
 242                                 $wgArticle = new CategoryPage( $wgTitle );
 243                         } else {
 244                                 $wgArticle = new Article( $wgTitle );
 245                         }
 246                         $wgArticle->view();
 247                 }
 248
 249                 $sk =& $wgUser->getSkin();
 250                 ob_start();
 251                 $sk->outputPage( $wgOut );
 252                 $text = ob_get_contents();
 253                 ob_end_clean();
 254
 255                 return $text;
 256         }
 257
 258         /** Returns image paths used in an XHTML document */
 259         function findImages( $text ) {
 260                 global $wgOutputEncoding, $wgDumpImages;
 261                 $parser = xml_parser_create( $wgOutputEncoding );
 262                 xml_set_element_handler( $parser, 'wfDumpStartTagHandler', 'wfDumpEndTagHandler' );
 263
 264                 $wgDumpImages = array();
 265                 xml_parse( $parser, $text );
 266                 xml_parser_free( $parser );
 267
 268                 return $wgDumpImages;
 269         }
 270
 271         /**
 272          * Copy images (or create symlinks) from commons to a static directory.
 273          * This is necessary even if you intend to distribute all of commons, because
 274          * the directory contents is used to work out which image description pages
 275          * are needed.
 276          */
 277         function copyImages( $images ) {
 278                 global $wgSharedUploadPath, $wgSharedUploadDirectory;
 279                 # Find shared uploads and copy them into the static directory
 280                 $sharedPathLength = strlen( $wgSharedUploadPath );
 281                 foreach ( $images as $image => $dummy ) {
 282                         # Is it shared?
 283                         if ( substr( $image, 0, $sharedPathLength ) == $wgSharedUploadPath ) {
 284                                 # Reconstruct full filename
 285                                 $rel = substr( $image, $sharedPathLength + 1 ); // +1 for slash
 286                                 $sourceLoc = "$wgSharedUploadDirectory/$rel";
 287                                 $staticLoc = "{$this->sharedStaticPath}/$rel";
 288                                 #print "Copying $sourceLoc to $staticLoc\n";
 289                                 # Copy to static directory
 290                                 if ( !file_exists( $staticLoc ) ) {
 291                                         wfMkdirParents( dirname( $staticLoc ), 0755 );
 292                                         if ( function_exists( 'symlink' ) ) {
 293                                                 symlink( $staticLoc, $sourceLoc );
 294                                         } else {
 295                                                 copy( $sourceLoc, $staticLoc );
 296                                         }
 297                                 }
 298
 299                                 if ( substr( $rel, 0, 6 ) == 'thumb/' ) {
 300                                         # That was a thumbnail
 301                                         # We will also copy the real image
 302                                         $parts = explode( '/', $rel );
 303                                         $rel = "{$parts[1]}/{$parts[2]}/{$parts[3]}";
 304                                         $sourceLoc = "$wgSharedUploadDirectory/$rel";
 305                                         $staticLoc = "{$this->sharedStaticPath}/$rel";
 306                                         #print "Copying $sourceLoc to $staticLoc\n";
 307                                         if ( !file_exists( $staticLoc ) ) {
 308                                                 wfMkdirParents( dirname( $staticLoc ), 0755 );
 309                                                 if ( function_exists( 'symlink' ) ) {
 310                                                         symlink( $staticLoc, $sourceLoc );
 311                                                 } else {
 312                                                         copy( $sourceLoc, $staticLoc );
 313                                                 }
 314                                         }
 315                                 }
 316                         }
 317                 }
 318         }
 319 }
 320
 321 /** XML parser callback */
 322 function wfDumpStartTagHandler( $parser, $name, $attribs ) {
 323         global $wgDumpImages;
 324
 325         if ( $name == 'IMG' && isset( $attribs['SRC'] ) ) {
 326                 $wgDumpImages[$attribs['SRC']] = true;
 327         }
 328 }
 329
 330 /** XML parser callback */
 331 function wfDumpEndTagHandler( $parser, $name ) {}
 332
 333 # vim: syn=php
 334 ?>