maintenance/generateSitemap.php

   1 <?php
   2 define( 'GS_MAIN', -2 );
   3 define( 'GS_TALK', -1 );
   4 /**
   5  * Creates a Google sitemap for the site
   6  *
   7  * @package MediaWiki
   8  * @subpackage Maintenance
   9  *
  10  * @copyright Copyright © 2005, Ævar Arnfjörð Bjarmason
  11  * @copyright Copyright © 2005, Jens Frank <jeluf@gmx.de>
  12  * @copyright Copyright © 2005, Brion Vibber <brion@pobox.com>
  13  *
  14  * @link http://www.google.com/webmasters/sitemaps/docs/en/about.html
  15  * @link http://www.google.com/schemas/sitemap/0.84/sitemap.xsd
  16  *
  17  * @license http://www.gnu.org/copyleft/gpl.html GNU General Public License 2.0 or later
  18  */
  19
  20 class GenerateSitemap {
  21         /**
  22          * The maximum amount of urls in a sitemap file
  23          *
  24          * @link http://www.google.com/schemas/sitemap/0.84/sitemap.xsd
  25          *
  26          * @var int
  27          */
  28         var $url_limit;
  29
  30         /**
  31          * The maximum size of a sitemap file
  32          *
  33          * @link http://www.google.com/webmasters/sitemaps/docs/en/protocol.html#faq_sitemap_size
  34          *
  35          * @var int
  36          */
  37         var $size_limit;
  38
  39         /**
  40          * The path to prepend to the filename
  41          *
  42          * @var string
  43          */
  44         var $fspath;
  45
  46         /**
  47          * The path to append to the domain name
  48          *
  49          * @var string
  50          */
  51         var $path;
  52
  53         /**
  54          * Whether or not to use compression
  55          *
  56          * @var bool
  57          */
  58         var $compress;
  59
  60         /**
  61          * The number of entries to save in each sitemap file
  62          *
  63          * @var array
  64          */
  65         var $limit = array();
  66
  67         /**
  68          * Key => value entries of namespaces and their priorities
  69          *
  70          * @var array
  71          */
  72         var $priorities = array(
  73                 // Custom main namespaces
  74                 GS_MAIN                 => '0.5',
  75                 // Custom talk namesspaces
  76                 GS_TALK                 => '0.1',
  77                 // MediaWiki standard namespaces
  78                 NS_MAIN                 => '1.0',
  79                 NS_TALK                 => '0.1',
  80                 NS_USER                 => '0.5',
  81                 NS_USER_TALK            => '0.1',
  82                 NS_PROJECT              => '0.5',
  83                 NS_PROJECT_TALK         => '0.1',
  84                 NS_IMAGE                => '0.5',
  85                 NS_IMAGE_TALK           => '0.1',
  86                 NS_MEDIAWIKI            => '0.0',
  87                 NS_MEDIAWIKI_TALK       => '0.1',
  88                 NS_TEMPLATE             => '0.0',
  89                 NS_TEMPLATE_TALK        => '0.1',
  90                 NS_HELP                 => '0.5',
  91                 NS_HELP_TALK            => '0.1',
  92                 NS_CATEGORY             => '0.5',
  93                 NS_CATEGORY_TALK        => '0.1',
  94         );
  95
  96         /**
  97          * A one-dimensional array of namespaces in the wiki
  98          *
  99          * @var array
 100          */
 101         var $namespaces = array();
 102
 103         /**
 104          * When this sitemap batch was generated
 105          *
 106          * @var string
 107          */
 108         var $timestamp;
 109
 110         /**
 111          * A database slave object
 112          *
 113          * @var object
 114          */
 115         var $dbr;
 116
 117         /**
 118          * A resource pointing to the sitemap index file
 119          *
 120          * @var resource
 121          */
 122         var $findex;
 123
 124
 125         /**
 126          * A resource pointing to a sitemap file
 127          *
 128          * @var resource
 129          */
 130         var $file;
 131
 132         /**
 133          * A resource pointing to php://stderr
 134          *
 135          * @var resource
 136          */
 137         var $stderr;
 138
 139         /**
 140          * Constructor
 141          *
 142          * @param string $fspath The path to prepend to the filenames, used to
 143          *                     save them somewhere else than in the root directory
 144          * @param string $path The path to append to the domain name
 145          * @param bool $compress Whether to compress the sitemap files
 146          */
 147         function GenerateSitemap( $fspath, $path, $compress ) {
 148                 global $wgDBname, $wgScriptPath;
 149
 150                 $this->url_limit = 50000;
 151                 $this->size_limit = pow( 2, 20 ) * 10;
 152                 $this->fspath = isset( $fspath ) ? $fspath : '';
 153                 $this->path = isset( $path ) ? $path : $wgScriptPath;
 154                 $this->compress = $compress;
 155
 156                 $this->stderr = fopen( 'php://stderr', 'wt' );
 157                 $this->dbr =& wfGetDB( DB_SLAVE );
 158                 $this->generateNamespaces();
 159                 $this->timestamp = wfTimestamp( TS_ISO_8601, wfTimestampNow() );
 160                 $this->findex = fopen( "{$this->fspath}sitemap-index-$wgDBname.xml", 'wb' );
 161         }
 162
 163         /**
 164          * Generate a one-dimensional array of existing namespaces
 165          */
 166         function generateNamespaces() {
 167                 $fname = 'GenerateSitemap::generateNamespaces';
 168
 169                 $res = $this->dbr->select( 'page',
 170                         array( 'page_namespace' ),
 171                         array(),
 172                         $fname,
 173                         array(
 174                                 'GROUP BY' => 'page_namespace',
 175                                 'ORDER BY' => 'page_namespace',
 176                         )
 177                 );
 178
 179                 while ( $row = $this->dbr->fetchObject( $res ) )
 180                         $this->namespaces[] = $row->page_namespace;
 181         }
 182
 183         /**
 184          * Get the priority of a given namespace
 185          *
 186          * @param int $namespace The namespace to get the priority for
 187          +
 188          * @return string
 189          */
 190
 191         function priority( $namespace ) {
 192                 return isset( $this->priorities[$namespace] ) ? $this->priorities[$namespace] : $this->guessPriority( $namespace );
 193         }
 194
 195         /**
 196          * If the namespace isn't listed on the priority list return the
 197          * default priority for the namespace, varies depending on whether it's
 198          * a talkpage or not.
 199          *
 200          * @param int $namespace The namespace to get the priority for
 201          *
 202          * @return string
 203          */
 204         function guessPriority( $namespace ) {
 205                 return Namespace::isMain( $namespace ) ? $this->priorities[GS_MAIN] : $this->priorities[GS_TALK];
 206         }
 207
 208         /**
 209          * Return a database resolution of all the pages in a given namespace
 210          *
 211          * @param int $namespace Limit the query to this namespace
 212          *
 213          * @return resource
 214          */
 215         function getPageRes( $namespace ) {
 216                 $fname = 'GenerateSitemap::getPageRes';
 217
 218                 return $this->dbr->select( 'page',
 219                         array(
 220                                 'page_namespace',
 221                                 'page_title',
 222                                 'page_touched',
 223                         ),
 224                         array( 'page_namespace' => $namespace ),
 225                         $fname
 226                 );
 227         }
 228
 229         /**
 230          * Main loop
 231          *
 232          * @access public
 233          */
 234         function main() {
 235                 global $wgDBname, $wgContLang;
 236
 237                 fwrite( $this->findex, $this->openIndex() );
 238
 239                 foreach ( $this->namespaces as $namespace ) {
 240                         $res = $this->getPageRes( $namespace );
 241                         $this->file = false;
 242                         $this->generateLimit( $namespace );
 243                         $length = $this->limit[0];
 244                         $i = $smcount = 0;
 245
 246                         $fns = $wgContLang->getFormattedNsText( $namespace );
 247                         $this->debug( "$namespace ($fns)" );
 248                         while ( $row = $this->dbr->fetchObject( $res ) ) {
 249                                 if ( $i++ === 0 || $i === $this->url_limit + 1 || $length + $this->limit[1] + $this->limit[2] > $this->size_limit ) {
 250                                         if ( $this->file !== false ) {
 251                                                 $this->write( $this->file, $this->closeFile() );
 252                                                 $this->close( $this->file );
 253                                         }
 254                                         $filename = $this->sitemapFilename( $namespace, $smcount++ );
 255                                         $this->file = $this->open( $this->fspath . $filename, 'wb' );
 256                                         $this->write( $this->file, $this->openFile() );
 257                                         fwrite( $this->findex, $this->indexEntry( $filename ) );
 258                                         $this->debug( "\t$filename" );
 259                                         $length = $this->limit[0];
 260                                         $i = 1;
 261                                 }
 262                                 $title = Title::makeTitle( $row->page_namespace, $row->page_title );
 263                                 $date = wfTimestamp( TS_ISO_8601, $row->page_touched );
 264                                 $entry = $this->fileEntry( $title->getFullURL(), $date, $this->priority( $namespace ) );
 265                                 $length += strlen( $entry );
 266                                 $this->write( $this->file, $entry );
 267                         }
 268                         if ( $this->file ) {
 269                                 $this->write( $this->file, $this->closeFile() );
 270                                 $this->close( $this->file );
 271                         }
 272                 }
 273                 fwrite( $this->findex, $this->closeIndex() );
 274                 fclose( $this->findex );
 275         }
 276
 277         /**
 278          * gzopen() / fopen() wrapper
 279          *
 280          * @return resource
 281          */
 282         function open( $file, $flags ) {
 283                 return $this->compress ? gzopen( $file, $flags ) : fopen( $file, $flags );
 284         }
 285
 286         /**
 287          * gzwrite() / fwrite() wrapper
 288          */
 289         function write( &$handle, $str ) {
 290                 if ( $this->compress )
 291                         gzwrite( $handle, $str );
 292                 else
 293                         fwrite( $handle, $str );
 294         }
 295
 296         /**
 297          * gzclose() / fclose() wrapper
 298          */
 299         function close( &$handle ) {
 300                 if ( $this->compress )
 301                         gzclose( $handle );
 302                 else
 303                         fclose( $handle );
 304         }
 305
 306         /**
 307          * Get a sitemap filename
 308          *
 309          * @static
 310          *
 311          * @param int $namespace The namespace
 312          * @param int $count The count
 313          *
 314          * @return string
 315          */
 316         function sitemapFilename( $namespace, $count ) {
 317                 global $wgDBname;
 318
 319                 $ext = $this->compress ? '.gz' : '';
 320
 321                 return "sitemap-$wgDBname-NS_$namespace-$count.xml$ext";
 322         }
 323
 324         /**
 325          * Return the XML required to open an XML file
 326          *
 327          * @static
 328          *
 329          * @return string
 330          */
 331         function xmlHead() {
 332                 return '<?xml version="1.0" encoding="UTF-8"?>' . "\n";
 333         }
 334
 335         /**
 336          * Return the XML schema being used
 337          *
 338          * @static
 339          *
 340          * @returns string
 341          */
 342         function xmlSchema() {
 343                 return 'http://www.google.com/schemas/sitemap/0.84';
 344         }
 345
 346         /**
 347          * Return the XML required to open a sitemap index file
 348          *
 349          * @return string
 350          */
 351         function openIndex() {
 352                 return $this->xmlHead() . '<sitemapindex xmlns="' . $this->xmlSchema() . '">' . "\n";
 353         }
 354
 355         /**
 356          * Return the XML for a single sitemap indexfile entry
 357          *
 358          * @static
 359          *
 360          * @param string $filename The filename of the sitemap file
 361          *
 362          * @return string
 363          */
 364         function indexEntry( $filename ) {
 365                 return
 366                         "\t<sitemap>\n" .
 367                         "\t\t<loc>$filename</loc>\n" .
 368                         "\t\t<lastmod>{$this->timestamp}</lastmod>\n" .
 369                         "\t</sitemap>\n";
 370         }
 371
 372         /**
 373          * Return the XML required to close a sitemap index file
 374          *
 375          * @static
 376          *
 377          * @return string
 378          */
 379         function closeIndex() {
 380                 return "</sitemapindex>\n";
 381         }
 382
 383         /**
 384          * Return the XML required to open a sitemap file
 385          *
 386          * @return string
 387          */
 388         function openFile() {
 389                 return $this->xmlHead() . '<urlset xmlns="' . $this->xmlSchema() . '">' . "\n";
 390         }
 391
 392         /**
 393          * Return the XML for a single sitemap entry
 394          *
 395          * @static
 396          *
 397          * @param string $url An RFC 2396 compilant URL
 398          * @param string $date A ISO 8601 date
 399          * @param string $priority A priority indicator, 0.0 - 1.0 inclusive with a 0.1 stepsize
 400          *
 401          * @return string
 402          */
 403         function fileEntry( $url, $date, $priority ) {
 404                 return
 405                         "\t<url>\n" .
 406                         "\t\t<loc>$url</loc>\n" .
 407                         "\t\t<lastmod>$date</lastmod>\n" .
 408                         "\t\t<priority>$priority</priority>\n" .
 409                         "\t</url>\n";
 410         }
 411
 412         /**
 413          * Return the XML required to close sitemap file
 414          *
 415          * @static
 416          * @return string
 417          */
 418         function closeFile() {
 419                 return "</urlset>\n";
 420         }
 421
 422         /**
 423          * Write a string to stderr followed by a UNIX newline
 424          */
 425         function debug( $str ) {
 426                 fwrite( $this->stderr, "$str\n" );
 427         }
 428
 429         /**
 430          * Populate $this->limit
 431          */
 432         function generateLimit( $namespace ) {
 433                 $title = Title::makeTitle( $namespace, str_repeat( "\xf0\xa8\xae\x81", 63 ) . "\xe5\x96\x83" );
 434
 435                 $this->limit = array(
 436                         strlen( $this->openFile() ),
 437                         strlen( $this->fileEntry( $title->getFullUrl(), wfTimestamp( TS_ISO_8601, wfTimestamp() ), $this->priority( $namespace ) ) ),
 438                         strlen( $this->closeFile() )
 439                 );
 440         }
 441 }
 442
 443 if ( in_array( '--help', $argv ) ) {
 444         echo
 445                 "Usage: php generateSitemap.php [host] [options]\n" .
 446                 "\thost = hostname\n" .
 447                 "\toptions:\n" .
 448                 "\t\t--help\tshow this message\n" .
 449                 "\t\t--fspath\tThe file system path to save to, e.g /tmp/sitemap/\n" .
 450                 "\t\t--path\tThe http path to use, e.g. /wiki\n" .
 451                 "\t\t--compress=[yes|no]\tcompress the sitemap files, default yes\n";
 452         die( -1 );
 453 }
 454
 455 if ( isset( $argv[1] ) && strpos( $argv[1], '--' ) !== 0 )
 456         $_SERVER['SERVER_NAME'] = $argv[1];
 457
 458 $optionsWithArgs = array( 'fspath', 'path', 'compress' );
 459 require_once 'commandLine.inc';
 460
 461 $gs = new GenerateSitemap( @$options['fspath'], @$options['path'], @$options['compress'] !== 'no' );
 462 $gs->main();
 463 ?>