maintenance/generateSitemap.php

   1 <?php
   2 define( 'GS_MAIN', -2 );
   3 define( 'GS_TALK', -1 );
   4 /**
   5  * Creates a sitemap for the site
   6  *
   7  * @ingroup Maintenance
   8  *
   9  * @copyright Copyright © 2005, Ævar Arnfjörð Bjarmason
  10  * @copyright Copyright © 2005, Jens Frank <jeluf@gmx.de>
  11  * @copyright Copyright © 2005, Brion Vibber <brion@pobox.com>
  12  *
  13  * @see http://www.sitemaps.org/
  14  * @see http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd
  15  *
  16  * @license http://www.gnu.org/copyleft/gpl.html GNU General Public License 2.0 or later
  17  */
  18
  19 class GenerateSitemap {
  20         /**
  21          * The maximum amount of urls in a sitemap file
  22          *
  23          * @link http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd
  24          *
  25          * @var int
  26          */
  27         var $url_limit;
  28
  29         /**
  30          * The maximum size of a sitemap file
  31          *
  32          * @link http://www.sitemaps.org/faq.php#faq_sitemap_size
  33          *
  34          * @var int
  35          */
  36         var $size_limit;
  37
  38         /**
  39          * The path to prepend to the filename
  40          *
  41          * @var string
  42          */
  43         var $fspath;
  44
  45         /**
  46          * The path to append to the domain name
  47          *
  48          * @var string
  49          */
  50         var $path;
  51
  52         /**
  53          * Whether or not to use compression
  54          *
  55          * @var bool
  56          */
  57         var $compress;
  58
  59         /**
  60          * The number of entries to save in each sitemap file
  61          *
  62          * @var array
  63          */
  64         var $limit = array();
  65
  66         /**
  67          * Key => value entries of namespaces and their priorities
  68          *
  69          * @var array
  70          */
  71         var $priorities = array(
  72                 // Custom main namespaces
  73                 GS_MAIN                 => '0.5',
  74                 // Custom talk namesspaces
  75                 GS_TALK                 => '0.1',
  76                 // MediaWiki standard namespaces
  77                 NS_MAIN                 => '1.0',
  78                 NS_TALK                 => '0.1',
  79                 NS_USER                 => '0.5',
  80                 NS_USER_TALK            => '0.1',
  81                 NS_PROJECT              => '0.5',
  82                 NS_PROJECT_TALK         => '0.1',
  83                 NS_IMAGE                => '0.5',
  84                 NS_IMAGE_TALK           => '0.1',
  85                 NS_MEDIAWIKI            => '0.0',
  86                 NS_MEDIAWIKI_TALK       => '0.1',
  87                 NS_TEMPLATE             => '0.0',
  88                 NS_TEMPLATE_TALK        => '0.1',
  89                 NS_HELP                 => '0.5',
  90                 NS_HELP_TALK            => '0.1',
  91                 NS_CATEGORY             => '0.5',
  92                 NS_CATEGORY_TALK        => '0.1',
  93         );
  94
  95         /**
  96          * A one-dimensional array of namespaces in the wiki
  97          *
  98          * @var array
  99          */
 100         var $namespaces = array();
 101
 102         /**
 103          * When this sitemap batch was generated
 104          *
 105          * @var string
 106          */
 107         var $timestamp;
 108
 109         /**
 110          * A database slave object
 111          *
 112          * @var object
 113          */
 114         var $dbr;
 115
 116         /**
 117          * A resource pointing to the sitemap index file
 118          *
 119          * @var resource
 120          */
 121         var $findex;
 122
 123
 124         /**
 125          * A resource pointing to a sitemap file
 126          *
 127          * @var resource
 128          */
 129         var $file;
 130
 131         /**
 132          * A resource pointing to php://stderr
 133          *
 134          * @var resource
 135          */
 136         var $stderr;
 137
 138         /**
 139          * Constructor
 140          *
 141          * @param string $fspath The path to prepend to the filenames, used to
 142          *                     save them somewhere else than in the root directory
 143          * @param string $path The path to append to the domain name
 144          * @param bool $compress Whether to compress the sitemap files
 145          */
 146         function GenerateSitemap( $fspath, $compress ) {
 147                 global $wgScriptPath;
 148
 149                 $this->url_limit = 50000;
 150                 $this->size_limit = pow( 2, 20 ) * 10;
 151                 $this->fspath = isset( $fspath ) ? $fspath : '';
 152                 $this->compress = $compress;
 153
 154                 $this->stderr = fopen( 'php://stderr', 'wt' );
 155                 $this->dbr = wfGetDB( DB_SLAVE );
 156                 $this->generateNamespaces();
 157                 $this->timestamp = wfTimestamp( TS_ISO_8601, wfTimestampNow() );
 158                 $this->findex = fopen( "{$this->fspath}sitemap-index-" . wfWikiID() . ".xml", 'wb' );
 159         }
 160
 161         /**
 162          * Generate a one-dimensional array of existing namespaces
 163          */
 164         function generateNamespaces() {
 165                 $fname = 'GenerateSitemap::generateNamespaces';
 166
 167                 // Only generate for specific namespaces if $wgSitemapNamespaces is an array.
 168                 global $wgSitemapNamespaces;
 169                 if( is_array( $wgSitemapNamespaces ) ) {
 170                         $this->namespaces = $wgSitemapNamespaces;
 171                         return;
 172                 }
 173
 174                 $res = $this->dbr->select( 'page',
 175                         array( 'page_namespace' ),
 176                         array(),
 177                         $fname,
 178                         array(
 179                                 'GROUP BY' => 'page_namespace',
 180                                 'ORDER BY' => 'page_namespace',
 181                         )
 182                 );
 183
 184                 while ( $row = $this->dbr->fetchObject( $res ) )
 185                         $this->namespaces[] = $row->page_namespace;
 186         }
 187
 188         /**
 189          * Get the priority of a given namespace
 190          *
 191          * @param int $namespace The namespace to get the priority for
 192          +
 193          * @return string
 194          */
 195
 196         function priority( $namespace ) {
 197                 return isset( $this->priorities[$namespace] ) ? $this->priorities[$namespace] : $this->guessPriority( $namespace );
 198         }
 199
 200         /**
 201          * If the namespace isn't listed on the priority list return the
 202          * default priority for the namespace, varies depending on whether it's
 203          * a talkpage or not.
 204          *
 205          * @param int $namespace The namespace to get the priority for
 206          *
 207          * @return string
 208          */
 209         function guessPriority( $namespace ) {
 210                 return MWNamespace::isMain( $namespace ) ? $this->priorities[GS_MAIN] : $this->priorities[GS_TALK];
 211         }
 212
 213         /**
 214          * Return a database resolution of all the pages in a given namespace
 215          *
 216          * @param int $namespace Limit the query to this namespace
 217          *
 218          * @return resource
 219          */
 220         function getPageRes( $namespace ) {
 221                 $fname = 'GenerateSitemap::getPageRes';
 222
 223                 return $this->dbr->select( 'page',
 224                         array(
 225                                 'page_namespace',
 226                                 'page_title',
 227                                 'page_touched',
 228                         ),
 229                         array( 'page_namespace' => $namespace ),
 230                         $fname
 231                 );
 232         }
 233
 234         /**
 235          * Main loop
 236          *
 237          * @access public
 238          */
 239         function main() {
 240                 global $wgContLang;
 241
 242                 fwrite( $this->findex, $this->openIndex() );
 243
 244                 foreach ( $this->namespaces as $namespace ) {
 245                         $res = $this->getPageRes( $namespace );
 246                         $this->file = false;
 247                         $this->generateLimit( $namespace );
 248                         $length = $this->limit[0];
 249                         $i = $smcount = 0;
 250
 251                         $fns = $wgContLang->getFormattedNsText( $namespace );
 252                         $this->debug( "$namespace ($fns)" );
 253                         while ( $row = $this->dbr->fetchObject( $res ) ) {
 254                                 if ( $i++ === 0 || $i === $this->url_limit + 1 || $length + $this->limit[1] + $this->limit[2] > $this->size_limit ) {
 255                                         if ( $this->file !== false ) {
 256                                                 $this->write( $this->file, $this->closeFile() );
 257                                                 $this->close( $this->file );
 258                                         }
 259                                         $filename = $this->sitemapFilename( $namespace, $smcount++ );
 260                                         $this->file = $this->open( $this->fspath . $filename, 'wb' );
 261                                         $this->write( $this->file, $this->openFile() );
 262                                         fwrite( $this->findex, $this->indexEntry( $filename ) );
 263                                         $this->debug( "\t$filename" );
 264                                         $length = $this->limit[0];
 265                                         $i = 1;
 266                                 }
 267                                 $title = Title::makeTitle( $row->page_namespace, $row->page_title );
 268                                 $date = wfTimestamp( TS_ISO_8601, $row->page_touched );
 269                                 $entry = $this->fileEntry( $title->getFullURL(), $date, $this->priority( $namespace ) );
 270                                 $length += strlen( $entry );
 271                                 $this->write( $this->file, $entry );
 272                                 // generate pages for language variants
 273                                 if($wgContLang->hasVariants()){
 274                                         $variants = $wgContLang->getVariants();
 275                                         foreach($variants as $vCode){
 276                                                 if($vCode==$wgContLang->getCode()) continue; // we don't want default variant
 277                                                 $entry = $this->fileEntry( $title->getFullURL('',$vCode), $date, $this->priority( $namespace ) );
 278                                                 $length += strlen( $entry );
 279                                                 $this->write( $this->file, $entry );
 280                                         }
 281                                 }
 282                         }
 283                         if ( $this->file ) {
 284                                 $this->write( $this->file, $this->closeFile() );
 285                                 $this->close( $this->file );
 286                         }
 287                 }
 288                 fwrite( $this->findex, $this->closeIndex() );
 289                 fclose( $this->findex );
 290         }
 291
 292         /**
 293          * gzopen() / fopen() wrapper
 294          *
 295          * @return resource
 296          */
 297         function open( $file, $flags ) {
 298                 return $this->compress ? gzopen( $file, $flags ) : fopen( $file, $flags );
 299         }
 300
 301         /**
 302          * gzwrite() / fwrite() wrapper
 303          */
 304         function write( &$handle, $str ) {
 305                 if ( $this->compress )
 306                         gzwrite( $handle, $str );
 307                 else
 308                         fwrite( $handle, $str );
 309         }
 310
 311         /**
 312          * gzclose() / fclose() wrapper
 313          */
 314         function close( &$handle ) {
 315                 if ( $this->compress )
 316                         gzclose( $handle );
 317                 else
 318                         fclose( $handle );
 319         }
 320
 321         /**
 322          * Get a sitemap filename
 323          *
 324          * @static
 325          *
 326          * @param int $namespace The namespace
 327          * @param int $count The count
 328          *
 329          * @return string
 330          */
 331         function sitemapFilename( $namespace, $count ) {
 332                 $ext = $this->compress ? '.gz' : '';
 333                 return "sitemap-".wfWikiID()."-NS_$namespace-$count.xml$ext";
 334         }
 335
 336         /**
 337          * Return the XML required to open an XML file
 338          *
 339          * @static
 340          *
 341          * @return string
 342          */
 343         function xmlHead() {
 344                 return '<?xml version="1.0" encoding="UTF-8"?>' . "\n";
 345         }
 346
 347         /**
 348          * Return the XML schema being used
 349          *
 350          * @static
 351          *
 352          * @returns string
 353          */
 354         function xmlSchema() {
 355                 return 'http://www.sitemaps.org/schemas/sitemap/0.9';
 356         }
 357
 358         /**
 359          * Return the XML required to open a sitemap index file
 360          *
 361          * @return string
 362          */
 363         function openIndex() {
 364                 return $this->xmlHead() . '<sitemapindex xmlns="' . $this->xmlSchema() . '">' . "\n";
 365         }
 366
 367         /**
 368          * Return the XML for a single sitemap indexfile entry
 369          *
 370          * @static
 371          *
 372          * @param string $filename The filename of the sitemap file
 373          *
 374          * @return string
 375          */
 376         function indexEntry( $filename ) {
 377                 return
 378                         "\t<sitemap>\n" .
 379                         "\t\t<loc>$filename</loc>\n" .
 380                         "\t\t<lastmod>{$this->timestamp}</lastmod>\n" .
 381                         "\t</sitemap>\n";
 382         }
 383
 384         /**
 385          * Return the XML required to close a sitemap index file
 386          *
 387          * @static
 388          *
 389          * @return string
 390          */
 391         function closeIndex() {
 392                 return "</sitemapindex>\n";
 393         }
 394
 395         /**
 396          * Return the XML required to open a sitemap file
 397          *
 398          * @return string
 399          */
 400         function openFile() {
 401                 return $this->xmlHead() . '<urlset xmlns="' . $this->xmlSchema() . '">' . "\n";
 402         }
 403
 404         /**
 405          * Return the XML for a single sitemap entry
 406          *
 407          * @static
 408          *
 409          * @param string $url An RFC 2396 compilant URL
 410          * @param string $date A ISO 8601 date
 411          * @param string $priority A priority indicator, 0.0 - 1.0 inclusive with a 0.1 stepsize
 412          *
 413          * @return string
 414          */
 415         function fileEntry( $url, $date, $priority ) {
 416                 return
 417                         "\t<url>\n" .
 418                         "\t\t<loc>$url</loc>\n" .
 419                         "\t\t<lastmod>$date</lastmod>\n" .
 420                         "\t\t<priority>$priority</priority>\n" .
 421                         "\t</url>\n";
 422         }
 423
 424         /**
 425          * Return the XML required to close sitemap file
 426          *
 427          * @static
 428          * @return string
 429          */
 430         function closeFile() {
 431                 return "</urlset>\n";
 432         }
 433
 434         /**
 435          * Write a string to stderr followed by a UNIX newline
 436          */
 437         function debug( $str ) {
 438                 fwrite( $this->stderr, "$str\n" );
 439         }
 440
 441         /**
 442          * Populate $this->limit
 443          */
 444         function generateLimit( $namespace ) {
 445                 $title = Title::makeTitle( $namespace, str_repeat( "\xf0\xa8\xae\x81", 63 ) . "\xe5\x96\x83" );
 446
 447                 $this->limit = array(
 448                         strlen( $this->openFile() ),
 449                         strlen( $this->fileEntry( $title->getFullUrl(), wfTimestamp( TS_ISO_8601, wfTimestamp() ), $this->priority( $namespace ) ) ),
 450                         strlen( $this->closeFile() )
 451                 );
 452         }
 453 }
 454
 455 if ( in_array( '--help', $argv ) ) {
 456         echo <<<EOT
 457 Usage: php generateSitemap.php [options]
 458         --help                  show this message
 459
 460         --fspath=<path>         The file system path to save to, e.g /tmp/sitemap/
 461
 462         --server=<server>       The protocol and server name to use in URLs, e.g.
 463                 http://en.wikipedia.org. This is sometimes necessary because
 464                 server name detection may fail in command line scripts.
 465
 466         --compress=[yes|no]     compress the sitemap files, default yes
 467
 468 EOT;
 469         die( -1 );
 470 }
 471
 472 $optionsWithArgs = array( 'fspath', 'server', 'compress' );
 473 require_once( dirname( __FILE__ ) . '/commandLine.inc' );
 474
 475 if ( isset( $options['server'] ) ) {
 476         $wgServer = $options['server'];
 477 }
 478
 479 $gs = new GenerateSitemap( @$options['fspath'], @$options['compress'] !== 'no' );
 480 $gs->main();
 481