maintenance/generateSitemap.php

   1 <?php
   2 define( 'GS_MAIN', -2 );
   3 define( 'GS_TALK', -1 );
   4 /**
   5  * Creates a Google sitemap for the site
   6  *
   7  * @addtogroup Maintenance
   8  *
   9  * @copyright Copyright © 2005, Ævar Arnfjörð Bjarmason
  10  * @copyright Copyright © 2005, Jens Frank <jeluf@gmx.de>
  11  * @copyright Copyright © 2005, Brion Vibber <brion@pobox.com>
  12  *
  13  * @link http://www.google.com/webmasters/sitemaps/docs/en/about.html
  14  * @link http://www.google.com/schemas/sitemap/0.84/sitemap.xsd
  15  *
  16  * @license http://www.gnu.org/copyleft/gpl.html GNU General Public License 2.0 or later
  17  */
  18
  19 class GenerateSitemap {
  20         /**
  21          * The maximum amount of urls in a sitemap file
  22          *
  23          * @link http://www.google.com/schemas/sitemap/0.84/sitemap.xsd
  24          *
  25          * @var int
  26          */
  27         var $url_limit;
  28
  29         /**
  30          * The maximum size of a sitemap file
  31          *
  32          * @link http://www.google.com/webmasters/sitemaps/docs/en/protocol.html#faq_sitemap_size
  33          *
  34          * @var int
  35          */
  36         var $size_limit;
  37
  38         /**
  39          * The path to prepend to the filename
  40          *
  41          * @var string
  42          */
  43         var $fspath;
  44
  45         /**
  46          * The path to append to the domain name
  47          *
  48          * @var string
  49          */
  50         var $path;
  51
  52         /**
  53          * Whether or not to use compression
  54          *
  55          * @var bool
  56          */
  57         var $compress;
  58
  59         /**
  60          * The number of entries to save in each sitemap file
  61          *
  62          * @var array
  63          */
  64         var $limit = array();
  65
  66         /**
  67          * Key => value entries of namespaces and their priorities
  68          *
  69          * @var array
  70          */
  71         var $priorities = array(
  72                 // Custom main namespaces
  73                 GS_MAIN                 => '0.5',
  74                 // Custom talk namesspaces
  75                 GS_TALK                 => '0.1',
  76                 // MediaWiki standard namespaces
  77                 NS_MAIN                 => '1.0',
  78                 NS_TALK                 => '0.1',
  79                 NS_USER                 => '0.5',
  80                 NS_USER_TALK            => '0.1',
  81                 NS_PROJECT              => '0.5',
  82                 NS_PROJECT_TALK         => '0.1',
  83                 NS_IMAGE                => '0.5',
  84                 NS_IMAGE_TALK           => '0.1',
  85                 NS_MEDIAWIKI            => '0.0',
  86                 NS_MEDIAWIKI_TALK       => '0.1',
  87                 NS_TEMPLATE             => '0.0',
  88                 NS_TEMPLATE_TALK        => '0.1',
  89                 NS_HELP                 => '0.5',
  90                 NS_HELP_TALK            => '0.1',
  91                 NS_CATEGORY             => '0.5',
  92                 NS_CATEGORY_TALK        => '0.1',
  93         );
  94
  95         /**
  96          * A one-dimensional array of namespaces in the wiki
  97          *
  98          * @var array
  99          */
 100         var $namespaces = array();
 101
 102         /**
 103          * When this sitemap batch was generated
 104          *
 105          * @var string
 106          */
 107         var $timestamp;
 108
 109         /**
 110          * A database slave object
 111          *
 112          * @var object
 113          */
 114         var $dbr;
 115
 116         /**
 117          * A resource pointing to the sitemap index file
 118          *
 119          * @var resource
 120          */
 121         var $findex;
 122
 123
 124         /**
 125          * A resource pointing to a sitemap file
 126          *
 127          * @var resource
 128          */
 129         var $file;
 130
 131         /**
 132          * A resource pointing to php://stderr
 133          *
 134          * @var resource
 135          */
 136         var $stderr;
 137
 138         /**
 139          * Constructor
 140          *
 141          * @param string $fspath The path to prepend to the filenames, used to
 142          *                     save them somewhere else than in the root directory
 143          * @param string $path The path to append to the domain name
 144          * @param bool $compress Whether to compress the sitemap files
 145          */
 146         function GenerateSitemap( $fspath, $path, $compress ) {
 147                 global $wgScriptPath;
 148
 149                 $this->url_limit = 50000;
 150                 $this->size_limit = pow( 2, 20 ) * 10;
 151                 $this->fspath = isset( $fspath ) ? $fspath : '';
 152                 $this->path = isset( $path ) ? $path : $wgScriptPath;
 153                 $this->compress = $compress;
 154
 155                 $this->stderr = fopen( 'php://stderr', 'wt' );
 156                 $this->dbr = wfGetDB( DB_SLAVE );
 157                 $this->generateNamespaces();
 158                 $this->timestamp = wfTimestamp( TS_ISO_8601, wfTimestampNow() );
 159                 $this->findex = fopen( "{$this->fspath}sitemap-index-" . wfWikiID() . ".xml", 'wb' );
 160         }
 161
 162         /**
 163          * Generate a one-dimensional array of existing namespaces
 164          */
 165         function generateNamespaces() {
 166                 $fname = 'GenerateSitemap::generateNamespaces';
 167
 168                 $res = $this->dbr->select( 'page',
 169                         array( 'page_namespace' ),
 170                         array(),
 171                         $fname,
 172                         array(
 173                                 'GROUP BY' => 'page_namespace',
 174                                 'ORDER BY' => 'page_namespace',
 175                         )
 176                 );
 177
 178                 while ( $row = $this->dbr->fetchObject( $res ) )
 179                         $this->namespaces[] = $row->page_namespace;
 180         }
 181
 182         /**
 183          * Get the priority of a given namespace
 184          *
 185          * @param int $namespace The namespace to get the priority for
 186          +
 187          * @return string
 188          */
 189
 190         function priority( $namespace ) {
 191                 return isset( $this->priorities[$namespace] ) ? $this->priorities[$namespace] : $this->guessPriority( $namespace );
 192         }
 193
 194         /**
 195          * If the namespace isn't listed on the priority list return the
 196          * default priority for the namespace, varies depending on whether it's
 197          * a talkpage or not.
 198          *
 199          * @param int $namespace The namespace to get the priority for
 200          *
 201          * @return string
 202          */
 203         function guessPriority( $namespace ) {
 204                 return Namespace::isMain( $namespace ) ? $this->priorities[GS_MAIN] : $this->priorities[GS_TALK];
 205         }
 206
 207         /**
 208          * Return a database resolution of all the pages in a given namespace
 209          *
 210          * @param int $namespace Limit the query to this namespace
 211          *
 212          * @return resource
 213          */
 214         function getPageRes( $namespace ) {
 215                 $fname = 'GenerateSitemap::getPageRes';
 216
 217                 return $this->dbr->select( 'page',
 218                         array(
 219                                 'page_namespace',
 220                                 'page_title',
 221                                 'page_touched',
 222                         ),
 223                         array( 'page_namespace' => $namespace ),
 224                         $fname
 225                 );
 226         }
 227
 228         /**
 229          * Main loop
 230          *
 231          * @access public
 232          */
 233         function main() {
 234                 global $wgContLang;
 235
 236                 fwrite( $this->findex, $this->openIndex() );
 237
 238                 foreach ( $this->namespaces as $namespace ) {
 239                         $res = $this->getPageRes( $namespace );
 240                         $this->file = false;
 241                         $this->generateLimit( $namespace );
 242                         $length = $this->limit[0];
 243                         $i = $smcount = 0;
 244
 245                         $fns = $wgContLang->getFormattedNsText( $namespace );
 246                         $this->debug( "$namespace ($fns)" );
 247                         while ( $row = $this->dbr->fetchObject( $res ) ) {
 248                                 if ( $i++ === 0 || $i === $this->url_limit + 1 || $length + $this->limit[1] + $this->limit[2] > $this->size_limit ) {
 249                                         if ( $this->file !== false ) {
 250                                                 $this->write( $this->file, $this->closeFile() );
 251                                                 $this->close( $this->file );
 252                                         }
 253                                         $filename = $this->sitemapFilename( $namespace, $smcount++ );
 254                                         $this->file = $this->open( $this->fspath . $filename, 'wb' );
 255                                         $this->write( $this->file, $this->openFile() );
 256                                         fwrite( $this->findex, $this->indexEntry( $filename ) );
 257                                         $this->debug( "\t$filename" );
 258                                         $length = $this->limit[0];
 259                                         $i = 1;
 260                                 }
 261                                 $title = Title::makeTitle( $row->page_namespace, $row->page_title );
 262                                 $date = wfTimestamp( TS_ISO_8601, $row->page_touched );
 263                                 $entry = $this->fileEntry( $title->getFullURL(), $date, $this->priority( $namespace ) );
 264                                 $length += strlen( $entry );
 265                                 $this->write( $this->file, $entry );
 266                                 // generate pages for language variants
 267                                 if($wgContLang->hasVariants()){
 268                                         $variants = $wgContLang->getVariants();
 269                                         foreach($variants as $vCode){
 270                                                 if($vCode==$wgContLang->getCode()) continue; // we don't want default variant
 271                                                 $entry = $this->fileEntry( $title->getFullURL('',$vCode), $date, $this->priority( $namespace ) );
 272                                                 $length += strlen( $entry );
 273                                                 $this->write( $this->file, $entry );
 274                                         }
 275                                 }
 276                         }
 277                         if ( $this->file ) {
 278                                 $this->write( $this->file, $this->closeFile() );
 279                                 $this->close( $this->file );
 280                         }
 281                 }
 282                 fwrite( $this->findex, $this->closeIndex() );
 283                 fclose( $this->findex );
 284         }
 285
 286         /**
 287          * gzopen() / fopen() wrapper
 288          *
 289          * @return resource
 290          */
 291         function open( $file, $flags ) {
 292                 return $this->compress ? gzopen( $file, $flags ) : fopen( $file, $flags );
 293         }
 294
 295         /**
 296          * gzwrite() / fwrite() wrapper
 297          */
 298         function write( &$handle, $str ) {
 299                 if ( $this->compress )
 300                         gzwrite( $handle, $str );
 301                 else
 302                         fwrite( $handle, $str );
 303         }
 304
 305         /**
 306          * gzclose() / fclose() wrapper
 307          */
 308         function close( &$handle ) {
 309                 if ( $this->compress )
 310                         gzclose( $handle );
 311                 else
 312                         fclose( $handle );
 313         }
 314
 315         /**
 316          * Get a sitemap filename
 317          *
 318          * @static
 319          *
 320          * @param int $namespace The namespace
 321          * @param int $count The count
 322          *
 323          * @return string
 324          */
 325         function sitemapFilename( $namespace, $count ) {
 326                 $ext = $this->compress ? '.gz' : '';
 327                 return "sitemap-".wfWikiID()."-NS_$namespace-$count.xml$ext";
 328         }
 329
 330         /**
 331          * Return the XML required to open an XML file
 332          *
 333          * @static
 334          *
 335          * @return string
 336          */
 337         function xmlHead() {
 338                 return '<?xml version="1.0" encoding="UTF-8"?>' . "\n";
 339         }
 340
 341         /**
 342          * Return the XML schema being used
 343          *
 344          * @static
 345          *
 346          * @returns string
 347          */
 348         function xmlSchema() {
 349                 return 'http://www.google.com/schemas/sitemap/0.84';
 350         }
 351
 352         /**
 353          * Return the XML required to open a sitemap index file
 354          *
 355          * @return string
 356          */
 357         function openIndex() {
 358                 return $this->xmlHead() . '<sitemapindex xmlns="' . $this->xmlSchema() . '">' . "\n";
 359         }
 360
 361         /**
 362          * Return the XML for a single sitemap indexfile entry
 363          *
 364          * @static
 365          *
 366          * @param string $filename The filename of the sitemap file
 367          *
 368          * @return string
 369          */
 370         function indexEntry( $filename ) {
 371                 return
 372                         "\t<sitemap>\n" .
 373                         "\t\t<loc>$filename</loc>\n" .
 374                         "\t\t<lastmod>{$this->timestamp}</lastmod>\n" .
 375                         "\t</sitemap>\n";
 376         }
 377
 378         /**
 379          * Return the XML required to close a sitemap index file
 380          *
 381          * @static
 382          *
 383          * @return string
 384          */
 385         function closeIndex() {
 386                 return "</sitemapindex>\n";
 387         }
 388
 389         /**
 390          * Return the XML required to open a sitemap file
 391          *
 392          * @return string
 393          */
 394         function openFile() {
 395                 return $this->xmlHead() . '<urlset xmlns="' . $this->xmlSchema() . '">' . "\n";
 396         }
 397
 398         /**
 399          * Return the XML for a single sitemap entry
 400          *
 401          * @static
 402          *
 403          * @param string $url An RFC 2396 compilant URL
 404          * @param string $date A ISO 8601 date
 405          * @param string $priority A priority indicator, 0.0 - 1.0 inclusive with a 0.1 stepsize
 406          *
 407          * @return string
 408          */
 409         function fileEntry( $url, $date, $priority ) {
 410                 return
 411                         "\t<url>\n" .
 412                         "\t\t<loc>$url</loc>\n" .
 413                         "\t\t<lastmod>$date</lastmod>\n" .
 414                         "\t\t<priority>$priority</priority>\n" .
 415                         "\t</url>\n";
 416         }
 417
 418         /**
 419          * Return the XML required to close sitemap file
 420          *
 421          * @static
 422          * @return string
 423          */
 424         function closeFile() {
 425                 return "</urlset>\n";
 426         }
 427
 428         /**
 429          * Write a string to stderr followed by a UNIX newline
 430          */
 431         function debug( $str ) {
 432                 fwrite( $this->stderr, "$str\n" );
 433         }
 434
 435         /**
 436          * Populate $this->limit
 437          */
 438         function generateLimit( $namespace ) {
 439                 $title = Title::makeTitle( $namespace, str_repeat( "\xf0\xa8\xae\x81", 63 ) . "\xe5\x96\x83" );
 440
 441                 $this->limit = array(
 442                         strlen( $this->openFile() ),
 443                         strlen( $this->fileEntry( $title->getFullUrl(), wfTimestamp( TS_ISO_8601, wfTimestamp() ), $this->priority( $namespace ) ) ),
 444                         strlen( $this->closeFile() )
 445                 );
 446         }
 447 }
 448
 449 if ( in_array( '--help', $argv ) ) {
 450         echo
 451                 "Usage: php generateSitemap.php [host] [options]\n" .
 452                 "\thost = hostname\n" .
 453                 "\toptions:\n" .
 454                 "\t\t--help\tshow this message\n" .
 455                 "\t\t--fspath\tThe file system path to save to, e.g /tmp/sitemap/\n" .
 456                 "\t\t--path\tThe http path to use, e.g. /wiki\n" .
 457                 "\t\t--compress=[yes|no]\tcompress the sitemap files, default yes\n";
 458         die( -1 );
 459 }
 460
 461 if ( isset( $argv[1] ) && strpos( $argv[1], '--' ) !== 0 )
 462         $_SERVER['SERVER_NAME'] = $argv[1];
 463
 464 $optionsWithArgs = array( 'fspath', 'path', 'compress' );
 465 require_once 'commandLine.inc';
 466
 467 $gs = new GenerateSitemap( @$options['fspath'], @$options['path'], @$options['compress'] !== 'no' );
 468 $gs->main();
 469 ?>