includes/job/JobQueue.php

   1 <?php
   2 /**
   3  * Job queue base code
   4  *
   5  * @file
   6  * @defgroup JobQueue JobQueue
   7  */
   8
   9 if ( !defined( 'MEDIAWIKI' ) ) {
  10         die( "This file is part of MediaWiki, it is not a valid entry point\n" );
  11 }
  12
  13 /**
  14  * Class to both describe a background job and handle jobs.
  15  *
  16  * @ingroup JobQueue
  17  */
  18 abstract class Job {
  19
  20         /**
  21          * @var Title
  22          */
  23         var $title;
  24
  25         var $command,
  26                 $params,
  27                 $id,
  28                 $removeDuplicates,
  29                 $error;
  30
  31         /*-------------------------------------------------------------------------
  32          * Abstract functions
  33          *------------------------------------------------------------------------*/
  34
  35         /**
  36          * Run the job
  37          * @return boolean success
  38          */
  39         abstract function run();
  40
  41         /*-------------------------------------------------------------------------
  42          * Static functions
  43          *------------------------------------------------------------------------*/
  44
  45         /**
  46          * Pop a job of a certain type.  This tries less hard than pop() to
  47          * actually find a job; it may be adversely affected by concurrent job
  48          * runners.
  49          */
  50         static function pop_type( $type ) {
  51                 wfProfilein( __METHOD__ );
  52
  53                 $dbw = wfGetDB( DB_MASTER );
  54
  55                 $row = $dbw->selectRow(
  56                         'job',
  57                         '*',
  58                         array( 'job_cmd' => $type ),
  59                         __METHOD__,
  60                         array( 'LIMIT' => 1 )
  61                 );
  62
  63                 if ( $row === false ) {
  64                         wfProfileOut( __METHOD__ );
  65                         return false;
  66                 }
  67
  68                 /* Ensure we "own" this row */
  69                 $dbw->delete( 'job', array( 'job_id' => $row->job_id ), __METHOD__ );
  70                 $affected = $dbw->affectedRows();
  71                 $dbw->commit();
  72
  73                 if ( $affected == 0 ) {
  74                         wfProfileOut( __METHOD__ );
  75                         return false;
  76                 }
  77
  78                 wfIncrStats( 'job-pop' );
  79                 $namespace = $row->job_namespace;
  80                 $dbkey = $row->job_title;
  81                 $title = Title::makeTitleSafe( $namespace, $dbkey );
  82                 $job = Job::factory( $row->job_cmd, $title, Job::extractBlob( $row->job_params ),
  83                         $row->job_id );
  84
  85                 $job->removeDuplicates();
  86
  87                 wfProfileOut( __METHOD__ );
  88                 return $job;
  89         }
  90
  91         /**
  92          * Pop a job off the front of the queue
  93          *
  94          * @param $offset Integer: Number of jobs to skip
  95          * @return Job or false if there's no jobs
  96          */
  97         static function pop( $offset = 0 ) {
  98                 global $wgJobTypesExcludedFromDefaultQueue;
  99                 wfProfileIn( __METHOD__ );
 100
 101                 $dbr = wfGetDB( DB_SLAVE );
 102
 103                 /* Get a job from the slave, start with an offset,
 104                         scan full set afterwards, avoid hitting purged rows
 105
 106                         NB: If random fetch previously was used, offset
 107                                 will always be ahead of few entries
 108                 */
 109                 $conditions = array();
 110                 if ( count( $wgJobTypesExcludedFromDefaultQueue ) != 0 ) {
 111                         foreach ( $wgJobTypesExcludedFromDefaultQueue as $cmdType ) {
 112                                 $conditions[] = "job_cmd != " . $dbr->addQuotes( $cmdType );
 113                         }
 114                 }
 115                 $offset = intval( $offset );
 116                 $row = $dbr->selectRow( 'job', '*', array_merge( $conditions, array( "job_id >= $offset" ) ) , __METHOD__,
 117                         array( 'ORDER BY' => 'job_id', 'LIMIT' => 1 )
 118                 );
 119
 120                 // Refetching without offset is needed as some of job IDs could have had delayed commits
 121                 // and have lower IDs than jobs already executed, blame concurrency :)
 122                 //
 123                 if ( $row === false ) {
 124                         if ( $offset != 0 ) {
 125                                 $row = $dbr->selectRow( 'job', '*', $conditions, __METHOD__,
 126                                         array( 'ORDER BY' => 'job_id', 'LIMIT' => 1 ) );
 127                         }
 128
 129                         if ( $row === false ) {
 130                                 wfProfileOut( __METHOD__ );
 131                                 return false;
 132                         }
 133                 }
 134
 135                 // Try to delete it from the master
 136                 $dbw = wfGetDB( DB_MASTER );
 137                 $dbw->delete( 'job', array( 'job_id' => $row->job_id ), __METHOD__ );
 138                 $affected = $dbw->affectedRows();
 139                 $dbw->commit();
 140
 141                 if ( !$affected ) {
 142                         // Failed, someone else beat us to it
 143                         // Try getting a random row
 144                         $row = $dbw->selectRow( 'job', array( 'MIN(job_id) as minjob',
 145                                 'MAX(job_id) as maxjob' ), '1=1', __METHOD__ );
 146                         if ( $row === false || is_null( $row->minjob ) || is_null( $row->maxjob ) ) {
 147                                 // No jobs to get
 148                                 wfProfileOut( __METHOD__ );
 149                                 return false;
 150                         }
 151                         // Get the random row
 152                         $row = $dbw->selectRow( 'job', '*',
 153                                 'job_id >= ' . mt_rand( $row->minjob, $row->maxjob ), __METHOD__ );
 154                         if ( $row === false ) {
 155                                 // Random job gone before we got the chance to select it
 156                                 // Give up
 157                                 wfProfileOut( __METHOD__ );
 158                                 return false;
 159                         }
 160                         // Delete the random row
 161                         $dbw->delete( 'job', array( 'job_id' => $row->job_id ), __METHOD__ );
 162                         $affected = $dbw->affectedRows();
 163                         $dbw->commit();
 164
 165                         if ( !$affected ) {
 166                                 // Random job gone before we exclusively deleted it
 167                                 // Give up
 168                                 wfProfileOut( __METHOD__ );
 169                                 return false;
 170                         }
 171                 }
 172
 173                 // If execution got to here, there's a row in $row that has been deleted from the database
 174                 // by this thread. Hence the concurrent pop was successful.
 175                 wfIncrStats( 'job-pop' );
 176                 $namespace = $row->job_namespace;
 177                 $dbkey = $row->job_title;
 178                 $title = Title::makeTitleSafe( $namespace, $dbkey );
 179                 $job = Job::factory( $row->job_cmd, $title, Job::extractBlob( $row->job_params ), $row->job_id );
 180
 181                 // Remove any duplicates it may have later in the queue
 182                 $job->removeDuplicates();
 183
 184                 wfProfileOut( __METHOD__ );
 185                 return $job;
 186         }
 187
 188         /**
 189          * Create the appropriate object to handle a specific job
 190          *
 191          * @param $command String: Job command
 192          * @param $title Title: Associated title
 193          * @param $params Array: Job parameters
 194          * @param $id Int: Job identifier
 195          * @return Job
 196          */
 197         static function factory( $command, $title, $params = false, $id = 0 ) {
 198                 global $wgJobClasses;
 199                 if( isset( $wgJobClasses[$command] ) ) {
 200                         $class = $wgJobClasses[$command];
 201                         return new $class( $title, $params, $id );
 202                 }
 203                 throw new MWException( "Invalid job command `{$command}`" );
 204         }
 205
 206         static function makeBlob( $params ) {
 207                 if ( $params !== false ) {
 208                         return serialize( $params );
 209                 } else {
 210                         return '';
 211                 }
 212         }
 213
 214         static function extractBlob( $blob ) {
 215                 if ( (string)$blob !== '' ) {
 216                         return unserialize( $blob );
 217                 } else {
 218                         return false;
 219                 }
 220         }
 221
 222         /**
 223          * Batch-insert a group of jobs into the queue.
 224          * This will be wrapped in a transaction with a forced commit.
 225          *
 226          * This may add duplicate at insert time, but they will be
 227          * removed later on, when the first one is popped.
 228          *
 229          * @param $jobs array of Job objects
 230          */
 231         static function batchInsert( $jobs ) {
 232                 if ( !count( $jobs ) ) {
 233                         return;
 234                 }
 235                 $dbw = wfGetDB( DB_MASTER );
 236                 $rows = array();
 237                 foreach ( $jobs as $job ) {
 238                         $rows[] = $job->insertFields();
 239                         if ( count( $rows ) >= 50 ) {
 240                                 # Do a small transaction to avoid slave lag
 241                                 $dbw->begin();
 242                                 $dbw->insert( 'job', $rows, __METHOD__, 'IGNORE' );
 243                                 $dbw->commit();
 244                                 $rows = array();
 245                         }
 246                 }
 247                 if ( $rows ) { // last chunk
 248                         $dbw->begin();
 249                         $dbw->insert( 'job', $rows, __METHOD__, 'IGNORE' );
 250                         $dbw->commit();
 251                 }
 252                 wfIncrStats( 'job-insert', count( $jobs ) );
 253         }
 254
 255         /**
 256          * Insert a group of jobs into the queue.
 257          *
 258          * Same as batchInsert() but does not commit and can thus
 259          * be rolled-back as part of a larger transaction. However,
 260          * large batches of jobs can cause slave lag.
 261          *
 262          * @param $jobs array of Job objects
 263          */
 264         static function safeBatchInsert( $jobs ) {
 265                 if ( !count( $jobs ) ) {
 266                         return;
 267                 }
 268                 $dbw = wfGetDB( DB_MASTER );
 269                 $rows = array();
 270                 foreach ( $jobs as $job ) {
 271                         $rows[] = $job->insertFields();
 272                         if ( count( $rows ) >= 500 ) {
 273                                 $dbw->insert( 'job', $rows, __METHOD__, 'IGNORE' );
 274                                 $rows = array();
 275                         }
 276                 }
 277                 if ( $rows ) { // last chunk
 278                         $dbw->insert( 'job', $rows, __METHOD__, 'IGNORE' );
 279                 }
 280                 wfIncrStats( 'job-insert', count( $jobs ) );
 281         }
 282
 283         /*-------------------------------------------------------------------------
 284          * Non-static functions
 285          *------------------------------------------------------------------------*/
 286
 287         /**
 288          * @param $command
 289          * @param $title
 290          * @param $params array
 291          * @param int $id
 292          */
 293         function __construct( $command, $title, $params = false, $id = 0 ) {
 294                 $this->command = $command;
 295                 $this->title = $title;
 296                 $this->params = $params;
 297                 $this->id = $id;
 298
 299                 // A bit of premature generalisation
 300                 // Oh well, the whole class is premature generalisation really
 301                 $this->removeDuplicates = true;
 302         }
 303
 304         /**
 305          * Insert a single job into the queue.
 306          * @return bool true on success
 307          */
 308         function insert() {
 309                 $fields = $this->insertFields();
 310
 311                 $dbw = wfGetDB( DB_MASTER );
 312
 313                 if ( $this->removeDuplicates ) {
 314                         $res = $dbw->select( 'job', array( '1' ), $fields, __METHOD__ );
 315                         if ( $dbw->numRows( $res ) ) {
 316                                 return;
 317                         }
 318                 }
 319                 wfIncrStats( 'job-insert' );
 320                 return $dbw->insert( 'job', $fields, __METHOD__ );
 321         }
 322
 323         protected function insertFields() {
 324                 $dbw = wfGetDB( DB_MASTER );
 325                 return array(
 326                         'job_id' => $dbw->nextSequenceValue( 'job_job_id_seq' ),
 327                         'job_cmd' => $this->command,
 328                         'job_namespace' => $this->title->getNamespace(),
 329                         'job_title' => $this->title->getDBkey(),
 330                         'job_params' => Job::makeBlob( $this->params )
 331                 );
 332         }
 333
 334         /**
 335          * Remove jobs in the job queue which are duplicates of this job.
 336          * This is deadlock-prone and so starts its own transaction.
 337          */
 338         function removeDuplicates() {
 339                 if ( !$this->removeDuplicates ) {
 340                         return;
 341                 }
 342
 343                 $fields = $this->insertFields();
 344                 unset( $fields['job_id'] );
 345                 $dbw = wfGetDB( DB_MASTER );
 346                 $dbw->begin();
 347                 $dbw->delete( 'job', $fields, __METHOD__ );
 348                 $affected = $dbw->affectedRows();
 349                 $dbw->commit();
 350                 if ( $affected ) {
 351                         wfIncrStats( 'job-dup-delete', $affected );
 352                 }
 353         }
 354
 355         function toString() {
 356                 $paramString = '';
 357                 if ( $this->params ) {
 358                         foreach ( $this->params as $key => $value ) {
 359                                 if ( $paramString != '' ) {
 360                                         $paramString .= ' ';
 361                                 }
 362                                 $paramString .= "$key=$value";
 363                         }
 364                 }
 365
 366                 if ( is_object( $this->title ) ) {
 367                         $s = "{$this->command} " . $this->title->getPrefixedDBkey();
 368                         if ( $paramString !== '' ) {
 369                                 $s .= ' ' . $paramString;
 370                         }
 371                         return $s;
 372                 } else {
 373                         return "{$this->command} $paramString";
 374                 }
 375         }
 376
 377         protected function setLastError( $error ) {
 378                 $this->error = $error;
 379         }
 380
 381         function getLastError() {
 382                 return $this->error;
 383         }
 384 }