6 * @defgroup JobQueue JobQueue
9 if ( !defined( 'MEDIAWIKI' ) ) {
10 die( "This file is part of MediaWiki, it is not a valid entry point\n" );
14 * Class to both describe a background job and handle jobs.
31 /*-------------------------------------------------------------------------
33 *------------------------------------------------------------------------*/
37 * @return boolean success
39 abstract function run();
41 /*-------------------------------------------------------------------------
43 *------------------------------------------------------------------------*/
46 * Pop a job of a certain type. This tries less hard than pop() to
47 * actually find a job; it may be adversely affected by concurrent job
54 static function pop_type( $type ) {
55 wfProfilein( __METHOD__
);
57 $dbw = wfGetDB( DB_MASTER
);
59 $dbw->begin( __METHOD__
);
61 $row = $dbw->selectRow(
64 array( 'job_cmd' => $type ),
66 array( 'LIMIT' => 1, 'FOR UPDATE' )
69 if ( $row === false ) {
70 $dbw->commit( __METHOD__
);
71 wfProfileOut( __METHOD__
);
75 /* Ensure we "own" this row */
76 $dbw->delete( 'job', array( 'job_id' => $row->job_id
), __METHOD__
);
77 $affected = $dbw->affectedRows();
78 $dbw->commit( __METHOD__
);
80 if ( $affected == 0 ) {
81 wfProfileOut( __METHOD__
);
85 wfIncrStats( 'job-pop' );
86 $namespace = $row->job_namespace
;
87 $dbkey = $row->job_title
;
88 $title = Title
::makeTitleSafe( $namespace, $dbkey );
89 $job = Job
::factory( $row->job_cmd
, $title, Job
::extractBlob( $row->job_params
),
92 $job->removeDuplicates();
94 wfProfileOut( __METHOD__
);
99 * Pop a job off the front of the queue
101 * @param $offset Integer: Number of jobs to skip
102 * @return Job or false if there's no jobs
104 static function pop( $offset = 0 ) {
105 global $wgJobTypesExcludedFromDefaultQueue;
106 wfProfileIn( __METHOD__
);
108 $dbr = wfGetDB( DB_SLAVE
);
110 /* Get a job from the slave, start with an offset,
111 scan full set afterwards, avoid hitting purged rows
113 NB: If random fetch previously was used, offset
114 will always be ahead of few entries
116 $conditions = array();
117 if ( count( $wgJobTypesExcludedFromDefaultQueue ) != 0 ) {
118 foreach ( $wgJobTypesExcludedFromDefaultQueue as $cmdType ) {
119 $conditions[] = "job_cmd != " . $dbr->addQuotes( $cmdType );
122 $offset = intval( $offset );
123 $options = array( 'ORDER BY' => 'job_id', 'USE INDEX' => 'PRIMARY' );
125 $row = $dbr->selectRow( 'job', '*',
126 array_merge( $conditions, array( "job_id >= $offset" ) ),
131 // Refetching without offset is needed as some of job IDs could have had delayed commits
132 // and have lower IDs than jobs already executed, blame concurrency :)
134 if ( $row === false ) {
135 if ( $offset != 0 ) {
136 $row = $dbr->selectRow( 'job', '*', $conditions, __METHOD__
, $options );
139 if ( $row === false ) {
140 wfProfileOut( __METHOD__
);
145 // Try to delete it from the master
146 $dbw = wfGetDB( DB_MASTER
);
147 $dbw->delete( 'job', array( 'job_id' => $row->job_id
), __METHOD__
);
148 $affected = $dbw->affectedRows();
149 $dbw->commit( __METHOD__
);
152 // Failed, someone else beat us to it
153 // Try getting a random row
154 $row = $dbw->selectRow( 'job', array( 'MIN(job_id) as minjob',
155 'MAX(job_id) as maxjob' ), '1=1', __METHOD__
);
156 if ( $row === false ||
is_null( $row->minjob
) ||
is_null( $row->maxjob
) ) {
158 wfProfileOut( __METHOD__
);
161 // Get the random row
162 $row = $dbw->selectRow( 'job', '*',
163 'job_id >= ' . mt_rand( $row->minjob
, $row->maxjob
), __METHOD__
);
164 if ( $row === false ) {
165 // Random job gone before we got the chance to select it
167 wfProfileOut( __METHOD__
);
170 // Delete the random row
171 $dbw->delete( 'job', array( 'job_id' => $row->job_id
), __METHOD__
);
172 $affected = $dbw->affectedRows();
173 $dbw->commit( __METHOD__
);
176 // Random job gone before we exclusively deleted it
178 wfProfileOut( __METHOD__
);
183 // If execution got to here, there's a row in $row that has been deleted from the database
184 // by this thread. Hence the concurrent pop was successful.
185 wfIncrStats( 'job-pop' );
186 $namespace = $row->job_namespace
;
187 $dbkey = $row->job_title
;
188 $title = Title
::makeTitleSafe( $namespace, $dbkey );
189 $job = Job
::factory( $row->job_cmd
, $title, Job
::extractBlob( $row->job_params
), $row->job_id
);
191 // Remove any duplicates it may have later in the queue
192 $job->removeDuplicates();
194 wfProfileOut( __METHOD__
);
199 * Create the appropriate object to handle a specific job
201 * @param $command String: Job command
202 * @param $title Title: Associated title
203 * @param $params Array: Job parameters
204 * @param $id Int: Job identifier
207 static function factory( $command, $title, $params = false, $id = 0 ) {
208 global $wgJobClasses;
209 if( isset( $wgJobClasses[$command] ) ) {
210 $class = $wgJobClasses[$command];
211 return new $class( $title, $params, $id );
213 throw new MWException( "Invalid job command `{$command}`" );
220 static function makeBlob( $params ) {
221 if ( $params !== false ) {
222 return serialize( $params );
232 static function extractBlob( $blob ) {
233 if ( (string)$blob !== '' ) {
234 return unserialize( $blob );
241 * Batch-insert a group of jobs into the queue.
242 * This will be wrapped in a transaction with a forced commit.
244 * This may add duplicate at insert time, but they will be
245 * removed later on, when the first one is popped.
247 * @param $jobs array of Job objects
249 static function batchInsert( $jobs ) {
250 if ( !count( $jobs ) ) {
253 $dbw = wfGetDB( DB_MASTER
);
259 foreach ( $jobs as $job ) {
260 $rows[] = $job->insertFields();
261 if ( count( $rows ) >= 50 ) {
262 # Do a small transaction to avoid slave lag
263 $dbw->begin( __METHOD__
);
264 $dbw->insert( 'job', $rows, __METHOD__
, 'IGNORE' );
265 $dbw->commit( __METHOD__
);
269 if ( $rows ) { // last chunk
270 $dbw->begin( __METHOD__
);
271 $dbw->insert( 'job', $rows, __METHOD__
, 'IGNORE' );
272 $dbw->commit( __METHOD__
);
274 wfIncrStats( 'job-insert', count( $jobs ) );
278 * Insert a group of jobs into the queue.
280 * Same as batchInsert() but does not commit and can thus
281 * be rolled-back as part of a larger transaction. However,
282 * large batches of jobs can cause slave lag.
284 * @param $jobs array of Job objects
286 static function safeBatchInsert( $jobs ) {
287 if ( !count( $jobs ) ) {
290 $dbw = wfGetDB( DB_MASTER
);
292 foreach ( $jobs as $job ) {
293 $rows[] = $job->insertFields();
294 if ( count( $rows ) >= 500 ) {
295 $dbw->insert( 'job', $rows, __METHOD__
, 'IGNORE' );
299 if ( $rows ) { // last chunk
300 $dbw->insert( 'job', $rows, __METHOD__
, 'IGNORE' );
302 wfIncrStats( 'job-insert', count( $jobs ) );
305 /*-------------------------------------------------------------------------
306 * Non-static functions
307 *------------------------------------------------------------------------*/
312 * @param $params array
315 function __construct( $command, $title, $params = false, $id = 0 ) {
316 $this->command
= $command;
317 $this->title
= $title;
318 $this->params
= $params;
321 // A bit of premature generalisation
322 // Oh well, the whole class is premature generalisation really
323 $this->removeDuplicates
= true;
327 * Insert a single job into the queue.
328 * @return bool true on success
331 $fields = $this->insertFields();
333 $dbw = wfGetDB( DB_MASTER
);
335 if ( $this->removeDuplicates
) {
336 $res = $dbw->select( 'job', array( '1' ), $fields, __METHOD__
);
337 if ( $dbw->numRows( $res ) ) {
341 wfIncrStats( 'job-insert' );
342 return $dbw->insert( 'job', $fields, __METHOD__
);
348 protected function insertFields() {
349 $dbw = wfGetDB( DB_MASTER
);
351 'job_id' => $dbw->nextSequenceValue( 'job_job_id_seq' ),
352 'job_cmd' => $this->command
,
353 'job_namespace' => $this->title
->getNamespace(),
354 'job_title' => $this->title
->getDBkey(),
355 'job_timestamp' => $dbw->timestamp(),
356 'job_params' => Job
::makeBlob( $this->params
)
361 * Remove jobs in the job queue which are duplicates of this job.
362 * This is deadlock-prone and so starts its own transaction.
364 function removeDuplicates() {
365 if ( !$this->removeDuplicates
) {
369 $fields = $this->insertFields();
370 unset( $fields['job_id'] );
371 $dbw = wfGetDB( DB_MASTER
);
372 $dbw->begin( __METHOD__
);
373 $dbw->delete( 'job', $fields, __METHOD__
);
374 $affected = $dbw->affectedRows();
375 $dbw->commit( __METHOD__
);
377 wfIncrStats( 'job-dup-delete', $affected );
384 function toString() {
386 if ( $this->params
) {
387 foreach ( $this->params
as $key => $value ) {
388 if ( $paramString != '' ) {
391 $paramString .= "$key=$value";
395 if ( is_object( $this->title
) ) {
396 $s = "{$this->command} " . $this->title
->getPrefixedDBkey();
397 if ( $paramString !== '' ) {
398 $s .= ' ' . $paramString;
402 return "{$this->command} $paramString";
406 protected function setLastError( $error ) {
407 $this->error
= $error;
410 function getLastError() {