6 * @defgroup JobQueue JobQueue
9 if ( !defined( 'MEDIAWIKI' ) ) {
10 die( "This file is part of MediaWiki, it is not a valid entry point\n" );
14 * Class to both describe a background job and handle jobs.
31 /*-------------------------------------------------------------------------
33 *------------------------------------------------------------------------*/
37 * @return boolean success
39 abstract function run();
41 /*-------------------------------------------------------------------------
43 *------------------------------------------------------------------------*/
46 * Pop a job of a certain type. This tries less hard than pop() to
47 * actually find a job; it may be adversely affected by concurrent job
50 static function pop_type( $type ) {
51 wfProfilein( __METHOD__
);
53 $dbw = wfGetDB( DB_MASTER
);
55 $row = $dbw->selectRow(
58 array( 'job_cmd' => $type ),
63 if ( $row === false ) {
64 wfProfileOut( __METHOD__
);
68 /* Ensure we "own" this row */
69 $dbw->delete( 'job', array( 'job_id' => $row->job_id
), __METHOD__
);
70 $affected = $dbw->affectedRows();
73 if ( $affected == 0 ) {
74 wfProfileOut( __METHOD__
);
78 wfIncrStats( 'job-pop' );
79 $namespace = $row->job_namespace
;
80 $dbkey = $row->job_title
;
81 $title = Title
::makeTitleSafe( $namespace, $dbkey );
82 $job = Job
::factory( $row->job_cmd
, $title, Job
::extractBlob( $row->job_params
),
85 $job->removeDuplicates();
87 wfProfileOut( __METHOD__
);
92 * Pop a job off the front of the queue
94 * @param $offset Integer: Number of jobs to skip
95 * @return Job or false if there's no jobs
97 static function pop( $offset = 0 ) {
98 wfProfileIn( __METHOD__
);
100 $dbr = wfGetDB( DB_SLAVE
);
102 /* Get a job from the slave, start with an offset,
103 scan full set afterwards, avoid hitting purged rows
105 NB: If random fetch previously was used, offset
106 will always be ahead of few entries
109 $row = $dbr->selectRow( 'job', '*', "job_id >= ${offset}", __METHOD__
,
110 array( 'ORDER BY' => 'job_id', 'LIMIT' => 1 ) );
112 // Refetching without offset is needed as some of job IDs could have had delayed commits
113 // and have lower IDs than jobs already executed, blame concurrency :)
115 if ( $row === false ) {
116 if ( $offset != 0 ) {
117 $row = $dbr->selectRow( 'job', '*', '', __METHOD__
,
118 array( 'ORDER BY' => 'job_id', 'LIMIT' => 1 ) );
121 if ( $row === false ) {
122 wfProfileOut( __METHOD__
);
127 // Try to delete it from the master
128 $dbw = wfGetDB( DB_MASTER
);
129 $dbw->delete( 'job', array( 'job_id' => $row->job_id
), __METHOD__
);
130 $affected = $dbw->affectedRows();
134 // Failed, someone else beat us to it
135 // Try getting a random row
136 $row = $dbw->selectRow( 'job', array( 'MIN(job_id) as minjob',
137 'MAX(job_id) as maxjob' ), '1=1', __METHOD__
);
138 if ( $row === false ||
is_null( $row->minjob
) ||
is_null( $row->maxjob
) ) {
140 wfProfileOut( __METHOD__
);
143 // Get the random row
144 $row = $dbw->selectRow( 'job', '*',
145 'job_id >= ' . mt_rand( $row->minjob
, $row->maxjob
), __METHOD__
);
146 if ( $row === false ) {
147 // Random job gone before we got the chance to select it
149 wfProfileOut( __METHOD__
);
152 // Delete the random row
153 $dbw->delete( 'job', array( 'job_id' => $row->job_id
), __METHOD__
);
154 $affected = $dbw->affectedRows();
158 // Random job gone before we exclusively deleted it
160 wfProfileOut( __METHOD__
);
165 // If execution got to here, there's a row in $row that has been deleted from the database
166 // by this thread. Hence the concurrent pop was successful.
167 wfIncrStats( 'job-pop' );
168 $namespace = $row->job_namespace
;
169 $dbkey = $row->job_title
;
170 $title = Title
::makeTitleSafe( $namespace, $dbkey );
171 $job = Job
::factory( $row->job_cmd
, $title, Job
::extractBlob( $row->job_params
), $row->job_id
);
173 // Remove any duplicates it may have later in the queue
174 $job->removeDuplicates();
176 wfProfileOut( __METHOD__
);
181 * Create the appropriate object to handle a specific job
183 * @param $command String: Job command
184 * @param $title Title: Associated title
185 * @param $params Array: Job parameters
186 * @param $id Int: Job identifier
189 static function factory( $command, $title, $params = false, $id = 0 ) {
190 global $wgJobClasses;
191 if( isset( $wgJobClasses[$command] ) ) {
192 $class = $wgJobClasses[$command];
193 return new $class( $title, $params, $id );
195 throw new MWException( "Invalid job command `{$command}`" );
198 static function makeBlob( $params ) {
199 if ( $params !== false ) {
200 return serialize( $params );
206 static function extractBlob( $blob ) {
207 if ( (string)$blob !== '' ) {
208 return unserialize( $blob );
215 * Batch-insert a group of jobs into the queue.
216 * This will be wrapped in a transaction with a forced commit.
218 * This may add duplicate at insert time, but they will be
219 * removed later on, when the first one is popped.
221 * @param $jobs array of Job objects
223 static function batchInsert( $jobs ) {
224 if ( !count( $jobs ) ) {
227 $dbw = wfGetDB( DB_MASTER
);
229 foreach ( $jobs as $job ) {
230 $rows[] = $job->insertFields();
231 if ( count( $rows ) >= 50 ) {
232 # Do a small transaction to avoid slave lag
234 $dbw->insert( 'job', $rows, __METHOD__
, 'IGNORE' );
239 if ( $rows ) { // last chunk
241 $dbw->insert( 'job', $rows, __METHOD__
, 'IGNORE' );
244 wfIncrStats( 'job-insert', count( $jobs ) );
248 * Insert a group of jobs into the queue.
250 * Same as batchInsert() but does not commit and can thus
251 * be rolled-back as part of a larger transaction. However,
252 * large batches of jobs can cause slave lag.
254 * @param $jobs array of Job objects
256 static function safeBatchInsert( $jobs ) {
257 if ( !count( $jobs ) ) {
260 $dbw = wfGetDB( DB_MASTER
);
262 foreach ( $jobs as $job ) {
263 $rows[] = $job->insertFields();
264 if ( count( $rows ) >= 500 ) {
265 $dbw->insert( 'job', $rows, __METHOD__
, 'IGNORE' );
269 if ( $rows ) { // last chunk
270 $dbw->insert( 'job', $rows, __METHOD__
, 'IGNORE' );
272 wfIncrStats( 'job-insert', count( $jobs ) );
275 /*-------------------------------------------------------------------------
276 * Non-static functions
277 *------------------------------------------------------------------------*/
282 * @param $params array
285 function __construct( $command, $title, $params = false, $id = 0 ) {
286 $this->command
= $command;
287 $this->title
= $title;
288 $this->params
= $params;
291 // A bit of premature generalisation
292 // Oh well, the whole class is premature generalisation really
293 $this->removeDuplicates
= true;
297 * Insert a single job into the queue.
298 * @return bool true on success
301 $fields = $this->insertFields();
303 $dbw = wfGetDB( DB_MASTER
);
305 if ( $this->removeDuplicates
) {
306 $res = $dbw->select( 'job', array( '1' ), $fields, __METHOD__
);
307 if ( $dbw->numRows( $res ) ) {
311 wfIncrStats( 'job-insert' );
312 return $dbw->insert( 'job', $fields, __METHOD__
);
315 protected function insertFields() {
316 $dbw = wfGetDB( DB_MASTER
);
318 'job_id' => $dbw->nextSequenceValue( 'job_job_id_seq' ),
319 'job_cmd' => $this->command
,
320 'job_namespace' => $this->title
->getNamespace(),
321 'job_title' => $this->title
->getDBkey(),
322 'job_params' => Job
::makeBlob( $this->params
)
327 * Remove jobs in the job queue which are duplicates of this job.
328 * This is deadlock-prone and so starts its own transaction.
330 function removeDuplicates() {
331 if ( !$this->removeDuplicates
) {
335 $fields = $this->insertFields();
336 unset( $fields['job_id'] );
337 $dbw = wfGetDB( DB_MASTER
);
339 $dbw->delete( 'job', $fields, __METHOD__
);
340 $affected = $dbw->affectedRows();
343 wfIncrStats( 'job-dup-delete', $affected );
347 function toString() {
349 if ( $this->params
) {
350 foreach ( $this->params
as $key => $value ) {
351 if ( $paramString != '' ) {
354 $paramString .= "$key=$value";
358 if ( is_object( $this->title
) ) {
359 $s = "{$this->command} " . $this->title
->getPrefixedDBkey();
360 if ( $paramString !== '' ) {
361 $s .= ' ' . $paramString;
365 return "{$this->command} $paramString";
369 protected function setLastError( $error ) {
370 $this->error
= $error;
373 function getLastError() {