3 * HTML cache invalidation of all pages linking to a given title.
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License along
16 * with this program; if not, write to the Free Software Foundation, Inc.,
17 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
18 * http://www.gnu.org/copyleft/gpl.html
25 * Class to invalidate the HTML cache of all the pages linking to a given title.
26 * Small numbers of links will be done immediately, large numbers are pushed onto
29 * This class is designed to work efficiently with small numbers of links, and
30 * to work reasonably well with up to ~10^5 links. Above ~10^6 links, the memory
31 * and time requirements of loading all backlinked IDs in doUpdate() might become
32 * prohibitive. The requirements measured at Wikimedia are approximately:
34 * memory: 48 bytes per row
35 * time: 16us per row for the query plus processing
37 * The reason this query is done is to support partitioning of the job
38 * by backlinked ID. The memory issue could be allieviated by doing this query in
39 * batches, but of course LIMIT with an offset is inefficient on the DB side.
41 * The class is nevertheless a vast improvement on the previous method of using
42 * File::getLinksTo() and Title::touchArray(), which uses about 2KB of memory per
47 class HTMLCacheUpdate
implements DeferrableUpdate
{
53 public $mTable, $mPrefix, $mStart, $mEnd;
54 public $mRowsPerJob, $mRowsPerQuery;
62 function __construct( $titleTo, $table, $start = false, $end = false ) {
63 global $wgUpdateRowsPerJob, $wgUpdateRowsPerQuery;
65 $this->mTitle
= $titleTo;
66 $this->mTable
= $table;
67 $this->mStart
= $start;
69 $this->mRowsPerJob
= $wgUpdateRowsPerJob;
70 $this->mRowsPerQuery
= $wgUpdateRowsPerQuery;
71 $this->mCache
= $this->mTitle
->getBacklinkCache();
74 public function doUpdate() {
75 if ( $this->mStart ||
$this->mEnd
) {
76 $this->doPartialUpdate();
80 # Get an estimate of the number of rows from the BacklinkCache
81 $numRows = $this->mCache
->getNumLinks( $this->mTable
);
82 if ( $numRows > $this->mRowsPerJob
* 2 ) {
83 # Do fast cached partition
86 # Get the links from the DB
87 $titleArray = $this->mCache
->getLinks( $this->mTable
);
88 # Check if the row count estimate was correct
89 if ( $titleArray->count() > $this->mRowsPerJob
* 2 ) {
90 # Not correct, do accurate partition
91 wfDebug( __METHOD__
.": row count estimate was incorrect, repartitioning\n" );
92 $this->insertJobsFromTitles( $titleArray );
94 $this->invalidateTitles( $titleArray );
100 * Update some of the backlinks, defined by a page ID range
102 protected function doPartialUpdate() {
103 $titleArray = $this->mCache
->getLinks( $this->mTable
, $this->mStart
, $this->mEnd
);
104 if ( $titleArray->count() <= $this->mRowsPerJob
* 2 ) {
105 # This partition is small enough, do the update
106 $this->invalidateTitles( $titleArray );
108 # Partitioning was excessively inaccurate. Divide the job further.
109 # This can occur when a large number of links are added in a short
110 # period of time, say by updating a heavily-used template.
111 $this->insertJobsFromTitles( $titleArray );
116 * Partition the current range given by $this->mStart and $this->mEnd,
117 * using a pre-calculated title array which gives the links in that range.
118 * Queue the resulting jobs.
120 * @param $titleArray array
122 protected function insertJobsFromTitles( $titleArray ) {
123 # We make subpartitions in the sense that the start of the first job
124 # will be the start of the parent partition, and the end of the last
125 # job will be the end of the parent partition.
127 $start = $this->mStart
; # start of the current job
129 foreach ( $titleArray as $title ) {
130 $id = $title->getArticleID();
131 # $numTitles is now the number of titles in the current job not
132 # including the current ID
133 if ( $numTitles >= $this->mRowsPerJob
) {
134 # Add a job up to but not including the current ID
136 'table' => $this->mTable
,
140 $jobs[] = new HTMLCacheUpdateJob( $this->mTitle
, $params );
148 'table' => $this->mTable
,
152 $jobs[] = new HTMLCacheUpdateJob( $this->mTitle
, $params );
153 wfDebug( __METHOD__
.": repartitioning into " . count( $jobs ) . " jobs\n" );
155 if ( count( $jobs ) < 2 ) {
156 # I don't think this is possible at present, but handling this case
157 # makes the code a bit more robust against future code updates and
158 # avoids a potential infinite loop of repartitioning
159 wfDebug( __METHOD__
.": repartitioning failed!\n" );
160 $this->invalidateTitles( $titleArray );
164 Job
::batchInsert( $jobs );
170 protected function insertJobs() {
171 $batches = $this->mCache
->partition( $this->mTable
, $this->mRowsPerJob
);
176 foreach ( $batches as $batch ) {
178 'table' => $this->mTable
,
179 'start' => $batch[0],
182 $jobs[] = new HTMLCacheUpdateJob( $this->mTitle
, $params );
184 Job
::batchInsert( $jobs );
188 * Invalidate an array (or iterator) of Title objects, right now
189 * @param $titleArray array
191 protected function invalidateTitles( $titleArray ) {
192 global $wgUseFileCache, $wgUseSquid;
194 $dbw = wfGetDB( DB_MASTER
);
195 $timestamp = $dbw->timestamp();
197 # Get all IDs in this query into an array
199 foreach ( $titleArray as $title ) {
200 $ids[] = $title->getArticleID();
207 # Update page_touched
208 $batches = array_chunk( $ids, $this->mRowsPerQuery
);
209 foreach ( $batches as $batch ) {
210 $dbw->update( 'page',
211 array( 'page_touched' => $timestamp ),
212 array( 'page_id' => $batch ),
219 $u = SquidUpdate
::newFromTitles( $titleArray );
224 if ( $wgUseFileCache ) {
225 foreach ( $titleArray as $title ) {
226 HTMLFileCache
::clearFileCache( $title );
234 * Job wrapper for HTMLCacheUpdate. Gets run whenever a related
235 * job gets called from the queue.
239 class HTMLCacheUpdateJob
extends Job
{
240 var $table, $start, $end;
244 * @param $title Title: the title linked to
245 * @param $params Array: job parameters (table, start and end page_ids)
246 * @param $id Integer: job id
248 function __construct( $title, $params, $id = 0 ) {
249 parent
::__construct( 'htmlCacheUpdate', $title, $params, $id );
250 $this->table
= $params['table'];
251 $this->start
= $params['start'];
252 $this->end
= $params['end'];
255 public function run() {
256 $update = new HTMLCacheUpdate( $this->title
, $this->table
, $this->start
, $this->end
);