3 require( dirname( __FILE__
) . '/../commandLine.inc' );
6 if ( count( $args ) < 1 ) {
7 echo "Usage: php trackBlobs.php <cluster> [... <cluster>]\n";
8 echo "Adds blobs from a given ES cluster to the blob_tracking table\n";
9 echo "Automatically deletes the tracking table and starts from the start again when restarted.\n";
13 $tracker = new TrackBlobs( $args );
18 var $clusters, $textClause;
20 var $trackedBlobs = array();
22 var $batchSize = 1000;
23 var $reportingInterval = 10;
25 function __construct( $clusters ) {
26 $this->clusters
= $clusters;
27 if ( extension_loaded( 'gmp' ) ) {
28 $this->doBlobOrphans
= true;
29 foreach ( $clusters as $cluster ) {
30 $this->trackedBlobs
[$cluster] = gmp_init( 0 );
33 echo "Warning: the gmp extension is needed to find orphan blobs\n";
38 $this->checkIntegrity();
39 $this->initTrackingTable();
40 $this->trackRevisions();
41 $this->trackOrphanText();
42 if ( $this->doBlobOrphans
) {
43 $this->findOrphanBlobs();
47 function checkIntegrity() {
48 echo "Doing integrity check...\n";
49 $dbr = wfGetDB( DB_SLAVE
);
51 // Scan for HistoryBlobStub objects in the text table (bug 20757)
53 $exists = $dbr->selectField( 'text', 1,
54 'old_flags LIKE \'%object%\' AND old_flags NOT LIKE \'%external%\' ' .
55 'AND LOWER(CONVERT(LEFT(old_text,22) USING latin1)) = \'o:15:"historyblobstub"\'',
60 echo "Integrity check failed: found HistoryBlobStub objects in your text table.\n" .
61 "This script could destroy these objects if it continued. Run resolveStubs.php\n" .
66 // Scan the archive table for HistoryBlobStub objects or external flags (bug 22624)
67 $flags = $dbr->selectField( 'archive', 'ar_flags',
68 'ar_flags LIKE \'%external%\' OR (' .
69 'ar_flags LIKE \'%object%\' ' .
70 'AND LOWER(CONVERT(LEFT(ar_text,22) USING latin1)) = \'o:15:"historyblobstub"\' )',
74 if ( strpos( $flags, 'external' ) !== false ) {
75 echo "Integrity check failed: found external storage pointers in your archive table.\n" .
76 "Run normaliseArchiveTable.php to fix this.\n";
79 echo "Integrity check failed: found HistoryBlobStub objects in your archive table.\n" .
80 "These objects are probably already broken, continuing would make them\n" .
81 "unrecoverable. Run \"normaliseArchiveTable.php --fix-cgz-bug\" to fix this.\n";
85 echo "Integrity check OK\n";
88 function initTrackingTable() {
89 $dbw = wfGetDB( DB_MASTER
);
90 if ( $dbw->tableExists( 'blob_tracking' ) ) {
91 $dbw->query( 'DROP TABLE ' . $dbw->tableName( 'blob_tracking' ) );
92 $dbw->query( 'DROP TABLE ' . $dbw->tableName( 'blob_orphans' ) );
94 $dbw->sourceFile( dirname( __FILE__
) . '/blob_tracking.sql' );
97 function getTextClause() {
98 if ( !$this->textClause
) {
99 $dbr = wfGetDB( DB_SLAVE
);
100 $this->textClause
= '';
101 foreach ( $this->clusters
as $cluster ) {
102 if ( $this->textClause
!= '' ) {
103 $this->textClause
.= ' OR ';
105 $this->textClause
.= 'old_text' . $dbr->buildLike( "DB://$cluster/", $dbr->anyString() );
108 return $this->textClause
;
111 function interpretPointer( $text ) {
112 if ( !preg_match( '!^DB://(\w+)/(\d+)(?:/([0-9a-fA-F]+)|)$!', $text, $m ) ) {
117 'id' => intval( $m[2] ),
118 'hash' => isset( $m[3] ) ?
$m[3] : null
123 * Scan the revision table for rows stored in the specified clusters
125 function trackRevisions() {
126 $dbw = wfGetDB( DB_MASTER
);
127 $dbr = wfGetDB( DB_SLAVE
);
129 $textClause = $this->getTextClause();
131 $endId = $dbr->selectField( 'revision', 'MAX(rev_id)', false, __METHOD__
);
135 echo "Finding revisions...\n";
138 $res = $dbr->select( array( 'revision', 'text' ),
139 array( 'rev_id', 'rev_page', 'old_id', 'old_flags', 'old_text' ),
141 'rev_id > ' . $dbr->addQuotes( $startId ),
142 'rev_text_id=old_id',
144 'old_flags ' . $dbr->buildLike( $dbr->anyString(), 'external', $dbr->anyString() ),
148 'ORDER BY' => 'rev_id',
149 'LIMIT' => $this->batchSize
152 if ( !$res->numRows() ) {
156 $insertBatch = array();
157 foreach ( $res as $row ) {
158 $startId = $row->rev_id
;
159 $info = $this->interpretPointer( $row->old_text
);
161 echo "Invalid DB:// URL in rev_id {$row->rev_id}\n";
164 if ( !in_array( $info['cluster'], $this->clusters
) ) {
165 echo "Invalid cluster returned in SQL query: {$info['cluster']}\n";
168 $insertBatch[] = array(
169 'bt_page' => $row->rev_page
,
170 'bt_rev_id' => $row->rev_id
,
171 'bt_text_id' => $row->old_id
,
172 'bt_cluster' => $info['cluster'],
173 'bt_blob_id' => $info['id'],
174 'bt_cgz_hash' => $info['hash']
176 if ( $this->doBlobOrphans
) {
177 gmp_setbit( $this->trackedBlobs
[$info['cluster']], $info['id'] );
180 $dbw->insert( 'blob_tracking', $insertBatch, __METHOD__
);
181 $rowsInserted +
= count( $insertBatch );
184 if ( $batchesDone >= $this->reportingInterval
) {
186 echo "$startId / $endId\n";
187 wfWaitForSlaves( 5 );
190 echo "Found $rowsInserted revisions\n";
194 * Scan the text table for orphan text
195 * Orphan text here does not imply DB corruption -- deleted text tracked by the
196 * archive table counts as orphan for our purposes.
198 function trackOrphanText() {
199 # Wait until the blob_tracking table is available in the slave
200 $dbw = wfGetDB( DB_MASTER
);
201 $dbr = wfGetDB( DB_SLAVE
);
202 $pos = $dbw->getMasterPos();
203 $dbr->masterPosWait( $pos, 100000 );
205 $textClause = $this->getTextClause( $this->clusters
);
207 $endId = $dbr->selectField( 'text', 'MAX(old_id)', false, __METHOD__
);
211 echo "Finding orphan text...\n";
213 # Scan the text table for orphan text
215 $res = $dbr->select( array( 'text', 'blob_tracking' ),
216 array( 'old_id', 'old_flags', 'old_text' ),
218 'old_id>' . $dbr->addQuotes( $startId ),
220 'old_flags ' . $dbr->buildLike( $dbr->anyString(), 'external', $dbr->anyString() ),
225 'ORDER BY' => 'old_id',
226 'LIMIT' => $this->batchSize
228 array( 'blob_tracking' => array( 'LEFT JOIN', 'bt_text_id=old_id' ) )
231 foreach ( $res as $row ) {
232 $ids[] = $row->old_id
;
235 if ( !$res->numRows() ) {
239 $insertBatch = array();
240 foreach ( $res as $row ) {
241 $startId = $row->old_id
;
242 $info = $this->interpretPointer( $row->old_text
);
244 echo "Invalid DB:// URL in old_id {$row->old_id}\n";
247 if ( !in_array( $info['cluster'], $this->clusters
) ) {
248 echo "Invalid cluster returned in SQL query\n";
252 $insertBatch[] = array(
255 'bt_text_id' => $row->old_id
,
256 'bt_cluster' => $info['cluster'],
257 'bt_blob_id' => $info['id'],
258 'bt_cgz_hash' => $info['hash']
260 if ( $this->doBlobOrphans
) {
261 gmp_setbit( $this->trackedBlobs
[$info['cluster']], $info['id'] );
264 $dbw->insert( 'blob_tracking', $insertBatch, __METHOD__
);
266 $rowsInserted +
= count( $insertBatch );
268 if ( $batchesDone >= $this->reportingInterval
) {
270 echo "$startId / $endId\n";
271 wfWaitForSlaves( 5 );
274 echo "Found $rowsInserted orphan text rows\n";
278 * Scan the blobs table for rows not registered in blob_tracking (and thus not
279 * registered in the text table).
281 * Orphan blobs are indicative of DB corruption. They are inaccessible and
282 * should probably be deleted.
284 function findOrphanBlobs() {
285 if ( !extension_loaded( 'gmp' ) ) {
286 echo "Can't find orphan blobs, need bitfield support provided by GMP.\n";
290 $dbw = wfGetDB( DB_MASTER
);
292 foreach ( $this->clusters
as $cluster ) {
293 echo "Searching for orphan blobs in $cluster...\n";
294 $lb = wfGetLBFactory()->getExternalLB( $cluster );
296 $extDB = $lb->getConnection( DB_SLAVE
);
297 } catch ( DBConnectionError
$e ) {
298 if ( strpos( $e->error
, 'Unknown database' ) !== false ) {
299 echo "No database on $cluster\n";
301 echo "Error on $cluster: " . $e->getMessage() . "\n";
305 $table = $extDB->getLBInfo( 'blobs table' );
306 if ( is_null( $table ) ) {
309 if ( !$extDB->tableExists( $table ) ) {
310 echo "No blobs table on cluster $cluster\n";
315 $actualBlobs = gmp_init( 0 );
316 $endId = $extDB->selectField( $table, 'MAX(blob_id)', false, __METHOD__
);
318 // Build a bitmap of actual blob rows
320 $res = $extDB->select( $table,
322 array( 'blob_id > ' . $extDB->addQuotes( $startId ) ),
324 array( 'LIMIT' => $this->batchSize
, 'ORDER BY' => 'blob_id' )
327 if ( !$res->numRows() ) {
331 foreach ( $res as $row ) {
332 gmp_setbit( $actualBlobs, $row->blob_id
);
334 $startId = $row->blob_id
;
337 if ( $batchesDone >= $this->reportingInterval
) {
339 echo "$startId / $endId\n";
343 // Find actual blobs that weren't tracked by the previous passes
344 // This is a set-theoretic difference A \ B, or in bitwise terms, A & ~B
345 $orphans = gmp_and( $actualBlobs, gmp_com( $this->trackedBlobs
[$cluster] ) );
347 // Traverse the orphan list
348 $insertBatch = array();
352 $id = gmp_scan1( $orphans, $id );
356 $insertBatch[] = array(
357 'bo_cluster' => $cluster,
360 if ( count( $insertBatch ) > $this->batchSize
) {
361 $dbw->insert( 'blob_orphans', $insertBatch, __METHOD__
);
362 $insertBatch = array();
368 if ( $insertBatch ) {
369 $dbw->insert( 'blob_orphans', $insertBatch, __METHOD__
);
371 echo "Found $numOrphans orphan(s) in $cluster\n";