diff options
Diffstat (limited to 'maintenance/storage')
| -rw-r--r-- | maintenance/storage/blobs.sql | 6 | ||||
| -rw-r--r-- | maintenance/storage/checkStorage.php | 12 | ||||
| -rw-r--r-- | maintenance/storage/compressOld.inc | 38 | ||||
| -rw-r--r-- | maintenance/storage/compressOld.php | 7 | ||||
| -rw-r--r-- | maintenance/storage/dumpRev.php | 9 | ||||
| -rw-r--r-- | maintenance/storage/moveToExternal.php | 128 | ||||
| -rw-r--r-- | maintenance/storage/resolveStubs.php | 17 |
7 files changed, 112 insertions, 105 deletions
diff --git a/maintenance/storage/blobs.sql b/maintenance/storage/blobs.sql index 5782ac47..b50865c6 100644 --- a/maintenance/storage/blobs.sql +++ b/maintenance/storage/blobs.sql @@ -1,8 +1,8 @@ -- Blobs table for external storage CREATE TABLE /*$wgDBprefix*/blobs ( - blob_id int(8) NOT NULL AUTO_INCREMENT, - blob_text mediumtext, + blob_id integer UNSIGNED NOT NULL AUTO_INCREMENT, + blob_text longblob, PRIMARY KEY (blob_id) -) TYPE=InnoDB; +) TYPE=MyISAM MAX_ROWS=1000000 AVG_ROW_LENGTH=1000000; diff --git a/maintenance/storage/checkStorage.php b/maintenance/storage/checkStorage.php index 579954d5..46b938ef 100644 --- a/maintenance/storage/checkStorage.php +++ b/maintenance/storage/checkStorage.php @@ -40,9 +40,9 @@ class CheckStorage function check( $fix = false, $xml = '' ) { $fname = 'checkStorage'; - $dbr =& wfGetDB( DB_SLAVE ); + $dbr = wfGetDB( DB_SLAVE ); if ( $fix ) { - $dbw =& wfGetDB( DB_MASTER ); + $dbw = wfGetDB( DB_MASTER ); print "Checking, will fix errors if possible...\n"; } else { print "Checking...\n"; @@ -197,6 +197,7 @@ class CheckStorage array( 'old_id IN (' . implode( ',', $objectRevs ) . ')' ), $fname ); while ( $row = $dbr->fetchObject( $res ) ) { $oldId = $row->old_id; + $matches = array(); if ( !preg_match( '/^O:(\d+):"(\w+)"/', $row->header, $matches ) ) { $this->error( 'restore text', "Error: invalid object header", $oldId ); continue; @@ -410,8 +411,8 @@ class CheckStorage return; } - $dbr =& wfGetDB( DB_SLAVE ); - $dbw =& wfGetDB( DB_MASTER ); + $dbr = wfGetDB( DB_SLAVE ); + $dbw = wfGetDB( DB_MASTER ); $dbr->ping(); $dbw->ping(); @@ -443,7 +444,7 @@ class CheckStorage } // Find text row again - $dbr =& wfGetDB( DB_SLAVE ); + $dbr = wfGetDB( DB_SLAVE ); $oldId = $dbr->selectField( 'revision', 'rev_text_id', array( 'rev_id' => $id ), $fname ); if ( !$oldId ) { echo "Missing revision row for rev_id $id\n"; @@ -454,6 +455,7 @@ class CheckStorage $flags = Revision::compressRevisionText( $text ); // Update the text row + $dbw = wfGetDB( DB_MASTER ); $dbw->update( 'text', array( 'old_flags' => $flags, 'old_text' => $text ), array( 'old_id' => $oldId ), diff --git a/maintenance/storage/compressOld.inc b/maintenance/storage/compressOld.inc index d38bb741..2da015b0 100644 --- a/maintenance/storage/compressOld.inc +++ b/maintenance/storage/compressOld.inc @@ -1,7 +1,6 @@ <?php /** - * @package MediaWiki - * @subpackage Maintenance + * @addtogroup Maintenance */ /** */ @@ -14,9 +13,8 @@ function compressOldPages( $start = 0, $extdb = '' ) { $chunksize = 50; print "Starting from old_id $start...\n"; - $dbw =& wfGetDB( DB_MASTER ); + $dbw = wfGetDB( DB_MASTER ); do { - $end = $start + $chunksize; $res = $dbw->select( 'text', array( 'old_id','old_flags','old_namespace','old_title','old_text' ), "old_id>=$start", $fname, array( 'ORDER BY' => 'old_id', 'LIMIT' => $chunksize, 'FOR UPDATE' ) ); if( $dbw->numRows( $res ) == 0 ) { @@ -41,7 +39,7 @@ function compressPage( $row, $extdb ) { #print "Already compressed row {$row->old_id}\n"; return false; } - $dbw =& wfGetDB( DB_MASTER ); + $dbw = wfGetDB( DB_MASTER ); $flags = $row->old_flags ? "{$row->old_flags},gzip" : "gzip"; $compress = gzdeflate( $row->old_text ); @@ -77,8 +75,8 @@ function compressWithConcat( $startId, $maxChunkSize, $maxChunkFactor, $factorTh $fname = 'compressWithConcat'; $loadStyle = LS_CHUNKED; - $dbr =& wfGetDB( DB_SLAVE ); - $dbw =& wfGetDB( DB_MASTER ); + $dbr = wfGetDB( DB_SLAVE ); + $dbw = wfGetDB( DB_MASTER ); # Set up external storage if ( $extdb != '' ) { @@ -112,9 +110,17 @@ function compressWithConcat( $startId, $maxChunkSize, $maxChunkFactor, $factorTh "old_flags NOT LIKE '%object%' AND old_flags NOT LIKE '%external%'"); if ( $beginDate ) { + if ( !preg_match( '/^\d{14}$/', $beginDate ) ) { + print "Invalid begin date \"$beginDate\"\n"; + return false; + } $conds[] = "rev_timestamp>'" . $beginDate . "'"; } if ( $endDate ) { + if ( !preg_match( '/^\d{14}$/', $endDate ) ) { + print "Invalid end date \"$endDate\"\n"; + return false; + } $conds[] = "rev_timestamp<'" . $endDate . "'"; } if ( $loadStyle == LS_CHUNKED ) { @@ -133,9 +139,6 @@ function compressWithConcat( $startId, $maxChunkSize, $maxChunkFactor, $factorTh #$tables[] = 'page'; #$conds[] = 'page_id=rev_page AND rev_id != page_latest'; - $oldReadsSinceLastSlaveWait = 0; #check slave lag periodically - $totalMatchingRevisions = 0; - $masterPos = false; for ( $pageId = $startId; $pageId <= $maxPageId; $pageId++ ) { wfWaitForSlaves( 5 ); @@ -155,26 +158,15 @@ function compressWithConcat( $startId, $maxChunkSize, $maxChunkFactor, $factorTh $titleObj = Title::makeTitle( $pageRow->page_namespace, $pageRow->page_title ); print "$pageId\t" . $titleObj->getPrefixedDBkey() . " "; - print_r( - array( - 'rev_page' => $pageRow->page_id, - # Don't operate on the current revision - # Use < instead of <> in case the current revision has changed - # since the page select, which wasn't locking - 'rev_id < ' . $pageRow->page_latest - ) + $conds - ); - exit; - # Load revisions $revRes = $dbw->select( $tables, $fields, - array( + array_merge( array( 'rev_page' => $pageRow->page_id, # Don't operate on the current revision # Use < instead of <> in case the current revision has changed # since the page select, which wasn't locking 'rev_id < ' . $pageRow->page_latest - ) + $conds, + ), $conds ), $fname, $revLoadOptions ); diff --git a/maintenance/storage/compressOld.php b/maintenance/storage/compressOld.php index d597f1df..87aebb78 100644 --- a/maintenance/storage/compressOld.php +++ b/maintenance/storage/compressOld.php @@ -2,8 +2,7 @@ /** * Compress the text of a wiki * - * @package MediaWiki - * @subpackage Maintenance + * @addtogroup Maintenance */ /** */ @@ -31,8 +30,8 @@ * */ -$optionsWithArgs = array( 't', 'c', 's', 'f', 'h', 'extdb', 'endid' ); -require_once( "../commandLine.inc" ); +$optionsWithArgs = array( 't', 'c', 's', 'f', 'h', 'extdb', 'endid', 'e' ); +require_once( dirname(__FILE__) . '/../commandLine.inc' ); require_once( "compressOld.inc" ); if( !function_exists( "gzdeflate" ) ) { diff --git a/maintenance/storage/dumpRev.php b/maintenance/storage/dumpRev.php index 4d0ccb58..d35af384 100644 --- a/maintenance/storage/dumpRev.php +++ b/maintenance/storage/dumpRev.php @@ -1,8 +1,9 @@ <?php -require_once( 'commandLine.inc' ); -$dbr =& wfGetDB( DB_SLAVE ); -$row = $dbr->selectRow( 'old', array( 'old_flags', 'old_text' ), array( 'old_id' => $args[0] ) ); +require_once( dirname(__FILE__) . '/../commandLine.inc' ); + +$dbr = wfGetDB( DB_SLAVE ); +$row = $dbr->selectRow( 'text', array( 'old_flags', 'old_text' ), array( 'old_id' => $args[0] ) ); $obj = unserialize( $row->old_text ); if ( get_class( $obj ) == 'concatenatedgziphistoryblob' ) { @@ -11,4 +12,4 @@ if ( get_class( $obj ) == 'concatenatedgziphistoryblob' ) { var_dump( $obj ); } -?> +?>
\ No newline at end of file diff --git a/maintenance/storage/moveToExternal.php b/maintenance/storage/moveToExternal.php index 0b46f70b..6d3ebfcb 100644 --- a/maintenance/storage/moveToExternal.php +++ b/maintenance/storage/moveToExternal.php @@ -1,96 +1,112 @@ <?php -define( 'REPORTING_INTERVAL', 100 ); +define( 'REPORTING_INTERVAL', 1 ); if ( !defined( 'MEDIAWIKI' ) ) { - $optionsWithArgs = array( 'm' ); + $optionsWithArgs = array( 'm', 's' ); - require_once( '../commandLine.inc' ); + require_once( dirname(__FILE__) . '/../commandLine.inc' ); require_once( 'ExternalStoreDB.php' ); require_once( 'resolveStubs.php' ); $fname = 'moveToExternal'; if ( !isset( $args[0] ) ) { - print "Usage: php moveToExternal.php [-m <maxid>] <cluster>\n"; + print "Usage: php moveToExternal.php [-s <startid>] [-e <endid>] <cluster>\n"; exit; } $cluster = $args[0]; - $dbw =& wfGetDB( DB_MASTER ); + $dbw = wfGetDB( DB_MASTER ); - if ( isset( $options['m'] ) ) { - $maxID = $options['m']; + if ( isset( $options['e'] ) ) { + $maxID = $options['e']; } else { $maxID = $dbw->selectField( 'text', 'MAX(old_id)', false, $fname ); } + $minID = isset( $options['s'] ) ? $options['s'] : 1; - moveToExternal( $cluster, $maxID ); + moveToExternal( $cluster, $maxID, $minID ); } -function moveToExternal( $cluster, $maxID ) { +function moveToExternal( $cluster, $maxID, $minID = 1 ) { $fname = 'moveToExternal'; - $dbw =& wfGetDB( DB_MASTER ); + $dbw = wfGetDB( DB_MASTER ); + $dbr = wfGetDB( DB_SLAVE ); - print "Moving $maxID text rows to external storage\n"; + $count = $maxID - $minID + 1; + $blockSize = 1000; + $numBlocks = ceil( $count / $blockSize ); + print "Moving text rows from $minID to $maxID to external storage\n"; $ext = new ExternalStoreDB; - for ( $id = 1; $id <= $maxID; $id++ ) { - if ( !($id % REPORTING_INTERVAL) ) { - print "$id\n"; - wfWaitForSlaves( 5 ); + $numMoved = 0; + $numStubs = 0; + + for ( $block = 0; $block < $numBlocks; $block++ ) { + $blockStart = $block * $blockSize + $minID; + $blockEnd = $blockStart + $blockSize - 1; + + if ( !($block % REPORTING_INTERVAL) ) { + print "oldid=$blockStart, moved=$numMoved\n"; + wfWaitForSlaves( 2 ); } - $row = $dbw->selectRow( 'text', array( 'old_flags', 'old_text' ), + + $res = $dbr->select( 'text', array( 'old_id', 'old_flags', 'old_text' ), array( - 'old_id' => $id, + "old_id BETWEEN $blockStart AND $blockEnd", "old_flags NOT LIKE '%external%'", ), $fname ); - if ( !$row ) { - # Non-existent or already done - continue; - } - - # Resolve stubs - $text = $row->old_text; - if ( $row->old_flags === '' ) { - $flags = 'external'; - } else { - $flags = "{$row->old_flags},external"; - } - - if ( strpos( $flags, 'object' ) !== false ) { - $obj = unserialize( $text ); - $className = strtolower( get_class( $obj ) ); - if ( $className == 'historyblobstub' ) { - resolveStub( $id, $row->old_text, $row->old_flags ); - continue; - } elseif ( $className == 'historyblobcurstub' ) { - $text = gzdeflate( $obj->getText() ); - $flags = 'utf-8,gzip,external'; - } elseif ( $className == 'concatenatedgziphistoryblob' ) { - // Do nothing + while ( $row = $dbr->fetchObject( $res ) ) { + # Resolve stubs + $text = $row->old_text; + $id = $row->old_id; + if ( $row->old_flags === '' ) { + $flags = 'external'; } else { - print "Warning: unrecognised object class \"$className\"\n"; - continue; + $flags = "{$row->old_flags},external"; + } + + if ( strpos( $flags, 'object' ) !== false ) { + $obj = unserialize( $text ); + $className = strtolower( get_class( $obj ) ); + if ( $className == 'historyblobstub' ) { + #resolveStub( $id, $row->old_text, $row->old_flags ); + #$numStubs++; + continue; + } elseif ( $className == 'historyblobcurstub' ) { + $text = gzdeflate( $obj->getText() ); + $flags = 'utf-8,gzip,external'; + } elseif ( $className == 'concatenatedgziphistoryblob' ) { + // Do nothing + } else { + print "Warning: unrecognised object class \"$className\"\n"; + continue; + } + } else { + $className = false; } - } - if ( strlen( $text ) < 100 ) { - // Don't move tiny revisions - continue; - } + if ( strlen( $text ) < 100 && $className === false ) { + // Don't move tiny revisions + continue; + } - #print "Storing " . strlen( $text ) . " bytes to $url\n"; + #print "Storing " . strlen( $text ) . " bytes to $url\n"; + #print "old_id=$id\n"; - $url = $ext->store( $cluster, $text ); - if ( !$url ) { - print "Error writing to external storage\n"; - exit; + $url = $ext->store( $cluster, $text ); + if ( !$url ) { + print "Error writing to external storage\n"; + exit; + } + $dbw->update( 'text', + array( 'old_flags' => $flags, 'old_text' => $url ), + array( 'old_id' => $id ), $fname ); + $numMoved++; } - $dbw->update( 'text', - array( 'old_flags' => $flags, 'old_text' => $url ), - array( 'old_id' => $id ), $fname ); + $dbr->freeResult( $res ); } } diff --git a/maintenance/storage/resolveStubs.php b/maintenance/storage/resolveStubs.php index e93d5c97..6836ae60 100644 --- a/maintenance/storage/resolveStubs.php +++ b/maintenance/storage/resolveStubs.php @@ -5,7 +5,7 @@ define( 'REPORTING_INTERVAL', 100 ); if ( !defined( 'MEDIAWIKI' ) ) { $optionsWithArgs = array( 'm' ); - require_once( '../commandLine.inc' ); + require_once( dirname(__FILE__) . '/../commandLine.inc' ); require_once( 'includes/ExternalStoreDB.php' ); resolveStubs(); @@ -18,21 +18,17 @@ if ( !defined( 'MEDIAWIKI' ) ) { function resolveStubs() { $fname = 'resolveStubs'; - $dbr =& wfGetDB( DB_SLAVE ); - $dbw =& wfGetDB( DB_MASTER ); + $dbr = wfGetDB( DB_SLAVE ); $maxID = $dbr->selectField( 'text', 'MAX(old_id)', false, $fname ); $blockSize = 10000; $numBlocks = intval( $maxID / $blockSize ) + 1; for ( $b = 0; $b < $numBlocks; $b++ ) { - wfWaitForSlaves( 5 ); + wfWaitForSlaves( 2 ); printf( "%5.2f%%\n", $b / $numBlocks * 100 ); $start = intval($maxID / $numBlocks) * $b + 1; $end = intval($maxID / $numBlocks) * ($b + 1); - $stubs = array(); - $flagsArray = array(); - $res = $dbr->select( 'text', array( 'old_id', 'old_text', 'old_flags' ), "old_id>=$start AND old_id<=$end " . @@ -40,7 +36,7 @@ function resolveStubs() { #"AND old_flags LIKE '%object%' AND old_flags NOT LIKE '%external%' ". "AND old_flags='object' " . - "AND old_text LIKE 'O:15:\"historyblobstub\"%'", $fname ); + "AND LOWER(LEFT(old_text,22)) = 'O:15:\"historyblobstub\"'", $fname ); while ( $row = $dbr->fetchObject( $res ) ) { resolveStub( $row->old_id, $row->old_text, $row->old_flags ); } @@ -60,8 +56,8 @@ function resolveStub( $id, $stubText, $flags ) { $stub = unserialize( $stubText ); $flags = explode( ',', $flags ); - $dbr =& wfGetDB( DB_SLAVE ); - $dbw =& wfGetDB( DB_MASTER ); + $dbr = wfGetDB( DB_SLAVE ); + $dbw = wfGetDB( DB_MASTER ); if ( strtolower( get_class( $stub ) ) !== 'historyblobstub' ) { print "Error found object of class " . get_class( $stub ) . ", expecting historyblobstub\n"; @@ -87,6 +83,7 @@ function resolveStub( $id, $stubText, $flags ) { } # Update the row + #print "oldid=$id\n"; $dbw->update( 'text', array( /* SET */ 'old_flags' => $newFlags, |