diff options
Diffstat (limited to 'maintenance/dumpTextPass.php')
| -rw-r--r-- | maintenance/dumpTextPass.php | 150 |
1 files changed, 146 insertions, 4 deletions
diff --git a/maintenance/dumpTextPass.php b/maintenance/dumpTextPass.php index 92ab4b4e..440f6d27 100644 --- a/maintenance/dumpTextPass.php +++ b/maintenance/dumpTextPass.php @@ -104,6 +104,13 @@ class TextPassDumper extends BackupDumper { var $failures = 0; var $maxFailures = 200; var $failureTimeout = 5; // Seconds to sleep after db failure + + var $php = "php"; + var $spawn = false; + var $spawnProc = false; + var $spawnWrite = false; + var $spawnRead = false; + var $spawnErr = false; function dump() { # This shouldn't happen if on console... ;) @@ -111,7 +118,8 @@ class TextPassDumper extends BackupDumper { # Notice messages will foul up your XML output even if they're # relatively harmless. -// ini_set( 'display_errors', false ); + if( ini_get( 'display_errors' ) ) + ini_set( 'display_errors', 'stderr' ); $this->initProgress( $this->history ); @@ -125,6 +133,10 @@ class TextPassDumper extends BackupDumper { if( WikiError::isError( $result ) ) { wfDie( $result->getMessage() ); } + + if( $this->spawnProc ) { + $this->closeSpawn(); + } $this->report( true ); } @@ -134,7 +146,8 @@ class TextPassDumper extends BackupDumper { switch( $opt ) { case 'prefetch': - require_once 'maintenance/backupPrefetch.inc'; + global $IP; + require_once "$IP/maintenance/backupPrefetch.inc"; $this->prefetch = new BaseDump( $url ); break; case 'stub': @@ -146,6 +159,12 @@ class TextPassDumper extends BackupDumper { case 'full': $this->history = WikiExporter::FULL; break; + case 'spawn': + $this->spawn = true; + if( $val ) { + $this->php = $val; + } + break; } } @@ -237,9 +256,26 @@ class TextPassDumper extends BackupDumper { return $text; } } + return $this->doGetText( $id ); + } + + private function doGetText( $id ) { + if( $this->spawn ) { + return $this->getTextSpawned( $id ); + } else { + return $this->getTextDbSafe( $id ); + } + } + + /** + * Fetch a text revision from the database, retrying in case of failure. + * This may survive some transitory errors by reconnecting, but + * may not survive a long-term server outage. + */ + private function getTextDbSafe( $id ) { while( true ) { try { - $text = $this->doGetText( $id ); + $text = $this->getTextDb( $id ); $ex = new MWException("Graceful storage failure"); } catch (DBQueryError $ex) { $text = false; @@ -263,7 +299,7 @@ class TextPassDumper extends BackupDumper { /** * May throw a database error if, say, the server dies during query. */ - private function doGetText( $id ) { + private function getTextDb( $id ) { $id = intval( $id ); $row = $this->db->selectRow( 'text', array( 'old_text', 'old_flags' ), @@ -277,6 +313,111 @@ class TextPassDumper extends BackupDumper { $normalized = UtfNormal::cleanUp( $stripped ); return $normalized; } + + private function getTextSpawned( $id ) { + wfSuppressWarnings(); + if( !$this->spawnProc ) { + // First time? + $this->openSpawn(); + } + while( true ) { + + $text = $this->getTextSpawnedOnce( $id ); + if( !is_string( $text ) ) { + $this->progress("Database subprocess failed. Respawning..."); + + $this->closeSpawn(); + sleep( $this->failureTimeout ); + $this->openSpawn(); + + continue; + } + wfRestoreWarnings(); + return $text; + } + } + + function openSpawn() { + global $IP, $wgDBname; + + $cmd = implode( " ", + array_map( 'wfEscapeShellArg', + array( + $this->php, + "$IP/maintenance/fetchText.php", + $wgDBname ) ) ); + $spec = array( + 0 => array( "pipe", "r" ), + 1 => array( "pipe", "w" ), + 2 => array( "file", "/dev/null", "a" ) ); + $pipes = array(); + + $this->progress( "Spawning database subprocess: $cmd" ); + $this->spawnProc = proc_open( $cmd, $spec, $pipes ); + if( !$this->spawnProc ) { + // shit + $this->progress( "Subprocess spawn failed." ); + return false; + } + list( + $this->spawnWrite, // -> stdin + $this->spawnRead, // <- stdout + ) = $pipes; + + return true; + } + + private function closeSpawn() { + wfSuppressWarnings(); + if( $this->spawnRead ) + fclose( $this->spawnRead ); + $this->spawnRead = false; + if( $this->spawnWrite ) + fclose( $this->spawnWrite ); + $this->spawnWrite = false; + if( $this->spawnErr ) + fclose( $this->spawnErr ); + $this->spawnErr = false; + if( $this->spawnProc ) + pclose( $this->spawnProc ); + $this->spawnProc = false; + wfRestoreWarnings(); + } + + private function getTextSpawnedOnce( $id ) { + $ok = fwrite( $this->spawnWrite, "$id\n" ); + //$this->progress( ">> $id" ); + if( !$ok ) return false; + + $ok = fflush( $this->spawnWrite ); + //$this->progress( ">> [flush]" ); + if( !$ok ) return false; + + $len = fgets( $this->spawnRead ); + //$this->progress( "<< " . trim( $len ) ); + if( $len === false ) return false; + + $nbytes = intval( $len ); + $text = ""; + + // Subprocess may not send everything at once, we have to loop. + while( $nbytes > strlen( $text ) ) { + $buffer = fread( $this->spawnRead, $nbytes - strlen( $text ) ); + if( $text === false ) break; + $text .= $buffer; + } + + $gotbytes = strlen( $text ); + if( $gotbytes != $nbytes ) { + $this->progress( "Expected $nbytes bytes from database subprocess, got $gotbytes "); + return false; + } + + // Do normalization in the dump thread... + $stripped = str_replace( "\r", "", $text ); + $normalized = UtfNormal::cleanUp( $stripped ); + return $normalized; + } function startElement( $parser, $name, $attribs ) { $this->clearOpenElement( null ); @@ -371,6 +512,7 @@ Options: (Default: 100) --server=h Force reading from MySQL server h --current Base ETA on number of pages in database instead of all revisions + --spawn Spawn a subprocess for loading text records END ); } |