diff options
Diffstat (limited to 'includes/filebackend')
21 files changed, 2405 insertions, 1234 deletions
diff --git a/includes/filebackend/FSFile.php b/includes/filebackend/FSFile.php index e07c99d4..7d0dbd52 100644 --- a/includes/filebackend/FSFile.php +++ b/includes/filebackend/FSFile.php @@ -28,11 +28,12 @@ */ class FSFile { protected $path; // path to file + private $sha1Base36 = null; // File Sha1Base36 /** * Sets up the file object * - * @param $path string Path to temporary file on local disk + * @param string $path Path to temporary file on local disk * @throws MWException */ public function __construct( $path ) { @@ -86,8 +87,8 @@ class FSFile { /** * Guess the MIME type from the file contents alone - * - * @return string + * + * @return string */ public function getMimeType() { return MimeMagic::singleton()->guessMimeType( $this->path, false ); @@ -104,7 +105,7 @@ class FSFile { */ public function getProps( $ext = true ) { wfProfileIn( __METHOD__ ); - wfDebug( __METHOD__.": Getting file info for $this->path\n" ); + wfDebug( __METHOD__ . ": Getting file info for $this->path\n" ); $info = self::placeholderProps(); $info['fileExists'] = $this->exists(); @@ -131,7 +132,7 @@ class FSFile { # Height, width and metadata $handler = MediaHandler::getHandler( $info['mime'] ); if ( $handler ) { - $tempImage = (object)array(); + $tempImage = (object)array(); // XXX (hack for File object) $info['metadata'] = $handler->getMetadata( $tempImage, $this->path ); $gis = $handler->getImageSize( $tempImage, $this->path, $info['metadata'] ); if ( is_array( $gis ) ) { @@ -140,9 +141,9 @@ class FSFile { } $info['sha1'] = $this->getSha1Base36(); - wfDebug(__METHOD__.": $this->path loaded, {$info['size']} bytes, {$info['mime']}.\n"); + wfDebug( __METHOD__ . ": $this->path loaded, {$info['size']} bytes, {$info['mime']}.\n" ); } else { - wfDebug(__METHOD__.": $this->path NOT FOUND!\n"); + wfDebug( __METHOD__ . ": $this->path NOT FOUND!\n" ); } wfProfileOut( __METHOD__ ); @@ -193,25 +194,32 @@ class FSFile { * 160 log 2 / log 36 = 30.95, so the 160-bit hash fills 31 digits in base 36 * fairly neatly. * + * @param $recache bool * @return bool|string False on failure */ - public function getSha1Base36() { + public function getSha1Base36( $recache = false ) { wfProfileIn( __METHOD__ ); + if ( $this->sha1Base36 !== null && !$recache ) { + wfProfileOut( __METHOD__ ); + return $this->sha1Base36; + } + wfSuppressWarnings(); - $hash = sha1_file( $this->path ); + $this->sha1Base36 = sha1_file( $this->path ); wfRestoreWarnings(); - if ( $hash !== false ) { - $hash = wfBaseConvert( $hash, 16, 36, 31 ); + + if ( $this->sha1Base36 !== false ) { + $this->sha1Base36 = wfBaseConvert( $this->sha1Base36, 16, 36, 31 ); } wfProfileOut( __METHOD__ ); - return $hash; + return $this->sha1Base36; } /** * Get the final file extension from a file system path - * + * * @param $path string * @return string */ @@ -223,7 +231,7 @@ class FSFile { /** * Get an associative array containing information about a file in the local filesystem. * - * @param $path String: absolute local filesystem path + * @param string $path absolute local filesystem path * @param $ext Mixed: the file extension, or true to extract it from the filename. * Set it to false to ignore the extension. * @@ -242,11 +250,18 @@ class FSFile { * fairly neatly. * * @param $path string + * @param $recache bool * * @return bool|string False on failure */ - public static function getSha1Base36FromPath( $path ) { - $fsFile = new self( $path ); - return $fsFile->getSha1Base36(); + public static function getSha1Base36FromPath( $path, $recache = false ) { + static $sha1Base36 = array(); + + if ( !isset( $sha1Base36[$path] ) || $recache ) { + $fsFile = new self( $path ); + $sha1Base36[$path] = $fsFile->getSha1Base36(); + } + + return $sha1Base36[$path]; } } diff --git a/includes/filebackend/FSFileBackend.php b/includes/filebackend/FSFileBackend.php index 93495340..c9769989 100644 --- a/includes/filebackend/FSFileBackend.php +++ b/includes/filebackend/FSFileBackend.php @@ -46,6 +46,7 @@ class FSFileBackend extends FileBackendStore { protected $fileOwner; // string; required OS username to own files protected $currentUser; // string; OS username running this script + /** @var Array */ protected $hadWarningErrors = array(); /** @@ -69,7 +70,7 @@ class FSFileBackend extends FileBackendStore { if ( isset( $config['containerPaths'] ) ) { $this->containerPaths = (array)$config['containerPaths']; foreach ( $this->containerPaths as &$path ) { - $path = rtrim( $path, '/' ); // remove trailing slash + $path = rtrim( $path, '/' ); // remove trailing slash } } @@ -101,7 +102,7 @@ class FSFileBackend extends FileBackendStore { /** * Sanity check a relative file system path for validity * - * @param $path string Normalized relative path + * @param string $path Normalized relative path * @return bool */ protected function isLegalRelPath( $path ) { @@ -136,7 +137,7 @@ class FSFileBackend extends FileBackendStore { /** * Get the absolute file system path for a storage path * - * @param $storagePath string Storage path + * @param string $storagePath Storage path * @return string|null */ protected function resolveToFSPath( $storagePath ) { @@ -144,7 +145,7 @@ class FSFileBackend extends FileBackendStore { if ( $relPath === null ) { return null; // invalid } - list( $b, $shortCont, $r ) = FileBackend::splitStoragePath( $storagePath ); + list( , $shortCont, ) = FileBackend::splitStoragePath( $storagePath ); $fsPath = $this->containerFSRoot( $shortCont, $fullCont ); // must be valid if ( $relPath != '' ) { $fsPath .= "/{$relPath}"; @@ -178,10 +179,10 @@ class FSFileBackend extends FileBackendStore { } /** - * @see FileBackendStore::doStoreInternal() + * @see FileBackendStore::doCreateInternal() * @return Status */ - protected function doStoreInternal( array $params ) { + protected function doCreateInternal( array $params ) { $status = Status::newGood(); $dest = $this->resolveToFSPath( $params['dst'] ); @@ -190,27 +191,74 @@ class FSFileBackend extends FileBackendStore { return $status; } - if ( file_exists( $dest ) ) { - if ( !empty( $params['overwrite'] ) ) { - $ok = unlink( $dest ); - if ( !$ok ) { - $status->fatal( 'backend-fail-delete', $params['dst'] ); - return $status; - } - } else { - $status->fatal( 'backend-fail-alreadyexists', $params['dst'] ); + if ( !empty( $params['async'] ) ) { // deferred + $tempFile = TempFSFile::factory( 'create_', 'tmp' ); + if ( !$tempFile ) { + $status->fatal( 'backend-fail-create', $params['dst'] ); + return $status; + } + $this->trapWarnings(); + $bytes = file_put_contents( $tempFile->getPath(), $params['content'] ); + $this->untrapWarnings(); + if ( $bytes === false ) { + $status->fatal( 'backend-fail-create', $params['dst'] ); + return $status; + } + $cmd = implode( ' ', array( + wfIsWindows() ? 'COPY /B /Y' : 'cp', // (binary, overwrite) + wfEscapeShellArg( $this->cleanPathSlashes( $tempFile->getPath() ) ), + wfEscapeShellArg( $this->cleanPathSlashes( $dest ) ) + ) ); + $status->value = new FSFileOpHandle( $this, $params, 'Create', $cmd, $dest ); + $tempFile->bind( $status->value ); + } else { // immediate write + $this->trapWarnings(); + $bytes = file_put_contents( $dest, $params['content'] ); + $this->untrapWarnings(); + if ( $bytes === false ) { + $status->fatal( 'backend-fail-create', $params['dst'] ); return $status; } + $this->chmod( $dest ); + } + + return $status; + } + + /** + * @see FSFileBackend::doExecuteOpHandlesInternal() + */ + protected function _getResponseCreate( $errors, Status $status, array $params, $cmd ) { + if ( $errors !== '' && !( wfIsWindows() && $errors[0] === " " ) ) { + $status->fatal( 'backend-fail-create', $params['dst'] ); + trigger_error( "$cmd\n$errors", E_USER_WARNING ); // command output + } + } + + /** + * @see FileBackendStore::doStoreInternal() + * @return Status + */ + protected function doStoreInternal( array $params ) { + $status = Status::newGood(); + + $dest = $this->resolveToFSPath( $params['dst'] ); + if ( $dest === null ) { + $status->fatal( 'backend-fail-invalidpath', $params['dst'] ); + return $status; } if ( !empty( $params['async'] ) ) { // deferred - $cmd = implode( ' ', array( wfIsWindows() ? 'COPY' : 'cp', + $cmd = implode( ' ', array( + wfIsWindows() ? 'COPY /B /Y' : 'cp', // (binary, overwrite) wfEscapeShellArg( $this->cleanPathSlashes( $params['src'] ) ), wfEscapeShellArg( $this->cleanPathSlashes( $dest ) ) ) ); $status->value = new FSFileOpHandle( $this, $params, 'Store', $cmd, $dest ); } else { // immediate write + $this->trapWarnings(); $ok = copy( $params['src'], $dest ); + $this->untrapWarnings(); // In some cases (at least over NFS), copy() returns true when it fails if ( !$ok || ( filesize( $params['src'] ) !== filesize( $dest ) ) ) { if ( $ok ) { // PHP bug @@ -255,31 +303,30 @@ class FSFileBackend extends FileBackendStore { return $status; } - if ( file_exists( $dest ) ) { - if ( !empty( $params['overwrite'] ) ) { - $ok = unlink( $dest ); - if ( !$ok ) { - $status->fatal( 'backend-fail-delete', $params['dst'] ); - return $status; - } - } else { - $status->fatal( 'backend-fail-alreadyexists', $params['dst'] ); - return $status; + if ( !is_file( $source ) ) { + if ( empty( $params['ignoreMissingSource'] ) ) { + $status->fatal( 'backend-fail-copy', $params['src'] ); } + return $status; // do nothing; either OK or bad status } if ( !empty( $params['async'] ) ) { // deferred - $cmd = implode( ' ', array( wfIsWindows() ? 'COPY' : 'cp', + $cmd = implode( ' ', array( + wfIsWindows() ? 'COPY /B /Y' : 'cp', // (binary, overwrite) wfEscapeShellArg( $this->cleanPathSlashes( $source ) ), wfEscapeShellArg( $this->cleanPathSlashes( $dest ) ) ) ); $status->value = new FSFileOpHandle( $this, $params, 'Copy', $cmd, $dest ); } else { // immediate write + $this->trapWarnings(); $ok = copy( $source, $dest ); + $this->untrapWarnings(); // In some cases (at least over NFS), copy() returns true when it fails if ( !$ok || ( filesize( $source ) !== filesize( $dest ) ) ) { if ( $ok ) { // PHP bug + $this->trapWarnings(); unlink( $dest ); // remove broken file + $this->untrapWarnings(); trigger_error( __METHOD__ . ": copy() failed but returned true." ); } $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] ); @@ -320,30 +367,24 @@ class FSFileBackend extends FileBackendStore { return $status; } - if ( file_exists( $dest ) ) { - if ( !empty( $params['overwrite'] ) ) { - // Windows does not support moving over existing files - if ( wfIsWindows() ) { - $ok = unlink( $dest ); - if ( !$ok ) { - $status->fatal( 'backend-fail-delete', $params['dst'] ); - return $status; - } - } - } else { - $status->fatal( 'backend-fail-alreadyexists', $params['dst'] ); - return $status; + if ( !is_file( $source ) ) { + if ( empty( $params['ignoreMissingSource'] ) ) { + $status->fatal( 'backend-fail-move', $params['src'] ); } + return $status; // do nothing; either OK or bad status } if ( !empty( $params['async'] ) ) { // deferred - $cmd = implode( ' ', array( wfIsWindows() ? 'MOVE' : 'mv', + $cmd = implode( ' ', array( + wfIsWindows() ? 'MOVE /Y' : 'mv', // (overwrite) wfEscapeShellArg( $this->cleanPathSlashes( $source ) ), wfEscapeShellArg( $this->cleanPathSlashes( $dest ) ) ) ); $status->value = new FSFileOpHandle( $this, $params, 'Move', $cmd ); } else { // immediate write + $this->trapWarnings(); $ok = rename( $source, $dest ); + $this->untrapWarnings(); clearstatcache(); // file no longer at source if ( !$ok ) { $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] ); @@ -385,12 +426,15 @@ class FSFileBackend extends FileBackendStore { } if ( !empty( $params['async'] ) ) { // deferred - $cmd = implode( ' ', array( wfIsWindows() ? 'DEL' : 'unlink', + $cmd = implode( ' ', array( + wfIsWindows() ? 'DEL' : 'unlink', wfEscapeShellArg( $this->cleanPathSlashes( $source ) ) ) ); $status->value = new FSFileOpHandle( $this, $params, 'Copy', $cmd ); } else { // immediate write + $this->trapWarnings(); $ok = unlink( $source ); + $this->untrapWarnings(); if ( !$ok ) { $status->fatal( 'backend-fail-delete', $params['src'] ); return $status; @@ -411,89 +455,27 @@ class FSFileBackend extends FileBackendStore { } /** - * @see FileBackendStore::doCreateInternal() - * @return Status - */ - protected function doCreateInternal( array $params ) { - $status = Status::newGood(); - - $dest = $this->resolveToFSPath( $params['dst'] ); - if ( $dest === null ) { - $status->fatal( 'backend-fail-invalidpath', $params['dst'] ); - return $status; - } - - if ( file_exists( $dest ) ) { - if ( !empty( $params['overwrite'] ) ) { - $ok = unlink( $dest ); - if ( !$ok ) { - $status->fatal( 'backend-fail-delete', $params['dst'] ); - return $status; - } - } else { - $status->fatal( 'backend-fail-alreadyexists', $params['dst'] ); - return $status; - } - } - - if ( !empty( $params['async'] ) ) { // deferred - $tempFile = TempFSFile::factory( 'create_', 'tmp' ); - if ( !$tempFile ) { - $status->fatal( 'backend-fail-create', $params['dst'] ); - return $status; - } - $bytes = file_put_contents( $tempFile->getPath(), $params['content'] ); - if ( $bytes === false ) { - $status->fatal( 'backend-fail-create', $params['dst'] ); - return $status; - } - $cmd = implode( ' ', array( wfIsWindows() ? 'COPY' : 'cp', - wfEscapeShellArg( $this->cleanPathSlashes( $tempFile->getPath() ) ), - wfEscapeShellArg( $this->cleanPathSlashes( $dest ) ) - ) ); - $status->value = new FSFileOpHandle( $this, $params, 'Create', $cmd, $dest ); - $tempFile->bind( $status->value ); - } else { // immediate write - $bytes = file_put_contents( $dest, $params['content'] ); - if ( $bytes === false ) { - $status->fatal( 'backend-fail-create', $params['dst'] ); - return $status; - } - $this->chmod( $dest ); - } - - return $status; - } - - /** - * @see FSFileBackend::doExecuteOpHandlesInternal() - */ - protected function _getResponseCreate( $errors, Status $status, array $params, $cmd ) { - if ( $errors !== '' && !( wfIsWindows() && $errors[0] === " " ) ) { - $status->fatal( 'backend-fail-create', $params['dst'] ); - trigger_error( "$cmd\n$errors", E_USER_WARNING ); // command output - } - } - - /** * @see FileBackendStore::doPrepareInternal() * @return Status */ protected function doPrepareInternal( $fullCont, $dirRel, array $params ) { $status = Status::newGood(); - list( $b, $shortCont, $r ) = FileBackend::splitStoragePath( $params['dir'] ); + list( , $shortCont, ) = FileBackend::splitStoragePath( $params['dir'] ); $contRoot = $this->containerFSRoot( $shortCont, $fullCont ); // must be valid $dir = ( $dirRel != '' ) ? "{$contRoot}/{$dirRel}" : $contRoot; $existed = is_dir( $dir ); // already there? - if ( !wfMkdirParents( $dir ) ) { // make directory and its parents + // Create the directory and its parents as needed... + $this->trapWarnings(); + if ( !wfMkdirParents( $dir ) ) { $status->fatal( 'directorycreateerror', $params['dir'] ); // fails on races } elseif ( !is_writable( $dir ) ) { $status->fatal( 'directoryreadonlyerror', $params['dir'] ); } elseif ( !is_readable( $dir ) ) { $status->fatal( 'directorynotreadableerror', $params['dir'] ); } + $this->untrapWarnings(); + // Respect any 'noAccess' or 'noListing' flags... if ( is_dir( $dir ) && !$existed ) { - // Respect any 'noAccess' or 'noListing' flags... $status->merge( $this->doSecureInternal( $fullCont, $dirRel, $params ) ); } return $status; @@ -505,24 +487,26 @@ class FSFileBackend extends FileBackendStore { */ protected function doSecureInternal( $fullCont, $dirRel, array $params ) { $status = Status::newGood(); - list( $b, $shortCont, $r ) = FileBackend::splitStoragePath( $params['dir'] ); + list( , $shortCont, ) = FileBackend::splitStoragePath( $params['dir'] ); $contRoot = $this->containerFSRoot( $shortCont, $fullCont ); // must be valid $dir = ( $dirRel != '' ) ? "{$contRoot}/{$dirRel}" : $contRoot; // Seed new directories with a blank index.html, to prevent crawling... if ( !empty( $params['noListing'] ) && !file_exists( "{$dir}/index.html" ) ) { + $this->trapWarnings(); $bytes = file_put_contents( "{$dir}/index.html", $this->indexHtmlPrivate() ); + $this->untrapWarnings(); if ( $bytes === false ) { $status->fatal( 'backend-fail-create', $params['dir'] . '/index.html' ); - return $status; } } // Add a .htaccess file to the root of the container... if ( !empty( $params['noAccess'] ) && !file_exists( "{$contRoot}/.htaccess" ) ) { + $this->trapWarnings(); $bytes = file_put_contents( "{$contRoot}/.htaccess", $this->htaccessPrivate() ); + $this->untrapWarnings(); if ( $bytes === false ) { $storeDir = "mwstore://{$this->name}/{$shortCont}"; $status->fatal( 'backend-fail-create', "{$storeDir}/.htaccess" ); - return $status; } } return $status; @@ -534,25 +518,27 @@ class FSFileBackend extends FileBackendStore { */ protected function doPublishInternal( $fullCont, $dirRel, array $params ) { $status = Status::newGood(); - list( $b, $shortCont, $r ) = FileBackend::splitStoragePath( $params['dir'] ); + list( , $shortCont, ) = FileBackend::splitStoragePath( $params['dir'] ); $contRoot = $this->containerFSRoot( $shortCont, $fullCont ); // must be valid $dir = ( $dirRel != '' ) ? "{$contRoot}/{$dirRel}" : $contRoot; // Unseed new directories with a blank index.html, to allow crawling... if ( !empty( $params['listing'] ) && is_file( "{$dir}/index.html" ) ) { $exists = ( file_get_contents( "{$dir}/index.html" ) === $this->indexHtmlPrivate() ); + $this->trapWarnings(); if ( $exists && !unlink( "{$dir}/index.html" ) ) { // reverse secure() $status->fatal( 'backend-fail-delete', $params['dir'] . '/index.html' ); - return $status; } + $this->untrapWarnings(); } // Remove the .htaccess file from the root of the container... if ( !empty( $params['access'] ) && is_file( "{$contRoot}/.htaccess" ) ) { $exists = ( file_get_contents( "{$contRoot}/.htaccess" ) === $this->htaccessPrivate() ); + $this->trapWarnings(); if ( $exists && !unlink( "{$contRoot}/.htaccess" ) ) { // reverse secure() $storeDir = "mwstore://{$this->name}/{$shortCont}"; $status->fatal( 'backend-fail-delete', "{$storeDir}/.htaccess" ); - return $status; } + $this->untrapWarnings(); } return $status; } @@ -563,14 +549,14 @@ class FSFileBackend extends FileBackendStore { */ protected function doCleanInternal( $fullCont, $dirRel, array $params ) { $status = Status::newGood(); - list( $b, $shortCont, $r ) = FileBackend::splitStoragePath( $params['dir'] ); + list( , $shortCont, ) = FileBackend::splitStoragePath( $params['dir'] ); $contRoot = $this->containerFSRoot( $shortCont, $fullCont ); // must be valid $dir = ( $dirRel != '' ) ? "{$contRoot}/{$dirRel}" : $contRoot; - wfSuppressWarnings(); + $this->trapWarnings(); if ( is_dir( $dir ) ) { rmdir( $dir ); // remove directory if empty } - wfRestoreWarnings(); + $this->untrapWarnings(); return $status; } @@ -612,7 +598,7 @@ class FSFileBackend extends FileBackendStore { * @return bool|null */ protected function doDirectoryExists( $fullCont, $dirRel, array $params ) { - list( $b, $shortCont, $r ) = FileBackend::splitStoragePath( $params['dir'] ); + list( , $shortCont, ) = FileBackend::splitStoragePath( $params['dir'] ); $contRoot = $this->containerFSRoot( $shortCont, $fullCont ); // must be valid $dir = ( $dirRel != '' ) ? "{$contRoot}/{$dirRel}" : $contRoot; @@ -628,7 +614,7 @@ class FSFileBackend extends FileBackendStore { * @return Array|null */ public function getDirectoryListInternal( $fullCont, $dirRel, array $params ) { - list( $b, $shortCont, $r ) = FileBackend::splitStoragePath( $params['dir'] ); + list( , $shortCont, ) = FileBackend::splitStoragePath( $params['dir'] ); $contRoot = $this->containerFSRoot( $shortCont, $fullCont ); // must be valid $dir = ( $dirRel != '' ) ? "{$contRoot}/{$dirRel}" : $contRoot; $exists = is_dir( $dir ); @@ -644,10 +630,10 @@ class FSFileBackend extends FileBackendStore { /** * @see FileBackendStore::getFileListInternal() - * @return array|FSFileBackendFileList|null + * @return Array|FSFileBackendFileList|null */ public function getFileListInternal( $fullCont, $dirRel, array $params ) { - list( $b, $shortCont, $r ) = FileBackend::splitStoragePath( $params['dir'] ); + list( , $shortCont, ) = FileBackend::splitStoragePath( $params['dir'] ); $contRoot = $this->containerFSRoot( $shortCont, $fullCont ); // must be valid $dir = ( $dirRel != '' ) ? "{$contRoot}/{$dirRel}" : $contRoot; $exists = is_dir( $dir ); @@ -662,44 +648,58 @@ class FSFileBackend extends FileBackendStore { } /** - * @see FileBackendStore::getLocalReference() - * @return FSFile|null + * @see FileBackendStore::doGetLocalReferenceMulti() + * @return Array */ - public function getLocalReference( array $params ) { - $source = $this->resolveToFSPath( $params['src'] ); - if ( $source === null ) { - return null; + protected function doGetLocalReferenceMulti( array $params ) { + $fsFiles = array(); // (path => FSFile) + + foreach ( $params['srcs'] as $src ) { + $source = $this->resolveToFSPath( $src ); + if ( $source === null || !is_file( $source ) ) { + $fsFiles[$src] = null; // invalid path or file does not exist + } else { + $fsFiles[$src] = new FSFile( $source ); + } } - return new FSFile( $source ); + + return $fsFiles; } /** - * @see FileBackendStore::getLocalCopy() - * @return null|TempFSFile + * @see FileBackendStore::doGetLocalCopyMulti() + * @return Array */ - public function getLocalCopy( array $params ) { - $source = $this->resolveToFSPath( $params['src'] ); - if ( $source === null ) { - return null; - } - - // Create a new temporary file with the same extension... - $ext = FileBackend::extensionFromPath( $params['src'] ); - $tmpFile = TempFSFile::factory( 'localcopy_', $ext ); - if ( !$tmpFile ) { - return null; - } - $tmpPath = $tmpFile->getPath(); + protected function doGetLocalCopyMulti( array $params ) { + $tmpFiles = array(); // (path => TempFSFile) - // Copy the source file over the temp file - $ok = copy( $source, $tmpPath ); - if ( !$ok ) { - return null; + foreach ( $params['srcs'] as $src ) { + $source = $this->resolveToFSPath( $src ); + if ( $source === null ) { + $tmpFiles[$src] = null; // invalid path + } else { + // Create a new temporary file with the same extension... + $ext = FileBackend::extensionFromPath( $src ); + $tmpFile = TempFSFile::factory( 'localcopy_', $ext ); + if ( !$tmpFile ) { + $tmpFiles[$src] = null; + } else { + $tmpPath = $tmpFile->getPath(); + // Copy the source file over the temp file + $this->trapWarnings(); + $ok = copy( $source, $tmpPath ); + $this->untrapWarnings(); + if ( !$ok ) { + $tmpFiles[$src] = null; + } else { + $this->chmod( $tmpPath ); + $tmpFiles[$src] = $tmpFile; + } + } + } } - $this->chmod( $tmpPath ); - - return $tmpFile; + return $tmpFiles; } /** @@ -747,13 +747,13 @@ class FSFileBackend extends FileBackendStore { /** * Chmod a file, suppressing the warnings * - * @param $path string Absolute file system path + * @param string $path Absolute file system path * @return bool Success */ protected function chmod( $path ) { - wfSuppressWarnings(); + $this->trapWarnings(); $ok = chmod( $path, $this->fileMode ); - wfRestoreWarnings(); + $this->untrapWarnings(); return $ok; } @@ -779,7 +779,7 @@ class FSFileBackend extends FileBackendStore { /** * Clean up directory separators for the given OS * - * @param $path string FS path + * @param string $path FS path * @return string */ protected function cleanPathSlashes( $path ) { @@ -789,12 +789,11 @@ class FSFileBackend extends FileBackendStore { /** * Listen for E_WARNING errors and track whether any happen * - * @return bool + * @return void */ protected function trapWarnings() { $this->hadWarningErrors[] = false; // push to stack set_error_handler( array( $this, 'handleWarning' ), E_WARNING ); - return false; // invoke normal PHP error handler } /** @@ -808,9 +807,12 @@ class FSFileBackend extends FileBackendStore { } /** + * @param $errno integer + * @param $errstr string * @return bool */ - private function handleWarning() { + private function handleWarning( $errno, $errstr ) { + wfDebugLog( 'FSFileBackend', $errstr ); // more detailed error logging $this->hadWarningErrors[count( $this->hadWarningErrors ) - 1] = true; return true; // suppress from PHP handler } @@ -855,16 +857,19 @@ abstract class FSFileBackendList implements Iterator { protected $params = array(); /** - * @param $dir string file system directory + * @param string $dir file system directory * @param $params array */ public function __construct( $dir, array $params ) { - $dir = realpath( $dir ); // normalize - $this->suffixStart = strlen( $dir ) + 1; // size of "path/to/dir/" + $path = realpath( $dir ); // normalize + if( $path === false ) { + $path = $dir; + } + $this->suffixStart = strlen( $path ) + 1; // size of "path/to/dir/" $this->params = $params; try { - $this->iter = $this->initIterator( $dir ); + $this->iter = $this->initIterator( $path ); } catch ( UnexpectedValueException $e ) { $this->iter = null; // bad permissions? deleted? } @@ -873,7 +878,7 @@ abstract class FSFileBackendList implements Iterator { /** * Return an appropriate iterator object to wrap * - * @param $dir string file system directory + * @param string $dir file system directory * @return Iterator */ protected function initIterator( $dir ) { @@ -956,8 +961,12 @@ abstract class FSFileBackendList implements Iterator { * @param $path string * @return string */ - protected function getRelPath( $path ) { - return strtr( substr( realpath( $path ), $this->suffixStart ), '\\', '/' ); + protected function getRelPath( $dir ) { + $path = realpath( $dir ); + if( $path === false ) { + $path = $dir; + } + return strtr( substr( $path, $this->suffixStart ), '\\', '/' ); } } diff --git a/includes/filebackend/FileBackend.php b/includes/filebackend/FileBackend.php index 76c761b0..f40b8c16 100644 --- a/includes/filebackend/FileBackend.php +++ b/includes/filebackend/FileBackend.php @@ -37,14 +37,18 @@ * Outside callers can assume that all backends will have these functions. * * All "storage paths" are of the format "mwstore://<backend>/<container>/<path>". - * The "<path>" portion is a relative path that uses UNIX file system (FS) - * notation, though any particular backend may not actually be using a local - * filesystem. Therefore, the relative paths are only virtual. + * The "backend" portion is unique name for MediaWiki to refer to a backend, while + * the "container" portion is a top-level directory of the backend. The "path" portion + * is a relative path that uses UNIX file system (FS) notation, though any particular + * backend may not actually be using a local filesystem. Therefore, the relative paths + * are only virtual. * * Backend contents are stored under wiki-specific container names by default. - * For legacy reasons, this has no effect for the FS backend class, and per-wiki - * segregation must be done by setting the container paths appropriately. + * Global (qualified) backends are achieved by configuring the "wiki ID" to a constant. + * For legacy reasons, the FSFileBackend class allows manually setting the paths of + * containers to ones that do not respect the "wiki ID". * + * In key/value stores, the container is the only hierarchy (the rest is emulated). * FS-based backends are somewhat more restrictive due to the existence of real * directory files; a regular file cannot have the same name as a directory. Other * backends with virtual directories may not have this limitation. Callers should @@ -75,9 +79,13 @@ abstract class FileBackend { * $config includes: * - name : The unique name of this backend. * This should consist of alphanumberic, '-', and '_' characters. - * This name should not be changed after use. - * - wikiId : Prefix to container names that is unique to this wiki. + * This name should not be changed after use (e.g. with journaling). + * Note that the name is *not* used in actual container names. + * - wikiId : Prefix to container names that is unique to this backend. + * If not provided, this defaults to the current wiki ID. * It should only consist of alphanumberic, '-', and '_' characters. + * This ID is what avoids collisions if multiple logical backends + * use the same storage system, so this should be set carefully. * - lockManager : Registered name of a file lock manager to use. * - fileJournal : File journal configuration; see FileJournal::factory(). * Journals simply log changes to files stored in the backend. @@ -100,7 +108,7 @@ abstract class FileBackend { : wfWikiID(); // e.g. "my_wiki-en_" $this->lockManager = ( $config['lockManager'] instanceof LockManager ) ? $config['lockManager'] - : LockManagerGroup::singleton()->get( $config['lockManager'] ); + : LockManagerGroup::singleton( $this->wikiId )->get( $config['lockManager'] ); $this->fileJournal = isset( $config['fileJournal'] ) ? ( ( $config['fileJournal'] instanceof FileJournal ) ? $config['fileJournal'] @@ -129,7 +137,8 @@ abstract class FileBackend { } /** - * Get the wiki identifier used for this backend (possibly empty) + * Get the wiki identifier used for this backend (possibly empty). + * Note that this might *not* be in the same format as wfWikiID(). * * @return string * @since 1.20 @@ -171,6 +180,7 @@ abstract class FileBackend { * - copy * - move * - delete + * - describe (since 1.21) * - null * * a) Create a new file in storage with the contents of a string @@ -181,7 +191,8 @@ abstract class FileBackend { * 'content' => <string of new file contents>, * 'overwrite' => <boolean>, * 'overwriteSame' => <boolean>, - * 'disposition' => <Content-Disposition header value> + * 'disposition' => <Content-Disposition header value>, + * 'headers' => <HTTP header name/value map> # since 1.21 * ); * @endcode * @@ -193,7 +204,8 @@ abstract class FileBackend { * 'dst' => <storage path>, * 'overwrite' => <boolean>, * 'overwriteSame' => <boolean>, - * 'disposition' => <Content-Disposition header value> + * 'disposition' => <Content-Disposition header value>, + * 'headers' => <HTTP header name/value map> # since 1.21 * ) * @endcode * @@ -205,6 +217,7 @@ abstract class FileBackend { * 'dst' => <storage path>, * 'overwrite' => <boolean>, * 'overwriteSame' => <boolean>, + * 'ignoreMissingSource' => <boolean>, # since 1.21 * 'disposition' => <Content-Disposition header value> * ) * @endcode @@ -217,6 +230,7 @@ abstract class FileBackend { * 'dst' => <storage path>, * 'overwrite' => <boolean>, * 'overwriteSame' => <boolean>, + * 'ignoreMissingSource' => <boolean>, # since 1.21 * 'disposition' => <Content-Disposition header value> * ) * @endcode @@ -230,7 +244,17 @@ abstract class FileBackend { * ) * @endcode * - * f) Do nothing (no-op) + * f) Update metadata for a file within storage + * @code + * array( + * 'op' => 'describe', + * 'src' => <storage path>, + * 'disposition' => <Content-Disposition header value>, + * 'headers' => <HTTP header name/value map> + * ) + * @endcode + * + * g) Do nothing (no-op) * @code * array( * 'op' => 'null', @@ -244,28 +268,32 @@ abstract class FileBackend { * - overwriteSame : An error will not be given if a file already * exists at the destination that has the same * contents as the new contents to be written there. - * - disposition : When supplied, the backend will add a Content-Disposition + * - disposition : If supplied, the backend will return a Content-Disposition * header when GETs/HEADs of the destination file are made. - * Backends that don't support file metadata will ignore this. - * See http://tools.ietf.org/html/rfc6266 (since 1.20). + * Backends that don't support metadata ignore this. + * See http://tools.ietf.org/html/rfc6266. (since 1.20) + * - headers : If supplied, the backend will return these headers when + * GETs/HEADs of the destination file are made. Header values + * should be smaller than 256 bytes, often options or numbers. + * Existing headers will remain, but these will replace any + * conflicting previous headers, and headers will be removed + * if they are set to an empty string. + * Backends that don't support metadata ignore this. (since 1.21) * * $opts is an associative of boolean flags, including: * - force : Operation precondition errors no longer trigger an abort. * Any remaining operations are still attempted. Unexpected - * failures may still cause remaning operations to be aborted. + * failures may still cause remaining operations to be aborted. * - nonLocking : No locks are acquired for the operations. * This can increase performance for non-critical writes. * This has no effect unless the 'force' flag is set. - * - allowStale : Don't require the latest available data. - * This can increase performance for non-critical writes. - * This has no effect unless the 'force' flag is set. * - nonJournaled : Don't log this operation batch in the file journal. * This limits the ability of recovery scripts. * - parallelize : Try to do operations in parallel when possible. - * - bypassReadOnly : Allow writes in read-only mode (since 1.20). + * - bypassReadOnly : Allow writes in read-only mode. (since 1.20) * - preserveCache : Don't clear the process cache before checking files. * This should only be used if all entries in the process - * cache were added after the files were already locked (since 1.20). + * cache were added after the files were already locked. (since 1.20) * * @remarks Remarks on locking: * File system paths given to operations should refer to files that are @@ -282,8 +310,8 @@ abstract class FileBackend { * - a) unexpected operation errors occurred (network partitions, disk full...) * - b) significant operation errors occurred and 'force' was not set * - * @param $ops Array List of operations to execute in order - * @param $opts Array Batch operation options + * @param array $ops List of operations to execute in order + * @param array $opts Batch operation options * @return Status */ final public function doOperations( array $ops, array $opts = array() ) { @@ -292,18 +320,8 @@ abstract class FileBackend { } if ( empty( $opts['force'] ) ) { // sanity unset( $opts['nonLocking'] ); - unset( $opts['allowStale'] ); - } - $opts['concurrency'] = 1; // off - if ( $this->parallelize === 'implicit' ) { - if ( !isset( $opts['parallelize'] ) || $opts['parallelize'] ) { - $opts['concurrency'] = $this->concurrency; - } - } elseif ( $this->parallelize === 'explicit' ) { - if ( !empty( $opts['parallelize'] ) ) { - $opts['concurrency'] = $this->concurrency; - } } + $scope = $this->getScopedPHPBehaviorForOps(); // try to ignore client aborts return $this->doOperationsInternal( $ops, $opts ); } @@ -319,8 +337,8 @@ abstract class FileBackend { * * @see FileBackend::doOperations() * - * @param $op Array Operation - * @param $opts Array Operation options + * @param array $op Operation + * @param array $opts Operation options * @return Status */ final public function doOperation( array $op, array $opts = array() ) { @@ -333,8 +351,8 @@ abstract class FileBackend { * * @see FileBackend::doOperation() * - * @param $params Array Operation parameters - * @param $opts Array Operation options + * @param array $params Operation parameters + * @param array $opts Operation options * @return Status */ final public function create( array $params, array $opts = array() ) { @@ -347,8 +365,8 @@ abstract class FileBackend { * * @see FileBackend::doOperation() * - * @param $params Array Operation parameters - * @param $opts Array Operation options + * @param array $params Operation parameters + * @param array $opts Operation options * @return Status */ final public function store( array $params, array $opts = array() ) { @@ -361,8 +379,8 @@ abstract class FileBackend { * * @see FileBackend::doOperation() * - * @param $params Array Operation parameters - * @param $opts Array Operation options + * @param array $params Operation parameters + * @param array $opts Operation options * @return Status */ final public function copy( array $params, array $opts = array() ) { @@ -375,8 +393,8 @@ abstract class FileBackend { * * @see FileBackend::doOperation() * - * @param $params Array Operation parameters - * @param $opts Array Operation options + * @param array $params Operation parameters + * @param array $opts Operation options * @return Status */ final public function move( array $params, array $opts = array() ) { @@ -389,8 +407,8 @@ abstract class FileBackend { * * @see FileBackend::doOperation() * - * @param $params Array Operation parameters - * @param $opts Array Operation options + * @param array $params Operation parameters + * @param array $opts Operation options * @return Status */ final public function delete( array $params, array $opts = array() ) { @@ -398,6 +416,21 @@ abstract class FileBackend { } /** + * Performs a single describe operation. + * This sets $params['op'] to 'describe' and passes it to doOperation(). + * + * @see FileBackend::doOperation() + * + * @param array $params Operation parameters + * @param array $opts Operation options + * @return Status + * @since 1.21 + */ + final public function describe( array $params, array $opts = array() ) { + return $this->doOperation( array( 'op' => 'describe' ) + $params, $opts ); + } + + /** * Perform a set of independent file operations on some files. * * This does no locking, nor journaling, and possibly no stat calls. @@ -410,6 +443,7 @@ abstract class FileBackend { * - copy * - move * - delete + * - describe (since 1.21) * - null * * a) Create a new file in storage with the contents of a string @@ -418,36 +452,44 @@ abstract class FileBackend { * 'op' => 'create', * 'dst' => <storage path>, * 'content' => <string of new file contents>, - * 'disposition' => <Content-Disposition header value> + * 'disposition' => <Content-Disposition header value>, + * 'headers' => <HTTP header name/value map> # since 1.21 * ) * @endcode + * * b) Copy a file system file into storage * @code * array( * 'op' => 'store', * 'src' => <file system path>, * 'dst' => <storage path>, - * 'disposition' => <Content-Disposition header value> + * 'disposition' => <Content-Disposition header value>, + * 'headers' => <HTTP header name/value map> # since 1.21 * ) * @endcode + * * c) Copy a file within storage * @code * array( * 'op' => 'copy', * 'src' => <storage path>, * 'dst' => <storage path>, + * 'ignoreMissingSource' => <boolean>, # since 1.21 * 'disposition' => <Content-Disposition header value> * ) * @endcode + * * d) Move a file within storage * @code * array( * 'op' => 'move', * 'src' => <storage path>, * 'dst' => <storage path>, + * 'ignoreMissingSource' => <boolean>, # since 1.21 * 'disposition' => <Content-Disposition header value> * ) * @endcode + * * e) Delete a file within storage * @code * array( @@ -456,7 +498,18 @@ abstract class FileBackend { * 'ignoreMissingSource' => <boolean> * ) * @endcode - * f) Do nothing (no-op) + * + * f) Update metadata for a file within storage + * @code + * array( + * 'op' => 'describe', + * 'src' => <storage path>, + * 'disposition' => <Content-Disposition header value>, + * 'headers' => <HTTP header name/value map> + * ) + * @endcode + * + * g) Do nothing (no-op) * @code * array( * 'op' => 'null', @@ -470,6 +523,13 @@ abstract class FileBackend { * header when GETs/HEADs of the destination file are made. * Backends that don't support file metadata will ignore this. * See http://tools.ietf.org/html/rfc6266 (since 1.20). + * - headers : If supplied with a header name/value map, the backend will + * reply with these headers when GETs/HEADs of the destination + * file are made. Header values should be smaller than 256 bytes. + * Existing headers will remain, but these will replace any + * conflicting previous headers, and headers will be removed + * if they are set to an empty string. + * Backends that don't support metadata ignore this. (since 1.21) * * $opts is an associative of boolean flags, including: * - bypassReadOnly : Allow writes in read-only mode (since 1.20) @@ -480,8 +540,8 @@ abstract class FileBackend { * will reflect each operation attempted for the given files. The status will be * considered "OK" as long as no fatal errors occurred. * - * @param $ops Array Set of operations to execute - * @param $opts Array Batch operation options + * @param array $ops Set of operations to execute + * @param array $opts Batch operation options * @return Status * @since 1.20 */ @@ -492,6 +552,7 @@ abstract class FileBackend { foreach ( $ops as &$op ) { $op['overwrite'] = true; // avoids RTTs in key/value stores } + $scope = $this->getScopedPHPBehaviorForOps(); // try to ignore client aborts return $this->doQuickOperationsInternal( $ops ); } @@ -507,7 +568,7 @@ abstract class FileBackend { * * @see FileBackend::doQuickOperations() * - * @param $op Array Operation + * @param array $op Operation * @return Status * @since 1.20 */ @@ -521,7 +582,7 @@ abstract class FileBackend { * * @see FileBackend::doQuickOperation() * - * @param $params Array Operation parameters + * @param array $params Operation parameters * @return Status * @since 1.20 */ @@ -535,7 +596,7 @@ abstract class FileBackend { * * @see FileBackend::doQuickOperation() * - * @param $params Array Operation parameters + * @param array $params Operation parameters * @return Status * @since 1.20 */ @@ -549,7 +610,7 @@ abstract class FileBackend { * * @see FileBackend::doQuickOperation() * - * @param $params Array Operation parameters + * @param array $params Operation parameters * @return Status * @since 1.20 */ @@ -563,7 +624,7 @@ abstract class FileBackend { * * @see FileBackend::doQuickOperation() * - * @param $params Array Operation parameters + * @param array $params Operation parameters * @return Status * @since 1.20 */ @@ -577,7 +638,7 @@ abstract class FileBackend { * * @see FileBackend::doQuickOperation() * - * @param $params Array Operation parameters + * @param array $params Operation parameters * @return Status * @since 1.20 */ @@ -586,15 +647,30 @@ abstract class FileBackend { } /** + * Performs a single quick describe operation. + * This sets $params['op'] to 'describe' and passes it to doQuickOperation(). + * + * @see FileBackend::doQuickOperation() + * + * @param array $params Operation parameters + * @return Status + * @since 1.21 + */ + final public function quickDescribe( array $params ) { + return $this->doQuickOperation( array( 'op' => 'describe' ) + $params ); + } + + /** * Concatenate a list of storage files into a single file system file. * The target path should refer to a file that is already locked or * otherwise safe from modification from other processes. Normally, * the file will be a new temp file, which should be adequate. * - * @param $params Array Operation parameters + * @param array $params Operation parameters * $params include: - * - srcs : ordered source storage paths (e.g. chunk1, chunk2, ...) - * - dst : file system path to 0-byte temp file + * - srcs : ordered source storage paths (e.g. chunk1, chunk2, ...) + * - dst : file system path to 0-byte temp file + * - parallelize : try to do operations in parallel when possible * @return Status */ abstract public function concatenate( array $params ); @@ -608,7 +684,7 @@ abstract class FileBackend { * except they are only applied *if* the directory/container had to be created. * These flags should always be set for directories that have private files. * - * @param $params Array + * @param array $params * $params include: * - dir : storage directory * - noAccess : try to deny file access (since 1.20) @@ -620,6 +696,7 @@ abstract class FileBackend { if ( empty( $params['bypassReadOnly'] ) && $this->isReadOnly() ) { return Status::newFatal( 'backend-fail-readonly', $this->name, $this->readOnly ); } + $scope = $this->getScopedPHPBehaviorForOps(); // try to ignore client aborts return $this->doPrepare( $params ); } @@ -635,7 +712,7 @@ abstract class FileBackend { * access to the storage user representing end-users in web requests. * This is not guaranteed to actually do anything. * - * @param $params Array + * @param array $params * $params include: * - dir : storage directory * - noAccess : try to deny file access @@ -647,6 +724,7 @@ abstract class FileBackend { if ( empty( $params['bypassReadOnly'] ) && $this->isReadOnly() ) { return Status::newFatal( 'backend-fail-readonly', $this->name, $this->readOnly ); } + $scope = $this->getScopedPHPBehaviorForOps(); // try to ignore client aborts return $this->doSecure( $params ); } @@ -662,7 +740,7 @@ abstract class FileBackend { * access to the storage user representing end-users in web requests. * This essentially can undo the result of secure() calls. * - * @param $params Array + * @param array $params * $params include: * - dir : storage directory * - access : try to allow file access @@ -675,6 +753,7 @@ abstract class FileBackend { if ( empty( $params['bypassReadOnly'] ) && $this->isReadOnly() ) { return Status::newFatal( 'backend-fail-readonly', $this->name, $this->readOnly ); } + $scope = $this->getScopedPHPBehaviorForOps(); // try to ignore client aborts return $this->doPublish( $params ); } @@ -688,7 +767,7 @@ abstract class FileBackend { * Backends using key/value stores may do nothing unless the directory * is that of an empty container, in which case it will be deleted. * - * @param $params Array + * @param array $params * $params include: * - dir : storage directory * - recursive : recursively delete empty subdirectories first (since 1.20) @@ -699,6 +778,7 @@ abstract class FileBackend { if ( empty( $params['bypassReadOnly'] ) && $this->isReadOnly() ) { return Status::newFatal( 'backend-fail-readonly', $this->name, $this->readOnly ); } + $scope = $this->getScopedPHPBehaviorForOps(); // try to ignore client aborts return $this->doClean( $params ); } @@ -708,10 +788,25 @@ abstract class FileBackend { abstract protected function doClean( array $params ); /** + * Enter file operation scope. + * This just makes PHP ignore user aborts/disconnects until the return + * value leaves scope. This returns null and does nothing in CLI mode. + * + * @return ScopedCallback|null + */ + final protected function getScopedPHPBehaviorForOps() { + if ( php_sapi_name() != 'cli' ) { // http://bugs.php.net/bug.php?id=47540 + $old = ignore_user_abort( true ); // avoid half-finished operations + return new ScopedCallback( function() use ( $old ) { ignore_user_abort( $old ); } ); + } + return null; + } + + /** * Check if a file exists at a storage path in the backend. * This returns false if only a directory exists at the path. * - * @param $params Array + * @param array $params * $params include: * - src : source storage path * - latest : use the latest available data @@ -722,7 +817,7 @@ abstract class FileBackend { /** * Get the last-modified timestamp of the file at a storage path. * - * @param $params Array + * @param array $params * $params include: * - src : source storage path * - latest : use the latest available data @@ -734,18 +829,40 @@ abstract class FileBackend { * Get the contents of a file at a storage path in the backend. * This should be avoided for potentially large files. * - * @param $params Array + * @param array $params * $params include: * - src : source storage path * - latest : use the latest available data * @return string|bool Returns false on failure */ - abstract public function getFileContents( array $params ); + final public function getFileContents( array $params ) { + $contents = $this->getFileContentsMulti( + array( 'srcs' => array( $params['src'] ) ) + $params ); + + return $contents[$params['src']]; + } + + /** + * Like getFileContents() except it takes an array of storage paths + * and returns a map of storage paths to strings (or null on failure). + * The map keys (paths) are in the same order as the provided list of paths. + * + * @see FileBackend::getFileContents() + * + * @param array $params + * $params include: + * - srcs : list of source storage paths + * - latest : use the latest available data + * - parallelize : try to do operations in parallel when possible + * @return Array Map of (path name => string or false on failure) + * @since 1.20 + */ + abstract public function getFileContentsMulti( array $params ); /** * Get the size (bytes) of a file at a storage path in the backend. * - * @param $params Array + * @param array $params * $params include: * - src : source storage path * - latest : use the latest available data @@ -761,7 +878,7 @@ abstract class FileBackend { * - size : the file size (bytes) * Additional values may be included for internal use only. * - * @param $params Array + * @param array $params * $params include: * - src : source storage path * - latest : use the latest available data @@ -772,7 +889,7 @@ abstract class FileBackend { /** * Get a SHA-1 hash of the file at a storage path in the backend. * - * @param $params Array + * @param array $params * $params include: * - src : source storage path * - latest : use the latest available data @@ -782,13 +899,13 @@ abstract class FileBackend { /** * Get the properties of the file at a storage path in the backend. - * Returns FSFile::placeholderProps() on failure. + * This gives the result of FSFile::getProps() on a local copy of the file. * - * @param $params Array + * @param array $params * $params include: * - src : source storage path * - latest : use the latest available data - * @return Array + * @return Array Returns FSFile::placeholderProps() on failure */ abstract public function getFileProps( array $params ); @@ -799,7 +916,7 @@ abstract class FileBackend { * will be sent if streaming began, while none will be sent otherwise. * Implementations should flush the output buffer before sending data. * - * @param $params Array + * @param array $params * $params include: * - src : source storage path * - headers : list of additional HTTP headers to send on success @@ -821,26 +938,89 @@ abstract class FileBackend { * In that later case, there are copies of the file that must stay in sync. * Additionally, further calls to this function may return the same file. * - * @param $params Array + * @param array $params * $params include: * - src : source storage path * - latest : use the latest available data * @return FSFile|null Returns null on failure */ - abstract public function getLocalReference( array $params ); + final public function getLocalReference( array $params ) { + $fsFiles = $this->getLocalReferenceMulti( + array( 'srcs' => array( $params['src'] ) ) + $params ); + + return $fsFiles[$params['src']]; + } + + /** + * Like getLocalReference() except it takes an array of storage paths + * and returns a map of storage paths to FSFile objects (or null on failure). + * The map keys (paths) are in the same order as the provided list of paths. + * + * @see FileBackend::getLocalReference() + * + * @param array $params + * $params include: + * - srcs : list of source storage paths + * - latest : use the latest available data + * - parallelize : try to do operations in parallel when possible + * @return Array Map of (path name => FSFile or null on failure) + * @since 1.20 + */ + abstract public function getLocalReferenceMulti( array $params ); /** * Get a local copy on disk of the file at a storage path in the backend. * The temporary copy will have the same file extension as the source. * Temporary files may be purged when the file object falls out of scope. * - * @param $params Array + * @param array $params * $params include: * - src : source storage path * - latest : use the latest available data * @return TempFSFile|null Returns null on failure */ - abstract public function getLocalCopy( array $params ); + final public function getLocalCopy( array $params ) { + $tmpFiles = $this->getLocalCopyMulti( + array( 'srcs' => array( $params['src'] ) ) + $params ); + + return $tmpFiles[$params['src']]; + } + + /** + * Like getLocalCopy() except it takes an array of storage paths and + * returns a map of storage paths to TempFSFile objects (or null on failure). + * The map keys (paths) are in the same order as the provided list of paths. + * + * @see FileBackend::getLocalCopy() + * + * @param array $params + * $params include: + * - srcs : list of source storage paths + * - latest : use the latest available data + * - parallelize : try to do operations in parallel when possible + * @return Array Map of (path name => TempFSFile or null on failure) + * @since 1.20 + */ + abstract public function getLocalCopyMulti( array $params ); + + /** + * Return an HTTP URL to a given file that requires no authentication to use. + * The URL may be pre-authenticated (via some token in the URL) and temporary. + * This will return null if the backend cannot make an HTTP URL for the file. + * + * This is useful for key/value stores when using scripts that seek around + * large files and those scripts (and the backend) support HTTP Range headers. + * Otherwise, one would need to use getLocalReference(), which involves loading + * the entire file on to local disk. + * + * @param array $params + * $params include: + * - src : source storage path + * - ttl : lifetime (seconds) if pre-authenticated; default is 1 day + * @return string|null + * @since 1.21 + */ + abstract public function getFileHttpUrl( array $params ); /** * Check if a directory exists at a given storage path. @@ -930,7 +1110,7 @@ abstract class FileBackend { * Preload persistent file stat and property cache into in-process cache. * This should be used when stat calls will be made on a known list of a many files. * - * @param $paths Array Storage paths + * @param array $paths Storage paths * @return void */ public function preloadCache( array $paths ) {} @@ -939,7 +1119,7 @@ abstract class FileBackend { * Invalidate any in-process file stat and property cache. * If $paths is given, then only the cache for those files will be cleared. * - * @param $paths Array Storage paths (optional) + * @param array $paths Storage paths (optional) * @return void */ public function clearCache( array $paths = null ) {} @@ -950,7 +1130,7 @@ abstract class FileBackend { * * Callers should consider using getScopedFileLocks() instead. * - * @param $paths Array Storage paths + * @param array $paths Storage paths * @param $type integer LockManager::LOCK_* constant * @return Status */ @@ -961,7 +1141,7 @@ abstract class FileBackend { /** * Unlock the files at the given storage paths in the backend. * - * @param $paths Array Storage paths + * @param array $paths Storage paths * @param $type integer LockManager::LOCK_* constant * @return Status */ @@ -977,7 +1157,7 @@ abstract class FileBackend { * Once the return value goes out scope, the locks will be released and * the status updated. Unlock fatals will not change the status "OK" value. * - * @param $paths Array Storage paths + * @param array $paths Storage paths * @param $type integer LockManager::LOCK_* constant * @param $status Status Status to update on lock/unlock * @return ScopedLock|null Returns null on failure @@ -997,7 +1177,7 @@ abstract class FileBackend { * * @see FileBackend::doOperations() * - * @param $ops Array List of file operations to FileBackend::doOperations() + * @param array $ops List of file operations to FileBackend::doOperations() * @param $status Status Status to update on lock/unlock * @return Array List of ScopedFileLocks or null values * @since 1.20 @@ -1016,6 +1196,17 @@ abstract class FileBackend { } /** + * Get the storage path for the given container for this backend + * + * @param string $container Container name + * @return string Storage path + * @since 1.21 + */ + final public function getContainerStoragePath( $container ) { + return $this->getRootStoragePath() . "/{$container}"; + } + + /** * Get the file journal object for this backend * * @return FileJournal @@ -1088,7 +1279,7 @@ abstract class FileBackend { */ final public static function parentStoragePath( $storagePath ) { $storagePath = dirname( $storagePath ); - list( $b, $cont, $rel ) = self::splitStoragePath( $storagePath ); + list( , , $rel ) = self::splitStoragePath( $storagePath ); return ( $rel === null ) ? null : $storagePath; } @@ -1117,8 +1308,9 @@ abstract class FileBackend { /** * Build a Content-Disposition header value per RFC 6266. * - * @param $type string One of (attachment, inline) - * @param $filename string Suggested file name (should not contain slashes) + * @param string $type One of (attachment, inline) + * @param string $filename Suggested file name (should not contain slashes) + * @throws MWException * @return string * @since 1.20 */ @@ -1145,7 +1337,7 @@ abstract class FileBackend { * * This uses the same traversal protection as Title::secureAndSplit(). * - * @param $path string Storage path relative to a container + * @param string $path Storage path relative to a container * @return string|null */ final protected static function normalizeContainerPath( $path ) { diff --git a/includes/filebackend/FileBackendGroup.php b/includes/filebackend/FileBackendGroup.php index 8bbc96d0..d790a996 100644 --- a/includes/filebackend/FileBackendGroup.php +++ b/includes/filebackend/FileBackendGroup.php @@ -87,6 +87,9 @@ class FileBackendGroup { $thumbDir = isset( $info['thumbDir'] ) ? $info['thumbDir'] : "{$directory}/thumb"; + $transcodedDir = isset( $info['transcodedDir'] ) + ? $info['transcodedDir'] + : "{$directory}/transcoded"; $fileMode = isset( $info['fileMode'] ) ? $info['fileMode'] : 0644; @@ -98,6 +101,7 @@ class FileBackendGroup { 'containerPaths' => array( "{$repoName}-public" => "{$directory}", "{$repoName}-thumb" => $thumbDir, + "{$repoName}-transcoded" => $transcodedDir, "{$repoName}-deleted" => $deletedDir, "{$repoName}-temp" => "{$directory}/temp" ), @@ -122,7 +126,9 @@ class FileBackendGroup { throw new MWException( "Cannot register a backend with no name." ); } $name = $config['name']; - if ( !isset( $config['class'] ) ) { + if ( isset( $this->backends[$name] ) ) { + throw new MWException( "Backend with name `{$name}` already registered." ); + } elseif ( !isset( $config['class'] ) ) { throw new MWException( "Cannot register backend `{$name}` with no class." ); } $class = $config['class']; @@ -178,7 +184,7 @@ class FileBackendGroup { * @return FileBackend|null Backend or null on failure */ public function backendFromPath( $storagePath ) { - list( $backend, $c, $p ) = FileBackend::splitStoragePath( $storagePath ); + list( $backend, , ) = FileBackend::splitStoragePath( $storagePath ); if ( $backend !== null && isset( $this->backends[$backend] ) ) { return $this->get( $backend ); } diff --git a/includes/filebackend/FileBackendMultiWrite.php b/includes/filebackend/FileBackendMultiWrite.php index 4be03231..939315d1 100644 --- a/includes/filebackend/FileBackendMultiWrite.php +++ b/includes/filebackend/FileBackendMultiWrite.php @@ -62,7 +62,7 @@ class FileBackendMultiWrite extends FileBackend { * Additional $config params include: * - backends : Array of backend config and multi-backend settings. * Each value is the config used in the constructor of a - * FileBackendStore class, but with these additional settings: + * FileBackendStore class, but with these additional settings: * - class : The name of the backend class * - isMultiMaster : This must be set for one backend. * - template: : If given a backend name, this will use @@ -179,10 +179,11 @@ class FileBackendMultiWrite extends FileBackend { // Actually attempt the operation batch on the master backend... $masterStatus = $mbe->doOperations( $realOps, $opts ); $status->merge( $masterStatus ); - // Propagate the operations to the clone backends if there were no fatal errors. - // If $ops only had one operation, this might avoid backend inconsistencies. - // This also avoids inconsistency for expected errors (like "file already exists"). - if ( !count( $masterStatus->getErrorsArray() ) ) { + // Propagate the operations to the clone backends if there were no unexpected errors + // and if there were either no expected errors or if the 'force' option was used. + // However, if nothing succeeded at all, then don't replicate any of the operations. + // If $ops only had one operation, this might avoid backend sync inconsistencies. + if ( $masterStatus->isOK() && $masterStatus->successCount > 0 ) { foreach ( $this->backends as $index => $backend ) { if ( $index !== $this->masterIndex ) { // not done already $realOps = $this->substOpBatchPaths( $ops, $backend ); @@ -203,7 +204,7 @@ class FileBackendMultiWrite extends FileBackend { /** * Check that a set of files are consistent across all internal backends * - * @param $paths Array List of storage paths + * @param array $paths List of storage paths * @return Status */ public function consistencyCheck( array $paths ) { @@ -269,7 +270,7 @@ class FileBackendMultiWrite extends FileBackend { /** * Check that a set of file paths are usable across all internal backends * - * @param $paths Array List of storage paths + * @param array $paths List of storage paths * @return Status */ public function accessibilityCheck( array $paths ) { @@ -294,7 +295,7 @@ class FileBackendMultiWrite extends FileBackend { * Check that a set of files are consistent across all internal backends * and re-synchronize those files againt the "multi master" if needed. * - * @param $paths Array List of storage paths + * @param array $paths List of storage paths * @return Status */ public function resyncFiles( array $paths ) { @@ -302,8 +303,8 @@ class FileBackendMultiWrite extends FileBackend { $mBackend = $this->backends[$this->masterIndex]; foreach ( $paths as $path ) { - $mPath = $this->substPaths( $path, $mBackend ); - $mSha1 = $mBackend->getFileSha1Base36( array( 'src' => $mPath ) ); + $mPath = $this->substPaths( $path, $mBackend ); + $mSha1 = $mBackend->getFileSha1Base36( array( 'src' => $mPath ) ); $mExist = $mBackend->fileExists( array( 'src' => $mPath ) ); // Check if the master backend is available... if ( $mExist === null ) { @@ -335,14 +336,20 @@ class FileBackendMultiWrite extends FileBackend { /** * Get a list of file storage paths to read or write for a list of operations * - * @param $ops Array Same format as doOperations() + * @param array $ops Same format as doOperations() * @return Array List of storage paths to files (does not include directories) */ protected function fileStoragePathsForOps( array $ops ) { $paths = array(); foreach ( $ops as $op ) { if ( isset( $op['src'] ) ) { - $paths[] = $op['src']; + // For things like copy/move/delete with "ignoreMissingSource" and there + // is no source file, nothing should happen and there should be no errors. + if ( empty( $op['ignoreMissingSource'] ) + || $this->fileExists( array( 'src' => $op['src'] ) ) ) + { + $paths[] = $op['src']; + } } if ( isset( $op['srcs'] ) ) { $paths = array_merge( $paths, $op['srcs'] ); @@ -351,14 +358,14 @@ class FileBackendMultiWrite extends FileBackend { $paths[] = $op['dst']; } } - return array_unique( array_filter( $paths, 'FileBackend::isStoragePath' ) ); + return array_values( array_unique( array_filter( $paths, 'FileBackend::isStoragePath' ) ) ); } /** * Substitute the backend name in storage path parameters * for a set of operations with that of a given internal backend. * - * @param $ops Array List of file operation arrays + * @param array $ops List of file operation arrays * @param $backend FileBackendStore * @return Array */ @@ -379,7 +386,7 @@ class FileBackendMultiWrite extends FileBackend { /** * Same as substOpBatchPaths() but for a single operation * - * @param $ops array File operation array + * @param array $ops File operation array * @param $backend FileBackendStore * @return Array */ @@ -391,7 +398,7 @@ class FileBackendMultiWrite extends FileBackend { /** * Substitute the backend of storage paths with an internal backend's name * - * @param $paths Array|string List of paths or single string path + * @param array|string $paths List of paths or single string path * @param $backend FileBackendStore * @return Array|string */ @@ -406,7 +413,7 @@ class FileBackendMultiWrite extends FileBackend { /** * Substitute the backend of internal storage paths with the proxy backend's name * - * @param $paths Array|string List of paths or single string path + * @param array|string $paths List of paths or single string path * @return Array|string */ protected function unsubstPaths( $paths ) { @@ -446,11 +453,11 @@ class FileBackendMultiWrite extends FileBackend { } /** - * @param $path string Storage path + * @param string $path Storage path * @return bool Path container should have dir changes pushed to all backends */ protected function replicateContainerDirChanges( $path ) { - list( $b, $shortCont, $r ) = self::splitStoragePath( $path ); + list( , $shortCont, ) = self::splitStoragePath( $path ); return !in_array( $shortCont, $this->noPushDirConts ); } @@ -535,6 +542,7 @@ class FileBackendMultiWrite extends FileBackend { /** * @see FileBackend::fileExists() * @param $params array + * @return bool|null */ public function fileExists( array $params ) { $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] ); @@ -572,13 +580,19 @@ class FileBackendMultiWrite extends FileBackend { } /** - * @see FileBackend::getFileContents() + * @see FileBackend::getFileContentsMulti() * @param $params array * @return bool|string */ - public function getFileContents( array $params ) { + public function getFileContentsMulti( array $params ) { $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] ); - return $this->backends[$this->masterIndex]->getFileContents( $realParams ); + $contentsM = $this->backends[$this->masterIndex]->getFileContentsMulti( $realParams ); + + $contents = array(); // (path => FSFile) mapping using the proxy backend's name + foreach ( $contentsM as $path => $data ) { + $contents[$this->unsubstPaths( $path )] = $data; + } + return $contents; } /** @@ -612,23 +626,44 @@ class FileBackendMultiWrite extends FileBackend { } /** - * @see FileBackend::getLocalReference() + * @see FileBackend::getLocalReferenceMulti() * @param $params array * @return FSFile|null */ - public function getLocalReference( array $params ) { + public function getLocalReferenceMulti( array $params ) { $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] ); - return $this->backends[$this->masterIndex]->getLocalReference( $realParams ); + $fsFilesM = $this->backends[$this->masterIndex]->getLocalReferenceMulti( $realParams ); + + $fsFiles = array(); // (path => FSFile) mapping using the proxy backend's name + foreach ( $fsFilesM as $path => $fsFile ) { + $fsFiles[$this->unsubstPaths( $path )] = $fsFile; + } + return $fsFiles; } /** - * @see FileBackend::getLocalCopy() + * @see FileBackend::getLocalCopyMulti() * @param $params array * @return null|TempFSFile */ - public function getLocalCopy( array $params ) { + public function getLocalCopyMulti( array $params ) { + $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] ); + $tempFilesM = $this->backends[$this->masterIndex]->getLocalCopyMulti( $realParams ); + + $tempFiles = array(); // (path => TempFSFile) mapping using the proxy backend's name + foreach ( $tempFilesM as $path => $tempFile ) { + $tempFiles[$this->unsubstPaths( $path )] = $tempFile; + } + return $tempFiles; + } + + /** + * @see FileBackend::getFileHttpUrl() + * @return string|null + */ + public function getFileHttpUrl( array $params ) { $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] ); - return $this->backends[$this->masterIndex]->getLocalCopy( $realParams ); + return $this->backends[$this->masterIndex]->getFileHttpUrl( $realParams ); } /** diff --git a/includes/filebackend/FileBackendStore.php b/includes/filebackend/FileBackendStore.php index 083dfea9..3f1d1857 100644 --- a/includes/filebackend/FileBackendStore.php +++ b/includes/filebackend/FileBackendStore.php @@ -48,6 +48,8 @@ abstract class FileBackendStore extends FileBackend { protected $maxFileSize = 4294967296; // integer bytes (4GiB) + const CACHE_TTL = 10; // integer; TTL in seconds for process cache entries + /** * @see FileBackend::__construct() * @@ -55,8 +57,8 @@ abstract class FileBackendStore extends FileBackend { */ public function __construct( array $config ) { parent::__construct( $config ); - $this->memCache = new EmptyBagOStuff(); // disabled by default - $this->cheapCache = new ProcessCacheLRU( 300 ); + $this->memCache = new EmptyBagOStuff(); // disabled by default + $this->cheapCache = new ProcessCacheLRU( 300 ); $this->expensiveCache = new ProcessCacheLRU( 5 ); } @@ -72,8 +74,9 @@ abstract class FileBackendStore extends FileBackend { } /** - * Check if a file can be created at a given storage path. - * FS backends should check if the parent directory exists and the file is writable. + * Check if a file can be created or changed at a given storage path. + * FS backends should check if the parent directory exists, files can be + * written under it, and that any file already there is writable. * Backends using key/value stores should check if the container exists. * * @param $storagePath string @@ -83,18 +86,21 @@ abstract class FileBackendStore extends FileBackend { /** * Create a file in the backend with the given contents. + * This will overwrite any file that exists at the destination. * Do not call this function from places outside FileBackend and FileOp. * * $params include: - * - content : the raw file contents - * - dst : destination storage path - * - overwrite : overwrite any file that exists at the destination - * - disposition : Content-Disposition header value for the destination - * - async : Status will be returned immediately if supported. - * If the status is OK, then its value field will be - * set to a FileBackendStoreOpHandle object. + * - content : the raw file contents + * - dst : destination storage path + * - disposition : Content-Disposition header value for the destination + * - headers : HTTP header name/value map + * - async : Status will be returned immediately if supported. + * If the status is OK, then its value field will be + * set to a FileBackendStoreOpHandle object. + * - dstExists : Whether a file exists at the destination (optimization). + * Callers can use "false" if no existing file is being changed. * - * @param $params Array + * @param array $params * @return Status */ final public function createInternal( array $params ) { @@ -106,7 +112,7 @@ abstract class FileBackendStore extends FileBackend { } else { $status = $this->doCreateInternal( $params ); $this->clearCache( array( $params['dst'] ) ); - if ( !empty( $params['overwrite'] ) ) { // file possibly mutated + if ( !isset( $params['dstExists'] ) || $params['dstExists'] ) { $this->deleteFileCache( $params['dst'] ); // persistent cache } } @@ -117,23 +123,27 @@ abstract class FileBackendStore extends FileBackend { /** * @see FileBackendStore::createInternal() + * @return Status */ abstract protected function doCreateInternal( array $params ); /** * Store a file into the backend from a file on disk. + * This will overwrite any file that exists at the destination. * Do not call this function from places outside FileBackend and FileOp. * * $params include: - * - src : source path on disk - * - dst : destination storage path - * - overwrite : overwrite any file that exists at the destination - * - disposition : Content-Disposition header value for the destination - * - async : Status will be returned immediately if supported. - * If the status is OK, then its value field will be - * set to a FileBackendStoreOpHandle object. + * - src : source path on disk + * - dst : destination storage path + * - disposition : Content-Disposition header value for the destination + * - headers : HTTP header name/value map + * - async : Status will be returned immediately if supported. + * If the status is OK, then its value field will be + * set to a FileBackendStoreOpHandle object. + * - dstExists : Whether a file exists at the destination (optimization). + * Callers can use "false" if no existing file is being changed. * - * @param $params Array + * @param array $params * @return Status */ final public function storeInternal( array $params ) { @@ -145,7 +155,7 @@ abstract class FileBackendStore extends FileBackend { } else { $status = $this->doStoreInternal( $params ); $this->clearCache( array( $params['dst'] ) ); - if ( !empty( $params['overwrite'] ) ) { // file possibly mutated + if ( !isset( $params['dstExists'] ) || $params['dstExists'] ) { $this->deleteFileCache( $params['dst'] ); // persistent cache } } @@ -156,23 +166,27 @@ abstract class FileBackendStore extends FileBackend { /** * @see FileBackendStore::storeInternal() + * @return Status */ abstract protected function doStoreInternal( array $params ); /** * Copy a file from one storage path to another in the backend. + * This will overwrite any file that exists at the destination. * Do not call this function from places outside FileBackend and FileOp. * * $params include: - * - src : source storage path - * - dst : destination storage path - * - overwrite : overwrite any file that exists at the destination - * - disposition : Content-Disposition header value for the destination - * - async : Status will be returned immediately if supported. - * If the status is OK, then its value field will be - * set to a FileBackendStoreOpHandle object. + * - src : source storage path + * - dst : destination storage path + * - ignoreMissingSource : do nothing if the source file does not exist + * - disposition : Content-Disposition header value for the destination + * - async : Status will be returned immediately if supported. + * If the status is OK, then its value field will be + * set to a FileBackendStoreOpHandle object. + * - dstExists : Whether a file exists at the destination (optimization). + * Callers can use "false" if no existing file is being changed. * - * @param $params Array + * @param array $params * @return Status */ final public function copyInternal( array $params ) { @@ -180,7 +194,7 @@ abstract class FileBackendStore extends FileBackend { wfProfileIn( __METHOD__ . '-' . $this->name ); $status = $this->doCopyInternal( $params ); $this->clearCache( array( $params['dst'] ) ); - if ( !empty( $params['overwrite'] ) ) { // file possibly mutated + if ( !isset( $params['dstExists'] ) || $params['dstExists'] ) { $this->deleteFileCache( $params['dst'] ); // persistent cache } wfProfileOut( __METHOD__ . '-' . $this->name ); @@ -190,6 +204,7 @@ abstract class FileBackendStore extends FileBackend { /** * @see FileBackendStore::copyInternal() + * @return Status */ abstract protected function doCopyInternal( array $params ); @@ -204,7 +219,7 @@ abstract class FileBackendStore extends FileBackend { * If the status is OK, then its value field will be * set to a FileBackendStoreOpHandle object. * - * @param $params Array + * @param array $params * @return Status */ final public function deleteInternal( array $params ) { @@ -220,23 +235,27 @@ abstract class FileBackendStore extends FileBackend { /** * @see FileBackendStore::deleteInternal() + * @return Status */ abstract protected function doDeleteInternal( array $params ); /** * Move a file from one storage path to another in the backend. + * This will overwrite any file that exists at the destination. * Do not call this function from places outside FileBackend and FileOp. * * $params include: - * - src : source storage path - * - dst : destination storage path - * - overwrite : overwrite any file that exists at the destination - * - disposition : Content-Disposition header value for the destination - * - async : Status will be returned immediately if supported. - * If the status is OK, then its value field will be - * set to a FileBackendStoreOpHandle object. + * - src : source storage path + * - dst : destination storage path + * - ignoreMissingSource : do nothing if the source file does not exist + * - disposition : Content-Disposition header value for the destination + * - async : Status will be returned immediately if supported. + * If the status is OK, then its value field will be + * set to a FileBackendStoreOpHandle object. + * - dstExists : Whether a file exists at the destination (optimization). + * Callers can use "false" if no existing file is being changed. * - * @param $params Array + * @param array $params * @return Status */ final public function moveInternal( array $params ) { @@ -245,7 +264,7 @@ abstract class FileBackendStore extends FileBackend { $status = $this->doMoveInternal( $params ); $this->clearCache( array( $params['src'], $params['dst'] ) ); $this->deleteFileCache( $params['src'] ); // persistent cache - if ( !empty( $params['overwrite'] ) ) { // file possibly mutated + if ( !isset( $params['dstExists'] ) || $params['dstExists'] ) { $this->deleteFileCache( $params['dst'] ); // persistent cache } wfProfileOut( __METHOD__ . '-' . $this->name ); @@ -270,10 +289,44 @@ abstract class FileBackendStore extends FileBackend { } /** + * Alter metadata for a file at the storage path. + * Do not call this function from places outside FileBackend and FileOp. + * + * $params include: + * - src : source storage path + * - disposition : Content-Disposition header value for the destination + * - headers : HTTP header name/value map + * - async : Status will be returned immediately if supported. + * If the status is OK, then its value field will be + * set to a FileBackendStoreOpHandle object. + * + * @param array $params + * @return Status + */ + final public function describeInternal( array $params ) { + wfProfileIn( __METHOD__ ); + wfProfileIn( __METHOD__ . '-' . $this->name ); + $status = $this->doDescribeInternal( $params ); + $this->clearCache( array( $params['src'] ) ); + $this->deleteFileCache( $params['src'] ); // persistent cache + wfProfileOut( __METHOD__ . '-' . $this->name ); + wfProfileOut( __METHOD__ ); + return $status; + } + + /** + * @see FileBackendStore::describeInternal() + * @return Status + */ + protected function doDescribeInternal( array $params ) { + return Status::newGood(); + } + + /** * No-op file operation that does nothing. * Do not call this function from places outside FileBackend and FileOp. * - * @param $params Array + * @param array $params * @return Status */ final public function nullInternal( array $params ) { @@ -314,31 +367,41 @@ abstract class FileBackendStore extends FileBackend { protected function doConcatenate( array $params ) { $status = Status::newGood(); $tmpPath = $params['dst']; // convenience + unset( $params['latest'] ); // sanity // Check that the specified temp file is valid... wfSuppressWarnings(); - $ok = ( is_file( $tmpPath ) && !filesize( $tmpPath ) ); + $ok = ( is_file( $tmpPath ) && filesize( $tmpPath ) == 0 ); wfRestoreWarnings(); if ( !$ok ) { // not present or not empty $status->fatal( 'backend-fail-opentemp', $tmpPath ); return $status; } - // Build up the temp file using the source chunks (in order)... + // Get local FS versions of the chunks needed for the concatenation... + $fsFiles = $this->getLocalReferenceMulti( $params ); + foreach ( $fsFiles as $path => &$fsFile ) { + if ( !$fsFile ) { // chunk failed to download? + $fsFile = $this->getLocalReference( array( 'src' => $path ) ); + if ( !$fsFile ) { // retry failed? + $status->fatal( 'backend-fail-read', $path ); + return $status; + } + } + } + unset( $fsFile ); // unset reference so we can reuse $fsFile + + // Get a handle for the destination temp file $tmpHandle = fopen( $tmpPath, 'ab' ); if ( $tmpHandle === false ) { $status->fatal( 'backend-fail-opentemp', $tmpPath ); return $status; } - foreach ( $params['srcs'] as $virtualSource ) { - // Get a local FS version of the chunk - $tmpFile = $this->getLocalReference( array( 'src' => $virtualSource ) ); - if ( !$tmpFile ) { - $status->fatal( 'backend-fail-read', $virtualSource ); - return $status; - } + + // Build up the temp file using the source chunks (in order)... + foreach ( $fsFiles as $virtualSource => $fsFile ) { // Get a handle to the local FS version - $sourceHandle = fopen( $tmpFile->getPath(), 'r' ); + $sourceHandle = fopen( $fsFile->getPath(), 'rb' ); if ( $sourceHandle === false ) { fclose( $tmpHandle ); $status->fatal( 'backend-fail-read', $virtualSource ); @@ -384,7 +447,7 @@ abstract class FileBackendStore extends FileBackend { $status->merge( $this->doPrepareInternal( $fullCont, $dir, $params ) ); } else { // directory is on several shards wfDebug( __METHOD__ . ": iterating over all container shards.\n" ); - list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] ); + list( , $shortCont, ) = self::splitStoragePath( $params['dir'] ); foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) { $status->merge( $this->doPrepareInternal( "{$fullCont}{$suffix}", $dir, $params ) ); } @@ -424,7 +487,7 @@ abstract class FileBackendStore extends FileBackend { $status->merge( $this->doSecureInternal( $fullCont, $dir, $params ) ); } else { // directory is on several shards wfDebug( __METHOD__ . ": iterating over all container shards.\n" ); - list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] ); + list( , $shortCont, ) = self::splitStoragePath( $params['dir'] ); foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) { $status->merge( $this->doSecureInternal( "{$fullCont}{$suffix}", $dir, $params ) ); } @@ -464,7 +527,7 @@ abstract class FileBackendStore extends FileBackend { $status->merge( $this->doPublishInternal( $fullCont, $dir, $params ) ); } else { // directory is on several shards wfDebug( __METHOD__ . ": iterating over all container shards.\n" ); - list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] ); + list( , $shortCont, ) = self::splitStoragePath( $params['dir'] ); foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) { $status->merge( $this->doPublishInternal( "{$fullCont}{$suffix}", $dir, $params ) ); } @@ -500,6 +563,7 @@ abstract class FileBackendStore extends FileBackend { $subDir = $params['dir'] . "/{$subDirRel}"; // full path $status->merge( $this->doClean( array( 'dir' => $subDir ) + $params ) ); } + unset( $subDirsRel ); // free directory for rmdir() on Windows (for FS backends) } } @@ -525,7 +589,7 @@ abstract class FileBackendStore extends FileBackend { $this->deleteContainerCache( $fullCont ); // purge cache } else { // directory is on several shards wfDebug( __METHOD__ . ": iterating over all container shards.\n" ); - list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] ); + list( , $shortCont, ) = self::splitStoragePath( $params['dir'] ); foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) { $status->merge( $this->doCleanInternal( "{$fullCont}{$suffix}", $dir, $params ) ); $this->deleteContainerCache( "{$fullCont}{$suffix}" ); // purge cache @@ -596,17 +660,25 @@ abstract class FileBackendStore extends FileBackend { wfProfileIn( __METHOD__ ); wfProfileIn( __METHOD__ . '-' . $this->name ); $latest = !empty( $params['latest'] ); // use latest data? - if ( !$this->cheapCache->has( $path, 'stat' ) ) { + if ( !$this->cheapCache->has( $path, 'stat', self::CACHE_TTL ) ) { $this->primeFileCache( array( $path ) ); // check persistent cache } - if ( $this->cheapCache->has( $path, 'stat' ) ) { + if ( $this->cheapCache->has( $path, 'stat', self::CACHE_TTL ) ) { $stat = $this->cheapCache->get( $path, 'stat' ); // If we want the latest data, check that this cached // value was in fact fetched with the latest available data. - if ( !$latest || $stat['latest'] ) { - wfProfileOut( __METHOD__ . '-' . $this->name ); - wfProfileOut( __METHOD__ ); - return $stat; + if ( is_array( $stat ) ) { + if ( !$latest || $stat['latest'] ) { + wfProfileOut( __METHOD__ . '-' . $this->name ); + wfProfileOut( __METHOD__ ); + return $stat; + } + } elseif ( in_array( $stat, array( 'NOT_EXIST', 'NOT_EXIST_LATEST' ) ) ) { + if ( !$latest || $stat === 'NOT_EXIST_LATEST' ) { + wfProfileOut( __METHOD__ . '-' . $this->name ); + wfProfileOut( __METHOD__ ); + return false; + } } } wfProfileIn( __METHOD__ . '-miss' ); @@ -614,7 +686,7 @@ abstract class FileBackendStore extends FileBackend { $stat = $this->doGetFileStat( $params ); wfProfileOut( __METHOD__ . '-miss-' . $this->name ); wfProfileOut( __METHOD__ . '-miss' ); - if ( is_array( $stat ) ) { // don't cache negatives + if ( is_array( $stat ) ) { // file exists $stat['latest'] = $latest; $this->cheapCache->set( $path, 'stat', $stat ); $this->setFileCache( $path, $stat ); // update persistent cache @@ -622,8 +694,11 @@ abstract class FileBackendStore extends FileBackend { $this->cheapCache->set( $path, 'sha1', array( 'hash' => $stat['sha1'], 'latest' => $latest ) ); } - } else { + } elseif ( $stat === false ) { // file does not exist + $this->cheapCache->set( $path, 'stat', $latest ? 'NOT_EXIST_LATEST' : 'NOT_EXIST' ); wfDebug( __METHOD__ . ": File $path does not exist.\n" ); + } else { // an error occurred + wfDebug( __METHOD__ . ": Could not stat file $path.\n" ); } wfProfileOut( __METHOD__ . '-' . $this->name ); wfProfileOut( __METHOD__ ); @@ -636,24 +711,33 @@ abstract class FileBackendStore extends FileBackend { abstract protected function doGetFileStat( array $params ); /** - * @see FileBackend::getFileContents() - * @return bool|string + * @see FileBackend::getFileContentsMulti() + * @return Array */ - public function getFileContents( array $params ) { + public function getFileContentsMulti( array $params ) { wfProfileIn( __METHOD__ ); wfProfileIn( __METHOD__ . '-' . $this->name ); - $tmpFile = $this->getLocalReference( $params ); - if ( !$tmpFile ) { - wfProfileOut( __METHOD__ . '-' . $this->name ); - wfProfileOut( __METHOD__ ); - return false; - } - wfSuppressWarnings(); - $data = file_get_contents( $tmpFile->getPath() ); - wfRestoreWarnings(); + + $params = $this->setConcurrencyFlags( $params ); + $contents = $this->doGetFileContentsMulti( $params ); + wfProfileOut( __METHOD__ . '-' . $this->name ); wfProfileOut( __METHOD__ ); - return $data; + return $contents; + } + + /** + * @see FileBackendStore::getFileContentsMulti() + * @return Array + */ + protected function doGetFileContentsMulti( array $params ) { + $contents = array(); + foreach ( $this->doGetLocalReferenceMulti( $params ) as $path => $fsFile ) { + wfSuppressWarnings(); + $contents[$path] = $fsFile ? file_get_contents( $fsFile->getPath() ) : false; + wfRestoreWarnings(); + } + return $contents; } /** @@ -668,7 +752,7 @@ abstract class FileBackendStore extends FileBackend { wfProfileIn( __METHOD__ ); wfProfileIn( __METHOD__ . '-' . $this->name ); $latest = !empty( $params['latest'] ); // use latest data? - if ( $this->cheapCache->has( $path, 'sha1' ) ) { + if ( $this->cheapCache->has( $path, 'sha1', self::CACHE_TTL ) ) { $stat = $this->cheapCache->get( $path, 'sha1' ); // If we want the latest data, check that this cached // value was in fact fetched with the latest available data. @@ -683,10 +767,7 @@ abstract class FileBackendStore extends FileBackend { $hash = $this->doGetFileSha1Base36( $params ); wfProfileOut( __METHOD__ . '-miss-' . $this->name ); wfProfileOut( __METHOD__ . '-miss' ); - if ( $hash ) { // don't cache negatives - $this->cheapCache->set( $path, 'sha1', - array( 'hash' => $hash, 'latest' => $latest ) ); - } + $this->cheapCache->set( $path, 'sha1', array( 'hash' => $hash, 'latest' => $latest ) ); wfProfileOut( __METHOD__ . '-' . $this->name ); wfProfileOut( __METHOD__ ); return $hash; @@ -720,35 +801,82 @@ abstract class FileBackendStore extends FileBackend { } /** - * @see FileBackend::getLocalReference() - * @return TempFSFile|null + * @see FileBackend::getLocalReferenceMulti() + * @return Array */ - public function getLocalReference( array $params ) { - $path = self::normalizeStoragePath( $params['src'] ); - if ( $path === null ) { - return null; // invalid storage path - } + final public function getLocalReferenceMulti( array $params ) { wfProfileIn( __METHOD__ ); wfProfileIn( __METHOD__ . '-' . $this->name ); + + $params = $this->setConcurrencyFlags( $params ); + + $fsFiles = array(); // (path => FSFile) $latest = !empty( $params['latest'] ); // use latest data? - if ( $this->expensiveCache->has( $path, 'localRef' ) ) { - $val = $this->expensiveCache->get( $path, 'localRef' ); - // If we want the latest data, check that this cached - // value was in fact fetched with the latest available data. - if ( !$latest || $val['latest'] ) { - wfProfileOut( __METHOD__ . '-' . $this->name ); - wfProfileOut( __METHOD__ ); - return $val['object']; + // Reuse any files already in process cache... + foreach ( $params['srcs'] as $src ) { + $path = self::normalizeStoragePath( $src ); + if ( $path === null ) { + $fsFiles[$src] = null; // invalid storage path + } elseif ( $this->expensiveCache->has( $path, 'localRef' ) ) { + $val = $this->expensiveCache->get( $path, 'localRef' ); + // If we want the latest data, check that this cached + // value was in fact fetched with the latest available data. + if ( !$latest || $val['latest'] ) { + $fsFiles[$src] = $val['object']; + } } } - $tmpFile = $this->getLocalCopy( $params ); - if ( $tmpFile ) { // don't cache negatives - $this->expensiveCache->set( $path, 'localRef', - array( 'object' => $tmpFile, 'latest' => $latest ) ); + // Fetch local references of any remaning files... + $params['srcs'] = array_diff( $params['srcs'], array_keys( $fsFiles ) ); + foreach ( $this->doGetLocalReferenceMulti( $params ) as $path => $fsFile ) { + $fsFiles[$path] = $fsFile; + if ( $fsFile ) { // update the process cache... + $this->expensiveCache->set( $path, 'localRef', + array( 'object' => $fsFile, 'latest' => $latest ) ); + } } + + wfProfileOut( __METHOD__ . '-' . $this->name ); + wfProfileOut( __METHOD__ ); + return $fsFiles; + } + + /** + * @see FileBackendStore::getLocalReferenceMulti() + * @return Array + */ + protected function doGetLocalReferenceMulti( array $params ) { + return $this->doGetLocalCopyMulti( $params ); + } + + /** + * @see FileBackend::getLocalCopyMulti() + * @return Array + */ + final public function getLocalCopyMulti( array $params ) { + wfProfileIn( __METHOD__ ); + wfProfileIn( __METHOD__ . '-' . $this->name ); + + $params = $this->setConcurrencyFlags( $params ); + $tmpFiles = $this->doGetLocalCopyMulti( $params ); + wfProfileOut( __METHOD__ . '-' . $this->name ); wfProfileOut( __METHOD__ ); - return $tmpFile; + return $tmpFiles; + } + + /** + * @see FileBackendStore::getLocalCopyMulti() + * @return Array + */ + abstract protected function doGetLocalCopyMulti( array $params ); + + /** + * @see FileBackend::getFileHttpUrl() + * @return string|null + */ + public function getFileHttpUrl( array $params ) { + return null; // not supported } /** @@ -776,6 +904,14 @@ abstract class FileBackendStore extends FileBackend { $status = $this->doStreamFile( $params ); wfProfileOut( __METHOD__ . '-send-' . $this->name ); wfProfileOut( __METHOD__ . '-send' ); + if ( !$status->isOK() ) { + // Per bug 41113, nasty things can happen if bad cache entries get + // stuck in cache. It's also possible that this error can come up + // with simple race conditions. Clear out the stat cache to be safe. + $this->clearCache( array( $params['src'] ) ); + $this->deleteFileCache( $params['src'] ); + trigger_error( "Bad stat cache or race condition for file {$params['src']}." ); + } } else { $status->fatal( 'backend-fail-stream', $params['src'] ); } @@ -815,7 +951,7 @@ abstract class FileBackendStore extends FileBackend { return $this->doDirectoryExists( $fullCont, $dir, $params ); } else { // directory is on several shards wfDebug( __METHOD__ . ": iterating over all container shards.\n" ); - list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] ); + list( , $shortCont, ) = self::splitStoragePath( $params['dir'] ); $res = false; // response foreach ( $this->getContainerSuffixes( $shortCont ) as $suffix ) { $exists = $this->doDirectoryExists( "{$fullCont}{$suffix}", $dir, $params ); @@ -833,9 +969,9 @@ abstract class FileBackendStore extends FileBackend { /** * @see FileBackendStore::directoryExists() * - * @param $container string Resolved container name - * @param $dir string Resolved path relative to container - * @param $params Array + * @param string $container Resolved container name + * @param string $dir Resolved path relative to container + * @param array $params * @return bool|null */ abstract protected function doDirectoryExists( $container, $dir, array $params ); @@ -855,7 +991,7 @@ abstract class FileBackendStore extends FileBackend { } else { wfDebug( __METHOD__ . ": iterating over all container shards.\n" ); // File listing spans multiple containers/shards - list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] ); + list( , $shortCont, ) = self::splitStoragePath( $params['dir'] ); return new FileBackendStoreShardDirIterator( $this, $fullCont, $dir, $this->getContainerSuffixes( $shortCont ), $params ); } @@ -866,9 +1002,9 @@ abstract class FileBackendStore extends FileBackend { * * @see FileBackendStore::getDirectoryList() * - * @param $container string Resolved container name - * @param $dir string Resolved path relative to container - * @param $params Array + * @param string $container Resolved container name + * @param string $dir Resolved path relative to container + * @param array $params * @return Traversable|Array|null Returns null on failure */ abstract public function getDirectoryListInternal( $container, $dir, array $params ); @@ -888,7 +1024,7 @@ abstract class FileBackendStore extends FileBackend { } else { wfDebug( __METHOD__ . ": iterating over all container shards.\n" ); // File listing spans multiple containers/shards - list( $b, $shortCont, $r ) = self::splitStoragePath( $params['dir'] ); + list( , $shortCont, ) = self::splitStoragePath( $params['dir'] ); return new FileBackendStoreShardFileIterator( $this, $fullCont, $dir, $this->getContainerSuffixes( $shortCont ), $params ); } @@ -899,9 +1035,9 @@ abstract class FileBackendStore extends FileBackend { * * @see FileBackendStore::getFileList() * - * @param $container string Resolved container name - * @param $dir string Resolved path relative to container - * @param $params Array + * @param string $container Resolved container name + * @param string $dir Resolved path relative to container + * @param array $params * @return Traversable|Array|null Returns null on failure */ abstract public function getFileListInternal( $container, $dir, array $params ); @@ -913,18 +1049,19 @@ abstract class FileBackendStore extends FileBackend { * The result must have the same number of items as the input. * An exception is thrown if an unsupported operation is requested. * - * @param $ops Array Same format as doOperations() + * @param array $ops Same format as doOperations() * @return Array List of FileOp objects * @throws MWException */ final public function getOperationsInternal( array $ops ) { $supportedOps = array( - 'store' => 'StoreFileOp', - 'copy' => 'CopyFileOp', - 'move' => 'MoveFileOp', - 'delete' => 'DeleteFileOp', - 'create' => 'CreateFileOp', - 'null' => 'NullFileOp' + 'store' => 'StoreFileOp', + 'copy' => 'CopyFileOp', + 'move' => 'MoveFileOp', + 'delete' => 'DeleteFileOp', + 'create' => 'CreateFileOp', + 'describe' => 'DescribeFileOp', + 'null' => 'NullFileOp' ); $performOps = array(); // array of FileOp objects @@ -949,8 +1086,9 @@ abstract class FileBackendStore extends FileBackend { * Get a list of storage paths to lock for a list of operations * Returns an array with 'sh' (shared) and 'ex' (exclusive) keys, * each corresponding to a list of storage paths to be locked. + * All returned paths are normalized. * - * @param $performOps Array List of FileOp objects + * @param array $performOps List of FileOp objects * @return Array ('sh' => list of paths, 'ex' => list of paths) */ final public function getPathsToLockForOpsInternal( array $performOps ) { @@ -989,6 +1127,9 @@ abstract class FileBackendStore extends FileBackend { wfProfileIn( __METHOD__ . '-' . $this->name ); $status = Status::newGood(); + // Fix up custom header name/value pairs... + $ops = array_map( array( $this, 'stripInvalidHeadersFromOp' ), $ops ); + // Build up a list of FileOps... $performOps = $this->getOperationsInternal( $ops ); @@ -1016,6 +1157,7 @@ abstract class FileBackendStore extends FileBackend { $this->primeContainerCache( $performOps ); // Actually attempt the operation batch... + $opts = $this->setConcurrencyFlags( $opts ); $subStatus = FileOpBatch::attempt( $performOps, $opts, $this->fileJournal ); // Merge errors into status fields @@ -1037,6 +1179,12 @@ abstract class FileBackendStore extends FileBackend { wfProfileIn( __METHOD__ . '-' . $this->name ); $status = Status::newGood(); + // Fix up custom header name/value pairs... + $ops = array_map( array( $this, 'stripInvalidHeadersFromOp' ), $ops ); + + // Clear any file cache entries + $this->clearCache(); + $supportedOps = array( 'create', 'store', 'copy', 'move', 'delete', 'null' ); $async = ( $this->parallelize === 'implicit' ); $maxConcurrency = $this->concurrency; // throttle @@ -1092,7 +1240,7 @@ abstract class FileBackendStore extends FileBackend { * The resulting Status object fields will correspond * to the order in which the handles where given. * - * @param $handles Array List of FileBackendStoreOpHandle objects + * @param array $handles List of FileBackendStoreOpHandle objects * @return Array Map of Status objects * @throws MWException */ @@ -1117,6 +1265,8 @@ abstract class FileBackendStore extends FileBackend { /** * @see FileBackendStore::executeOpHandlesInternal() + * @param array $fileOpHandles + * @throws MWException * @return Array List of corresponding Status objects */ protected function doExecuteOpHandlesInternal( array $fileOpHandles ) { @@ -1127,12 +1277,32 @@ abstract class FileBackendStore extends FileBackend { } /** + * Strip long HTTP headers from a file operation + * + * @param array $op Same format as doOperation() + * @return Array + */ + protected function stripInvalidHeadersFromOp( array $op ) { + if ( isset( $op['headers'] ) ) { + foreach ( $op['headers'] as $name => $value ) { + if ( strlen( $name ) > 255 || strlen( $value ) > 255 ) { + trigger_error( "Header '$name: $value' is too long." ); + unset( $op['headers'][$name] ); + } elseif ( !strlen( $value ) ) { + $op['headers'][$name] = ''; // null/false => "" + } + } + } + return $op; + } + + /** * @see FileBackend::preloadCache() */ final public function preloadCache( array $paths ) { $fullConts = array(); // full container names foreach ( $paths as $path ) { - list( $fullCont, $r, $s ) = $this->resolveStoragePath( $path ); + list( $fullCont, , ) = $this->resolveStoragePath( $path ); $fullConts[] = $fullCont; } // Load from the persistent file and container caches @@ -1165,7 +1335,7 @@ abstract class FileBackendStore extends FileBackend { * * @see FileBackend::clearCache() * - * @param $paths Array Storage paths (optional) + * @param array $paths Storage paths (optional) * @return void */ protected function doClearCache( array $paths = null ) {} @@ -1254,8 +1424,8 @@ abstract class FileBackendStore extends FileBackend { * Get the container name shard suffix for a given path. * Any empty suffix means the container is not sharded. * - * @param $container string Container name - * @param $relPath string Storage path relative to the container + * @param string $container Container name + * @param string $relPath Storage path relative to the container * @return string|null Returns null if shard could not be determined */ final protected function getContainerShard( $container, $relPath ) { @@ -1291,11 +1461,11 @@ abstract class FileBackendStore extends FileBackend { * Container dirs like "a", where the container shards on "x/xy", * can reside on several shards. Such paths are tricky to handle. * - * @param $storagePath string Storage path + * @param string $storagePath Storage path * @return bool */ final public function isSingleShardPathInternal( $storagePath ) { - list( $c, $r, $shard ) = $this->resolveStoragePath( $storagePath ); + list( , , $shard ) = $this->resolveStoragePath( $storagePath ); return ( $shard !== null ); } @@ -1371,8 +1541,8 @@ abstract class FileBackendStore extends FileBackend { * getting absolute paths (e.g. FS based backends). Note that the relative path * may be the empty string (e.g. the path is simply to the container). * - * @param $container string Container name - * @param $relStoragePath string Storage path relative to the container + * @param string $container Container name + * @param string $relStoragePath Storage path relative to the container * @return string|null Path or null if not valid */ protected function resolveContainerPath( $container, $relStoragePath ) { @@ -1382,7 +1552,7 @@ abstract class FileBackendStore extends FileBackend { /** * Get the cache key for a container * - * @param $container string Resolved container name + * @param string $container Resolved container name * @return string */ private function containerCacheKey( $container ) { @@ -1392,7 +1562,7 @@ abstract class FileBackendStore extends FileBackend { /** * Set the cached info for a container * - * @param $container string Resolved container name + * @param string $container Resolved container name * @param $val mixed Information to cache */ final protected function setContainerCache( $container, $val ) { @@ -1403,7 +1573,7 @@ abstract class FileBackendStore extends FileBackend { * Delete the cached info for a container. * The cache key is salted for a while to prevent race conditions. * - * @param $container string Resolved container name + * @param string $container Resolved container name */ final protected function deleteContainerCache( $container ) { if ( !$this->memCache->set( $this->containerCacheKey( $container ), 'PURGED', 300 ) ) { @@ -1414,6 +1584,7 @@ abstract class FileBackendStore extends FileBackend { /** * Do a batch lookup from cache for container stats for all containers * used in a list of container names, storage paths, or FileOp objects. + * This loads the persistent cache values into the process cache. * * @param $items Array * @return void @@ -1437,7 +1608,7 @@ abstract class FileBackendStore extends FileBackend { } // Get all the corresponding cache keys for paths... foreach ( $paths as $path ) { - list( $fullCont, $r, $s ) = $this->resolveStoragePath( $path ); + list( $fullCont, , ) = $this->resolveStoragePath( $path ); if ( $fullCont !== null ) { // valid path for this backend $contNames[$this->containerCacheKey( $fullCont )] = $fullCont; } @@ -1462,7 +1633,7 @@ abstract class FileBackendStore extends FileBackend { * resolved container names and their corresponding cached info. * Only containers that actually exist should appear in the map. * - * @param $containerInfo Array Map of resolved container names to cached info + * @param array $containerInfo Map of resolved container names to cached info * @return void */ protected function doPrimeContainerCache( array $containerInfo ) {} @@ -1470,7 +1641,7 @@ abstract class FileBackendStore extends FileBackend { /** * Get the cache key for a file path * - * @param $path string Storage path + * @param string $path Normalized storage path * @return string */ private function fileCacheKey( $path ) { @@ -1482,20 +1653,30 @@ abstract class FileBackendStore extends FileBackend { * Negatives (404s) are not cached. By not caching negatives, we can skip cache * salting for the case when a file is created at a path were there was none before. * - * @param $path string Storage path + * @param string $path Storage path * @param $val mixed Information to cache */ final protected function setFileCache( $path, $val ) { + $path = FileBackend::normalizeStoragePath( $path ); + if ( $path === null ) { + return; // invalid storage path + } $this->memCache->add( $this->fileCacheKey( $path ), $val, 7*86400 ); } /** * Delete the cached stat info for a file path. * The cache key is salted for a while to prevent race conditions. + * Since negatives (404s) are not cached, this does not need to be called when + * a file is created at a path were there was none before. * - * @param $path string Storage path + * @param string $path Storage path */ final protected function deleteFileCache( $path ) { + $path = FileBackend::normalizeStoragePath( $path ); + if ( $path === null ) { + return; // invalid storage path + } if ( !$this->memCache->set( $this->fileCacheKey( $path ), 'PURGED', 300 ) ) { trigger_error( "Unable to delete stat cache for file $path." ); } @@ -1504,8 +1685,9 @@ abstract class FileBackendStore extends FileBackend { /** * Do a batch lookup from cache for file stats for all paths * used in a list of storage paths or FileOp objects. + * This loads the persistent cache values into the process cache. * - * @param $items Array List of storage paths or FileOps + * @param array $items List of storage paths or FileOps * @return void */ final protected function primeFileCache( array $items ) { @@ -1520,12 +1702,14 @@ abstract class FileBackendStore extends FileBackend { $paths = array_merge( $paths, $item->storagePathsRead() ); $paths = array_merge( $paths, $item->storagePathsChanged() ); } elseif ( self::isStoragePath( $item ) ) { - $paths[] = $item; + $paths[] = FileBackend::normalizeStoragePath( $item ); } } + // Get rid of any paths that failed normalization... + $paths = array_filter( $paths, 'strlen' ); // remove nulls // Get all the corresponding cache keys for paths... foreach ( $paths as $path ) { - list( $cont, $rel, $s ) = $this->resolveStoragePath( $path ); + list( , $rel, ) = $this->resolveStoragePath( $path ); if ( $rel !== null ) { // valid path for this backend $pathNames[$this->fileCacheKey( $path )] = $path; } @@ -1546,6 +1730,26 @@ abstract class FileBackendStore extends FileBackend { wfProfileOut( __METHOD__ . '-' . $this->name ); wfProfileOut( __METHOD__ ); } + + /** + * Set the 'concurrency' option from a list of operation options + * + * @param array $opts Map of operation options + * @return Array + */ + final protected function setConcurrencyFlags( array $opts ) { + $opts['concurrency'] = 1; // off + if ( $this->parallelize === 'implicit' ) { + if ( !isset( $opts['parallelize'] ) || $opts['parallelize'] ) { + $opts['concurrency'] = $this->concurrency; + } + } elseif ( $this->parallelize === 'explicit' ) { + if ( !empty( $opts['parallelize'] ) ) { + $opts['concurrency'] = $this->concurrency; + } + } + return $opts; + } } /** @@ -1602,10 +1806,10 @@ abstract class FileBackendStoreShardListIterator implements Iterator { /** * @param $backend FileBackendStore - * @param $container string Full storage container name - * @param $dir string Storage directory relative to container - * @param $suffixes Array List of container shard suffixes - * @param $params Array + * @param string $container Full storage container name + * @param string $dir Storage directory relative to container + * @param array $suffixes List of container shard suffixes + * @param array $params */ public function __construct( FileBackendStore $backend, $container, $dir, array $suffixes, array $params @@ -1731,9 +1935,9 @@ abstract class FileBackendStoreShardListIterator implements Iterator { /** * Get the list for a given container shard * - * @param $container string Resolved container name - * @param $dir string Resolved path relative to container - * @param $params Array + * @param string $container Resolved container name + * @param string $dir Resolved path relative to container + * @param array $params * @return Traversable|Array|null */ abstract protected function listFromShard( $container, $dir, array $params ); diff --git a/includes/filebackend/FileOp.php b/includes/filebackend/FileOp.php index 7c43c489..bb0ab578 100644 --- a/includes/filebackend/FileOp.php +++ b/includes/filebackend/FileOp.php @@ -42,11 +42,12 @@ abstract class FileOp { protected $state = self::STATE_NEW; // integer protected $failed = false; // boolean protected $async = false; // boolean - protected $useLatest = true; // boolean protected $batchId; // string + protected $doOperation = true; // boolean; operation is not a no-op protected $sourceSha1; // string protected $destSameAsSource; // boolean + protected $destExists; // boolean /* Object life-cycle */ const STATE_NEW = 1; @@ -65,37 +66,61 @@ abstract class FileOp { list( $required, $optional ) = $this->allowedParams(); foreach ( $required as $name ) { if ( isset( $params[$name] ) ) { - $this->params[$name] = $params[$name]; + $this->params[$name] = self::normalizeAnyStoragePaths( $params[$name] ); } else { throw new MWException( "File operation missing parameter '$name'." ); } } foreach ( $optional as $name ) { if ( isset( $params[$name] ) ) { - $this->params[$name] = $params[$name]; + $this->params[$name] = self::normalizeAnyStoragePaths( $params[$name] ); } } $this->params = $params; } /** - * Set the batch UUID this operation belongs to + * Normalize $item or anything in $item that is a valid storage path * - * @param $batchId string - * @return void + * @param $item string|array + * @return string|Array + */ + protected function normalizeAnyStoragePaths( $item ) { + if ( is_array( $item ) ) { + $res = array(); + foreach ( $item as $k => $v ) { + $k = self::normalizeIfValidStoragePath( $k ); + $v = self::normalizeIfValidStoragePath( $v ); + $res[$k] = $v; + } + return $res; + } else { + return self::normalizeIfValidStoragePath( $item ); + } + } + + /** + * Normalize a string if it is a valid storage path + * + * @param $path string + * @return string */ - final public function setBatchId( $batchId ) { - $this->batchId = $batchId; + protected static function normalizeIfValidStoragePath( $path ) { + if ( FileBackend::isStoragePath( $path ) ) { + $res = FileBackend::normalizeStoragePath( $path ); + return ( $res !== null ) ? $res : $path; + } + return $path; } /** - * Whether to allow stale data for file reads and stat checks + * Set the batch UUID this operation belongs to * - * @param $allowStale bool + * @param $batchId string * @return void */ - final public function allowStaleReads( $allowStale ) { - $this->useLatest = !$allowStale; + final public function setBatchId( $batchId ) { + $this->batchId = $batchId; } /** @@ -138,11 +163,11 @@ abstract class FileOp { /** * Update a dependency tracking array to account for this operation * - * @param $deps Array Prior path reads/writes; format of FileOp::newPredicates() + * @param array $deps Prior path reads/writes; format of FileOp::newPredicates() * @return Array */ final public function applyDependencies( array $deps ) { - $deps['read'] += array_fill_keys( $this->storagePathsRead(), 1 ); + $deps['read'] += array_fill_keys( $this->storagePathsRead(), 1 ); $deps['write'] += array_fill_keys( $this->storagePathsChanged(), 1 ); return $deps; } @@ -150,7 +175,7 @@ abstract class FileOp { /** * Check if this operation changes files listed in $paths * - * @param $paths Array Prior path reads/writes; format of FileOp::newPredicates() + * @param array $paths Prior path reads/writes; format of FileOp::newPredicates() * @return boolean */ final public function dependsOn( array $deps ) { @@ -170,16 +195,19 @@ abstract class FileOp { /** * Get the file journal entries for this file operation * - * @param $oPredicates Array Pre-op info about files (format of FileOp::newPredicates) - * @param $nPredicates Array Post-op info about files (format of FileOp::newPredicates) + * @param array $oPredicates Pre-op info about files (format of FileOp::newPredicates) + * @param array $nPredicates Post-op info about files (format of FileOp::newPredicates) * @return Array */ final public function getJournalEntries( array $oPredicates, array $nPredicates ) { + if ( !$this->doOperation ) { + return array(); // this is a no-op + } $nullEntries = array(); $updateEntries = array(); $deleteEntries = array(); $pathsUsed = array_merge( $this->storagePathsRead(), $this->storagePathsChanged() ); - foreach ( $pathsUsed as $path ) { + foreach ( array_unique( $pathsUsed ) as $path ) { $nullEntries[] = array( // assertion for recovery 'op' => 'null', 'path' => $path, @@ -205,7 +233,9 @@ abstract class FileOp { } /** - * Check preconditions of the operation without writing anything + * Check preconditions of the operation without writing anything. + * This must update $predicates for each path that the op can change + * except when a failing status object is returned. * * @param $predicates Array * @return Status @@ -241,10 +271,14 @@ abstract class FileOp { return Status::newFatal( 'fileop-fail-attempt-precheck' ); } $this->state = self::STATE_ATTEMPTED; - $status = $this->doAttempt(); - if ( !$status->isOK() ) { - $this->failed = true; - $this->logFailure( 'attempt' ); + if ( $this->doOperation ) { + $status = $this->doAttempt(); + if ( !$status->isOK() ) { + $this->failed = true; + $this->logFailure( 'attempt' ); + } + } else { // no-op + $status = Status::newGood(); } return $status; } @@ -292,15 +326,7 @@ abstract class FileOp { * * @return Array */ - final public function storagePathsRead() { - return array_map( 'FileBackend::normalizeStoragePath', $this->doStoragePathsRead() ); - } - - /** - * @see FileOp::storagePathsRead() - * @return Array - */ - protected function doStoragePathsRead() { + public function storagePathsRead() { return array(); } @@ -309,21 +335,13 @@ abstract class FileOp { * * @return Array */ - final public function storagePathsChanged() { - return array_map( 'FileBackend::normalizeStoragePath', $this->doStoragePathsChanged() ); - } - - /** - * @see FileOp::storagePathsChanged() - * @return Array - */ - protected function doStoragePathsChanged() { + public function storagePathsChanged() { return array(); } /** * Check for errors with regards to the destination file already existing. - * This also updates the destSameAsSource and sourceSha1 member variables. + * Also set the destExists, destSameAsSource and sourceSha1 member variables. * A bad status will be returned if there is no chance it can be overwritten. * * @param $predicates Array @@ -337,7 +355,8 @@ abstract class FileOp { $this->sourceSha1 = $this->fileSha1( $this->params['src'], $predicates ); } $this->destSameAsSource = false; - if ( $this->fileExists( $this->params['dst'], $predicates ) ) { + $this->destExists = $this->fileExists( $this->params['dst'], $predicates ); + if ( $this->destExists ) { if ( $this->getParam( 'overwrite' ) ) { return $status; // OK } elseif ( $this->getParam( 'overwriteSame' ) ) { @@ -373,7 +392,7 @@ abstract class FileOp { /** * Check if a file will exist in storage when this operation is attempted * - * @param $source string Storage path + * @param string $source Storage path * @param $predicates Array * @return bool */ @@ -381,7 +400,7 @@ abstract class FileOp { if ( isset( $predicates['exists'][$source] ) ) { return $predicates['exists'][$source]; // previous op assures this } else { - $params = array( 'src' => $source, 'latest' => $this->useLatest ); + $params = array( 'src' => $source, 'latest' => true ); return $this->backend->fileExists( $params ); } } @@ -389,15 +408,17 @@ abstract class FileOp { /** * Get the SHA-1 of a file in storage when this operation is attempted * - * @param $source string Storage path + * @param string $source Storage path * @param $predicates Array * @return string|bool False on failure */ final protected function fileSha1( $source, array $predicates ) { if ( isset( $predicates['sha1'][$source] ) ) { return $predicates['sha1'][$source]; // previous op assures this + } elseif ( isset( $predicates['exists'][$source] ) && !$predicates['exists'][$source] ) { + return false; // previous op assures this } else { - $params = array( 'src' => $source, 'latest' => $this->useLatest ); + $params = array( 'src' => $source, 'latest' => true ); return $this->backend->getFileSha1Base36( $params ); } } @@ -430,42 +451,32 @@ abstract class FileOp { } /** - * Store a file into the backend from a file on the file system. + * Create a file in the backend with the given content. * Parameters for this operation are outlined in FileBackend::doOperations(). */ -class StoreFileOp extends FileOp { - /** - * @return array - */ +class CreateFileOp extends FileOp { protected function allowedParams() { - return array( array( 'src', 'dst' ), - array( 'overwrite', 'overwriteSame', 'disposition' ) ); + return array( array( 'content', 'dst' ), + array( 'overwrite', 'overwriteSame', 'disposition', 'headers' ) ); } - /** - * @param $predicates array - * @return Status - */ protected function doPrecheck( array &$predicates ) { $status = Status::newGood(); - // Check if the source file exists on the file system - if ( !is_file( $this->params['src'] ) ) { - $status->fatal( 'backend-fail-notexists', $this->params['src'] ); - return $status; - // Check if the source file is too big - } elseif ( filesize( $this->params['src'] ) > $this->backend->maxFileSizeInternal() ) { + // Check if the source data is too big + if ( strlen( $this->getParam( 'content' ) ) > $this->backend->maxFileSizeInternal() ) { $status->fatal( 'backend-fail-maxsize', $this->params['dst'], $this->backend->maxFileSizeInternal() ); - $status->fatal( 'backend-fail-store', $this->params['src'], $this->params['dst'] ); + $status->fatal( 'backend-fail-create', $this->params['dst'] ); return $status; - // Check if a file can be placed at the destination + // Check if a file can be placed/changed at the destination } elseif ( !$this->backend->isPathUsableInternal( $this->params['dst'] ) ) { $status->fatal( 'backend-fail-usable', $this->params['dst'] ); - $status->fatal( 'backend-fail-store', $this->params['src'], $this->params['dst'] ); + $status->fatal( 'backend-fail-create', $this->params['dst'] ); return $status; } // Check if destination file exists $status->merge( $this->precheckDestExistence( $predicates ) ); + $this->params['dstExists'] = $this->destExists; // see FileBackendStore::setFileCache() if ( $status->isOK() ) { // Update file existence predicates $predicates['exists'][$this->params['dst']] = true; @@ -478,57 +489,66 @@ class StoreFileOp extends FileOp { * @return Status */ protected function doAttempt() { - // Store the file at the destination if ( !$this->destSameAsSource ) { - return $this->backend->storeInternal( $this->setFlags( $this->params ) ); + // Create the file at the destination + return $this->backend->createInternal( $this->setFlags( $this->params ) ); } return Status::newGood(); } /** - * @return bool|string + * @return bool|String */ protected function getSourceSha1Base36() { - wfSuppressWarnings(); - $hash = sha1_file( $this->params['src'] ); - wfRestoreWarnings(); - if ( $hash !== false ) { - $hash = wfBaseConvert( $hash, 16, 36, 31 ); - } - return $hash; + return wfBaseConvert( sha1( $this->params['content'] ), 16, 36, 31 ); } - protected function doStoragePathsChanged() { + /** + * @return array + */ + public function storagePathsChanged() { return array( $this->params['dst'] ); } } /** - * Create a file in the backend with the given content. + * Store a file into the backend from a file on the file system. * Parameters for this operation are outlined in FileBackend::doOperations(). */ -class CreateFileOp extends FileOp { +class StoreFileOp extends FileOp { + /** + * @return array + */ protected function allowedParams() { - return array( array( 'content', 'dst' ), - array( 'overwrite', 'overwriteSame', 'disposition' ) ); + return array( array( 'src', 'dst' ), + array( 'overwrite', 'overwriteSame', 'disposition', 'headers' ) ); } + /** + * @param $predicates array + * @return Status + */ protected function doPrecheck( array &$predicates ) { $status = Status::newGood(); - // Check if the source data is too big - if ( strlen( $this->getParam( 'content' ) ) > $this->backend->maxFileSizeInternal() ) { + // Check if the source file exists on the file system + if ( !is_file( $this->params['src'] ) ) { + $status->fatal( 'backend-fail-notexists', $this->params['src'] ); + return $status; + // Check if the source file is too big + } elseif ( filesize( $this->params['src'] ) > $this->backend->maxFileSizeInternal() ) { $status->fatal( 'backend-fail-maxsize', $this->params['dst'], $this->backend->maxFileSizeInternal() ); - $status->fatal( 'backend-fail-create', $this->params['dst'] ); + $status->fatal( 'backend-fail-store', $this->params['src'], $this->params['dst'] ); return $status; - // Check if a file can be placed at the destination + // Check if a file can be placed/changed at the destination } elseif ( !$this->backend->isPathUsableInternal( $this->params['dst'] ) ) { $status->fatal( 'backend-fail-usable', $this->params['dst'] ); - $status->fatal( 'backend-fail-create', $this->params['dst'] ); + $status->fatal( 'backend-fail-store', $this->params['src'], $this->params['dst'] ); return $status; } // Check if destination file exists $status->merge( $this->precheckDestExistence( $predicates ) ); + $this->params['dstExists'] = $this->destExists; // see FileBackendStore::setFileCache() if ( $status->isOK() ) { // Update file existence predicates $predicates['exists'][$this->params['dst']] = true; @@ -541,24 +561,27 @@ class CreateFileOp extends FileOp { * @return Status */ protected function doAttempt() { + // Store the file at the destination if ( !$this->destSameAsSource ) { - // Create the file at the destination - return $this->backend->createInternal( $this->setFlags( $this->params ) ); + return $this->backend->storeInternal( $this->setFlags( $this->params ) ); } return Status::newGood(); } /** - * @return bool|String + * @return bool|string */ protected function getSourceSha1Base36() { - return wfBaseConvert( sha1( $this->params['content'] ), 16, 36, 31 ); + wfSuppressWarnings(); + $hash = sha1_file( $this->params['src'] ); + wfRestoreWarnings(); + if ( $hash !== false ) { + $hash = wfBaseConvert( $hash, 16, 36, 31 ); + } + return $hash; } - /** - * @return array - */ - protected function doStoragePathsChanged() { + public function storagePathsChanged() { return array( $this->params['dst'] ); } } @@ -573,7 +596,7 @@ class CopyFileOp extends FileOp { */ protected function allowedParams() { return array( array( 'src', 'dst' ), - array( 'overwrite', 'overwriteSame', 'disposition' ) ); + array( 'overwrite', 'overwriteSame', 'ignoreMissingSource', 'disposition' ) ); } /** @@ -584,9 +607,17 @@ class CopyFileOp extends FileOp { $status = Status::newGood(); // Check if the source file exists if ( !$this->fileExists( $this->params['src'], $predicates ) ) { - $status->fatal( 'backend-fail-notexists', $this->params['src'] ); - return $status; - // Check if a file can be placed at the destination + if ( $this->getParam( 'ignoreMissingSource' ) ) { + $this->doOperation = false; // no-op + // Update file existence predicates (cache 404s) + $predicates['exists'][$this->params['src']] = false; + $predicates['sha1'][$this->params['src']] = false; + return $status; // nothing to do + } else { + $status->fatal( 'backend-fail-notexists', $this->params['src'] ); + return $status; + } + // Check if a file can be placed/changed at the destination } elseif ( !$this->backend->isPathUsableInternal( $this->params['dst'] ) ) { $status->fatal( 'backend-fail-usable', $this->params['dst'] ); $status->fatal( 'backend-fail-copy', $this->params['src'], $this->params['dst'] ); @@ -594,6 +625,7 @@ class CopyFileOp extends FileOp { } // Check if destination file exists $status->merge( $this->precheckDestExistence( $predicates ) ); + $this->params['dstExists'] = $this->destExists; // see FileBackendStore::setFileCache() if ( $status->isOK() ) { // Update file existence predicates $predicates['exists'][$this->params['dst']] = true; @@ -619,14 +651,14 @@ class CopyFileOp extends FileOp { /** * @return array */ - protected function doStoragePathsRead() { + public function storagePathsRead() { return array( $this->params['src'] ); } /** * @return array */ - protected function doStoragePathsChanged() { + public function storagePathsChanged() { return array( $this->params['dst'] ); } } @@ -641,7 +673,7 @@ class MoveFileOp extends FileOp { */ protected function allowedParams() { return array( array( 'src', 'dst' ), - array( 'overwrite', 'overwriteSame', 'disposition' ) ); + array( 'overwrite', 'overwriteSame', 'ignoreMissingSource', 'disposition' ) ); } /** @@ -652,9 +684,17 @@ class MoveFileOp extends FileOp { $status = Status::newGood(); // Check if the source file exists if ( !$this->fileExists( $this->params['src'], $predicates ) ) { - $status->fatal( 'backend-fail-notexists', $this->params['src'] ); - return $status; - // Check if a file can be placed at the destination + if ( $this->getParam( 'ignoreMissingSource' ) ) { + $this->doOperation = false; // no-op + // Update file existence predicates (cache 404s) + $predicates['exists'][$this->params['src']] = false; + $predicates['sha1'][$this->params['src']] = false; + return $status; // nothing to do + } else { + $status->fatal( 'backend-fail-notexists', $this->params['src'] ); + return $status; + } + // Check if a file can be placed/changed at the destination } elseif ( !$this->backend->isPathUsableInternal( $this->params['dst'] ) ) { $status->fatal( 'backend-fail-usable', $this->params['dst'] ); $status->fatal( 'backend-fail-move', $this->params['src'], $this->params['dst'] ); @@ -662,6 +702,7 @@ class MoveFileOp extends FileOp { } // Check if destination file exists $status->merge( $this->precheckDestExistence( $predicates ) ); + $this->params['dstExists'] = $this->destExists; // see FileBackendStore::setFileCache() if ( $status->isOK() ) { // Update file existence predicates $predicates['exists'][$this->params['src']] = false; @@ -693,14 +734,14 @@ class MoveFileOp extends FileOp { /** * @return array */ - protected function doStoragePathsRead() { + public function storagePathsRead() { return array( $this->params['src'] ); } /** * @return array */ - protected function doStoragePathsChanged() { + public function storagePathsChanged() { return array( $this->params['src'], $this->params['dst'] ); } } @@ -717,21 +758,29 @@ class DeleteFileOp extends FileOp { return array( array( 'src' ), array( 'ignoreMissingSource' ) ); } - protected $needsDelete = true; - /** - * @param array $predicates + * @param $predicates array * @return Status */ protected function doPrecheck( array &$predicates ) { $status = Status::newGood(); // Check if the source file exists if ( !$this->fileExists( $this->params['src'], $predicates ) ) { - if ( !$this->getParam( 'ignoreMissingSource' ) ) { + if ( $this->getParam( 'ignoreMissingSource' ) ) { + $this->doOperation = false; // no-op + // Update file existence predicates (cache 404s) + $predicates['exists'][$this->params['src']] = false; + $predicates['sha1'][$this->params['src']] = false; + return $status; // nothing to do + } else { $status->fatal( 'backend-fail-notexists', $this->params['src'] ); return $status; } - $this->needsDelete = false; + // Check if a file can be placed/changed at the source + } elseif ( !$this->backend->isPathUsableInternal( $this->params['src'] ) ) { + $status->fatal( 'backend-fail-usable', $this->params['src'] ); + $status->fatal( 'backend-fail-delete', $this->params['src'] ); + return $status; } // Update file existence predicates $predicates['exists'][$this->params['src']] = false; @@ -743,17 +792,66 @@ class DeleteFileOp extends FileOp { * @return Status */ protected function doAttempt() { - if ( $this->needsDelete ) { - // Delete the source file - return $this->backend->deleteInternal( $this->setFlags( $this->params ) ); + // Delete the source file + return $this->backend->deleteInternal( $this->setFlags( $this->params ) ); + } + + /** + * @return array + */ + public function storagePathsChanged() { + return array( $this->params['src'] ); + } +} + +/** + * Change metadata for a file at the given storage path in the backend. + * Parameters for this operation are outlined in FileBackend::doOperations(). + */ +class DescribeFileOp extends FileOp { + /** + * @return array + */ + protected function allowedParams() { + return array( array( 'src' ), array( 'disposition', 'headers' ) ); + } + + /** + * @param $predicates array + * @return Status + */ + protected function doPrecheck( array &$predicates ) { + $status = Status::newGood(); + // Check if the source file exists + if ( !$this->fileExists( $this->params['src'], $predicates ) ) { + $status->fatal( 'backend-fail-notexists', $this->params['src'] ); + return $status; + // Check if a file can be placed/changed at the source + } elseif ( !$this->backend->isPathUsableInternal( $this->params['src'] ) ) { + $status->fatal( 'backend-fail-usable', $this->params['src'] ); + $status->fatal( 'backend-fail-describe', $this->params['src'] ); + return $status; } - return Status::newGood(); + // Update file existence predicates + $predicates['exists'][$this->params['src']] = + $this->fileExists( $this->params['src'], $predicates ); + $predicates['sha1'][$this->params['src']] = + $this->fileSha1( $this->params['src'], $predicates ); + return $status; // safe to call attempt() + } + + /** + * @return Status + */ + protected function doAttempt() { + // Update the source file's metadata + return $this->backend->describeInternal( $this->setFlags( $this->params ) ); } /** * @return array */ - protected function doStoragePathsChanged() { + public function storagePathsChanged() { return array( $this->params['src'] ); } } diff --git a/includes/filebackend/FileOpBatch.php b/includes/filebackend/FileOpBatch.php index 33558725..fc51d78a 100644 --- a/includes/filebackend/FileOpBatch.php +++ b/includes/filebackend/FileOpBatch.php @@ -42,9 +42,6 @@ class FileOpBatch { * $opts is an array of options, including: * - force : Errors that would normally cause a rollback do not. * The remaining operations are still attempted if any fail. - * - allowStale : Don't require the latest available data. - * This can increase performance for non-critical writes. - * This has no effect unless the 'force' flag is set. * - nonJournaled : Don't log this operation batch in the file journal. * - concurrency : Try to do this many operations in parallel when possible. * @@ -52,8 +49,8 @@ class FileOpBatch { * - a) unexpected operation errors occurred (network partitions, disk full...) * - b) significant operation errors occurred and 'force' was not set * - * @param $performOps Array List of FileOp operations - * @param $opts Array Batch operation options + * @param array $performOps List of FileOp operations + * @param array $opts Batch operation options * @param $journal FileJournal Journal to log operations to * @return Status */ @@ -69,7 +66,6 @@ class FileOpBatch { } $batchId = $journal->getTimestampedUUID(); - $allowStale = !empty( $opts['allowStale'] ); $ignoreErrors = !empty( $opts['force'] ); $journaled = empty( $opts['nonJournaled'] ); $maxConcurrency = isset( $opts['concurrency'] ) ? $opts['concurrency'] : 1; @@ -84,7 +80,6 @@ class FileOpBatch { foreach ( $performOps as $index => $fileOp ) { $backendName = $fileOp->getBackend()->getName(); $fileOp->setBatchId( $batchId ); // transaction ID - $fileOp->allowStaleReads( $allowStale ); // consistency level // Decide if this op can be done concurrently within this sub-batch // or if a new concurrent sub-batch must be started after this one... if ( $fileOp->dependsOn( $curBatchDeps ) @@ -136,49 +131,13 @@ class FileOpBatch { } // Attempt each operation (in parallel if allowed and possible)... - if ( count( $pPerformOps ) < count( $performOps ) ) { - self::runBatchParallel( $pPerformOps, $status ); - } else { - self::runBatchSeries( $performOps, $status ); - } + self::runParallelBatches( $pPerformOps, $status ); wfProfileOut( __METHOD__ ); return $status; } /** - * Attempt a list of file operations in series. - * This will abort remaining ops on failure. - * - * @param $performOps Array - * @param $status Status - * @return bool Success - */ - protected static function runBatchSeries( array $performOps, Status $status ) { - foreach ( $performOps as $index => $fileOp ) { - if ( $fileOp->failed() ) { - continue; // nothing to do - } - $subStatus = $fileOp->attempt(); - $status->merge( $subStatus ); - if ( $subStatus->isOK() ) { - $status->success[$index] = true; - ++$status->successCount; - } else { - $status->success[$index] = false; - ++$status->failCount; - // We can't continue (even with $ignoreErrors) as $predicates is wrong. - // Log the remaining ops as failed for recovery... - for ( $i = ($index + 1); $i < count( $performOps ); $i++ ) { - $performOps[$i]->logFailure( 'attempt_aborted' ); - } - return false; // bail out - } - } - return true; - } - - /** * Attempt a list of file operations sub-batches in series. * * The operations *in* each sub-batch will be done in parallel. @@ -190,8 +149,8 @@ class FileOpBatch { * @param $status Status * @return bool Success */ - protected static function runBatchParallel( array $pPerformOps, Status $status ) { - $aborted = false; + protected static function runParallelBatches( array $pPerformOps, Status $status ) { + $aborted = false; // set to true on unexpected errors foreach ( $pPerformOps as $performOpsBatch ) { if ( $aborted ) { // check batch op abort flag... // We can't continue (even with $ignoreErrors) as $predicates is wrong. @@ -205,11 +164,16 @@ class FileOpBatch { $opHandles = array(); // Get the backend; all sub-batch ops belong to a single backend $backend = reset( $performOpsBatch )->getBackend(); - // If attemptAsync() returns synchronously, it was either an - // error Status or the backend just doesn't support async ops. + // Get the operation handles or actually do it if there is just one. + // If attemptAsync() returns a Status, it was either due to an error + // or the backend does not support async ops and did it synchronously. foreach ( $performOpsBatch as $i => $fileOp ) { if ( !$fileOp->failed() ) { // failed => already has Status - $subStatus = $fileOp->attemptAsync(); + // If the batch is just one operation, it's faster to avoid + // pipelining as that can involve creating new TCP connections. + $subStatus = ( count( $performOpsBatch ) > 1 ) + ? $fileOp->attemptAsync() + : $fileOp->attempt(); if ( $subStatus->value instanceof FileBackendStoreOpHandle ) { $opHandles[$i] = $subStatus->value; // deferred } else { diff --git a/includes/filebackend/README b/includes/filebackend/README new file mode 100644 index 00000000..6ab54810 --- /dev/null +++ b/includes/filebackend/README @@ -0,0 +1,208 @@ +/*! +\ingroup FileBackend +\page file_backend_design File backend design + +Some notes on the FileBackend architecture. + +\section intro Introduction + +To abstract away the differences among different types of storage media, +MediaWiki is providing an interface known as FileBackend. Any MediaWiki +interaction with stored files should thus use a FileBackend object. + +Different types of backing storage media are supported (ranging from local +file system to distributed object stores). The types include: + +* FSFileBackend (used for mounted file systems) +* SwiftFileBackend (used for Swift or Ceph Rados+RGW object stores) +* FileBackendMultiWrite (useful for transitioning from one backend to another) + +Configuration documentation for each type of backend is to be found in their +__construct() inline documentation. + + +\section setup Setup + +File backends are registered in LocalSettings.php via the global variable +$wgFileBackends. To access one of those defined backends, one would use +FileBackendStore::get( <name> ) which will bring back a FileBackend object +handle. Such handles are reused for any subsequent get() call (via singleton). +The FileBackends objects are caching request calls such as file stats, +SHA1 requests or TCP connection handles. + +\par Note: +Some backends may require additional PHP extensions to be enabled or can rely on a +MediaWiki extension. This is often the case when a FileBackend subclass makes use of an +upstream client API for communicating with the backing store. + + +\section fileoperations File operations + +The MediaWiki FileBackend API supports various operations on either files or +directories. See FileBackend.php for full documentation for each function. + + +\subsection reading Reading + +The following basic operations are supported for reading from a backend: + +On files: +* state a file for basic information (timestamp, size) +* read a file into a string or several files into a map of path names to strings +* download a file or set of files to a temporary file (on a mounted file system) +* get the SHA1 hash of a file +* get various properties of a file (stat information, content time, mime information, ...) + +On directories: +* get a list of files directly under a directory +* get a recursive list of files under a directory +* get a list of directories directly under a directory +* get a recursive list of directories under a directory + +\par Note: +Backend handles should return directory listings as iterators, all though in some cases +they may just be simple arrays (which can still be iterated over). Iterators allow for +callers to traverse a large number of file listings without consuming excessive RAM in +the process. Either the memory consumed is flatly bounded (if the iterator does paging) +or it is proportional to the depth of the portion of the directory tree being traversed +(if the iterator works via recursion). + + +\subsection writing Writing + +The following basic operations are supported for writing or changing in the backend: + +On files: +* store (copying a mounted file system file into storage) +* create (creating a file within storage from a string) +* copy (within storage) +* move (within storage) +* delete (within storage) +* lock/unlock (lock or unlock a file in storage) + +The following operations are supported for writing directories in the backend: +* prepare (create parent container and directories for a path) +* secure (try to lock-down access to a container) +* publish (try to reverse the effects of secure) +* clean (remove empty containers or directories) + + +\subsection invokingoperation Invoking an operation + +Generally, callers should use doOperations() or doQuickOperations() when doing +batches of changes, rather than making a suite of single operation calls. This +makes the system tolerate high latency much better by pipelining operations +when possible. + +doOperations() should be used for working on important original data, i.e. when +consistency is important. The former will only pipeline operations that do not +depend on each other. It is best if the operations that do not depend on each +other occur in consecutive groups. This function can also log file changes to +a journal (see FileJournal), which can be used to sync two backend instances. +One might use this function for user uploads of file for example. + +doQuickOperations() is more geared toward ephemeral items that can be easily +regenerated from original data. It will always pipeline without checking for +dependencies within the operation batch. One might use this function for +creating and purging generated thumbnails of original files for example. + + +\section consistency Consistency + +Not all backing stores are sequentially consistent by default. Various FileBackend +functions offer a "latest" option that can be passed in to assure (or try to assure) +that the latest version of the file is read. Some backing stores are consistent by +default, but callers should always assume that without this option, stale data may +be read. This is actually true for stores that have eventual consistency. + +Note that file listing functions have no "latest" flag, and thus some systems may +return stale data. Thus callers should avoid assuming that listings contain changes +made my the current client or any other client from a very short time ago. For example, +creating a file under a directory and then immediately doing a file listing operation +on that directory may result in a listing that does not include that file. + + +\section locking Locking + +Locking is effective if and only if a proper lock manager is registered and is +actually being used by the backend. Lock managers can be registered in LocalSettings.php +using the $wgLockManagers global configuration variable. + +For object stores, locking is not generally useful for avoiding partially +written or read objects, since most stores use Multi Version Concurrency +Control (MVCC) to avoid this. However, locking can be important when: +* One or more operations must be done without objects changing in the meantime. +* It can also be useful when a file read is used to determine a file write or DB change. + For example, doOperations() first checks that there will be no "file already exists" + or "file does not exist" type errors before attempting an operation batch. This works + by stating the files first, and is only safe if the files are locked in the meantime. + +When locking, callers should use the latest available file data for reads. +Also, one should always lock the file *before* reading it, not after. If stale data is +used to determine a write, there will be some data corruption, even when reads of the +original file finally start returning the updated data without needing the "latest" +option (eventual consistency). The "scoped" lock functions are preferable since +there is not the problem of forgetting to unlock due to early returns or exceptions. + +Since acquiring locks can fail, and lock managers can be non-blocking, callers should: +* Acquire all required locks up font +* Be prepared for the case where locks fail to be acquired +* Possible retry acquiring certain locks + +MVCC is also a useful pattern to use on top of the backend interface, because operations +are not atomic, even with doOperations(), so doing complex batch file changes or changing +files and updating a database row can result in partially written "transactions". Thus one +should avoid changing files once they have been stored, except perhaps with ephemeral data +that are tolerant of some degree of inconsistency. + +Callers can use their own locking (e.g. SELECT FOR UPDATE) if it is more convenient, but +note that all callers that change any of the files should then go through functions that +acquire these locks. For example, if a caller just directly uses the file backend store() +function, it will ignore any custom "FOR UPDATE" locks, which can cause problems. + +\section objectstore Object stores + +Support for object stores (like Amazon S3/Swift) drive much of the API and design +decisions of FileBackend, but using any POSIX compliant file systems works fine. +The system essentially stores "files" in "containers". For a mounted file system +as a backing store, "files" will just be files under directories. For an object store +as a backing store, the "files" will be objects stored in actual containers. + + +\section file_obj_diffs File system and Object store differences + +An advantage of object stores is the reduced Round-Trip Times. This is +achieved by avoiding the need to create each parent directory before placing a +file somewhere. It gets worse the deeper the directory hierarchy is. Another +advantage of object stores is that object listings tend to use databases, which +scale better than the linked list directories that file sytems sometimes use. +File systems like btrfs and xfs use tree structures, which scale better. +For both object stores and file systems, using "/" in filenames will allow for the +intuitive use of directory functions. For example, creating a file in Swift +called "container/a/b/file1" will mean that: +- a "directory listing" of "container/a" will contain "b", +- and a "file listing" of "b" will contain "file1" + +This means that switching from an object store to a file system and vise versa +using the FileBackend interface will generally be harmless. However, one must be +aware of some important differences: + +* In a file system, you cannot have a file and a directory within the same path + whereas it is possible in an object stores. Calling code should avoid any layouts + which allow files and directories at the same path. +* Some file systems have file name length restrictions or overall path length + restrictions that others do not. The same goes with object stores which might + have a maximum object length or a limitation regarding the number of files + under a container or volume. +* Latency varies among systems, certain access patterns may not be tolerable for + certain backends but may hold up for others. Some backend subclasses use + MediaWiki's object caching for serving stat requests, which can greatly + reduce latency. Making sure that the backend has pipelining (see the + "parallelize" and "concurrency" settings) enabled can also mask latency in + batch operation scenarios. +* File systems may implement directories as linked-lists or other structures + with poor scalability, so calling code should use layouts that shard the data. + Instead of storing files like "container/file.txt", one can store files like + "container/<x>/<y>/file.txt". It is best if "sharding" optional or configurable. + +*/ diff --git a/includes/filebackend/SwiftFileBackend.php b/includes/filebackend/SwiftFileBackend.php index b6f0aa60..0f3d97a3 100644 --- a/includes/filebackend/SwiftFileBackend.php +++ b/includes/filebackend/SwiftFileBackend.php @@ -40,11 +40,16 @@ class SwiftFileBackend extends FileBackendStore { /** @var CF_Authentication */ protected $auth; // Swift authentication handler protected $authTTL; // integer seconds + protected $swiftTempUrlKey; // string; shared secret value for making temp urls protected $swiftAnonUser; // string; username to handle unauthenticated requests protected $swiftUseCDN; // boolean; whether CloudFiles CDN is enabled protected $swiftCDNExpiry; // integer; how long to cache things in the CDN protected $swiftCDNPurgable; // boolean; whether object CDN purging is enabled + // Rados Gateway specific options + protected $rgwS3AccessKey; // string; S3 access key + protected $rgwS3SecretKey; // string; S3 authentication key + /** @var CF_Connection */ protected $conn; // Swift connection handle protected $sessionStarted = 0; // integer UNIX timestamp @@ -66,6 +71,8 @@ class SwiftFileBackend extends FileBackendStore { * - swiftUser : Swift user used by MediaWiki (account:username) * - swiftKey : Swift authentication key for the above user * - swiftAuthTTL : Swift authentication TTL (seconds) + * - swiftTempUrlKey : Swift "X-Account-Meta-Temp-URL-Key" value on the account. + * Do not set this until it has been set in the backend. * - swiftAnonUser : Swift user used for end-user requests (account:username). * If set, then views of public containers are assumed to go * through this user. If not set, then public containers are @@ -84,6 +91,16 @@ class SwiftFileBackend extends FileBackendStore { * - cacheAuthInfo : Whether to cache authentication tokens in APC, XCache, ect. * If those are not available, then the main cache will be used. * This is probably insecure in shared hosting environments. + * - rgwS3AccessKey : Ragos Gateway S3 "access key" value on the account. + * Do not set this until it has been set in the backend. + * This is used for generating expiring pre-authenticated URLs. + * Only use this when using rgw and to work around + * http://tracker.newdream.net/issues/3454. + * - rgwS3SecretKey : Ragos Gateway S3 "secret key" value on the account. + * Do not set this until it has been set in the backend. + * This is used for generating expiring pre-authenticated URLs. + * Only use this when using rgw and to work around + * http://tracker.newdream.net/issues/3454. */ public function __construct( array $config ) { parent::__construct( $config ); @@ -104,6 +121,9 @@ class SwiftFileBackend extends FileBackendStore { $this->swiftAnonUser = isset( $config['swiftAnonUser'] ) ? $config['swiftAnonUser'] : ''; + $this->swiftTempUrlKey = isset( $config['swiftTempUrlKey'] ) + ? $config['swiftTempUrlKey'] + : ''; $this->shardViaHashLevels = isset( $config['shardViaHashLevels'] ) ? $config['shardViaHashLevels'] : ''; @@ -116,13 +136,19 @@ class SwiftFileBackend extends FileBackendStore { $this->swiftCDNPurgable = isset( $config['swiftCDNPurgable'] ) ? $config['swiftCDNPurgable'] : true; + $this->rgwS3AccessKey = isset( $config['rgwS3AccessKey'] ) + ? $config['rgwS3AccessKey'] + : ''; + $this->rgwS3SecretKey = isset( $config['rgwS3SecretKey'] ) + ? $config['rgwS3SecretKey'] + : ''; // Cache container information to mask latency $this->memCache = wfGetMainCache(); // Process cache for container info $this->connContainerCache = new ProcessCacheLRU( 300 ); // Cache auth token information to avoid RTTs if ( !empty( $config['cacheAuthInfo'] ) ) { - if ( php_sapi_name() === 'cli' ) { + if ( PHP_SAPI === 'cli' ) { $this->srvCache = wfGetMainCache(); // preferrably memcached } else { try { // look for APC, XCache, WinCache, ect... @@ -168,14 +194,14 @@ class SwiftFileBackend extends FileBackendStore { } /** - * @param $disposition string Content-Disposition header value + * @param string $disposition Content-Disposition header value * @return string Truncated Content-Disposition header value to meet Swift limits */ protected function truncDisp( $disposition ) { $res = ''; foreach ( explode( ';', $disposition ) as $part ) { $part = trim( $part ); - $new = ( $res === '' ) ? $part : "{$res};{$part}"; + $new = ( $res === '' ) ? $part : "{$res};{$part}"; if ( strlen( $new ) <= 255 ) { $res = $new; } else { @@ -201,12 +227,6 @@ class SwiftFileBackend extends FileBackendStore { // (a) Check the destination container and object try { $dContObj = $this->getContainer( $dstCont ); - if ( empty( $params['overwrite'] ) && - $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) ) - { - $status->fatal( 'backend-fail-alreadyexists', $params['dst'] ); - return $status; - } } catch ( NoSuchContainerException $e ) { $status->fatal( 'backend-fail-create', $params['dst'] ); return $status; @@ -223,8 +243,7 @@ class SwiftFileBackend extends FileBackendStore { // Create a fresh CF_Object with no fields preloaded. // We don't want to preserve headers, metadata, and such. $obj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD - // Note: metadata keys stored as [Upper case char][[Lower case char]...] - $obj->metadata = array( 'Sha1base36' => $sha1Hash ); + $obj->setMetadataValues( array( 'Sha1base36' => $sha1Hash ) ); // Manually set the ETag (https://github.com/rackspace/php-cloudfiles/issues/59). // The MD5 here will be checked within Swift against its own MD5. $obj->set_etag( md5( $params['content'] ) ); @@ -237,17 +256,17 @@ class SwiftFileBackend extends FileBackendStore { if ( isset( $params['disposition'] ) ) { $obj->headers['Content-Disposition'] = $this->truncDisp( $params['disposition'] ); } + // Set any other custom headers if requested + if ( isset( $params['headers'] ) ) { + $obj->headers += $params['headers']; + } if ( !empty( $params['async'] ) ) { // deferred $op = $obj->write_async( $params['content'] ); $status->value = new SwiftFileOpHandle( $this, $params, 'Create', $op ); - if ( !empty( $params['overwrite'] ) ) { // file possibly mutated - $status->value->affectedObjects[] = $obj; - } + $status->value->affectedObjects[] = $obj; } else { // actually write the object in Swift $obj->write( $params['content'] ); - if ( !empty( $params['overwrite'] ) ) { // file possibly mutated - $this->purgeCDNCache( array( $obj ) ); - } + $this->purgeCDNCache( array( $obj ) ); } } catch ( CDNNotEnabledException $e ) { // CDN not enabled; nothing to see here @@ -287,12 +306,6 @@ class SwiftFileBackend extends FileBackendStore { // (a) Check the destination container and object try { $dContObj = $this->getContainer( $dstCont ); - if ( empty( $params['overwrite'] ) && - $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) ) - { - $status->fatal( 'backend-fail-alreadyexists', $params['dst'] ); - return $status; - } } catch ( NoSuchContainerException $e ) { $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] ); return $status; @@ -302,7 +315,9 @@ class SwiftFileBackend extends FileBackendStore { } // (b) Get a SHA-1 hash of the object + wfSuppressWarnings(); $sha1Hash = sha1_file( $params['src'] ); + wfRestoreWarnings(); if ( $sha1Hash === false ) { // source doesn't exist? $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] ); return $status; @@ -314,8 +329,7 @@ class SwiftFileBackend extends FileBackendStore { // Create a fresh CF_Object with no fields preloaded. // We don't want to preserve headers, metadata, and such. $obj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD - // Note: metadata keys stored as [Upper case char][[Lower case char]...] - $obj->metadata = array( 'Sha1base36' => $sha1Hash ); + $obj->setMetadataValues( array( 'Sha1base36' => $sha1Hash ) ); // The MD5 here will be checked within Swift against its own MD5. $obj->set_etag( md5_file( $params['src'] ) ); // Use the same content type as StreamFile for security @@ -327,6 +341,10 @@ class SwiftFileBackend extends FileBackendStore { if ( isset( $params['disposition'] ) ) { $obj->headers['Content-Disposition'] = $this->truncDisp( $params['disposition'] ); } + // Set any other custom headers if requested + if ( isset( $params['headers'] ) ) { + $obj->headers += $params['headers']; + } if ( !empty( $params['async'] ) ) { // deferred wfSuppressWarnings(); $fp = fopen( $params['src'], 'rb' ); @@ -337,15 +355,11 @@ class SwiftFileBackend extends FileBackendStore { $op = $obj->write_async( $fp, filesize( $params['src'] ), true ); $status->value = new SwiftFileOpHandle( $this, $params, 'Store', $op ); $status->value->resourcesToClose[] = $fp; - if ( !empty( $params['overwrite'] ) ) { // file possibly mutated - $status->value->affectedObjects[] = $obj; - } + $status->value->affectedObjects[] = $obj; } } else { // actually write the object in Swift $obj->load_from_filename( $params['src'], true ); // calls $obj->write() - if ( !empty( $params['overwrite'] ) ) { // file possibly mutated - $this->purgeCDNCache( array( $obj ) ); - } + $this->purgeCDNCache( array( $obj ) ); } } catch ( CDNNotEnabledException $e ) { // CDN not enabled; nothing to see here @@ -396,14 +410,10 @@ class SwiftFileBackend extends FileBackendStore { try { $sContObj = $this->getContainer( $srcCont ); $dContObj = $this->getContainer( $dstCont ); - if ( empty( $params['overwrite'] ) && - $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) ) - { - $status->fatal( 'backend-fail-alreadyexists', $params['dst'] ); - return $status; - } } catch ( NoSuchContainerException $e ) { - $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] ); + if ( empty( $params['ignoreMissingSource'] ) || isset( $sContObj ) ) { + $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] ); + } return $status; } catch ( CloudFilesException $e ) { // some other exception? $this->handleException( $e, $status, __METHOD__, $params ); @@ -420,19 +430,17 @@ class SwiftFileBackend extends FileBackendStore { if ( !empty( $params['async'] ) ) { // deferred $op = $sContObj->copy_object_to_async( $srcRel, $dContObj, $dstRel, null, $hdrs ); $status->value = new SwiftFileOpHandle( $this, $params, 'Copy', $op ); - if ( !empty( $params['overwrite'] ) ) { // file possibly mutated - $status->value->affectedObjects[] = $dstObj; - } + $status->value->affectedObjects[] = $dstObj; } else { // actually write the object in Swift $sContObj->copy_object_to( $srcRel, $dContObj, $dstRel, null, $hdrs ); - if ( !empty( $params['overwrite'] ) ) { // file possibly mutated - $this->purgeCDNCache( array( $dstObj ) ); - } + $this->purgeCDNCache( array( $dstObj ) ); } } catch ( CDNNotEnabledException $e ) { // CDN not enabled; nothing to see here } catch ( NoSuchObjectException $e ) { // source object does not exist - $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] ); + if ( empty( $params['ignoreMissingSource'] ) ) { + $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] ); + } } catch ( CloudFilesException $e ) { // some other exception? $this->handleException( $e, $status, __METHOD__, $params ); } @@ -474,14 +482,10 @@ class SwiftFileBackend extends FileBackendStore { try { $sContObj = $this->getContainer( $srcCont ); $dContObj = $this->getContainer( $dstCont ); - if ( empty( $params['overwrite'] ) && - $this->fileExists( array( 'src' => $params['dst'], 'latest' => 1 ) ) ) - { - $status->fatal( 'backend-fail-alreadyexists', $params['dst'] ); - return $status; - } } catch ( NoSuchContainerException $e ) { - $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] ); + if ( empty( $params['ignoreMissingSource'] ) || isset( $sContObj ) ) { + $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] ); + } return $status; } catch ( CloudFilesException $e ) { // some other exception? $this->handleException( $e, $status, __METHOD__, $params ); @@ -500,20 +504,18 @@ class SwiftFileBackend extends FileBackendStore { $op = $sContObj->move_object_to_async( $srcRel, $dContObj, $dstRel, null, $hdrs ); $status->value = new SwiftFileOpHandle( $this, $params, 'Move', $op ); $status->value->affectedObjects[] = $srcObj; - if ( !empty( $params['overwrite'] ) ) { // file possibly mutated - $status->value->affectedObjects[] = $dstObj; - } + $status->value->affectedObjects[] = $dstObj; } else { // actually write the object in Swift $sContObj->move_object_to( $srcRel, $dContObj, $dstRel, null, $hdrs ); $this->purgeCDNCache( array( $srcObj ) ); - if ( !empty( $params['overwrite'] ) ) { // file possibly mutated - $this->purgeCDNCache( array( $dstObj ) ); - } + $this->purgeCDNCache( array( $dstObj ) ); } } catch ( CDNNotEnabledException $e ) { // CDN not enabled; nothing to see here } catch ( NoSuchObjectException $e ) { // source object does not exist - $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] ); + if ( empty( $params['ignoreMissingSource'] ) ) { + $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] ); + } } catch ( CloudFilesException $e ) { // some other exception? $this->handleException( $e, $status, __METHOD__, $params ); } @@ -559,7 +561,9 @@ class SwiftFileBackend extends FileBackendStore { } catch ( CDNNotEnabledException $e ) { // CDN not enabled; nothing to see here } catch ( NoSuchContainerException $e ) { - $status->fatal( 'backend-fail-delete', $params['src'] ); + if ( empty( $params['ignoreMissingSource'] ) ) { + $status->fatal( 'backend-fail-delete', $params['src'] ); + } } catch ( NoSuchObjectException $e ) { if ( empty( $params['ignoreMissingSource'] ) ) { $status->fatal( 'backend-fail-delete', $params['src'] ); @@ -587,6 +591,47 @@ class SwiftFileBackend extends FileBackendStore { } /** + * @see FileBackendStore::doDescribeInternal() + * @return Status + */ + protected function doDescribeInternal( array $params ) { + $status = Status::newGood(); + + list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] ); + if ( $srcRel === null ) { + $status->fatal( 'backend-fail-invalidpath', $params['src'] ); + return $status; + } + + $hdrs = isset( $params['headers'] ) ? $params['headers'] : array(); + // Set the Content-Disposition header if requested + if ( isset( $params['disposition'] ) ) { + $hdrs['Content-Disposition'] = $this->truncDisp( $params['disposition'] ); + } + + try { + $sContObj = $this->getContainer( $srcCont ); + // Get the latest version of the current metadata + $srcObj = $sContObj->get_object( $srcRel, + $this->headersFromParams( array( 'latest' => true ) ) ); + // Merge in the metadata updates... + $srcObj->headers = $hdrs + $srcObj->headers; + $srcObj->sync_metadata(); // save to Swift + $this->purgeCDNCache( array( $srcObj ) ); + } catch ( CDNNotEnabledException $e ) { + // CDN not enabled; nothing to see here + } catch ( NoSuchContainerException $e ) { + $status->fatal( 'backend-fail-describe', $params['src'] ); + } catch ( NoSuchObjectException $e ) { + $status->fatal( 'backend-fail-describe', $params['src'] ); + } catch ( CloudFilesException $e ) { // some other exception? + $this->handleException( $e, $status, __METHOD__, $params ); + } + + return $status; + } + + /** * @see FileBackendStore::doPrepareInternal() * @return Status */ @@ -595,7 +640,7 @@ class SwiftFileBackend extends FileBackendStore { // (a) Check if container already exists try { - $contObj = $this->getContainer( $fullCont ); + $this->getContainer( $fullCont ); // NoSuchContainerException not thrown: container must exist return $status; // already exists } catch ( NoSuchContainerException $e ) { @@ -761,7 +806,7 @@ class SwiftFileBackend extends FileBackendStore { // Convert dates like "Tue, 03 Jan 2012 22:01:04 GMT" to TS_MW 'mtime' => wfTimestamp( TS_MW, $srcObj->last_modified ), 'size' => (int)$srcObj->content_length, - 'sha1' => $srcObj->metadata['Sha1base36'] + 'sha1' => $srcObj->getMetadataValue( 'Sha1base36' ) ); } catch ( NoSuchContainerException $e ) { } catch ( NoSuchObjectException $e ) { @@ -777,60 +822,106 @@ class SwiftFileBackend extends FileBackendStore { * Fill in any missing object metadata and save it to Swift * * @param $obj CF_Object - * @param $path string Storage path to object + * @param string $path Storage path to object * @return bool Success * @throws Exception cloudfiles exceptions */ protected function addMissingMetadata( CF_Object $obj, $path ) { - if ( isset( $obj->metadata['Sha1base36'] ) ) { + if ( $obj->getMetadataValue( 'Sha1base36' ) !== null ) { return true; // nothing to do } wfProfileIn( __METHOD__ ); + trigger_error( "$path was not stored with SHA-1 metadata.", E_USER_WARNING ); $status = Status::newGood(); $scopeLockS = $this->getScopedFileLocks( array( $path ), LockManager::LOCK_UW, $status ); if ( $status->isOK() ) { - # Do not stat the file in getLocalCopy() to avoid infinite loops - $tmpFile = $this->getLocalCopy( array( 'src' => $path, 'latest' => 1, 'nostat' => 1 ) ); + $tmpFile = $this->getLocalCopy( array( 'src' => $path, 'latest' => 1 ) ); if ( $tmpFile ) { $hash = $tmpFile->getSha1Base36(); if ( $hash !== false ) { - $obj->metadata['Sha1base36'] = $hash; + $obj->setMetadataValues( array( 'Sha1base36' => $hash ) ); $obj->sync_metadata(); // save to Swift wfProfileOut( __METHOD__ ); return true; // success } } } - $obj->metadata['Sha1base36'] = false; + trigger_error( "Unable to set SHA-1 metadata for $path", E_USER_WARNING ); + $obj->setMetadataValues( array( 'Sha1base36' => false ) ); wfProfileOut( __METHOD__ ); return false; // failed } /** - * @see FileBackend::getFileContents() - * @return bool|null|string + * @see FileBackendStore::doGetFileContentsMulti() + * @return Array */ - public function getFileContents( array $params ) { - list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] ); - if ( $srcRel === null ) { - return false; // invalid storage path - } - - if ( !$this->fileExists( $params ) ) { - return null; - } + protected function doGetFileContentsMulti( array $params ) { + $contents = array(); + + $ep = array_diff_key( $params, array( 'srcs' => 1 ) ); // for error logging + // Blindly create tmp files and stream to them, catching any exception if the file does + // not exist. Doing stats here is useless and will loop infinitely in addMissingMetadata(). + foreach ( array_chunk( $params['srcs'], $params['concurrency'] ) as $pathBatch ) { + $cfOps = array(); // (path => CF_Async_Op) + + foreach ( $pathBatch as $path ) { // each path in this concurrent batch + list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $path ); + if ( $srcRel === null ) { + $contents[$path] = false; + continue; + } + $data = false; + try { + $sContObj = $this->getContainer( $srcCont ); + $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD + // Create a new temporary memory file... + $handle = fopen( 'php://temp', 'wb' ); + if ( $handle ) { + $headers = $this->headersFromParams( $params ); + if ( count( $pathBatch ) > 1 ) { + $cfOps[$path] = $obj->stream_async( $handle, $headers ); + $cfOps[$path]->_file_handle = $handle; // close this later + } else { + $obj->stream( $handle, $headers ); + rewind( $handle ); // start from the beginning + $data = stream_get_contents( $handle ); + fclose( $handle ); + } + } else { + $data = false; + } + } catch ( NoSuchContainerException $e ) { + $data = false; + } catch ( NoSuchObjectException $e ) { + $data = false; + } catch ( CloudFilesException $e ) { // some other exception? + $data = false; + $this->handleException( $e, null, __METHOD__, array( 'src' => $path ) + $ep ); + } + $contents[$path] = $data; + } - $data = false; - try { - $sContObj = $this->getContainer( $srcCont ); - $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD - $data = $obj->read( $this->headersFromParams( $params ) ); - } catch ( NoSuchContainerException $e ) { - } catch ( CloudFilesException $e ) { // some other exception? - $this->handleException( $e, null, __METHOD__, $params ); + $batch = new CF_Async_Op_Batch( $cfOps ); + $cfOps = $batch->execute(); + foreach ( $cfOps as $path => $cfOp ) { + try { + $cfOp->getLastResponse(); + rewind( $cfOp->_file_handle ); // start from the beginning + $contents[$path] = stream_get_contents( $cfOp->_file_handle ); + } catch ( NoSuchContainerException $e ) { + $contents[$path] = false; + } catch ( NoSuchObjectException $e ) { + $contents[$path] = false; + } catch ( CloudFilesException $e ) { // some other exception? + $contents[$path] = false; + $this->handleException( $e, null, __METHOD__, array( 'src' => $path ) + $ep ); + } + fclose( $cfOp->_file_handle ); // close open handle + } } - return $data; + return $contents; } /** @@ -871,11 +962,11 @@ class SwiftFileBackend extends FileBackendStore { /** * Do not call this function outside of SwiftFileBackendFileList * - * @param $fullCont string Resolved container name - * @param $dir string Resolved storage directory with no trailing slash - * @param $after string|null Storage path of file to list items after + * @param string $fullCont Resolved container name + * @param string $dir Resolved storage directory with no trailing slash + * @param string|null $after Storage path of file to list items after * @param $limit integer Max number of items to list - * @param $params Array Includes flag for 'topOnly' + * @param array $params Includes flag for 'topOnly' * @return Array List of relative paths of dirs directly under $dir */ public function getDirListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) { @@ -903,7 +994,7 @@ class SwiftFileBackend extends FileBackendStore { $objects = $container->list_objects( $limit, $after, $prefix ); foreach ( $objects as $object ) { // files $objectDir = $this->getParentDir( $object ); // directory of object - if ( $objectDir !== false ) { // file has a parent dir + if ( $objectDir !== false && $objectDir !== $dir ) { // Swift stores paths in UTF-8, using binary sorting. // See function "create_container_table" in common/db.py. // If a directory is not "greater" than the last one, @@ -944,11 +1035,11 @@ class SwiftFileBackend extends FileBackendStore { /** * Do not call this function outside of SwiftFileBackendFileList * - * @param $fullCont string Resolved container name - * @param $dir string Resolved storage directory with no trailing slash - * @param $after string|null Storage path of file to list items after + * @param string $fullCont Resolved container name + * @param string $dir Resolved storage directory with no trailing slash + * @param string|null $after Storage path of file to list items after * @param $limit integer Max number of items to list - * @param $params Array Includes flag for 'topOnly' + * @param array $params Includes flag for 'topOnly' * @return Array List of relative paths of files under $dir */ public function getFileListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) { @@ -1038,44 +1129,125 @@ class SwiftFileBackend extends FileBackendStore { } /** - * @see FileBackendStore::getLocalCopy() + * @see FileBackendStore::doGetLocalCopyMulti() * @return null|TempFSFile */ - public function getLocalCopy( array $params ) { - list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] ); - if ( $srcRel === null ) { - return null; - } + protected function doGetLocalCopyMulti( array $params ) { + $tmpFiles = array(); + + $ep = array_diff_key( $params, array( 'srcs' => 1 ) ); // for error logging + // Blindly create tmp files and stream to them, catching any exception if the file does + // not exist. Doing a stat here is useless causes infinite loops in addMissingMetadata(). + foreach ( array_chunk( $params['srcs'], $params['concurrency'] ) as $pathBatch ) { + $cfOps = array(); // (path => CF_Async_Op) + + foreach ( $pathBatch as $path ) { // each path in this concurrent batch + list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $path ); + if ( $srcRel === null ) { + $tmpFiles[$path] = null; + continue; + } + $tmpFile = null; + try { + $sContObj = $this->getContainer( $srcCont ); + $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD + // Get source file extension + $ext = FileBackend::extensionFromPath( $path ); + // Create a new temporary file... + $tmpFile = TempFSFile::factory( 'localcopy_', $ext ); + if ( $tmpFile ) { + $handle = fopen( $tmpFile->getPath(), 'wb' ); + if ( $handle ) { + $headers = $this->headersFromParams( $params ); + if ( count( $pathBatch ) > 1 ) { + $cfOps[$path] = $obj->stream_async( $handle, $headers ); + $cfOps[$path]->_file_handle = $handle; // close this later + } else { + $obj->stream( $handle, $headers ); + fclose( $handle ); + } + } else { + $tmpFile = null; + } + } + } catch ( NoSuchContainerException $e ) { + $tmpFile = null; + } catch ( NoSuchObjectException $e ) { + $tmpFile = null; + } catch ( CloudFilesException $e ) { // some other exception? + $tmpFile = null; + $this->handleException( $e, null, __METHOD__, array( 'src' => $path ) + $ep ); + } + $tmpFiles[$path] = $tmpFile; + } - // Blindly create a tmp file and stream to it, catching any exception if the file does - // not exist. Also, doing a stat here will cause infinite loops when filling metadata. - $tmpFile = null; - try { - $sContObj = $this->getContainer( $srcCont ); - $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD - // Get source file extension - $ext = FileBackend::extensionFromPath( $srcRel ); - // Create a new temporary file... - $tmpFile = TempFSFile::factory( 'localcopy_', $ext ); - if ( $tmpFile ) { - $handle = fopen( $tmpFile->getPath(), 'wb' ); - if ( $handle ) { - $obj->stream( $handle, $this->headersFromParams( $params ) ); - fclose( $handle ); - } else { - $tmpFile = null; // couldn't open temp file + $batch = new CF_Async_Op_Batch( $cfOps ); + $cfOps = $batch->execute(); + foreach ( $cfOps as $path => $cfOp ) { + try { + $cfOp->getLastResponse(); + } catch ( NoSuchContainerException $e ) { + $tmpFiles[$path] = null; + } catch ( NoSuchObjectException $e ) { + $tmpFiles[$path] = null; + } catch ( CloudFilesException $e ) { // some other exception? + $tmpFiles[$path] = null; + $this->handleException( $e, null, __METHOD__, array( 'src' => $path ) + $ep ); } + fclose( $cfOp->_file_handle ); // close open handle } - } catch ( NoSuchContainerException $e ) { - $tmpFile = null; - } catch ( NoSuchObjectException $e ) { - $tmpFile = null; - } catch ( CloudFilesException $e ) { // some other exception? - $tmpFile = null; - $this->handleException( $e, null, __METHOD__, $params ); } - return $tmpFile; + return $tmpFiles; + } + + /** + * @see FileBackendStore::getFileHttpUrl() + * @return string|null + */ + public function getFileHttpUrl( array $params ) { + if ( $this->swiftTempUrlKey != '' || + ( $this->rgwS3AccessKey != '' && $this->rgwS3SecretKey != '' ) ) + { + list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] ); + if ( $srcRel === null ) { + return null; // invalid path + } + try { + $ttl = isset( $params['ttl'] ) ? $params['ttl'] : 86400; + $sContObj = $this->getContainer( $srcCont ); + $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD + if ( $this->swiftTempUrlKey != '' ) { + return $obj->get_temp_url( $this->swiftTempUrlKey, $ttl, "GET" ); + } else { // give S3 API URL for rgw + $expires = time() + $ttl; + // Path for signature starts with the bucket + $spath = '/' . rawurlencode( $srcCont ) . '/' . + str_replace( '%2F', '/', rawurlencode( $srcRel ) ); + // Calculate the hash + $signature = base64_encode( hash_hmac( + 'sha1', + "GET\n\n\n{$expires}\n{$spath}", + $this->rgwS3SecretKey, + true // raw + ) ); + // See http://s3.amazonaws.com/doc/s3-developer-guide/RESTAuthentication.html. + // Note: adding a newline for empty CanonicalizedAmzHeaders does not work. + return wfAppendQuery( + str_replace( '/swift/v1', '', // S3 API is the rgw default + $sContObj->cfs_http->getStorageUrl() . $spath ), + array( + 'Signature' => $signature, + 'Expires' => $expires, + 'AWSAccessKeyId' => $this->rgwS3AccessKey ) + ); + } + } catch ( NoSuchContainerException $e ) { + } catch ( CloudFilesException $e ) { // some other exception? + $this->handleException( $e, null, __METHOD__, $params ); + } + } + return null; } /** @@ -1091,7 +1263,7 @@ class SwiftFileBackend extends FileBackendStore { * on a FileBackend params array, e.g. that of getLocalCopy(). * $params is currently only checked for a 'latest' flag. * - * @param $params Array + * @param array $params * @return Array */ protected function headersFromParams( array $params ) { @@ -1118,8 +1290,8 @@ class SwiftFileBackend extends FileBackendStore { $cfOps = $batch->execute(); foreach ( $cfOps as $index => $cfOp ) { $status = Status::newGood(); + $function = '_getResponse' . $fileOpHandles[$index]->call; try { // catch exceptions; update status - $function = '_getResponse' . $fileOpHandles[$index]->call; $this->$function( $cfOp, $status, $fileOpHandles[$index]->params ); $this->purgeCDNCache( $fileOpHandles[$index]->affectedObjects ); } catch ( CloudFilesException $e ) { // some other exception? @@ -1137,12 +1309,12 @@ class SwiftFileBackend extends FileBackendStore { * * $readGrps is a list of the possible criteria for a request to have * access to read a container. Each item is one of the following formats: - * - account:user : Grants access if the request is by the given user - * - .r:<regex> : Grants access if the request is from a referrer host that - * matches the expression and the request is not for a listing. - * Setting this to '*' effectively makes a container public. - * - .rlistings:<regex> : Grants access if the request is from a referrer host that - * matches the expression and the request for a listing. + * - account:user : Grants access if the request is by the given user + * - ".r:<regex>" : Grants access if the request is from a referrer host that + * matches the expression and the request is not for a listing. + * Setting this to '*' effectively makes a container public. + * -".rlistings:<regex>" : Grants access if the request is from a referrer host that + * matches the expression and the request for a listing. * * $writeGrps is a list of the possible criteria for a request to have * access to write to a container. Each item is of the following format: @@ -1154,8 +1326,8 @@ class SwiftFileBackend extends FileBackendStore { * (lists are truncated to 10000 item with no way to page), and is just a performance risk. * * @param $contObj CF_Container Swift container - * @param $readGrps Array List of read access routes - * @param $writeGrps Array List of write access routes + * @param array $readGrps List of read access routes + * @param array $writeGrps List of write access routes * @return Status */ protected function setContainerAccess( @@ -1178,7 +1350,7 @@ class SwiftFileBackend extends FileBackendStore { * Purge the CDN cache of affected objects if CDN caching is enabled. * This is for Rackspace/Akamai CDNs. * - * @param $objects Array List of CF_Object items + * @param array $objects List of CF_Object items * @return void */ public function purgeCDNCache( array $objects ) { @@ -1199,8 +1371,9 @@ class SwiftFileBackend extends FileBackendStore { /** * Get an authenticated connection handle to the Swift proxy * - * @return CF_Connection|bool False on failure * @throws CloudFilesException + * @throws CloudFilesException|Exception + * @return CF_Connection|bool False on failure */ protected function getConnection() { if ( $this->connException instanceof CloudFilesException ) { @@ -1251,6 +1424,7 @@ class SwiftFileBackend extends FileBackendStore { protected function closeConnection() { if ( $this->conn ) { $this->conn->close(); // close active cURL handles in CF_Http object + $this->conn = null; $this->sessionStarted = 0; $this->connContainerCache->clear(); } @@ -1267,18 +1441,11 @@ class SwiftFileBackend extends FileBackendStore { } /** - * @see FileBackendStore::doClearCache() - */ - protected function doClearCache( array $paths = null ) { - $this->connContainerCache->clear(); // clear container object cache - } - - /** * Get a Swift container object, possibly from process cache. * Use $reCache if the file count or byte count is needed. * - * @param $container string Container name - * @param $bypassCache bool Bypass all caches and load from Swift + * @param string $container Container name + * @param bool $bypassCache Bypass all caches and load from Swift * @return CF_Container * @throws CloudFilesException */ @@ -1305,7 +1472,7 @@ class SwiftFileBackend extends FileBackendStore { /** * Create a Swift container * - * @param $container string Container name + * @param string $container Container name * @return CF_Container * @throws CloudFilesException */ @@ -1319,7 +1486,7 @@ class SwiftFileBackend extends FileBackendStore { /** * Delete a Swift container * - * @param $container string Container name + * @param string $container Container name * @return void * @throws CloudFilesException */ @@ -1353,7 +1520,7 @@ class SwiftFileBackend extends FileBackendStore { * @param $e Exception * @param $status Status|null * @param $func string - * @param $params Array + * @param array $params * @return void */ protected function handleException( Exception $e, $status, $func, array $params ) { @@ -1420,9 +1587,9 @@ abstract class SwiftFileBackendList implements Iterator { /** * @param $backend SwiftFileBackend - * @param $fullCont string Resolved container name - * @param $dir string Resolved directory relative to container - * @param $params Array + * @param string $fullCont Resolved container name + * @param string $dir Resolved directory relative to container + * @param array $params */ public function __construct( SwiftFileBackend $backend, $fullCont, $dir, array $params ) { $this->backend = $backend; @@ -1491,11 +1658,11 @@ abstract class SwiftFileBackendList implements Iterator { /** * Get the given list portion (page) * - * @param $container string Resolved container name - * @param $dir string Resolved path relative to container + * @param string $container Resolved container name + * @param string $dir Resolved path relative to container * @param $after string|null * @param $limit integer - * @param $params Array + * @param array $params * @return Traversable|Array|null Returns null on failure */ abstract protected function pageFromList( $container, $dir, &$after, $limit, array $params ); diff --git a/includes/filebackend/TempFSFile.php b/includes/filebackend/TempFSFile.php index 5032bf68..11e125c1 100644 --- a/includes/filebackend/TempFSFile.php +++ b/includes/filebackend/TempFSFile.php @@ -82,30 +82,37 @@ class TempFSFile extends FSFile { * Clean up the temporary file only after an object goes out of scope * * @param $object Object - * @return void + * @return TempFSFile This object */ public function bind( $object ) { if ( is_object( $object ) ) { + if ( !isset( $object->tempFSFileReferences ) ) { + // Init first since $object might use __get() and return only a copy variable + $object->tempFSFileReferences = array(); + } $object->tempFSFileReferences[] = $this; } + return $this; } /** * Set flag to not clean up after the temporary file * - * @return void + * @return TempFSFile This object */ public function preserve() { $this->canDelete = false; + return $this; } /** * Set flag clean up after the temporary file * - * @return void + * @return TempFSFile This object */ public function autocollect() { $this->canDelete = true; + return $this; } /** diff --git a/includes/filebackend/filejournal/DBFileJournal.php b/includes/filebackend/filejournal/DBFileJournal.php index f6268c25..73f29a95 100644 --- a/includes/filebackend/filejournal/DBFileJournal.php +++ b/includes/filebackend/filejournal/DBFileJournal.php @@ -75,6 +75,9 @@ class DBFileJournal extends FileJournal { try { $dbw->insert( 'filejournal', $data, __METHOD__ ); + if ( mt_rand( 0, 99 ) == 0 ) { + $this->purgeOldLogs(); // occasionally delete old logs + } } catch ( DBError $e ) { $status->fatal( 'filejournal-fail-dbquery', $this->backend ); return $status; @@ -84,6 +87,35 @@ class DBFileJournal extends FileJournal { } /** + * @see FileJournal::doGetCurrentPosition() + * @return integer|false + */ + protected function doGetCurrentPosition() { + $dbw = $this->getMasterDB(); + + return $dbw->selectField( 'filejournal', 'MAX(fj_id)', + array( 'fj_backend' => $this->backend ), + __METHOD__ + ); + } + + /** + * @see FileJournal::doGetPositionAtTime() + * @param $time integer|string timestamp + * @return integer|false + */ + protected function doGetPositionAtTime( $time ) { + $dbw = $this->getMasterDB(); + + $encTimestamp = $dbw->addQuotes( $dbw->timestamp( $time ) ); + return $dbw->selectField( 'filejournal', 'fj_id', + array( 'fj_backend' => $this->backend, "fj_timestamp <= $encTimestamp" ), + __METHOD__, + array( 'ORDER BY' => 'fj_timestamp DESC' ) + ); + } + + /** * @see FileJournal::doGetChangeEntries() * @return Array * @throws DBError diff --git a/includes/filebackend/filejournal/FileJournal.php b/includes/filebackend/filejournal/FileJournal.php index ce029bbe..a1b7a459 100644 --- a/includes/filebackend/filejournal/FileJournal.php +++ b/includes/filebackend/filejournal/FileJournal.php @@ -54,7 +54,7 @@ abstract class FileJournal { * Create an appropriate FileJournal object from config * * @param $config Array - * @param $backend string A registered file backend name + * @param string $backend A registered file backend name * @throws MWException * @return FileJournal */ @@ -85,13 +85,13 @@ abstract class FileJournal { /** * Log changes made by a batch file operation. * $entries is an array of log entries, each of which contains: - * op : Basic operation name (create, store, copy, delete) + * op : Basic operation name (create, update, delete) * path : The storage path of the file * newSha1 : The final base 36 SHA-1 of the file * Note that 'false' should be used as the SHA-1 for non-existing files. * - * @param $entries Array List of file operations (each an array of parameters) - * @param $batchId string UUID string that identifies the operation batch + * @param array $entries List of file operations (each an array of parameters) + * @param string $batchId UUID string that identifies the operation batch * @return Status */ final public function logChangeBatch( array $entries, $batchId ) { @@ -104,13 +104,45 @@ abstract class FileJournal { /** * @see FileJournal::logChangeBatch() * - * @param $entries Array List of file operations (each an array of parameters) - * @param $batchId string UUID string that identifies the operation batch + * @param array $entries List of file operations (each an array of parameters) + * @param string $batchId UUID string that identifies the operation batch * @return Status */ abstract protected function doLogChangeBatch( array $entries, $batchId ); /** + * Get the position ID of the latest journal entry + * + * @return integer|false + */ + final public function getCurrentPosition() { + return $this->doGetCurrentPosition(); + } + + /** + * @see FileJournal::getCurrentPosition() + * @return integer|false + */ + abstract protected function doGetCurrentPosition(); + + /** + * Get the position ID of the latest journal entry at some point in time + * + * @param $time integer|string timestamp + * @return integer|false + */ + final public function getPositionAtTime( $time ) { + return $this->doGetPositionAtTime( $time ); + } + + /** + * @see FileJournal::getPositionAtTime() + * @param $time integer|string timestamp + * @return integer|false + */ + abstract protected function doGetPositionAtTime( $time ); + + /** * Get an array of file change log entries. * A starting change ID and/or limit can be specified. * @@ -169,7 +201,7 @@ abstract class FileJournal { */ class NullFileJournal extends FileJournal { /** - * @see FileJournal::logChangeBatch() + * @see FileJournal::doLogChangeBatch() * @param $entries array * @param $batchId string * @return Status @@ -179,6 +211,23 @@ class NullFileJournal extends FileJournal { } /** + * @see FileJournal::doGetCurrentPosition() + * @return integer|false + */ + protected function doGetCurrentPosition() { + return false; + } + + /** + * @see FileJournal::doGetPositionAtTime() + * @param $time integer|string timestamp + * @return integer|false + */ + protected function doGetPositionAtTime( $time ) { + return false; + } + + /** * @see FileJournal::doGetChangeEntries() * @return Array */ @@ -187,7 +236,7 @@ class NullFileJournal extends FileJournal { } /** - * @see FileJournal::purgeOldLogs() + * @see FileJournal::doPurgeOldLogs() * @return Status */ protected function doPurgeOldLogs() { diff --git a/includes/filebackend/lockmanager/DBLockManager.php b/includes/filebackend/lockmanager/DBLockManager.php index a8fe258b..f02387dc 100644 --- a/includes/filebackend/lockmanager/DBLockManager.php +++ b/includes/filebackend/lockmanager/DBLockManager.php @@ -22,10 +22,9 @@ */ /** - * Version of LockManager based on using DB table locks. + * Version of LockManager based on using named/row DB locks. + * * This is meant for multi-wiki systems that may share files. - * All locks are blocking, so it might be useful to set a small - * lock-wait timeout via server config to curtail deadlocks. * * All lock requests for a resource, identified by a hash string, will map * to one bucket. Each bucket maps to one or several peer DBs, each on their @@ -37,7 +36,7 @@ * @ingroup LockManager * @since 1.19 */ -class DBLockManager extends QuorumLockManager { +abstract class DBLockManager extends QuorumLockManager { /** @var Array Map of DB names to server config */ protected $dbServers; // (DB name => server config array) /** @var BagOStuff */ @@ -67,11 +66,12 @@ class DBLockManager extends QuorumLockManager { * each having an odd-numbered list of DB names (peers) as values. * Any DB named 'localDBMaster' will automatically use the DB master * settings for this wiki (without the need for a dbServers entry). + * Only use 'localDBMaster' if the domain is a valid wiki ID. * - lockExpiry : Lock timeout (seconds) for dropped connections. [optional] * This tells the DB server how long to wait before assuming * connection failure and releasing all the locks for a session. * - * @param Array $config + * @param array $config */ public function __construct( array $config ) { parent::__construct( $config ); @@ -111,65 +111,6 @@ class DBLockManager extends QuorumLockManager { } /** - * Get a connection to a lock DB and acquire locks on $paths. - * This does not use GET_LOCK() per http://bugs.mysql.com/bug.php?id=1118. - * - * @see QuorumLockManager::getLocksOnServer() - * @return Status - */ - protected function getLocksOnServer( $lockSrv, array $paths, $type ) { - $status = Status::newGood(); - - if ( $type == self::LOCK_EX ) { // writer locks - try { - $keys = array_unique( array_map( 'LockManager::sha1Base36', $paths ) ); - # Build up values for INSERT clause - $data = array(); - foreach ( $keys as $key ) { - $data[] = array( 'fle_key' => $key ); - } - # Wait on any existing writers and block new ones if we get in - $db = $this->getConnection( $lockSrv ); // checked in isServerUp() - $db->insert( 'filelocks_exclusive', $data, __METHOD__ ); - } catch ( DBError $e ) { - foreach ( $paths as $path ) { - $status->fatal( 'lockmanager-fail-acquirelock', $path ); - } - } - } - - return $status; - } - - /** - * @see QuorumLockManager::freeLocksOnServer() - * @return Status - */ - protected function freeLocksOnServer( $lockSrv, array $paths, $type ) { - return Status::newGood(); // not supported - } - - /** - * @see QuorumLockManager::releaseAllLocks() - * @return Status - */ - protected function releaseAllLocks() { - $status = Status::newGood(); - - foreach ( $this->conns as $lockDb => $db ) { - if ( $db->trxLevel() ) { // in transaction - try { - $db->rollback( __METHOD__ ); // finish transaction and kill any rows - } catch ( DBError $e ) { - $status->fatal( 'lockmanager-fail-db-release', $lockDb ); - } - } - } - - return $status; - } - - /** * @see QuorumLockManager::isServerUp() * @return bool */ @@ -197,8 +138,8 @@ class DBLockManager extends QuorumLockManager { if ( !isset( $this->conns[$lockDb] ) ) { $db = null; if ( $lockDb === 'localDBMaster' ) { - $lb = wfGetLBFactory()->newMainLB(); - $db = $lb->getConnection( DB_MASTER ); + $lb = wfGetLBFactory()->getMainLB( $this->domain ); + $db = $lb->getConnection( DB_MASTER, array(), $this->domain ); } elseif ( isset( $this->dbServers[$lockDb] ) ) { $config = $this->dbServers[$lockDb]; $db = DatabaseBase::factory( $config['type'], $config ); @@ -274,14 +215,8 @@ class DBLockManager extends QuorumLockManager { * Make sure remaining locks get cleared for sanity */ function __destruct() { + $this->releaseAllLocks(); foreach ( $this->conns as $db ) { - if ( $db->trxLevel() ) { // in transaction - try { - $db->rollback( __METHOD__ ); // finish transaction and kill any rows - } catch ( DBError $e ) { - // oh well - } - } $db->close(); } } @@ -321,27 +256,38 @@ class MySqlLockManager extends DBLockManager { $status = Status::newGood(); $db = $this->getConnection( $lockSrv ); // checked in isServerUp() - $keys = array_unique( array_map( 'LockManager::sha1Base36', $paths ) ); + + $keys = array(); // list of hash keys for the paths + $data = array(); // list of rows to insert + $checkEXKeys = array(); // list of hash keys that this has no EX lock on # Build up values for INSERT clause - $data = array(); - foreach ( $keys as $key ) { + foreach ( $paths as $path ) { + $key = $this->sha1Base36Absolute( $path ); + $keys[] = $key; $data[] = array( 'fls_key' => $key, 'fls_session' => $this->session ); + if ( !isset( $this->locksHeld[$path][self::LOCK_EX] ) ) { + $checkEXKeys[] = $key; + } } - # Block new writers... + + # Block new writers (both EX and SH locks leave entries here)... $db->insert( 'filelocks_shared', $data, __METHOD__, array( 'IGNORE' ) ); # Actually do the locking queries... if ( $type == self::LOCK_SH ) { // reader locks + $blocked = false; # Bail if there are any existing writers... - $blocked = $db->selectField( 'filelocks_exclusive', '1', - array( 'fle_key' => $keys ), - __METHOD__ - ); - # Prospective writers that haven't yet updated filelocks_exclusive - # will recheck filelocks_shared after doing so and bail due to our entry. + if ( count( $checkEXKeys ) ) { + $blocked = $db->selectField( 'filelocks_exclusive', '1', + array( 'fle_key' => $checkEXKeys ), + __METHOD__ + ); + } + # Other prospective writers that haven't yet updated filelocks_exclusive + # will recheck filelocks_shared after doing so and bail due to this entry. } else { // writer locks $encSession = $db->addQuotes( $this->session ); # Bail if there are any existing writers... - # The may detect readers, but the safe check for them is below. + # This may detect readers, but the safe check for them is below. # Note: if two writers come at the same time, both bail :) $blocked = $db->selectField( 'filelocks_shared', '1', array( 'fls_key' => $keys, "fls_session != $encSession" ), @@ -371,4 +317,117 @@ class MySqlLockManager extends DBLockManager { return $status; } + + /** + * @see QuorumLockManager::freeLocksOnServer() + * @return Status + */ + protected function freeLocksOnServer( $lockSrv, array $paths, $type ) { + return Status::newGood(); // not supported + } + + /** + * @see QuorumLockManager::releaseAllLocks() + * @return Status + */ + protected function releaseAllLocks() { + $status = Status::newGood(); + + foreach ( $this->conns as $lockDb => $db ) { + if ( $db->trxLevel() ) { // in transaction + try { + $db->rollback( __METHOD__ ); // finish transaction and kill any rows + } catch ( DBError $e ) { + $status->fatal( 'lockmanager-fail-db-release', $lockDb ); + } + } + } + + return $status; + } +} + +/** + * PostgreSQL version of DBLockManager that supports shared locks. + * All locks are non-blocking, which avoids deadlocks. + * + * @ingroup LockManager + */ +class PostgreSqlLockManager extends DBLockManager { + /** @var Array Mapping of lock types to the type actually used */ + protected $lockTypeMap = array( + self::LOCK_SH => self::LOCK_SH, + self::LOCK_UW => self::LOCK_SH, + self::LOCK_EX => self::LOCK_EX + ); + + protected function getLocksOnServer( $lockSrv, array $paths, $type ) { + $status = Status::newGood(); + if ( !count( $paths ) ) { + return $status; // nothing to lock + } + + $db = $this->getConnection( $lockSrv ); // checked in isServerUp() + $bigints = array_unique( array_map( + function( $key ) { return wfBaseConvert( substr( $key, 0, 15 ), 16, 10 ); }, + array_map( array( $this, 'sha1Base16Absolute' ), $paths ) + ) ); + + // Try to acquire all the locks... + $fields = array(); + foreach ( $bigints as $bigint ) { + $fields[] = ( $type == self::LOCK_SH ) + ? "pg_try_advisory_lock_shared({$db->addQuotes( $bigint )}) AS K$bigint" + : "pg_try_advisory_lock({$db->addQuotes( $bigint )}) AS K$bigint"; + } + $res = $db->query( 'SELECT ' . implode( ', ', $fields ), __METHOD__ ); + $row = (array)$res->fetchObject(); + + if ( in_array( 'f', $row ) ) { + // Release any acquired locks if some could not be acquired... + $fields = array(); + foreach ( $row as $kbigint => $ok ) { + if ( $ok === 't' ) { // locked + $bigint = substr( $kbigint, 1 ); // strip off the "K" + $fields[] = ( $type == self::LOCK_SH ) + ? "pg_advisory_unlock_shared({$db->addQuotes( $bigint )})" + : "pg_advisory_unlock({$db->addQuotes( $bigint )})"; + } + } + if ( count( $fields ) ) { + $db->query( 'SELECT ' . implode( ', ', $fields ), __METHOD__ ); + } + foreach ( $paths as $path ) { + $status->fatal( 'lockmanager-fail-acquirelock', $path ); + } + } + + return $status; + } + + /** + * @see QuorumLockManager::freeLocksOnServer() + * @return Status + */ + protected function freeLocksOnServer( $lockSrv, array $paths, $type ) { + return Status::newGood(); // not supported + } + + /** + * @see QuorumLockManager::releaseAllLocks() + * @return Status + */ + protected function releaseAllLocks() { + $status = Status::newGood(); + + foreach ( $this->conns as $lockDb => $db ) { + try { + $db->query( "SELECT pg_advisory_unlock_all()", __METHOD__ ); + } catch ( DBError $e ) { + $status->fatal( 'lockmanager-fail-db-release', $lockDb ); + } + } + + return $status; + } } diff --git a/includes/filebackend/lockmanager/FSLockManager.php b/includes/filebackend/lockmanager/FSLockManager.php index 9a6206fd..eacba704 100644 --- a/includes/filebackend/lockmanager/FSLockManager.php +++ b/includes/filebackend/lockmanager/FSLockManager.php @@ -43,7 +43,7 @@ class FSLockManager extends LockManager { protected $lockDir; // global dir for all servers - /** @var Array Map of (locked key => lock type => lock file handle) */ + /** @var Array Map of (locked key => lock file handle) */ protected $handles = array(); /** @@ -115,12 +115,16 @@ class FSLockManager extends LockManager { } elseif ( isset( $this->locksHeld[$path][self::LOCK_EX] ) ) { $this->locksHeld[$path][$type] = 1; } else { - wfSuppressWarnings(); - $handle = fopen( $this->getLockPath( $path ), 'a+' ); - wfRestoreWarnings(); - if ( !$handle ) { // lock dir missing? - wfMkdirParents( $this->lockDir ); - $handle = fopen( $this->getLockPath( $path ), 'a+' ); // try again + if ( isset( $this->handles[$path] ) ) { + $handle = $this->handles[$path]; + } else { + wfSuppressWarnings(); + $handle = fopen( $this->getLockPath( $path ), 'a+' ); + wfRestoreWarnings(); + if ( !$handle ) { // lock dir missing? + wfMkdirParents( $this->lockDir ); + $handle = fopen( $this->getLockPath( $path ), 'a+' ); // try again + } } if ( $handle ) { // Either a shared or exclusive lock @@ -128,7 +132,7 @@ class FSLockManager extends LockManager { if ( flock( $handle, $lock | LOCK_NB ) ) { // Record this lock as active $this->locksHeld[$path][$type] = 1; - $this->handles[$path][$type] = $handle; + $this->handles[$path] = $handle; } else { fclose( $handle ); $status->fatal( 'lockmanager-fail-acquirelock', $path ); @@ -160,24 +164,13 @@ class FSLockManager extends LockManager { --$this->locksHeld[$path][$type]; if ( $this->locksHeld[$path][$type] <= 0 ) { unset( $this->locksHeld[$path][$type] ); - // If a LOCK_SH comes in while we have a LOCK_EX, we don't - // actually add a handler, so check for handler existence. - if ( isset( $this->handles[$path][$type] ) ) { - if ( $type === self::LOCK_EX - && isset( $this->locksHeld[$path][self::LOCK_SH] ) - && !isset( $this->handles[$path][self::LOCK_SH] ) ) - { - // EX lock came first: move this handle to the SH one - $this->handles[$path][self::LOCK_SH] = $this->handles[$path][$type]; - } else { - // Mark this handle to be unlocked and closed - $handlesToClose[] = $this->handles[$path][$type]; - } - unset( $this->handles[$path][$type] ); - } } if ( !count( $this->locksHeld[$path] ) ) { unset( $this->locksHeld[$path] ); // no locks on this path + if ( isset( $this->handles[$path] ) ) { + $handlesToClose[] = $this->handles[$path]; + unset( $this->handles[$path] ); + } } // Unlock handles to release locks and delete // any lock files that end up with no locks on them... @@ -237,8 +230,7 @@ class FSLockManager extends LockManager { * @return string */ protected function getLockPath( $path ) { - $hash = self::sha1Base36( $path ); - return "{$this->lockDir}/{$hash}.lock"; + return "{$this->lockDir}/{$this->sha1Base36Absolute( $path )}.lock"; } /** diff --git a/includes/filebackend/lockmanager/LSLockManager.php b/includes/filebackend/lockmanager/LSLockManager.php index 89428182..97de8dca 100644 --- a/includes/filebackend/lockmanager/LSLockManager.php +++ b/includes/filebackend/lockmanager/LSLockManager.php @@ -66,7 +66,7 @@ class LSLockManager extends QuorumLockManager { * each having an odd-numbered list of server names (peers) as values. * - connTimeout : Lock server connection attempt timeout. [optional] * - * @param Array $config + * @param array $config */ public function __construct( array $config ) { parent::__construct( $config ); @@ -94,7 +94,7 @@ class LSLockManager extends QuorumLockManager { // Send out the command and get the response... $type = ( $type == self::LOCK_SH ) ? 'SH' : 'EX'; - $keys = array_unique( array_map( 'LockManager::sha1Base36', $paths ) ); + $keys = array_unique( array_map( array( $this, 'sha1Base36Absolute' ), $paths ) ); $response = $this->sendCommand( $lockSrv, 'ACQUIRE', $type, $keys ); if ( $response !== 'ACQUIRED' ) { @@ -115,7 +115,7 @@ class LSLockManager extends QuorumLockManager { // Send out the command and get the response... $type = ( $type == self::LOCK_SH ) ? 'SH' : 'EX'; - $keys = array_unique( array_map( 'LockManager::sha1Base36', $paths ) ); + $keys = array_unique( array_map( array( $this, 'sha1Base36Absolute' ), $paths ) ); $response = $this->sendCommand( $lockSrv, 'RELEASE', $type, $keys ); if ( $response !== 'RELEASED' ) { @@ -169,7 +169,7 @@ class LSLockManager extends QuorumLockManager { $authKey = $this->lockServers[$lockSrv]['authKey']; // Build of the command as a flat string... $values = implode( '|', $values ); - $key = sha1( $this->session . $action . $type . $values . $authKey ); + $key = hash_hmac( 'sha1', "{$this->session}\n{$action}\n{$type}\n{$values}", $authKey ); // Send out the command... if ( fwrite( $conn, "{$this->session}:$key:$action:$type:$values\n" ) === false ) { return false; diff --git a/includes/filebackend/lockmanager/LockManager.php b/includes/filebackend/lockmanager/LockManager.php index 07853f87..0512a01b 100644 --- a/includes/filebackend/lockmanager/LockManager.php +++ b/includes/filebackend/lockmanager/LockManager.php @@ -53,6 +53,9 @@ abstract class LockManager { /** @var Array Map of (resource path => lock type => count) */ protected $locksHeld = array(); + protected $domain; // string; domain (usually wiki ID) + protected $lockTTL; // integer; maximum time locks can be held + /* Lock types; stronger locks have higher values */ const LOCK_SH = 1; // shared lock (for reads) const LOCK_UW = 2; // shared lock (for reads used to write elsewhere) @@ -61,14 +64,29 @@ abstract class LockManager { /** * Construct a new instance from configuration * + * $config paramaters include: + * - domain : Domain (usually wiki ID) that all resources are relative to [optional] + * - lockTTL : Age (in seconds) at which resource locks should expire. + * This only applies if locks are not tied to a connection/process. + * * @param $config Array */ - public function __construct( array $config ) {} + public function __construct( array $config ) { + $this->domain = isset( $config['domain'] ) ? $config['domain'] : wfWikiID(); + if ( isset( $config['lockTTL'] ) ) { + $this->lockTTL = max( 1, $config['lockTTL'] ); + } elseif ( PHP_SAPI === 'cli' ) { + $this->lockTTL = 2*3600; + } else { + $met = ini_get( 'max_execution_time' ); // this is 0 in CLI mode + $this->lockTTL = max( 5*60, 2*(int)$met ); + } + } /** * Lock the resources at the given abstract paths * - * @param $paths Array List of resource names + * @param array $paths List of resource names * @param $type integer LockManager::LOCK_* constant * @return Status */ @@ -82,7 +100,7 @@ abstract class LockManager { /** * Unlock the resources at the given abstract paths * - * @param $paths Array List of storage paths + * @param array $paths List of storage paths * @param $type integer LockManager::LOCK_* constant * @return Status */ @@ -94,308 +112,46 @@ abstract class LockManager { } /** - * Get the base 36 SHA-1 of a string, padded to 31 digits + * Get the base 36 SHA-1 of a string, padded to 31 digits. + * Before hashing, the path will be prefixed with the domain ID. + * This should be used interally for lock key or file names. * * @param $path string * @return string */ - final protected static function sha1Base36( $path ) { - return wfBaseConvert( sha1( $path ), 16, 36, 31 ); + final protected function sha1Base36Absolute( $path ) { + return wfBaseConvert( sha1( "{$this->domain}:{$path}" ), 16, 36, 31 ); } /** - * Lock resources with the given keys and lock type + * Get the base 16 SHA-1 of a string, padded to 31 digits. + * Before hashing, the path will be prefixed with the domain ID. + * This should be used interally for lock key or file names. * - * @param $paths Array List of storage paths - * @param $type integer LockManager::LOCK_* constant + * @param $path string * @return string */ - abstract protected function doLock( array $paths, $type ); + final protected function sha1Base16Absolute( $path ) { + return sha1( "{$this->domain}:{$path}" ); + } /** - * Unlock resources with the given keys and lock type + * Lock resources with the given keys and lock type * - * @param $paths Array List of storage paths + * @param array $paths List of storage paths * @param $type integer LockManager::LOCK_* constant * @return string */ - abstract protected function doUnlock( array $paths, $type ); -} - -/** - * Self-releasing locks - * - * LockManager helper class to handle scoped locks, which - * release when an object is destroyed or goes out of scope. - * - * @ingroup LockManager - * @since 1.19 - */ -class ScopedLock { - /** @var LockManager */ - protected $manager; - /** @var Status */ - protected $status; - /** @var Array List of resource paths*/ - protected $paths; - - protected $type; // integer lock type - - /** - * @param $manager LockManager - * @param $paths Array List of storage paths - * @param $type integer LockManager::LOCK_* constant - * @param $status Status - */ - protected function __construct( - LockManager $manager, array $paths, $type, Status $status - ) { - $this->manager = $manager; - $this->paths = $paths; - $this->status = $status; - $this->type = $type; - } + abstract protected function doLock( array $paths, $type ); /** - * Get a ScopedLock object representing a lock on resource paths. - * Any locks are released once this object goes out of scope. - * The status object is updated with any errors or warnings. + * Unlock resources with the given keys and lock type * - * @param $manager LockManager - * @param $paths Array List of storage paths + * @param array $paths List of storage paths * @param $type integer LockManager::LOCK_* constant - * @param $status Status - * @return ScopedLock|null Returns null on failure - */ - public static function factory( - LockManager $manager, array $paths, $type, Status $status - ) { - $lockStatus = $manager->lock( $paths, $type ); - $status->merge( $lockStatus ); - if ( $lockStatus->isOK() ) { - return new self( $manager, $paths, $type, $status ); - } - return null; - } - - function __destruct() { - $wasOk = $this->status->isOK(); - $this->status->merge( $this->manager->unlock( $this->paths, $this->type ) ); - if ( $wasOk ) { - // Make sure status is OK, despite any unlockFiles() fatals - $this->status->setResult( true, $this->status->value ); - } - } -} - -/** - * Version of LockManager that uses a quorum from peer servers for locks. - * The resource space can also be sharded into separate peer groups. - * - * @ingroup LockManager - * @since 1.20 - */ -abstract class QuorumLockManager extends LockManager { - /** @var Array Map of bucket indexes to peer server lists */ - protected $srvsByBucket = array(); // (bucket index => (lsrv1, lsrv2, ...)) - - /** - * @see LockManager::doLock() - * @param $paths array - * @param $type int - * @return Status - */ - final protected function doLock( array $paths, $type ) { - $status = Status::newGood(); - - $pathsToLock = array(); // (bucket => paths) - // Get locks that need to be acquired (buckets => locks)... - foreach ( $paths as $path ) { - if ( isset( $this->locksHeld[$path][$type] ) ) { - ++$this->locksHeld[$path][$type]; - } elseif ( isset( $this->locksHeld[$path][self::LOCK_EX] ) ) { - $this->locksHeld[$path][$type] = 1; - } else { - $bucket = $this->getBucketFromKey( $path ); - $pathsToLock[$bucket][] = $path; - } - } - - $lockedPaths = array(); // files locked in this attempt - // Attempt to acquire these locks... - foreach ( $pathsToLock as $bucket => $paths ) { - // Try to acquire the locks for this bucket - $status->merge( $this->doLockingRequestBucket( $bucket, $paths, $type ) ); - if ( !$status->isOK() ) { - $status->merge( $this->doUnlock( $lockedPaths, $type ) ); - return $status; - } - // Record these locks as active - foreach ( $paths as $path ) { - $this->locksHeld[$path][$type] = 1; // locked - } - // Keep track of what locks were made in this attempt - $lockedPaths = array_merge( $lockedPaths, $paths ); - } - - return $status; - } - - /** - * @see LockManager::doUnlock() - * @param $paths array - * @param $type int - * @return Status - */ - final protected function doUnlock( array $paths, $type ) { - $status = Status::newGood(); - - $pathsToUnlock = array(); - foreach ( $paths as $path ) { - if ( !isset( $this->locksHeld[$path][$type] ) ) { - $status->warning( 'lockmanager-notlocked', $path ); - } else { - --$this->locksHeld[$path][$type]; - // Reference count the locks held and release locks when zero - if ( $this->locksHeld[$path][$type] <= 0 ) { - unset( $this->locksHeld[$path][$type] ); - $bucket = $this->getBucketFromKey( $path ); - $pathsToUnlock[$bucket][] = $path; - } - if ( !count( $this->locksHeld[$path] ) ) { - unset( $this->locksHeld[$path] ); // no SH or EX locks left for key - } - } - } - - // Remove these specific locks if possible, or at least release - // all locks once this process is currently not holding any locks. - foreach ( $pathsToUnlock as $bucket => $paths ) { - $status->merge( $this->doUnlockingRequestBucket( $bucket, $paths, $type ) ); - } - if ( !count( $this->locksHeld ) ) { - $status->merge( $this->releaseAllLocks() ); - } - - return $status; - } - - /** - * Attempt to acquire locks with the peers for a bucket. - * This is all or nothing; if any key is locked then this totally fails. - * - * @param $bucket integer - * @param $paths Array List of resource keys to lock - * @param $type integer LockManager::LOCK_EX or LockManager::LOCK_SH - * @return Status - */ - final protected function doLockingRequestBucket( $bucket, array $paths, $type ) { - $status = Status::newGood(); - - $yesVotes = 0; // locks made on trustable servers - $votesLeft = count( $this->srvsByBucket[$bucket] ); // remaining peers - $quorum = floor( $votesLeft/2 + 1 ); // simple majority - // Get votes for each peer, in order, until we have enough... - foreach ( $this->srvsByBucket[$bucket] as $lockSrv ) { - if ( !$this->isServerUp( $lockSrv ) ) { - --$votesLeft; - $status->warning( 'lockmanager-fail-svr-acquire', $lockSrv ); - continue; // server down? - } - // Attempt to acquire the lock on this peer - $status->merge( $this->getLocksOnServer( $lockSrv, $paths, $type ) ); - if ( !$status->isOK() ) { - return $status; // vetoed; resource locked - } - ++$yesVotes; // success for this peer - if ( $yesVotes >= $quorum ) { - return $status; // lock obtained - } - --$votesLeft; - $votesNeeded = $quorum - $yesVotes; - if ( $votesNeeded > $votesLeft ) { - break; // short-circuit - } - } - // At this point, we must not have met the quorum - $status->setResult( false ); - - return $status; - } - - /** - * Attempt to release locks with the peers for a bucket - * - * @param $bucket integer - * @param $paths Array List of resource keys to lock - * @param $type integer LockManager::LOCK_EX or LockManager::LOCK_SH - * @return Status - */ - final protected function doUnlockingRequestBucket( $bucket, array $paths, $type ) { - $status = Status::newGood(); - - foreach ( $this->srvsByBucket[$bucket] as $lockSrv ) { - if ( !$this->isServerUp( $lockSrv ) ) { - $status->fatal( 'lockmanager-fail-svr-release', $lockSrv ); - // Attempt to release the lock on this peer - } else { - $status->merge( $this->freeLocksOnServer( $lockSrv, $paths, $type ) ); - } - } - - return $status; - } - - /** - * Get the bucket for resource path. - * This should avoid throwing any exceptions. - * - * @param $path string - * @return integer - */ - protected function getBucketFromKey( $path ) { - $prefix = substr( sha1( $path ), 0, 2 ); // first 2 hex chars (8 bits) - return (int)base_convert( $prefix, 16, 10 ) % count( $this->srvsByBucket ); - } - - /** - * Check if a lock server is up - * - * @param $lockSrv string - * @return bool - */ - abstract protected function isServerUp( $lockSrv ); - - /** - * Get a connection to a lock server and acquire locks on $paths - * - * @param $lockSrv string - * @param $paths array - * @param $type integer - * @return Status - */ - abstract protected function getLocksOnServer( $lockSrv, array $paths, $type ); - - /** - * Get a connection to a lock server and release locks on $paths. - * - * Subclasses must effectively implement this or releaseAllLocks(). - * - * @param $lockSrv string - * @param $paths array - * @param $type integer - * @return Status - */ - abstract protected function freeLocksOnServer( $lockSrv, array $paths, $type ); - - /** - * Release all locks that this session is holding. - * - * Subclasses must effectively implement this or freeLocksOnServer(). - * - * @return Status + * @return string */ - abstract protected function releaseAllLocks(); + abstract protected function doUnlock( array $paths, $type ); } /** diff --git a/includes/filebackend/lockmanager/LockManagerGroup.php b/includes/filebackend/lockmanager/LockManagerGroup.php index 8c8c940a..ac0bd49b 100644 --- a/includes/filebackend/lockmanager/LockManagerGroup.php +++ b/includes/filebackend/lockmanager/LockManagerGroup.php @@ -29,33 +29,41 @@ * @since 1.19 */ class LockManagerGroup { - /** - * @var LockManagerGroup - */ - protected static $instance = null; + /** @var Array (domain => LockManager) */ + protected static $instances = array(); + + protected $domain; // string; domain (usually wiki ID) - /** @var Array of (name => ('class' =>, 'config' =>, 'instance' =>)) */ + /** @var Array of (name => ('class' => ..., 'config' => ..., 'instance' => ...)) */ protected $managers = array(); - protected function __construct() {} + /** + * @param string $domain Domain (usually wiki ID) + */ + protected function __construct( $domain ) { + $this->domain = $domain; + } /** + * @param string $domain Domain (usually wiki ID) * @return LockManagerGroup */ - public static function singleton() { - if ( self::$instance == null ) { - self::$instance = new self(); - self::$instance->initFromGlobals(); + public static function singleton( $domain = false ) { + $domain = ( $domain === false ) ? wfWikiID() : $domain; + if ( !isset( self::$instances[$domain] ) ) { + self::$instances[$domain] = new self( $domain ); + self::$instances[$domain]->initFromGlobals(); } - return self::$instance; + return self::$instances[$domain]; } /** - * Destroy the singleton instance, so that a new one will be created next - * time singleton() is called. + * Destroy the singleton instances + * + * @return void */ - public static function destroySingleton() { - self::$instance = null; + public static function destroySingletons() { + self::$instances = array(); } /** @@ -78,6 +86,7 @@ class LockManagerGroup { */ protected function register( array $configs ) { foreach ( $configs as $config ) { + $config['domain'] = $this->domain; if ( !isset( $config['name'] ) ) { throw new MWException( "Cannot register a lock manager with no name." ); } @@ -116,6 +125,21 @@ class LockManagerGroup { } /** + * Get the config array for a lock manager object with a given name + * + * @param $name string + * @return Array + * @throws MWException + */ + public function config( $name ) { + if ( !isset( $this->managers[$name] ) ) { + throw new MWException( "No lock manager defined with the name `$name`." ); + } + $class = $this->managers[$name]['class']; + return array( 'class' => $class ) + $this->managers[$name]['config']; + } + + /** * Get the default lock manager configured for the site. * Returns NullLockManager if no lock manager could be found. * diff --git a/includes/filebackend/lockmanager/MemcLockManager.php b/includes/filebackend/lockmanager/MemcLockManager.php index 57c0463d..fafc588a 100644 --- a/includes/filebackend/lockmanager/MemcLockManager.php +++ b/includes/filebackend/lockmanager/MemcLockManager.php @@ -28,8 +28,8 @@ * This is meant for multi-wiki systems that may share files. * All locks are non-blocking, which avoids deadlocks. * - * All lock requests for a resource, identified by a hash string, will map - * to one bucket. Each bucket maps to one or several peer servers, each running memcached. + * All lock requests for a resource, identified by a hash string, will map to one + * bucket. Each bucket maps to one or several peer servers, each running memcached. * A majority of peers must agree for a lock to be acquired. * * @ingroup LockManager @@ -48,9 +48,7 @@ class MemcLockManager extends QuorumLockManager { /** @var Array */ protected $serversUp = array(); // (server name => bool) - protected $lockExpiry; // integer; maximum time locks can be held - protected $session = ''; // string; random SHA-1 UUID - protected $wikiId = ''; // string + protected $session = ''; // string; random UUID /** * Construct a new instance from configuration. @@ -61,9 +59,9 @@ class MemcLockManager extends QuorumLockManager { * each having an odd-numbered list of server names (peers) as values. * - memcConfig : Configuration array for ObjectCache::newFromParams. [optional] * If set, this must use one of the memcached classes. - * - wikiId : Wiki ID string that all resources are relative to. [optional] * - * @param Array $config + * @param array $config + * @throws MWException */ public function __construct( array $config ) { parent::__construct( $config ); @@ -87,11 +85,6 @@ class MemcLockManager extends QuorumLockManager { } } - $this->wikiId = isset( $config['wikiId'] ) ? $config['wikiId'] : wfWikiID(); - - $met = ini_get( 'max_execution_time' ); // this is 0 in CLI mode - $this->lockExpiry = $met ? 2*(int)$met : 2*3600; - $this->session = wfRandomString( 32 ); } @@ -110,7 +103,7 @@ class MemcLockManager extends QuorumLockManager { foreach ( $paths as $path ) { $status->fatal( 'lockmanager-fail-acquirelock', $path ); } - return; + return $status; } // Fetch all the existing lock records... @@ -121,8 +114,8 @@ class MemcLockManager extends QuorumLockManager { foreach ( $paths as $path ) { $locksKey = $this->recordKeyForPath( $path ); $locksHeld = isset( $lockRecords[$locksKey] ) - ? $lockRecords[$locksKey] - : array( self::LOCK_SH => array(), self::LOCK_EX => array() ); // init + ? self::sanitizeLockArray( $lockRecords[$locksKey] ) + : self::newLockArray(); // init foreach ( $locksHeld[self::LOCK_EX] as $session => $expiry ) { if ( $expiry < $now ) { // stale? unset( $locksHeld[self::LOCK_EX][$session] ); @@ -141,7 +134,7 @@ class MemcLockManager extends QuorumLockManager { } if ( $status->isOK() ) { // Register the session in the lock record array - $locksHeld[$type][$this->session] = $now + $this->lockExpiry; + $locksHeld[$type][$this->session] = $now + $this->lockTTL; // We will update this record if none of the other locks conflict $lockRecords[$locksKey] = $locksHeld; } @@ -149,9 +142,15 @@ class MemcLockManager extends QuorumLockManager { // If there were no lock conflicts, update all the lock records... if ( $status->isOK() ) { - foreach ( $lockRecords as $locksKey => $locksHeld ) { - $memc->set( $locksKey, $locksHeld ); - wfDebug( __METHOD__ . ": acquired lock on key $locksKey.\n" ); + foreach ( $paths as $path ) { + $locksKey = $this->recordKeyForPath( $path ); + $locksHeld = $lockRecords[$locksKey]; + $ok = $memc->set( $locksKey, $locksHeld, 7*86400 ); + if ( !$ok ) { + $status->fatal( 'lockmanager-fail-acquirelock', $path ); + } else { + wfDebug( __METHOD__ . ": acquired lock on key $locksKey.\n" ); + } } } @@ -186,17 +185,22 @@ class MemcLockManager extends QuorumLockManager { foreach ( $paths as $path ) { $locksKey = $this->recordKeyForPath( $path ); // lock record if ( !isset( $lockRecords[$locksKey] ) ) { + $status->warning( 'lockmanager-fail-releaselock', $path ); continue; // nothing to do } - $locksHeld = $lockRecords[$locksKey]; - if ( is_array( $locksHeld ) && isset( $locksHeld[$type] ) ) { - unset( $locksHeld[$type][$this->session] ); - $ok = $memc->set( $locksKey, $locksHeld ); + $locksHeld = self::sanitizeLockArray( $lockRecords[$locksKey] ); + if ( isset( $locksHeld[$type][$this->session] ) ) { + unset( $locksHeld[$type][$this->session] ); // unregister this session + if ( $locksHeld === self::newLockArray() ) { + $ok = $memc->delete( $locksKey ); + } else { + $ok = $memc->set( $locksKey, $locksHeld ); + } + if ( !$ok ) { + $status->fatal( 'lockmanager-fail-releaselock', $path ); + } } else { - $ok = true; - } - if ( !$ok ) { - $status->fatal( 'lockmanager-fail-releaselock', $path ); + $status->warning( 'lockmanager-fail-releaselock', $path ); } wfDebug( __METHOD__ . ": released lock on key $locksKey.\n" ); } @@ -226,7 +230,7 @@ class MemcLockManager extends QuorumLockManager { /** * Get the MemcachedBagOStuff object for a $lockSrv * - * @param $lockSrv string Server name + * @param string $lockSrv Server name * @return MemcachedBagOStuff|null */ protected function getCache( $lockSrv ) { @@ -234,7 +238,7 @@ class MemcLockManager extends QuorumLockManager { if ( isset( $this->bagOStuffs[$lockSrv] ) ) { $memc = $this->bagOStuffs[$lockSrv]; if ( !isset( $this->serversUp[$lockSrv] ) ) { - $this->serversUp[$lockSrv] = $memc->set( 'MemcLockManager:ping', 1, 1 ); + $this->serversUp[$lockSrv] = $memc->set( __CLASS__ . ':ping', 1, 1 ); if ( !$this->serversUp[$lockSrv] ) { trigger_error( __METHOD__ . ": Could not contact $lockSrv.", E_USER_WARNING ); } @@ -251,14 +255,32 @@ class MemcLockManager extends QuorumLockManager { * @return string */ protected function recordKeyForPath( $path ) { - $hash = LockManager::sha1Base36( $path ); - list( $db, $prefix ) = wfSplitWikiID( $this->wikiId ); - return wfForeignMemcKey( $db, $prefix, __CLASS__, 'locks', $hash ); + return implode( ':', array( __CLASS__, 'locks', $this->sha1Base36Absolute( $path ) ) ); + } + + /** + * @return Array An empty lock structure for a key + */ + protected static function newLockArray() { + return array( self::LOCK_SH => array(), self::LOCK_EX => array() ); + } + + /** + * @param $a array + * @return Array An empty lock structure for a key + */ + protected static function sanitizeLockArray( $a ) { + if ( is_array( $a ) && isset( $a[self::LOCK_EX] ) && isset( $a[self::LOCK_SH] ) ) { + return $a; + } else { + trigger_error( __METHOD__ . ": reset invalid lock array.", E_USER_WARNING ); + return self::newLockArray(); + } } /** * @param $memc MemcachedBagOStuff - * @param $keys Array List of keys to acquire + * @param array $keys List of keys to acquire * @return bool */ protected function acquireMutexes( MemcachedBagOStuff $memc, array $keys ) { @@ -284,10 +306,10 @@ class MemcLockManager extends QuorumLockManager { continue; // acquire in order } } - } while ( count( $lockedKeys ) < count( $keys ) && ( microtime( true ) - $start ) <= 6 ); + } while ( count( $lockedKeys ) < count( $keys ) && ( microtime( true ) - $start ) <= 3 ); if ( count( $lockedKeys ) != count( $keys ) ) { - $this->releaseMutexes( $lockedKeys ); // failed; release what was locked + $this->releaseMutexes( $memc, $lockedKeys ); // failed; release what was locked return false; } @@ -296,7 +318,7 @@ class MemcLockManager extends QuorumLockManager { /** * @param $memc MemcachedBagOStuff - * @param $keys Array List of acquired keys + * @param array $keys List of acquired keys * @return void */ protected function releaseMutexes( MemcachedBagOStuff $memc, array $keys ) { diff --git a/includes/filebackend/lockmanager/QuorumLockManager.php b/includes/filebackend/lockmanager/QuorumLockManager.php new file mode 100644 index 00000000..b331b540 --- /dev/null +++ b/includes/filebackend/lockmanager/QuorumLockManager.php @@ -0,0 +1,230 @@ +<?php +/** + * Version of LockManager that uses a quorum from peer servers for locks. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup LockManager + */ + +/** + * Version of LockManager that uses a quorum from peer servers for locks. + * The resource space can also be sharded into separate peer groups. + * + * @ingroup LockManager + * @since 1.20 + */ +abstract class QuorumLockManager extends LockManager { + /** @var Array Map of bucket indexes to peer server lists */ + protected $srvsByBucket = array(); // (bucket index => (lsrv1, lsrv2, ...)) + + /** + * @see LockManager::doLock() + * @param $paths array + * @param $type int + * @return Status + */ + final protected function doLock( array $paths, $type ) { + $status = Status::newGood(); + + $pathsToLock = array(); // (bucket => paths) + // Get locks that need to be acquired (buckets => locks)... + foreach ( $paths as $path ) { + if ( isset( $this->locksHeld[$path][$type] ) ) { + ++$this->locksHeld[$path][$type]; + } else { + $bucket = $this->getBucketFromPath( $path ); + $pathsToLock[$bucket][] = $path; + } + } + + $lockedPaths = array(); // files locked in this attempt + // Attempt to acquire these locks... + foreach ( $pathsToLock as $bucket => $paths ) { + // Try to acquire the locks for this bucket + $status->merge( $this->doLockingRequestBucket( $bucket, $paths, $type ) ); + if ( !$status->isOK() ) { + $status->merge( $this->doUnlock( $lockedPaths, $type ) ); + return $status; + } + // Record these locks as active + foreach ( $paths as $path ) { + $this->locksHeld[$path][$type] = 1; // locked + } + // Keep track of what locks were made in this attempt + $lockedPaths = array_merge( $lockedPaths, $paths ); + } + + return $status; + } + + /** + * @see LockManager::doUnlock() + * @param $paths array + * @param $type int + * @return Status + */ + final protected function doUnlock( array $paths, $type ) { + $status = Status::newGood(); + + $pathsToUnlock = array(); + foreach ( $paths as $path ) { + if ( !isset( $this->locksHeld[$path][$type] ) ) { + $status->warning( 'lockmanager-notlocked', $path ); + } else { + --$this->locksHeld[$path][$type]; + // Reference count the locks held and release locks when zero + if ( $this->locksHeld[$path][$type] <= 0 ) { + unset( $this->locksHeld[$path][$type] ); + $bucket = $this->getBucketFromPath( $path ); + $pathsToUnlock[$bucket][] = $path; + } + if ( !count( $this->locksHeld[$path] ) ) { + unset( $this->locksHeld[$path] ); // no SH or EX locks left for key + } + } + } + + // Remove these specific locks if possible, or at least release + // all locks once this process is currently not holding any locks. + foreach ( $pathsToUnlock as $bucket => $paths ) { + $status->merge( $this->doUnlockingRequestBucket( $bucket, $paths, $type ) ); + } + if ( !count( $this->locksHeld ) ) { + $status->merge( $this->releaseAllLocks() ); + } + + return $status; + } + + /** + * Attempt to acquire locks with the peers for a bucket. + * This is all or nothing; if any key is locked then this totally fails. + * + * @param $bucket integer + * @param array $paths List of resource keys to lock + * @param $type integer LockManager::LOCK_EX or LockManager::LOCK_SH + * @return Status + */ + final protected function doLockingRequestBucket( $bucket, array $paths, $type ) { + $status = Status::newGood(); + + $yesVotes = 0; // locks made on trustable servers + $votesLeft = count( $this->srvsByBucket[$bucket] ); // remaining peers + $quorum = floor( $votesLeft/2 + 1 ); // simple majority + // Get votes for each peer, in order, until we have enough... + foreach ( $this->srvsByBucket[$bucket] as $lockSrv ) { + if ( !$this->isServerUp( $lockSrv ) ) { + --$votesLeft; + $status->warning( 'lockmanager-fail-svr-acquire', $lockSrv ); + continue; // server down? + } + // Attempt to acquire the lock on this peer + $status->merge( $this->getLocksOnServer( $lockSrv, $paths, $type ) ); + if ( !$status->isOK() ) { + return $status; // vetoed; resource locked + } + ++$yesVotes; // success for this peer + if ( $yesVotes >= $quorum ) { + return $status; // lock obtained + } + --$votesLeft; + $votesNeeded = $quorum - $yesVotes; + if ( $votesNeeded > $votesLeft ) { + break; // short-circuit + } + } + // At this point, we must not have met the quorum + $status->setResult( false ); + + return $status; + } + + /** + * Attempt to release locks with the peers for a bucket + * + * @param $bucket integer + * @param array $paths List of resource keys to lock + * @param $type integer LockManager::LOCK_EX or LockManager::LOCK_SH + * @return Status + */ + final protected function doUnlockingRequestBucket( $bucket, array $paths, $type ) { + $status = Status::newGood(); + + foreach ( $this->srvsByBucket[$bucket] as $lockSrv ) { + if ( !$this->isServerUp( $lockSrv ) ) { + $status->fatal( 'lockmanager-fail-svr-release', $lockSrv ); + // Attempt to release the lock on this peer + } else { + $status->merge( $this->freeLocksOnServer( $lockSrv, $paths, $type ) ); + } + } + + return $status; + } + + /** + * Get the bucket for resource path. + * This should avoid throwing any exceptions. + * + * @param $path string + * @return integer + */ + protected function getBucketFromPath( $path ) { + $prefix = substr( sha1( $path ), 0, 2 ); // first 2 hex chars (8 bits) + return (int)base_convert( $prefix, 16, 10 ) % count( $this->srvsByBucket ); + } + + /** + * Check if a lock server is up + * + * @param $lockSrv string + * @return bool + */ + abstract protected function isServerUp( $lockSrv ); + + /** + * Get a connection to a lock server and acquire locks on $paths + * + * @param $lockSrv string + * @param $paths array + * @param $type integer + * @return Status + */ + abstract protected function getLocksOnServer( $lockSrv, array $paths, $type ); + + /** + * Get a connection to a lock server and release locks on $paths. + * + * Subclasses must effectively implement this or releaseAllLocks(). + * + * @param $lockSrv string + * @param $paths array + * @param $type integer + * @return Status + */ + abstract protected function freeLocksOnServer( $lockSrv, array $paths, $type ); + + /** + * Release all locks that this session is holding. + * + * Subclasses must effectively implement this or freeLocksOnServer(). + * + * @return Status + */ + abstract protected function releaseAllLocks(); +} diff --git a/includes/filebackend/lockmanager/ScopedLock.php b/includes/filebackend/lockmanager/ScopedLock.php new file mode 100644 index 00000000..edcb1d65 --- /dev/null +++ b/includes/filebackend/lockmanager/ScopedLock.php @@ -0,0 +1,102 @@ +<?php +/** + * Resource locking handling. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup LockManager + * @author Aaron Schulz + */ + +/** + * Self-releasing locks + * + * LockManager helper class to handle scoped locks, which + * release when an object is destroyed or goes out of scope. + * + * @ingroup LockManager + * @since 1.19 + */ +class ScopedLock { + /** @var LockManager */ + protected $manager; + /** @var Status */ + protected $status; + /** @var Array List of resource paths*/ + protected $paths; + + protected $type; // integer lock type + + /** + * @param $manager LockManager + * @param array $paths List of storage paths + * @param $type integer LockManager::LOCK_* constant + * @param $status Status + */ + protected function __construct( + LockManager $manager, array $paths, $type, Status $status + ) { + $this->manager = $manager; + $this->paths = $paths; + $this->status = $status; + $this->type = $type; + } + + /** + * Get a ScopedLock object representing a lock on resource paths. + * Any locks are released once this object goes out of scope. + * The status object is updated with any errors or warnings. + * + * @param $manager LockManager + * @param array $paths List of storage paths + * @param $type integer LockManager::LOCK_* constant + * @param $status Status + * @return ScopedLock|null Returns null on failure + */ + public static function factory( + LockManager $manager, array $paths, $type, Status $status + ) { + $lockStatus = $manager->lock( $paths, $type ); + $status->merge( $lockStatus ); + if ( $lockStatus->isOK() ) { + return new self( $manager, $paths, $type, $status ); + } + return null; + } + + /** + * Release a scoped lock and set any errors in the attatched Status object. + * This is useful for early release of locks before function scope is destroyed. + * This is the same as setting the lock object to null. + * + * @param ScopedLock $lock + * @return void + * @since 1.21 + */ + public static function release( ScopedLock &$lock = null ) { + $lock = null; + } + + function __destruct() { + $wasOk = $this->status->isOK(); + $this->status->merge( $this->manager->unlock( $this->paths, $this->type ) ); + if ( $wasOk ) { + // Make sure status is OK, despite any unlockFiles() fatals + $this->status->setResult( true, $this->status->value ); + } + } +} |