From 4ac9fa081a7c045f6a9f1cfc529d82423f485b2e Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 8 Dec 2013 09:55:49 +0100 Subject: Update to MediaWiki 1.22.0 --- includes/filebackend/FSFile.php | 29 +- includes/filebackend/FSFileBackend.php | 117 +---- includes/filebackend/FileBackend.php | 143 +++-- includes/filebackend/FileBackendGroup.php | 28 +- includes/filebackend/FileBackendMultiWrite.php | 179 ++----- includes/filebackend/FileBackendStore.php | 576 ++++++--------------- includes/filebackend/FileOp.php | 193 +++---- includes/filebackend/FileOpBatch.php | 6 +- includes/filebackend/README | 2 +- includes/filebackend/SwiftFileBackend.php | 262 +++++----- includes/filebackend/TempFSFile.php | 6 +- includes/filebackend/filejournal/DBFileJournal.php | 10 +- includes/filebackend/lockmanager/DBLockManager.php | 37 +- includes/filebackend/lockmanager/LockManager.php | 125 ++++- .../filebackend/lockmanager/LockManagerGroup.php | 4 +- .../filebackend/lockmanager/MemcLockManager.php | 41 +- .../filebackend/lockmanager/QuorumLockManager.php | 140 ++--- .../filebackend/lockmanager/RedisLockManager.php | 288 +++++++++++ includes/filebackend/lockmanager/ScopedLock.php | 44 +- 19 files changed, 1106 insertions(+), 1124 deletions(-) create mode 100644 includes/filebackend/lockmanager/RedisLockManager.php (limited to 'includes/filebackend') diff --git a/includes/filebackend/FSFile.php b/includes/filebackend/FSFile.php index 7d0dbd52..8f0a1334 100644 --- a/includes/filebackend/FSFile.php +++ b/includes/filebackend/FSFile.php @@ -28,7 +28,7 @@ */ class FSFile { protected $path; // path to file - private $sha1Base36 = null; // File Sha1Base36 + protected $sha1Base36; // file SHA-1 in base 36 /** * Sets up the file object @@ -98,7 +98,7 @@ class FSFile { * Get an associative array containing information about * a file with the given storage path. * - * @param $ext Mixed: the file extension, or true to extract it from the filename. + * @param Mixed $ext: the file extension, or true to extract it from the filename. * Set it to false to ignore the extension. * * @return array @@ -171,7 +171,7 @@ class FSFile { /** * Exract image size information * - * @param $gis array + * @param array $gis * @return Array */ protected function extractImageSizeInfo( array $gis ) { @@ -194,7 +194,7 @@ class FSFile { * 160 log 2 / log 36 = 30.95, so the 160-bit hash fills 31 digits in base 36 * fairly neatly. * - * @param $recache bool + * @param bool $recache * @return bool|string False on failure */ public function getSha1Base36( $recache = false ) { @@ -220,7 +220,7 @@ class FSFile { /** * Get the final file extension from a file system path * - * @param $path string + * @param string $path * @return string */ public static function extensionFromPath( $path ) { @@ -232,9 +232,8 @@ class FSFile { * Get an associative array containing information about a file in the local filesystem. * * @param string $path absolute local filesystem path - * @param $ext Mixed: the file extension, or true to extract it from the filename. + * @param Mixed $ext: the file extension, or true to extract it from the filename. * Set it to false to ignore the extension. - * * @return array */ public static function getPropsFromPath( $path, $ext = true ) { @@ -249,19 +248,11 @@ class FSFile { * 160 log 2 / log 36 = 30.95, so the 160-bit hash fills 31 digits in base 36 * fairly neatly. * - * @param $path string - * @param $recache bool - * + * @param string $path * @return bool|string False on failure */ - public static function getSha1Base36FromPath( $path, $recache = false ) { - static $sha1Base36 = array(); - - if ( !isset( $sha1Base36[$path] ) || $recache ) { - $fsFile = new self( $path ); - $sha1Base36[$path] = $fsFile->getSha1Base36(); - } - - return $sha1Base36[$path]; + public static function getSha1Base36FromPath( $path ) { + $fsFile = new self( $path ); + return $fsFile->getSha1Base36(); } } diff --git a/includes/filebackend/FSFileBackend.php b/includes/filebackend/FSFileBackend.php index c9769989..6d642162 100644 --- a/includes/filebackend/FSFileBackend.php +++ b/includes/filebackend/FSFileBackend.php @@ -82,12 +82,6 @@ class FSFileBackend extends FileBackendStore { } } - /** - * @see FileBackendStore::resolveContainerPath() - * @param $container string - * @param $relStoragePath string - * @return null|string - */ protected function resolveContainerPath( $container, $relStoragePath ) { // Check that container has a root directory if ( isset( $this->containerPaths[$container] ) || isset( $this->basePath ) ) { @@ -121,8 +115,8 @@ class FSFileBackend extends FileBackendStore { * Given the short (unresolved) and full (resolved) name of * a container, return the file system path of the container. * - * @param $shortCont string - * @param $fullCont string + * @param string $shortCont + * @param string $fullCont * @return string|null */ protected function containerFSRoot( $shortCont, $fullCont ) { @@ -153,10 +147,6 @@ class FSFileBackend extends FileBackendStore { return $fsPath; } - /** - * @see FileBackendStore::isPathUsableInternal() - * @return bool - */ public function isPathUsableInternal( $storagePath ) { $fsPath = $this->resolveToFSPath( $storagePath ); if ( $fsPath === null ) { @@ -178,10 +168,6 @@ class FSFileBackend extends FileBackendStore { return $ok; } - /** - * @see FileBackendStore::doCreateInternal() - * @return Status - */ protected function doCreateInternal( array $params ) { $status = Status::newGood(); @@ -235,10 +221,6 @@ class FSFileBackend extends FileBackendStore { } } - /** - * @see FileBackendStore::doStoreInternal() - * @return Status - */ protected function doStoreInternal( array $params ) { $status = Status::newGood(); @@ -284,10 +266,6 @@ class FSFileBackend extends FileBackendStore { } } - /** - * @see FileBackendStore::doCopyInternal() - * @return Status - */ protected function doCopyInternal( array $params ) { $status = Status::newGood(); @@ -319,7 +297,7 @@ class FSFileBackend extends FileBackendStore { $status->value = new FSFileOpHandle( $this, $params, 'Copy', $cmd, $dest ); } else { // immediate write $this->trapWarnings(); - $ok = copy( $source, $dest ); + $ok = ( $source === $dest ) ? true : copy( $source, $dest ); $this->untrapWarnings(); // In some cases (at least over NFS), copy() returns true when it fails if ( !$ok || ( filesize( $source ) !== filesize( $dest ) ) ) { @@ -348,10 +326,6 @@ class FSFileBackend extends FileBackendStore { } } - /** - * @see FileBackendStore::doMoveInternal() - * @return Status - */ protected function doMoveInternal( array $params ) { $status = Status::newGood(); @@ -383,7 +357,7 @@ class FSFileBackend extends FileBackendStore { $status->value = new FSFileOpHandle( $this, $params, 'Move', $cmd ); } else { // immediate write $this->trapWarnings(); - $ok = rename( $source, $dest ); + $ok = ( $source === $dest ) ? true : rename( $source, $dest ); $this->untrapWarnings(); clearstatcache(); // file no longer at source if ( !$ok ) { @@ -405,10 +379,6 @@ class FSFileBackend extends FileBackendStore { } } - /** - * @see FileBackendStore::doDeleteInternal() - * @return Status - */ protected function doDeleteInternal( array $params ) { $status = Status::newGood(); @@ -454,10 +424,6 @@ class FSFileBackend extends FileBackendStore { } } - /** - * @see FileBackendStore::doPrepareInternal() - * @return Status - */ protected function doPrepareInternal( $fullCont, $dirRel, array $params ) { $status = Status::newGood(); list( , $shortCont, ) = FileBackend::splitStoragePath( $params['dir'] ); @@ -481,10 +447,6 @@ class FSFileBackend extends FileBackendStore { return $status; } - /** - * @see FileBackendStore::doSecureInternal() - * @return Status - */ protected function doSecureInternal( $fullCont, $dirRel, array $params ) { $status = Status::newGood(); list( , $shortCont, ) = FileBackend::splitStoragePath( $params['dir'] ); @@ -512,10 +474,6 @@ class FSFileBackend extends FileBackendStore { return $status; } - /** - * @see FileBackendStore::doPublishInternal() - * @return Status - */ protected function doPublishInternal( $fullCont, $dirRel, array $params ) { $status = Status::newGood(); list( , $shortCont, ) = FileBackend::splitStoragePath( $params['dir'] ); @@ -543,10 +501,6 @@ class FSFileBackend extends FileBackendStore { return $status; } - /** - * @see FileBackendStore::doCleanInternal() - * @return Status - */ protected function doCleanInternal( $fullCont, $dirRel, array $params ) { $status = Status::newGood(); list( , $shortCont, ) = FileBackend::splitStoragePath( $params['dir'] ); @@ -560,10 +514,6 @@ class FSFileBackend extends FileBackendStore { return $status; } - /** - * @see FileBackendStore::doFileExists() - * @return array|bool|null - */ protected function doGetFileStat( array $params ) { $source = $this->resolveToFSPath( $params['src'] ); if ( $source === null ) { @@ -577,7 +527,7 @@ class FSFileBackend extends FileBackendStore { if ( $stat ) { return array( 'mtime' => wfTimestamp( TS_MW, $stat['mtime'] ), - 'size' => $stat['size'] + 'size' => $stat['size'] ); } elseif ( !$hadError ) { return false; // file does not exist @@ -593,10 +543,6 @@ class FSFileBackend extends FileBackendStore { clearstatcache(); // clear the PHP file stat cache } - /** - * @see FileBackendStore::doDirectoryExists() - * @return bool|null - */ protected function doDirectoryExists( $fullCont, $dirRel, array $params ) { list( , $shortCont, ) = FileBackend::splitStoragePath( $params['dir'] ); $contRoot = $this->containerFSRoot( $shortCont, $fullCont ); // must be valid @@ -647,10 +593,6 @@ class FSFileBackend extends FileBackendStore { return new FSFileBackendFileList( $dir, $params ); } - /** - * @see FileBackendStore::doGetLocalReferenceMulti() - * @return Array - */ protected function doGetLocalReferenceMulti( array $params ) { $fsFiles = array(); // (path => FSFile) @@ -666,10 +608,6 @@ class FSFileBackend extends FileBackendStore { return $fsFiles; } - /** - * @see FileBackendStore::doGetLocalCopyMulti() - * @return Array - */ protected function doGetLocalCopyMulti( array $params ) { $tmpFiles = array(); // (path => TempFSFile) @@ -702,18 +640,10 @@ class FSFileBackend extends FileBackendStore { return $tmpFiles; } - /** - * @see FileBackendStore::directoriesAreVirtual() - * @return bool - */ protected function directoriesAreVirtual() { return false; } - /** - * @see FileBackendStore::doExecuteOpHandlesInternal() - * @return Array List of corresponding Status objects - */ protected function doExecuteOpHandlesInternal( array $fileOpHandles ) { $statuses = array(); @@ -807,11 +737,12 @@ class FSFileBackend extends FileBackendStore { } /** - * @param $errno integer - * @param $errstr string + * @param integer $errno + * @param string $errstr * @return bool + * @access private */ - private function handleWarning( $errno, $errstr ) { + public function handleWarning( $errno, $errstr ) { wfDebugLog( 'FSFileBackend', $errstr ); // more detailed error logging $this->hadWarningErrors[count( $this->hadWarningErrors ) - 1] = true; return true; // suppress from PHP handler @@ -826,13 +757,15 @@ class FSFileOpHandle extends FileBackendStoreOpHandle { public $chmodPath; // string; file to chmod /** - * @param $backend - * @param $params array - * @param $call - * @param $cmd - * @param $chmodPath null + * @param FSFileBackend $backend + * @param array $params + * @param string $call + * @param string $cmd + * @param integer|null $chmodPath */ - public function __construct( $backend, array $params, $call, $cmd, $chmodPath = null ) { + public function __construct( + FSFileBackend $backend, array $params, $call, $cmd, $chmodPath = null + ) { $this->backend = $backend; $this->params = $params; $this->call = $call; @@ -858,11 +791,11 @@ abstract class FSFileBackendList implements Iterator { /** * @param string $dir file system directory - * @param $params array + * @param array $params */ public function __construct( $dir, array $params ) { $path = realpath( $dir ); // normalize - if( $path === false ) { + if ( $path === false ) { $path = $dir; } $this->suffixStart = strlen( $path ) + 1; // size of "path/to/dir/" @@ -921,8 +854,8 @@ abstract class FSFileBackendList implements Iterator { try { $this->iter->next(); $this->filterViaNext(); - } catch ( UnexpectedValueException $e ) { - $this->iter = null; + } catch ( UnexpectedValueException $e ) { // bad permissions? deleted? + throw new FileBackendError( "File iterator gave UnexpectedValueException." ); } ++$this->pos; } @@ -936,8 +869,8 @@ abstract class FSFileBackendList implements Iterator { try { $this->iter->rewind(); $this->filterViaNext(); - } catch ( UnexpectedValueException $e ) { - $this->iter = null; + } catch ( UnexpectedValueException $e ) { // bad permissions? deleted? + throw new FileBackendError( "File iterator gave UnexpectedValueException." ); } } @@ -958,12 +891,12 @@ abstract class FSFileBackendList implements Iterator { * Return only the relative path and normalize slashes to FileBackend-style. * Uses the "real path" since the suffix is based upon that. * - * @param $path string + * @param string $path * @return string */ protected function getRelPath( $dir ) { $path = realpath( $dir ); - if( $path === false ) { + if ( $path === false ) { $path = $dir; } return strtr( substr( $path, $this->suffixStart ), '\\', '/' ); diff --git a/includes/filebackend/FileBackend.php b/includes/filebackend/FileBackend.php index f40b8c16..f586578b 100644 --- a/includes/filebackend/FileBackend.php +++ b/includes/filebackend/FileBackend.php @@ -1,7 +1,6 @@ , * 'overwrite' => , * 'overwriteSame' => , - * 'disposition' => , * 'headers' => # since 1.21 * ); * @endcode @@ -204,7 +202,6 @@ abstract class FileBackend { * 'dst' => , * 'overwrite' => , * 'overwriteSame' => , - * 'disposition' => , * 'headers' => # since 1.21 * ) * @endcode @@ -218,7 +215,7 @@ abstract class FileBackend { * 'overwrite' => , * 'overwriteSame' => , * 'ignoreMissingSource' => , # since 1.21 - * 'disposition' => + * 'headers' => # since 1.21 * ) * @endcode * @@ -231,7 +228,7 @@ abstract class FileBackend { * 'overwrite' => , * 'overwriteSame' => , * 'ignoreMissingSource' => , # since 1.21 - * 'disposition' => + * 'headers' => # since 1.21 * ) * @endcode * @@ -249,7 +246,6 @@ abstract class FileBackend { * array( * 'op' => 'describe', * 'src' => , - * 'disposition' => , * 'headers' => * ) * @endcode @@ -265,19 +261,19 @@ abstract class FileBackend { * - ignoreMissingSource : The operation will simply succeed and do * nothing if the source file does not exist. * - overwrite : Any destination file will be overwritten. - * - overwriteSame : An error will not be given if a file already - * exists at the destination that has the same - * contents as the new contents to be written there. - * - disposition : If supplied, the backend will return a Content-Disposition - * header when GETs/HEADs of the destination file are made. - * Backends that don't support metadata ignore this. - * See http://tools.ietf.org/html/rfc6266. (since 1.20) - * - headers : If supplied, the backend will return these headers when - * GETs/HEADs of the destination file are made. Header values - * should be smaller than 256 bytes, often options or numbers. - * Existing headers will remain, but these will replace any - * conflicting previous headers, and headers will be removed - * if they are set to an empty string. + * - overwriteSame : If a file already exists at the destination with the + * same contents, then do nothing to the destination file + * instead of giving an error. This does not compare headers. + * This option is ignored if 'overwrite' is already provided. + * - headers : If supplied, the result of merging these headers with any + * existing source file headers (replacing conflicting ones) + * will be set as the destination file headers. Headers are + * deleted if their value is set to the empty string. When a + * file has headers they are included in responses to GET and + * HEAD requests to the backing store for that file. + * Header values should be no larger than 255 bytes, except for + * Content-Disposition. The system might ignore or truncate any + * headers that are too long to store (exact limits will vary). * Backends that don't support metadata ignore this. (since 1.21) * * $opts is an associative of boolean flags, including: @@ -318,9 +314,17 @@ abstract class FileBackend { if ( empty( $opts['bypassReadOnly'] ) && $this->isReadOnly() ) { return Status::newFatal( 'backend-fail-readonly', $this->name, $this->readOnly ); } + if ( !count( $ops ) ) { + return Status::newGood(); // nothing to do + } if ( empty( $opts['force'] ) ) { // sanity unset( $opts['nonLocking'] ); } + foreach ( $ops as &$op ) { + if ( isset( $op['disposition'] ) ) { // b/c (MW 1.20) + $op['headers']['Content-Disposition'] = $op['disposition']; + } + } $scope = $this->getScopedPHPBehaviorForOps(); // try to ignore client aborts return $this->doOperationsInternal( $ops, $opts ); } @@ -452,7 +456,6 @@ abstract class FileBackend { * 'op' => 'create', * 'dst' => , * 'content' => , - * 'disposition' => , * 'headers' => # since 1.21 * ) * @endcode @@ -463,7 +466,6 @@ abstract class FileBackend { * 'op' => 'store', * 'src' => , * 'dst' => , - * 'disposition' => , * 'headers' => # since 1.21 * ) * @endcode @@ -475,7 +477,7 @@ abstract class FileBackend { * 'src' => , * 'dst' => , * 'ignoreMissingSource' => , # since 1.21 - * 'disposition' => + * 'headers' => # since 1.21 * ) * @endcode * @@ -486,7 +488,7 @@ abstract class FileBackend { * 'src' => , * 'dst' => , * 'ignoreMissingSource' => , # since 1.21 - * 'disposition' => + * 'headers' => # since 1.21 * ) * @endcode * @@ -504,7 +506,6 @@ abstract class FileBackend { * array( * 'op' => 'describe', * 'src' => , - * 'disposition' => , * 'headers' => * ) * @endcode @@ -519,13 +520,11 @@ abstract class FileBackend { * @par Boolean flags for operations (operation-specific): * - ignoreMissingSource : The operation will simply succeed and do * nothing if the source file does not exist. - * - disposition : When supplied, the backend will add a Content-Disposition - * header when GETs/HEADs of the destination file are made. - * Backends that don't support file metadata will ignore this. - * See http://tools.ietf.org/html/rfc6266 (since 1.20). * - headers : If supplied with a header name/value map, the backend will * reply with these headers when GETs/HEADs of the destination * file are made. Header values should be smaller than 256 bytes. + * Content-Disposition headers can be longer, though the system + * might ignore or truncate ones that are too long to store. * Existing headers will remain, but these will replace any * conflicting previous headers, and headers will be removed * if they are set to an empty string. @@ -549,8 +548,14 @@ abstract class FileBackend { if ( empty( $opts['bypassReadOnly'] ) && $this->isReadOnly() ) { return Status::newFatal( 'backend-fail-readonly', $this->name, $this->readOnly ); } + if ( !count( $ops ) ) { + return Status::newGood(); // nothing to do + } foreach ( $ops as &$op ) { $op['overwrite'] = true; // avoids RTTs in key/value stores + if ( isset( $op['disposition'] ) ) { // b/c (MW 1.20) + $op['headers']['Content-Disposition'] = $op['disposition']; + } } $scope = $this->getScopedPHPBehaviorForOps(); // try to ignore client aborts return $this->doQuickOperationsInternal( $ops ); @@ -683,6 +688,8 @@ abstract class FileBackend { * The 'noAccess' and 'noListing' parameters works the same as in secure(), * except they are only applied *if* the directory/container had to be created. * These flags should always be set for directories that have private files. + * However, setting them is not guaranteed to actually do anything. + * Additional server configuration may be needed to achieve the desired effect. * * @param array $params * $params include: @@ -710,7 +717,9 @@ abstract class FileBackend { * the container it belongs to. FS backends might add .htaccess * files whereas key/value store backends might revoke container * access to the storage user representing end-users in web requests. - * This is not guaranteed to actually do anything. + * + * This is not guaranteed to actually make files or listings publically hidden. + * Additional server configuration may be needed to achieve the desired effect. * * @param array $params * $params include: @@ -740,6 +749,9 @@ abstract class FileBackend { * access to the storage user representing end-users in web requests. * This essentially can undo the result of secure() calls. * + * This is not guaranteed to actually make files or listings publically viewable. + * Additional server configuration may be needed to achieve the desired effect. + * * @param array $params * $params include: * - dir : storage directory @@ -797,7 +809,9 @@ abstract class FileBackend { final protected function getScopedPHPBehaviorForOps() { if ( php_sapi_name() != 'cli' ) { // http://bugs.php.net/bug.php?id=47540 $old = ignore_user_abort( true ); // avoid half-finished operations - return new ScopedCallback( function() use ( $old ) { ignore_user_abort( $old ); } ); + return new ScopedCallback( function() use ( $old ) { + ignore_user_abort( $old ); + } ); } return null; } @@ -1029,7 +1043,7 @@ abstract class FileBackend { * * Storage backends with eventual consistency might return stale data. * - * @param $params array + * @param array $params * $params include: * - dir : storage directory * @return bool|null Returns null on failure @@ -1047,7 +1061,9 @@ abstract class FileBackend { * * Storage backends with eventual consistency might return stale data. * - * @param $params array + * Failures during iteration can result in FileBackendError exceptions (since 1.22). + * + * @param array $params * $params include: * - dir : storage directory * - topOnly : only return direct child dirs of the directory @@ -1062,7 +1078,9 @@ abstract class FileBackend { * * Storage backends with eventual consistency might return stale data. * - * @param $params array + * Failures during iteration can result in FileBackendError exceptions (since 1.22). + * + * @param array $params * $params include: * - dir : storage directory * @return Traversable|Array|null Returns null on failure @@ -1082,10 +1100,13 @@ abstract class FileBackend { * * Storage backends with eventual consistency might return stale data. * - * @param $params array + * Failures during iteration can result in FileBackendError exceptions (since 1.22). + * + * @param array $params * $params include: - * - dir : storage directory - * - topOnly : only return direct child files of the directory (since 1.20) + * - dir : storage directory + * - topOnly : only return direct child files of the directory (since 1.20) + * - adviseStat : set to true if stat requests will be made on the files (since 1.22) * @return Traversable|Array|null Returns null on failure */ abstract public function getFileList( array $params ); @@ -1096,9 +1117,12 @@ abstract class FileBackend { * * Storage backends with eventual consistency might return stale data. * - * @param $params array + * Failures during iteration can result in FileBackendError exceptions (since 1.22). + * + * @param array $params * $params include: - * - dir : storage directory + * - dir : storage directory + * - adviseStat : set to true if stat requests will be made on the files (since 1.22) * @return Traversable|Array|null Returns null on failure * @since 1.20 */ @@ -1131,10 +1155,11 @@ abstract class FileBackend { * Callers should consider using getScopedFileLocks() instead. * * @param array $paths Storage paths - * @param $type integer LockManager::LOCK_* constant + * @param integer $type LockManager::LOCK_* constant * @return Status */ final public function lockFiles( array $paths, $type ) { + $paths = array_map( 'FileBackend::normalizeStoragePath', $paths ); return $this->lockManager->lock( $paths, $type ); } @@ -1142,10 +1167,11 @@ abstract class FileBackend { * Unlock the files at the given storage paths in the backend. * * @param array $paths Storage paths - * @param $type integer LockManager::LOCK_* constant + * @param integer $type LockManager::LOCK_* constant * @return Status */ final public function unlockFiles( array $paths, $type ) { + $paths = array_map( 'FileBackend::normalizeStoragePath', $paths ); return $this->lockManager->unlock( $paths, $type ); } @@ -1157,12 +1183,21 @@ abstract class FileBackend { * Once the return value goes out scope, the locks will be released and * the status updated. Unlock fatals will not change the status "OK" value. * - * @param array $paths Storage paths - * @param $type integer LockManager::LOCK_* constant - * @param $status Status Status to update on lock/unlock + * @see ScopedLock::factory() + * + * @param array $paths List of storage paths or map of lock types to path lists + * @param integer|string $type LockManager::LOCK_* constant or "mixed" + * @param Status $status Status to update on lock/unlock * @return ScopedLock|null Returns null on failure */ final public function getScopedFileLocks( array $paths, $type, Status $status ) { + if ( $type === 'mixed' ) { + foreach ( $paths as &$typePaths ) { + $typePaths = array_map( 'FileBackend::normalizeStoragePath', $typePaths ); + } + } else { + $paths = array_map( 'FileBackend::normalizeStoragePath', $paths ); + } return ScopedLock::factory( $this->lockManager, $paths, $type, $status ); } @@ -1178,7 +1213,7 @@ abstract class FileBackend { * @see FileBackend::doOperations() * * @param array $ops List of file operations to FileBackend::doOperations() - * @param $status Status Status to update on lock/unlock + * @param Status $status Status to update on lock/unlock * @return Array List of ScopedFileLocks or null values * @since 1.20 */ @@ -1219,7 +1254,7 @@ abstract class FileBackend { * Check if a given path is a "mwstore://" path. * This does not do any further validation or any existence checks. * - * @param $path string + * @param string $path * @return bool */ final public static function isStoragePath( $path ) { @@ -1231,7 +1266,7 @@ abstract class FileBackend { * and a relative file path. The relative path may be the empty string. * This does not do any path normalization or traversal checks. * - * @param $storagePath string + * @param string $storagePath * @return Array (backend, container, rel object) or (null, null, null) */ final public static function splitStoragePath( $storagePath ) { @@ -1253,7 +1288,7 @@ abstract class FileBackend { * Normalize a storage path by cleaning up directory separators. * Returns null if the path is not of the format of a valid storage path. * - * @param $storagePath string + * @param string $storagePath * @return string|null */ final public static function normalizeStoragePath( $storagePath ) { @@ -1274,7 +1309,7 @@ abstract class FileBackend { * This returns a path like "mwstore://backend/container", * "mwstore://backend/container/...", or null if there is no parent. * - * @param $storagePath string + * @param string $storagePath * @return string|null */ final public static function parentStoragePath( $storagePath ) { @@ -1286,7 +1321,7 @@ abstract class FileBackend { /** * Get the final extension from a storage or FS path * - * @param $path string + * @param string $path * @return string */ final public static function extensionFromPath( $path ) { @@ -1297,7 +1332,7 @@ abstract class FileBackend { /** * Check if a relative path has no directory traversals * - * @param $path string + * @param string $path * @return bool * @since 1.20 */ @@ -1363,3 +1398,9 @@ abstract class FileBackend { return $path; } } + +/** + * @ingroup FileBackend + * @since 1.22 + */ +class FileBackendError extends MWException {} diff --git a/includes/filebackend/FileBackendGroup.php b/includes/filebackend/FileBackendGroup.php index d790a996..be8a2076 100644 --- a/includes/filebackend/FileBackendGroup.php +++ b/includes/filebackend/FileBackendGroup.php @@ -95,17 +95,17 @@ class FileBackendGroup { : 0644; // Get the FS backend configuration $autoBackends[] = array( - 'name' => $backendName, - 'class' => 'FSFileBackend', - 'lockManager' => 'fsLockManager', + 'name' => $backendName, + 'class' => 'FSFileBackend', + 'lockManager' => 'fsLockManager', 'containerPaths' => array( - "{$repoName}-public" => "{$directory}", - "{$repoName}-thumb" => $thumbDir, - "{$repoName}-transcoded" => $transcodedDir, + "{$repoName}-public" => "{$directory}", + "{$repoName}-thumb" => $thumbDir, + "{$repoName}-transcoded" => $transcodedDir, "{$repoName}-deleted" => $deletedDir, - "{$repoName}-temp" => "{$directory}/temp" + "{$repoName}-temp" => "{$directory}/temp" ), - 'fileMode' => $fileMode, + 'fileMode' => $fileMode, ); } @@ -116,7 +116,7 @@ class FileBackendGroup { /** * Register an array of file backend configurations * - * @param $configs Array + * @param Array $configs * @return void * @throws MWException */ @@ -135,8 +135,8 @@ class FileBackendGroup { unset( $config['class'] ); // backend won't need this $this->backends[$name] = array( - 'class' => $class, - 'config' => $config, + 'class' => $class, + 'config' => $config, 'instance' => null ); } @@ -145,7 +145,7 @@ class FileBackendGroup { /** * Get the backend object with a given name * - * @param $name string + * @param string $name * @return FileBackend * @throws MWException */ @@ -165,7 +165,7 @@ class FileBackendGroup { /** * Get the config array for a backend object with a given name * - * @param $name string + * @param string $name * @return Array * @throws MWException */ @@ -180,7 +180,7 @@ class FileBackendGroup { /** * Get an appropriate backend object from a storage path * - * @param $storagePath string + * @param string $storagePath * @return FileBackend|null Backend or null on failure */ public function backendFromPath( $storagePath ) { diff --git a/includes/filebackend/FileBackendMultiWrite.php b/includes/filebackend/FileBackendMultiWrite.php index 939315d1..97584a71 100644 --- a/includes/filebackend/FileBackendMultiWrite.php +++ b/includes/filebackend/FileBackendMultiWrite.php @@ -75,10 +75,13 @@ class FileBackendMultiWrite extends FileBackend { * - autoResync : Automatically resync the clone backends to the master backend * when pre-operation sync checks fail. This should only be used * if the master backend is stable and not missing any files. + * Use "conservative" to limit resyncing to copying newer master + * backend files over older (or non-existing) clone backend files. + * Cases that cannot be handled will result in operation abortion. * - noPushQuickOps : (hack) Only apply doQuickOperations() to the master backend. * - noPushDirConts : (hack) Only apply directory functions to the master backend. * - * @param $config Array + * @param Array $config * @throws MWException */ public function __construct( array $config ) { @@ -86,7 +89,9 @@ class FileBackendMultiWrite extends FileBackend { $this->syncChecks = isset( $config['syncChecks'] ) ? $config['syncChecks'] : self::CHECK_SIZE; - $this->autoResync = !empty( $config['autoResync'] ); + $this->autoResync = isset( $config['autoResync'] ) + ? $config['autoResync'] + : false; $this->noPushQuickOps = isset( $config['noPushQuickOps'] ) ? $config['noPushQuickOps'] : false; @@ -131,26 +136,15 @@ class FileBackendMultiWrite extends FileBackend { } } - /** - * @see FileBackend::doOperationsInternal() - * @return Status - */ final protected function doOperationsInternal( array $ops, array $opts ) { $status = Status::newGood(); $mbe = $this->backends[$this->masterIndex]; // convenience - // Get the paths to lock from the master backend - $realOps = $this->substOpBatchPaths( $ops, $mbe ); - $paths = $mbe->getPathsToLockForOpsInternal( $mbe->getOperationsInternal( $realOps ) ); - // Get the paths under the proxy backend's name - $paths['sh'] = $this->unsubstPaths( $paths['sh'] ); - $paths['ex'] = $this->unsubstPaths( $paths['ex'] ); // Try to lock those files for the scope of this function... if ( empty( $opts['nonLocking'] ) ) { // Try to lock those files for the scope of this function... - $scopeLockS = $this->getScopedFileLocks( $paths['sh'], LockManager::LOCK_UW, $status ); - $scopeLockE = $this->getScopedFileLocks( $paths['ex'], LockManager::LOCK_EX, $status ); + $scopeLock = $this->getScopedLocksForOps( $ops, $status ); if ( !$status->isOK() ) { return $status; // abort } @@ -177,6 +171,7 @@ class FileBackendMultiWrite extends FileBackend { } } // Actually attempt the operation batch on the master backend... + $realOps = $this->substOpBatchPaths( $ops, $mbe ); $masterStatus = $mbe->doOperations( $realOps, $opts ); $status->merge( $masterStatus ); // Propagate the operations to the clone backends if there were no unexpected errors @@ -304,11 +299,11 @@ class FileBackendMultiWrite extends FileBackend { $mBackend = $this->backends[$this->masterIndex]; foreach ( $paths as $path ) { $mPath = $this->substPaths( $path, $mBackend ); - $mSha1 = $mBackend->getFileSha1Base36( array( 'src' => $mPath ) ); - $mExist = $mBackend->fileExists( array( 'src' => $mPath ) ); - // Check if the master backend is available... - if ( $mExist === null ) { + $mSha1 = $mBackend->getFileSha1Base36( array( 'src' => $mPath, 'latest' => true ) ); + $mStat = $mBackend->getFileStat( array( 'src' => $mPath, 'latest' => true ) ); + if ( $mStat === null || ( $mSha1 !== false && !$mStat ) ) { // sanity $status->fatal( 'backend-fail-internal', $this->name ); + continue; // file is not available on the master backend... } // Check of all clone backends agree with the master... foreach ( $this->backends as $index => $cBackend ) { @@ -316,15 +311,31 @@ class FileBackendMultiWrite extends FileBackend { continue; // master } $cPath = $this->substPaths( $path, $cBackend ); - $cSha1 = $cBackend->getFileSha1Base36( array( 'src' => $cPath ) ); + $cSha1 = $cBackend->getFileSha1Base36( array( 'src' => $cPath, 'latest' => true ) ); + $cStat = $cBackend->getFileStat( array( 'src' => $cPath, 'latest' => true ) ); + if ( $cStat === null || ( $cSha1 !== false && !$cStat ) ) { // sanity + $status->fatal( 'backend-fail-internal', $cBackend->getName() ); + continue; // file is not available on the clone backend... + } if ( $mSha1 === $cSha1 ) { // already synced; nothing to do - } elseif ( $mSha1 ) { // file is in master - $fsFile = $mBackend->getLocalReference( array( 'src' => $mPath ) ); + } elseif ( $mSha1 !== false ) { // file is in master + if ( $this->autoResync === 'conservative' + && $cStat && $cStat['mtime'] > $mStat['mtime'] ) + { + $status->fatal( 'backend-fail-synced', $path ); + continue; // don't rollback data + } + $fsFile = $mBackend->getLocalReference( + array( 'src' => $mPath, 'latest' => true ) ); $status->merge( $cBackend->quickStore( array( 'src' => $fsFile->getPath(), 'dst' => $cPath ) ) ); - } elseif ( $mExist === false ) { // file is not in master + } elseif ( $mStat === false ) { // file is not in master + if ( $this->autoResync === 'conservative' ) { + $status->fatal( 'backend-fail-synced', $path ); + continue; // don't delete data + } $status->merge( $cBackend->quickDelete( array( 'src' => $cPath ) ) ); } } @@ -366,7 +377,7 @@ class FileBackendMultiWrite extends FileBackend { * for a set of operations with that of a given internal backend. * * @param array $ops List of file operation arrays - * @param $backend FileBackendStore + * @param FileBackendStore $backend * @return Array */ protected function substOpBatchPaths( array $ops, FileBackendStore $backend ) { @@ -387,7 +398,7 @@ class FileBackendMultiWrite extends FileBackend { * Same as substOpBatchPaths() but for a single operation * * @param array $ops File operation array - * @param $backend FileBackendStore + * @param FileBackendStore $backend * @return Array */ protected function substOpPaths( array $ops, FileBackendStore $backend ) { @@ -399,7 +410,7 @@ class FileBackendMultiWrite extends FileBackend { * Substitute the backend of storage paths with an internal backend's name * * @param array|string $paths List of paths or single string path - * @param $backend FileBackendStore + * @param FileBackendStore $backend * @return Array|string */ protected function substPaths( $paths, FileBackendStore $backend ) { @@ -424,10 +435,6 @@ class FileBackendMultiWrite extends FileBackend { ); } - /** - * @see FileBackend::doQuickOperationsInternal() - * @return Status - */ protected function doQuickOperationsInternal( array $ops ) { $status = Status::newGood(); // Do the operations on the master backend; setting Status fields... @@ -457,14 +464,10 @@ class FileBackendMultiWrite extends FileBackend { * @return bool Path container should have dir changes pushed to all backends */ protected function replicateContainerDirChanges( $path ) { - list( , $shortCont, ) = self::splitStoragePath( $path ); + list( , $shortCont, ) = self::splitStoragePath( $path ); return !in_array( $shortCont, $this->noPushDirConts ); } - /** - * @see FileBackend::doPrepare() - * @return Status - */ protected function doPrepare( array $params ) { $status = Status::newGood(); $replicate = $this->replicateContainerDirChanges( $params['dir'] ); @@ -477,11 +480,6 @@ class FileBackendMultiWrite extends FileBackend { return $status; } - /** - * @see FileBackend::doSecure() - * @param $params array - * @return Status - */ protected function doSecure( array $params ) { $status = Status::newGood(); $replicate = $this->replicateContainerDirChanges( $params['dir'] ); @@ -494,11 +492,6 @@ class FileBackendMultiWrite extends FileBackend { return $status; } - /** - * @see FileBackend::doPublish() - * @param $params array - * @return Status - */ protected function doPublish( array $params ) { $status = Status::newGood(); $replicate = $this->replicateContainerDirChanges( $params['dir'] ); @@ -511,11 +504,6 @@ class FileBackendMultiWrite extends FileBackend { return $status; } - /** - * @see FileBackend::doClean() - * @param $params array - * @return Status - */ protected function doClean( array $params ) { $status = Status::newGood(); $replicate = $this->replicateContainerDirChanges( $params['dir'] ); @@ -528,62 +516,32 @@ class FileBackendMultiWrite extends FileBackend { return $status; } - /** - * @see FileBackend::concatenate() - * @param $params array - * @return Status - */ public function concatenate( array $params ) { // We are writing to an FS file, so we don't need to do this per-backend $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] ); return $this->backends[$this->masterIndex]->concatenate( $realParams ); } - /** - * @see FileBackend::fileExists() - * @param $params array - * @return bool|null - */ public function fileExists( array $params ) { $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] ); return $this->backends[$this->masterIndex]->fileExists( $realParams ); } - /** - * @see FileBackend::getFileTimestamp() - * @param $params array - * @return bool|string - */ public function getFileTimestamp( array $params ) { $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] ); return $this->backends[$this->masterIndex]->getFileTimestamp( $realParams ); } - /** - * @see FileBackend::getFileSize() - * @param $params array - * @return bool|int - */ public function getFileSize( array $params ) { $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] ); return $this->backends[$this->masterIndex]->getFileSize( $realParams ); } - /** - * @see FileBackend::getFileStat() - * @param $params array - * @return Array|bool|null - */ public function getFileStat( array $params ) { $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] ); return $this->backends[$this->masterIndex]->getFileStat( $realParams ); } - /** - * @see FileBackend::getFileContentsMulti() - * @param $params array - * @return bool|string - */ public function getFileContentsMulti( array $params ) { $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] ); $contentsM = $this->backends[$this->masterIndex]->getFileContentsMulti( $realParams ); @@ -595,41 +553,21 @@ class FileBackendMultiWrite extends FileBackend { return $contents; } - /** - * @see FileBackend::getFileSha1Base36() - * @param $params array - * @return bool|string - */ public function getFileSha1Base36( array $params ) { $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] ); return $this->backends[$this->masterIndex]->getFileSha1Base36( $realParams ); } - /** - * @see FileBackend::getFileProps() - * @param $params array - * @return Array - */ public function getFileProps( array $params ) { $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] ); return $this->backends[$this->masterIndex]->getFileProps( $realParams ); } - /** - * @see FileBackend::streamFile() - * @param $params array - * @return \Status - */ public function streamFile( array $params ) { $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] ); return $this->backends[$this->masterIndex]->streamFile( $realParams ); } - /** - * @see FileBackend::getLocalReferenceMulti() - * @param $params array - * @return FSFile|null - */ public function getLocalReferenceMulti( array $params ) { $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] ); $fsFilesM = $this->backends[$this->masterIndex]->getLocalReferenceMulti( $realParams ); @@ -641,11 +579,6 @@ class FileBackendMultiWrite extends FileBackend { return $fsFiles; } - /** - * @see FileBackend::getLocalCopyMulti() - * @param $params array - * @return null|TempFSFile - */ public function getLocalCopyMulti( array $params ) { $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] ); $tempFilesM = $this->backends[$this->masterIndex]->getLocalCopyMulti( $realParams ); @@ -657,48 +590,26 @@ class FileBackendMultiWrite extends FileBackend { return $tempFiles; } - /** - * @see FileBackend::getFileHttpUrl() - * @return string|null - */ public function getFileHttpUrl( array $params ) { $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] ); return $this->backends[$this->masterIndex]->getFileHttpUrl( $realParams ); } - /** - * @see FileBackend::directoryExists() - * @param $params array - * @return bool|null - */ public function directoryExists( array $params ) { $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] ); return $this->backends[$this->masterIndex]->directoryExists( $realParams ); } - /** - * @see FileBackend::getSubdirectoryList() - * @param $params array - * @return Array|null|Traversable - */ public function getDirectoryList( array $params ) { $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] ); return $this->backends[$this->masterIndex]->getDirectoryList( $realParams ); } - /** - * @see FileBackend::getFileList() - * @param $params array - * @return Array|null|\Traversable - */ public function getFileList( array $params ) { $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] ); return $this->backends[$this->masterIndex]->getFileList( $realParams ); } - /** - * @see FileBackend::clearCache() - */ public function clearCache( array $paths = null ) { foreach ( $this->backends as $backend ) { $realPaths = is_array( $paths ) ? $this->substPaths( $paths, $backend ) : null; @@ -706,19 +617,17 @@ class FileBackendMultiWrite extends FileBackend { } } - /** - * @see FileBackend::getScopedLocksForOps() - */ public function getScopedLocksForOps( array $ops, Status $status ) { - $fileOps = $this->backends[$this->masterIndex]->getOperationsInternal( $ops ); + $realOps = $this->substOpBatchPaths( $ops, $this->backends[$this->masterIndex] ); + $fileOps = $this->backends[$this->masterIndex]->getOperationsInternal( $realOps ); // Get the paths to lock from the master backend $paths = $this->backends[$this->masterIndex]->getPathsToLockForOpsInternal( $fileOps ); // Get the paths under the proxy backend's name - $paths['sh'] = $this->unsubstPaths( $paths['sh'] ); - $paths['ex'] = $this->unsubstPaths( $paths['ex'] ); - return array( - $this->getScopedFileLocks( $paths['sh'], LockManager::LOCK_UW, $status ), - $this->getScopedFileLocks( $paths['ex'], LockManager::LOCK_EX, $status ) + $pbPaths = array( + LockManager::LOCK_UW => $this->unsubstPaths( $paths[LockManager::LOCK_UW] ), + LockManager::LOCK_EX => $this->unsubstPaths( $paths[LockManager::LOCK_EX] ) ); + // Actually acquire the locks + return array( $this->getScopedFileLocks( $pbPaths, 'mixed', $status ) ); } } diff --git a/includes/filebackend/FileBackendStore.php b/includes/filebackend/FileBackendStore.php index 3f1d1857..0921e99f 100644 --- a/includes/filebackend/FileBackendStore.php +++ b/includes/filebackend/FileBackendStore.php @@ -38,28 +38,43 @@ abstract class FileBackendStore extends FileBackend { /** @var BagOStuff */ protected $memCache; - /** @var ProcessCacheLRU */ - protected $cheapCache; // Map of paths to small (RAM/disk) cache items - /** @var ProcessCacheLRU */ - protected $expensiveCache; // Map of paths to large (RAM/disk) cache items + /** @var ProcessCacheLRU Map of paths to small (RAM/disk) cache items */ + protected $cheapCache; + /** @var ProcessCacheLRU Map of paths to large (RAM/disk) cache items */ + protected $expensiveCache; - /** @var Array Map of container names to sharding settings */ - protected $shardViaHashLevels = array(); // (container name => config array) + /** @var Array Map of container names to sharding config */ + protected $shardViaHashLevels = array(); + + /** @var callback Method to get the MIME type of files */ + protected $mimeCallback; protected $maxFileSize = 4294967296; // integer bytes (4GiB) const CACHE_TTL = 10; // integer; TTL in seconds for process cache entries + const CACHE_CHEAP_SIZE = 300; // integer; max entries in "cheap cache" + const CACHE_EXPENSIVE_SIZE = 5; // integer; max entries in "expensive cache" /** * @see FileBackend::__construct() + * Additional $config params include: + * - mimeCallback : Callback that takes (storage path, content, file system path) and + * returns the MIME type of the file or 'unknown/unknown'. The file + * system path parameter should be used if the content one is null. * - * @param $config Array + * @param array $config */ public function __construct( array $config ) { parent::__construct( $config ); + $this->mimeCallback = isset( $config['mimeCallback'] ) + ? $config['mimeCallback'] + : function( $storagePath, $content, $fsPath ) { + // @TODO: handle the case of extension-less files using the contents + return StreamFile::contentTypeFromPath( $storagePath ) ?: 'unknown/unknown'; + }; $this->memCache = new EmptyBagOStuff(); // disabled by default - $this->cheapCache = new ProcessCacheLRU( 300 ); - $this->expensiveCache = new ProcessCacheLRU( 5 ); + $this->cheapCache = new ProcessCacheLRU( self::CACHE_CHEAP_SIZE ); + $this->expensiveCache = new ProcessCacheLRU( self::CACHE_EXPENSIVE_SIZE ); } /** @@ -79,7 +94,7 @@ abstract class FileBackendStore extends FileBackend { * written under it, and that any file already there is writable. * Backends using key/value stores should check if the container exists. * - * @param $storagePath string + * @param string $storagePath * @return bool */ abstract public function isPathUsableInternal( $storagePath ); @@ -92,7 +107,6 @@ abstract class FileBackendStore extends FileBackend { * $params include: * - content : the raw file contents * - dst : destination storage path - * - disposition : Content-Disposition header value for the destination * - headers : HTTP header name/value map * - async : Status will be returned immediately if supported. * If the status is OK, then its value field will be @@ -104,8 +118,7 @@ abstract class FileBackendStore extends FileBackend { * @return Status */ final public function createInternal( array $params ) { - wfProfileIn( __METHOD__ ); - wfProfileIn( __METHOD__ . '-' . $this->name ); + $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); if ( strlen( $params['content'] ) > $this->maxFileSizeInternal() ) { $status = Status::newFatal( 'backend-fail-maxsize', $params['dst'], $this->maxFileSizeInternal() ); @@ -116,8 +129,6 @@ abstract class FileBackendStore extends FileBackend { $this->deleteFileCache( $params['dst'] ); // persistent cache } } - wfProfileOut( __METHOD__ . '-' . $this->name ); - wfProfileOut( __METHOD__ ); return $status; } @@ -135,7 +146,6 @@ abstract class FileBackendStore extends FileBackend { * $params include: * - src : source path on disk * - dst : destination storage path - * - disposition : Content-Disposition header value for the destination * - headers : HTTP header name/value map * - async : Status will be returned immediately if supported. * If the status is OK, then its value field will be @@ -147,8 +157,7 @@ abstract class FileBackendStore extends FileBackend { * @return Status */ final public function storeInternal( array $params ) { - wfProfileIn( __METHOD__ ); - wfProfileIn( __METHOD__ . '-' . $this->name ); + $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); if ( filesize( $params['src'] ) > $this->maxFileSizeInternal() ) { $status = Status::newFatal( 'backend-fail-maxsize', $params['dst'], $this->maxFileSizeInternal() ); @@ -159,8 +168,6 @@ abstract class FileBackendStore extends FileBackend { $this->deleteFileCache( $params['dst'] ); // persistent cache } } - wfProfileOut( __METHOD__ . '-' . $this->name ); - wfProfileOut( __METHOD__ ); return $status; } @@ -179,7 +186,7 @@ abstract class FileBackendStore extends FileBackend { * - src : source storage path * - dst : destination storage path * - ignoreMissingSource : do nothing if the source file does not exist - * - disposition : Content-Disposition header value for the destination + * - headers : HTTP header name/value map * - async : Status will be returned immediately if supported. * If the status is OK, then its value field will be * set to a FileBackendStoreOpHandle object. @@ -190,15 +197,12 @@ abstract class FileBackendStore extends FileBackend { * @return Status */ final public function copyInternal( array $params ) { - wfProfileIn( __METHOD__ ); - wfProfileIn( __METHOD__ . '-' . $this->name ); + $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); $status = $this->doCopyInternal( $params ); $this->clearCache( array( $params['dst'] ) ); if ( !isset( $params['dstExists'] ) || $params['dstExists'] ) { $this->deleteFileCache( $params['dst'] ); // persistent cache } - wfProfileOut( __METHOD__ . '-' . $this->name ); - wfProfileOut( __METHOD__ ); return $status; } @@ -223,13 +227,10 @@ abstract class FileBackendStore extends FileBackend { * @return Status */ final public function deleteInternal( array $params ) { - wfProfileIn( __METHOD__ ); - wfProfileIn( __METHOD__ . '-' . $this->name ); + $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); $status = $this->doDeleteInternal( $params ); $this->clearCache( array( $params['src'] ) ); $this->deleteFileCache( $params['src'] ); // persistent cache - wfProfileOut( __METHOD__ . '-' . $this->name ); - wfProfileOut( __METHOD__ ); return $status; } @@ -248,7 +249,7 @@ abstract class FileBackendStore extends FileBackend { * - src : source storage path * - dst : destination storage path * - ignoreMissingSource : do nothing if the source file does not exist - * - disposition : Content-Disposition header value for the destination + * - headers : HTTP header name/value map * - async : Status will be returned immediately if supported. * If the status is OK, then its value field will be * set to a FileBackendStoreOpHandle object. @@ -259,16 +260,13 @@ abstract class FileBackendStore extends FileBackend { * @return Status */ final public function moveInternal( array $params ) { - wfProfileIn( __METHOD__ ); - wfProfileIn( __METHOD__ . '-' . $this->name ); + $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); $status = $this->doMoveInternal( $params ); $this->clearCache( array( $params['src'], $params['dst'] ) ); $this->deleteFileCache( $params['src'] ); // persistent cache if ( !isset( $params['dstExists'] ) || $params['dstExists'] ) { $this->deleteFileCache( $params['dst'] ); // persistent cache } - wfProfileOut( __METHOD__ . '-' . $this->name ); - wfProfileOut( __METHOD__ ); return $status; } @@ -278,10 +276,12 @@ abstract class FileBackendStore extends FileBackend { */ protected function doMoveInternal( array $params ) { unset( $params['async'] ); // two steps, won't work here :) + $nsrc = FileBackend::normalizeStoragePath( $params['src'] ); + $ndst = FileBackend::normalizeStoragePath( $params['dst'] ); // Copy source to dest $status = $this->copyInternal( $params ); - if ( $status->isOK() ) { - // Delete source (only fails due to races or medium going down) + if ( $nsrc !== $ndst && $status->isOK() ) { + // Delete source (only fails due to races or network problems) $status->merge( $this->deleteInternal( array( 'src' => $params['src'] ) ) ); $status->setResult( true, $status->value ); // ignore delete() errors } @@ -294,7 +294,6 @@ abstract class FileBackendStore extends FileBackend { * * $params include: * - src : source storage path - * - disposition : Content-Disposition header value for the destination * - headers : HTTP header name/value map * - async : Status will be returned immediately if supported. * If the status is OK, then its value field will be @@ -304,13 +303,14 @@ abstract class FileBackendStore extends FileBackend { * @return Status */ final public function describeInternal( array $params ) { - wfProfileIn( __METHOD__ ); - wfProfileIn( __METHOD__ . '-' . $this->name ); - $status = $this->doDescribeInternal( $params ); - $this->clearCache( array( $params['src'] ) ); - $this->deleteFileCache( $params['src'] ); // persistent cache - wfProfileOut( __METHOD__ . '-' . $this->name ); - wfProfileOut( __METHOD__ ); + $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); + if ( count( $params['headers'] ) ) { + $status = $this->doDescribeInternal( $params ); + $this->clearCache( array( $params['src'] ) ); + $this->deleteFileCache( $params['src'] ); // persistent cache + } else { + $status = Status::newGood(); // nothing to do + } return $status; } @@ -333,13 +333,8 @@ abstract class FileBackendStore extends FileBackend { return Status::newGood(); } - /** - * @see FileBackend::concatenate() - * @return Status - */ final public function concatenate( array $params ) { - wfProfileIn( __METHOD__ ); - wfProfileIn( __METHOD__ . '-' . $this->name ); + $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); $status = Status::newGood(); // Try to lock the source files for the scope of this function @@ -355,8 +350,6 @@ abstract class FileBackendStore extends FileBackend { } } - wfProfileOut( __METHOD__ . '-' . $this->name ); - wfProfileOut( __METHOD__ ); return $status; } @@ -426,20 +419,13 @@ abstract class FileBackendStore extends FileBackend { return $status; } - /** - * @see FileBackend::doPrepare() - * @return Status - */ final protected function doPrepare( array $params ) { - wfProfileIn( __METHOD__ ); - wfProfileIn( __METHOD__ . '-' . $this->name ); - + $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); $status = Status::newGood(); + list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] ); if ( $dir === null ) { $status->fatal( 'backend-fail-invalidpath', $params['dir'] ); - wfProfileOut( __METHOD__ . '-' . $this->name ); - wfProfileOut( __METHOD__ ); return $status; // invalid storage path } @@ -453,8 +439,6 @@ abstract class FileBackendStore extends FileBackend { } } - wfProfileOut( __METHOD__ . '-' . $this->name ); - wfProfileOut( __METHOD__ ); return $status; } @@ -466,20 +450,13 @@ abstract class FileBackendStore extends FileBackend { return Status::newGood(); } - /** - * @see FileBackend::doSecure() - * @return Status - */ final protected function doSecure( array $params ) { - wfProfileIn( __METHOD__ ); - wfProfileIn( __METHOD__ . '-' . $this->name ); + $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); $status = Status::newGood(); list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] ); if ( $dir === null ) { $status->fatal( 'backend-fail-invalidpath', $params['dir'] ); - wfProfileOut( __METHOD__ . '-' . $this->name ); - wfProfileOut( __METHOD__ ); return $status; // invalid storage path } @@ -493,8 +470,6 @@ abstract class FileBackendStore extends FileBackend { } } - wfProfileOut( __METHOD__ . '-' . $this->name ); - wfProfileOut( __METHOD__ ); return $status; } @@ -506,20 +481,13 @@ abstract class FileBackendStore extends FileBackend { return Status::newGood(); } - /** - * @see FileBackend::doPublish() - * @return Status - */ final protected function doPublish( array $params ) { - wfProfileIn( __METHOD__ ); - wfProfileIn( __METHOD__ . '-' . $this->name ); + $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); $status = Status::newGood(); list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] ); if ( $dir === null ) { $status->fatal( 'backend-fail-invalidpath', $params['dir'] ); - wfProfileOut( __METHOD__ . '-' . $this->name ); - wfProfileOut( __METHOD__ ); return $status; // invalid storage path } @@ -533,8 +501,6 @@ abstract class FileBackendStore extends FileBackend { } } - wfProfileOut( __METHOD__ . '-' . $this->name ); - wfProfileOut( __METHOD__ ); return $status; } @@ -546,13 +512,8 @@ abstract class FileBackendStore extends FileBackend { return Status::newGood(); } - /** - * @see FileBackend::doClean() - * @return Status - */ final protected function doClean( array $params ) { - wfProfileIn( __METHOD__ ); - wfProfileIn( __METHOD__ . '-' . $this->name ); + $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); $status = Status::newGood(); // Recursive: first delete all empty subdirs recursively @@ -570,8 +531,6 @@ abstract class FileBackendStore extends FileBackend { list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] ); if ( $dir === null ) { $status->fatal( 'backend-fail-invalidpath', $params['dir'] ); - wfProfileOut( __METHOD__ . '-' . $this->name ); - wfProfileOut( __METHOD__ ); return $status; // invalid storage path } @@ -579,8 +538,6 @@ abstract class FileBackendStore extends FileBackend { $filesLockEx = array( $params['dir'] ); $scopedLockE = $this->getScopedFileLocks( $filesLockEx, LockManager::LOCK_EX, $status ); if ( !$status->isOK() ) { - wfProfileOut( __METHOD__ . '-' . $this->name ); - wfProfileOut( __METHOD__ ); return $status; // abort } @@ -596,8 +553,6 @@ abstract class FileBackendStore extends FileBackend { } } - wfProfileOut( __METHOD__ . '-' . $this->name ); - wfProfileOut( __METHOD__ ); return $status; } @@ -609,56 +564,30 @@ abstract class FileBackendStore extends FileBackend { return Status::newGood(); } - /** - * @see FileBackend::fileExists() - * @return bool|null - */ final public function fileExists( array $params ) { - wfProfileIn( __METHOD__ ); - wfProfileIn( __METHOD__ . '-' . $this->name ); + $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); $stat = $this->getFileStat( $params ); - wfProfileOut( __METHOD__ . '-' . $this->name ); - wfProfileOut( __METHOD__ ); return ( $stat === null ) ? null : (bool)$stat; // null => failure } - /** - * @see FileBackend::getFileTimestamp() - * @return bool - */ final public function getFileTimestamp( array $params ) { - wfProfileIn( __METHOD__ ); - wfProfileIn( __METHOD__ . '-' . $this->name ); + $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); $stat = $this->getFileStat( $params ); - wfProfileOut( __METHOD__ . '-' . $this->name ); - wfProfileOut( __METHOD__ ); return $stat ? $stat['mtime'] : false; } - /** - * @see FileBackend::getFileSize() - * @return bool - */ final public function getFileSize( array $params ) { - wfProfileIn( __METHOD__ ); - wfProfileIn( __METHOD__ . '-' . $this->name ); + $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); $stat = $this->getFileStat( $params ); - wfProfileOut( __METHOD__ . '-' . $this->name ); - wfProfileOut( __METHOD__ ); return $stat ? $stat['size'] : false; } - /** - * @see FileBackend::getFileStat() - * @return bool - */ final public function getFileStat( array $params ) { $path = self::normalizeStoragePath( $params['src'] ); if ( $path === null ) { return false; // invalid storage path } - wfProfileIn( __METHOD__ ); - wfProfileIn( __METHOD__ . '-' . $this->name ); + $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); $latest = !empty( $params['latest'] ); // use latest data? if ( !$this->cheapCache->has( $path, 'stat', self::CACHE_TTL ) ) { $this->primeFileCache( array( $path ) ); // check persistent cache @@ -669,14 +598,10 @@ abstract class FileBackendStore extends FileBackend { // value was in fact fetched with the latest available data. if ( is_array( $stat ) ) { if ( !$latest || $stat['latest'] ) { - wfProfileOut( __METHOD__ . '-' . $this->name ); - wfProfileOut( __METHOD__ ); return $stat; } } elseif ( in_array( $stat, array( 'NOT_EXIST', 'NOT_EXIST_LATEST' ) ) ) { if ( !$latest || $stat === 'NOT_EXIST_LATEST' ) { - wfProfileOut( __METHOD__ . '-' . $this->name ); - wfProfileOut( __METHOD__ ); return false; } } @@ -696,12 +621,12 @@ abstract class FileBackendStore extends FileBackend { } } elseif ( $stat === false ) { // file does not exist $this->cheapCache->set( $path, 'stat', $latest ? 'NOT_EXIST_LATEST' : 'NOT_EXIST' ); + $this->cheapCache->set( $path, 'sha1', // the SHA-1 must be false too + array( 'hash' => false, 'latest' => $latest ) ); wfDebug( __METHOD__ . ": File $path does not exist.\n" ); } else { // an error occurred wfDebug( __METHOD__ . ": Could not stat file $path.\n" ); } - wfProfileOut( __METHOD__ . '-' . $this->name ); - wfProfileOut( __METHOD__ ); return $stat; } @@ -710,19 +635,12 @@ abstract class FileBackendStore extends FileBackend { */ abstract protected function doGetFileStat( array $params ); - /** - * @see FileBackend::getFileContentsMulti() - * @return Array - */ public function getFileContentsMulti( array $params ) { - wfProfileIn( __METHOD__ ); - wfProfileIn( __METHOD__ . '-' . $this->name ); + $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); $params = $this->setConcurrencyFlags( $params ); $contents = $this->doGetFileContentsMulti( $params ); - wfProfileOut( __METHOD__ . '-' . $this->name ); - wfProfileOut( __METHOD__ ); return $contents; } @@ -740,25 +658,18 @@ abstract class FileBackendStore extends FileBackend { return $contents; } - /** - * @see FileBackend::getFileSha1Base36() - * @return bool|string - */ final public function getFileSha1Base36( array $params ) { $path = self::normalizeStoragePath( $params['src'] ); if ( $path === null ) { return false; // invalid storage path } - wfProfileIn( __METHOD__ ); - wfProfileIn( __METHOD__ . '-' . $this->name ); + $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); $latest = !empty( $params['latest'] ); // use latest data? if ( $this->cheapCache->has( $path, 'sha1', self::CACHE_TTL ) ) { $stat = $this->cheapCache->get( $path, 'sha1' ); // If we want the latest data, check that this cached // value was in fact fetched with the latest available data. if ( !$latest || $stat['latest'] ) { - wfProfileOut( __METHOD__ . '-' . $this->name ); - wfProfileOut( __METHOD__ ); return $stat['hash']; } } @@ -768,8 +679,6 @@ abstract class FileBackendStore extends FileBackend { wfProfileOut( __METHOD__ . '-miss-' . $this->name ); wfProfileOut( __METHOD__ . '-miss' ); $this->cheapCache->set( $path, 'sha1', array( 'hash' => $hash, 'latest' => $latest ) ); - wfProfileOut( __METHOD__ . '-' . $this->name ); - wfProfileOut( __METHOD__ ); return $hash; } @@ -786,27 +695,15 @@ abstract class FileBackendStore extends FileBackend { } } - /** - * @see FileBackend::getFileProps() - * @return Array - */ final public function getFileProps( array $params ) { - wfProfileIn( __METHOD__ ); - wfProfileIn( __METHOD__ . '-' . $this->name ); + $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); $fsFile = $this->getLocalReference( $params ); $props = $fsFile ? $fsFile->getProps() : FSFile::placeholderProps(); - wfProfileOut( __METHOD__ . '-' . $this->name ); - wfProfileOut( __METHOD__ ); return $props; } - /** - * @see FileBackend::getLocalReferenceMulti() - * @return Array - */ final public function getLocalReferenceMulti( array $params ) { - wfProfileIn( __METHOD__ ); - wfProfileIn( __METHOD__ . '-' . $this->name ); + $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); $params = $this->setConcurrencyFlags( $params ); @@ -836,8 +733,6 @@ abstract class FileBackendStore extends FileBackend { } } - wfProfileOut( __METHOD__ . '-' . $this->name ); - wfProfileOut( __METHOD__ ); return $fsFiles; } @@ -849,19 +744,12 @@ abstract class FileBackendStore extends FileBackend { return $this->doGetLocalCopyMulti( $params ); } - /** - * @see FileBackend::getLocalCopyMulti() - * @return Array - */ final public function getLocalCopyMulti( array $params ) { - wfProfileIn( __METHOD__ ); - wfProfileIn( __METHOD__ . '-' . $this->name ); + $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); $params = $this->setConcurrencyFlags( $params ); $tmpFiles = $this->doGetLocalCopyMulti( $params ); - wfProfileOut( __METHOD__ . '-' . $this->name ); - wfProfileOut( __METHOD__ ); return $tmpFiles; } @@ -879,13 +767,8 @@ abstract class FileBackendStore extends FileBackend { return null; // not supported } - /** - * @see FileBackend::streamFile() - * @return Status - */ final public function streamFile( array $params ) { - wfProfileIn( __METHOD__ ); - wfProfileIn( __METHOD__ . '-' . $this->name ); + $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); $status = Status::newGood(); $info = $this->getFileStat( $params ); @@ -916,8 +799,6 @@ abstract class FileBackendStore extends FileBackend { $status->fatal( 'backend-fail-stream', $params['src'] ); } - wfProfileOut( __METHOD__ . '-' . $this->name ); - wfProfileOut( __METHOD__ ); return $status; } @@ -938,10 +819,6 @@ abstract class FileBackendStore extends FileBackend { return $status; } - /** - * @see FileBackend::directoryExists() - * @return bool|null - */ final public function directoryExists( array $params ) { list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] ); if ( $dir === null ) { @@ -976,10 +853,6 @@ abstract class FileBackendStore extends FileBackend { */ abstract protected function doDirectoryExists( $container, $dir, array $params ); - /** - * @see FileBackend::getDirectoryList() - * @return Traversable|Array|null Returns null on failure - */ final public function getDirectoryList( array $params ) { list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] ); if ( $dir === null ) { // invalid storage path @@ -1009,10 +882,6 @@ abstract class FileBackendStore extends FileBackend { */ abstract public function getDirectoryListInternal( $container, $dir, array $params ); - /** - * @see FileBackend::getFileList() - * @return Traversable|Array|null Returns null on failure - */ final public function getFileList( array $params ) { list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] ); if ( $dir === null ) { // invalid storage path @@ -1055,13 +924,13 @@ abstract class FileBackendStore extends FileBackend { */ final public function getOperationsInternal( array $ops ) { $supportedOps = array( - 'store' => 'StoreFileOp', - 'copy' => 'CopyFileOp', - 'move' => 'MoveFileOp', - 'delete' => 'DeleteFileOp', - 'create' => 'CreateFileOp', + 'store' => 'StoreFileOp', + 'copy' => 'CopyFileOp', + 'move' => 'MoveFileOp', + 'delete' => 'DeleteFileOp', + 'create' => 'CreateFileOp', 'describe' => 'DescribeFileOp', - 'null' => 'NullFileOp' + 'null' => 'NullFileOp' ); $performOps = array(); // array of FileOp objects @@ -1084,12 +953,13 @@ abstract class FileBackendStore extends FileBackend { /** * Get a list of storage paths to lock for a list of operations - * Returns an array with 'sh' (shared) and 'ex' (exclusive) keys, - * each corresponding to a list of storage paths to be locked. - * All returned paths are normalized. + * Returns an array with LockManager::LOCK_UW (shared locks) and + * LockManager::LOCK_EX (exclusive locks) keys, each corresponding + * to a list of storage paths to be locked. All returned paths are + * normalized. * * @param array $performOps List of FileOp objects - * @return Array ('sh' => list of paths, 'ex' => list of paths) + * @return Array (LockManager::LOCK_UW => path list, LockManager::LOCK_EX => path list) */ final public function getPathsToLockForOpsInternal( array $performOps ) { // Build up a list of files to lock... @@ -1103,28 +973,19 @@ abstract class FileBackendStore extends FileBackend { // Get a shared lock on the parent directory of each path changed $paths['sh'] = array_merge( $paths['sh'], array_map( 'dirname', $paths['ex'] ) ); - return $paths; + return array( + LockManager::LOCK_UW => $paths['sh'], + LockManager::LOCK_EX => $paths['ex'] + ); } - /** - * @see FileBackend::getScopedLocksForOps() - * @return Array - */ public function getScopedLocksForOps( array $ops, Status $status ) { $paths = $this->getPathsToLockForOpsInternal( $this->getOperationsInternal( $ops ) ); - return array( - $this->getScopedFileLocks( $paths['sh'], LockManager::LOCK_UW, $status ), - $this->getScopedFileLocks( $paths['ex'], LockManager::LOCK_EX, $status ) - ); + return array( $this->getScopedFileLocks( $paths, 'mixed', $status ) ); } - /** - * @see FileBackend::doOperationsInternal() - * @return Status - */ final protected function doOperationsInternal( array $ops, array $opts ) { - wfProfileIn( __METHOD__ ); - wfProfileIn( __METHOD__ . '-' . $this->name ); + $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); $status = Status::newGood(); // Fix up custom header name/value pairs... @@ -1138,11 +999,8 @@ abstract class FileBackendStore extends FileBackend { // Build up a list of files to lock... $paths = $this->getPathsToLockForOpsInternal( $performOps ); // Try to lock those files for the scope of this function... - $scopeLockS = $this->getScopedFileLocks( $paths['sh'], LockManager::LOCK_UW, $status ); - $scopeLockE = $this->getScopedFileLocks( $paths['ex'], LockManager::LOCK_EX, $status ); + $scopeLock = $this->getScopedFileLocks( $paths, 'mixed', $status ); if ( !$status->isOK() ) { - wfProfileOut( __METHOD__ . '-' . $this->name ); - wfProfileOut( __METHOD__ ); return $status; // abort } } @@ -1164,19 +1022,11 @@ abstract class FileBackendStore extends FileBackend { $status->merge( $subStatus ); $status->success = $subStatus->success; // not done in merge() - wfProfileOut( __METHOD__ . '-' . $this->name ); - wfProfileOut( __METHOD__ ); return $status; } - /** - * @see FileBackend::doQuickOperationsInternal() - * @return Status - * @throws MWException - */ final protected function doQuickOperationsInternal( array $ops ) { - wfProfileIn( __METHOD__ ); - wfProfileIn( __METHOD__ . '-' . $this->name ); + $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); $status = Status::newGood(); // Fix up custom header name/value pairs... @@ -1186,7 +1036,7 @@ abstract class FileBackendStore extends FileBackend { $this->clearCache(); $supportedOps = array( 'create', 'store', 'copy', 'move', 'delete', 'null' ); - $async = ( $this->parallelize === 'implicit' ); + $async = ( $this->parallelize === 'implicit' && count( $ops ) > 1 ); $maxConcurrency = $this->concurrency; // throttle $statuses = array(); // array of (index => Status) @@ -1195,8 +1045,6 @@ abstract class FileBackendStore extends FileBackend { // Perform the sync-only ops and build up op handles for the async ops... foreach ( $ops as $index => $params ) { if ( !in_array( $params['op'], $supportedOps ) ) { - wfProfileOut( __METHOD__ . '-' . $this->name ); - wfProfileOut( __METHOD__ ); throw new MWException( "Operation '{$params['op']}' is not supported." ); } $method = $params['op'] . 'Internal'; // e.g. "storeInternal" @@ -1230,8 +1078,6 @@ abstract class FileBackendStore extends FileBackend { } } - wfProfileOut( __METHOD__ . '-' . $this->name ); - wfProfileOut( __METHOD__ ); return $status; } @@ -1245,8 +1091,7 @@ abstract class FileBackendStore extends FileBackend { * @throws MWException */ final public function executeOpHandlesInternal( array $fileOpHandles ) { - wfProfileIn( __METHOD__ ); - wfProfileIn( __METHOD__ . '-' . $this->name ); + $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); foreach ( $fileOpHandles as $fileOpHandle ) { if ( !( $fileOpHandle instanceof FileBackendStoreOpHandle ) ) { throw new MWException( "Given a non-FileBackendStoreOpHandle object." ); @@ -1258,8 +1103,6 @@ abstract class FileBackendStore extends FileBackend { foreach ( $fileOpHandles as $fileOpHandle ) { $fileOpHandle->closeResources(); } - wfProfileOut( __METHOD__ . '-' . $this->name ); - wfProfileOut( __METHOD__ ); return $res; } @@ -1277,15 +1120,20 @@ abstract class FileBackendStore extends FileBackend { } /** - * Strip long HTTP headers from a file operation + * Strip long HTTP headers from a file operation. + * Most headers are just numbers, but some are allowed to be long. + * This function is useful for cleaning up headers and avoiding backend + * specific errors, especially in the middle of batch file operations. * * @param array $op Same format as doOperation() * @return Array */ protected function stripInvalidHeadersFromOp( array $op ) { - if ( isset( $op['headers'] ) ) { + static $longs = array( 'Content-Disposition' ); + if ( isset( $op['headers'] ) ) { // op sets HTTP headers foreach ( $op['headers'] as $name => $value ) { - if ( strlen( $name ) > 255 || strlen( $value ) > 255 ) { + $maxHVLen = in_array( $name, $longs ) ? INF : 255; + if ( strlen( $name ) > 255 || strlen( $value ) > $maxHVLen ) { trigger_error( "Header '$name: $value' is too long." ); unset( $op['headers'][$name] ); } elseif ( !strlen( $value ) ) { @@ -1296,9 +1144,6 @@ abstract class FileBackendStore extends FileBackend { return $op; } - /** - * @see FileBackend::preloadCache() - */ final public function preloadCache( array $paths ) { $fullConts = array(); // full container names foreach ( $paths as $path ) { @@ -1310,9 +1155,6 @@ abstract class FileBackendStore extends FileBackend { $this->primeFileCache( $paths ); } - /** - * @see FileBackend::clearCache() - */ final public function clearCache( array $paths = null ) { if ( is_array( $paths ) ) { $paths = array_map( 'FileBackend::normalizeStoragePath', $paths ); @@ -1353,7 +1195,7 @@ abstract class FileBackendStore extends FileBackend { * Check if a container name is valid. * This checks for for length and illegal characters. * - * @param $container string + * @param string $container * @return bool */ final protected static function isValidContainerName( $container ) { @@ -1375,7 +1217,7 @@ abstract class FileBackendStore extends FileBackend { * this means that the path can only refer to a directory and can only * be scanned by looking in all the container shards. * - * @param $storagePath string + * @param string $storagePath * @return Array (container, path, container suffix) or (null, null, null) if invalid */ final protected function resolveStoragePath( $storagePath ) { @@ -1405,16 +1247,22 @@ abstract class FileBackendStore extends FileBackend { /** * Like resolveStoragePath() except null values are returned if - * the container is sharded and the shard could not be determined. + * the container is sharded and the shard could not be determined + * or if the path ends with '/'. The later case is illegal for FS + * backends and can confuse listings for object store backends. + * + * This function is used when resolving paths that must be valid + * locations for files. Directory and listing functions should + * generally just use resolveStoragePath() instead. * * @see FileBackendStore::resolveStoragePath() * - * @param $storagePath string + * @param string $storagePath * @return Array (container, path) or (null, null) if invalid */ final protected function resolveStoragePathReal( $storagePath ) { list( $container, $relPath, $cShard ) = $this->resolveStoragePath( $storagePath ); - if ( $cShard !== null ) { + if ( $cShard !== null && substr( $relPath, -1 ) !== '/' ) { return array( $container, $relPath ); } return array( null, null ); @@ -1474,7 +1322,7 @@ abstract class FileBackendStore extends FileBackend { * If greater than 0, then all file storage paths within * the container are required to be hashed accordingly. * - * @param $container string + * @param string $container * @return Array (integer levels, integer base, repeat flag) or (0, 0, false) */ final protected function getContainerHashLevels( $container ) { @@ -1494,7 +1342,7 @@ abstract class FileBackendStore extends FileBackend { /** * Get a list of full container shard suffixes for a container * - * @param $container string + * @param string $container * @return Array */ final protected function getContainerSuffixes( $container ) { @@ -1512,7 +1360,7 @@ abstract class FileBackendStore extends FileBackend { /** * Get the full container name, including the wiki ID prefix * - * @param $container string + * @param string $container * @return string */ final protected function fullContainerName( $container ) { @@ -1528,7 +1376,7 @@ abstract class FileBackendStore extends FileBackend { * This is intended for internal use, such as encoding illegal chars. * Subclasses can override this to be more restrictive. * - * @param $container string + * @param string $container * @return string|null */ protected function resolveContainerName( $container ) { @@ -1563,10 +1411,11 @@ abstract class FileBackendStore extends FileBackend { * Set the cached info for a container * * @param string $container Resolved container name - * @param $val mixed Information to cache + * @param array $val Information to cache + * @return void */ - final protected function setContainerCache( $container, $val ) { - $this->memCache->add( $this->containerCacheKey( $container ), $val, 14*86400 ); + final protected function setContainerCache( $container, array $val ) { + $this->memCache->add( $this->containerCacheKey( $container ), $val, 14 * 86400 ); } /** @@ -1574,6 +1423,7 @@ abstract class FileBackendStore extends FileBackend { * The cache key is salted for a while to prevent race conditions. * * @param string $container Resolved container name + * @return void */ final protected function deleteContainerCache( $container ) { if ( !$this->memCache->set( $this->containerCacheKey( $container ), 'PURGED', 300 ) ) { @@ -1586,12 +1436,11 @@ abstract class FileBackendStore extends FileBackend { * used in a list of container names, storage paths, or FileOp objects. * This loads the persistent cache values into the process cache. * - * @param $items Array + * @param Array $items * @return void */ final protected function primeContainerCache( array $items ) { - wfProfileIn( __METHOD__ ); - wfProfileIn( __METHOD__ . '-' . $this->name ); + $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); $paths = array(); // list of storage paths $contNames = array(); // (cache key => resolved container name) @@ -1623,9 +1472,6 @@ abstract class FileBackendStore extends FileBackend { // Populate the container process cache for the backend... $this->doPrimeContainerCache( array_filter( $contInfo, 'is_array' ) ); - - wfProfileOut( __METHOD__ . '-' . $this->name ); - wfProfileOut( __METHOD__ ); } /** @@ -1654,14 +1500,17 @@ abstract class FileBackendStore extends FileBackend { * salting for the case when a file is created at a path were there was none before. * * @param string $path Storage path - * @param $val mixed Information to cache + * @param array $val Stat information to cache + * @return void */ - final protected function setFileCache( $path, $val ) { + final protected function setFileCache( $path, array $val ) { $path = FileBackend::normalizeStoragePath( $path ); if ( $path === null ) { return; // invalid storage path } - $this->memCache->add( $this->fileCacheKey( $path ), $val, 7*86400 ); + $age = time() - wfTimestamp( TS_UNIX, $val['mtime'] ); + $ttl = min( 7 * 86400, max( 300, floor( .1 * $age ) ) ); + $this->memCache->add( $this->fileCacheKey( $path ), $val, $ttl ); } /** @@ -1671,6 +1520,7 @@ abstract class FileBackendStore extends FileBackend { * a file is created at a path were there was none before. * * @param string $path Storage path + * @return void */ final protected function deleteFileCache( $path ) { $path = FileBackend::normalizeStoragePath( $path ); @@ -1691,8 +1541,7 @@ abstract class FileBackendStore extends FileBackend { * @return void */ final protected function primeFileCache( array $items ) { - wfProfileIn( __METHOD__ ); - wfProfileIn( __METHOD__ . '-' . $this->name ); + $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); $paths = array(); // list of storage paths $pathNames = array(); // (cache key => storage path) @@ -1726,9 +1575,6 @@ abstract class FileBackendStore extends FileBackend { } } } - - wfProfileOut( __METHOD__ . '-' . $this->name ); - wfProfileOut( __METHOD__ ); } /** @@ -1750,6 +1596,18 @@ abstract class FileBackendStore extends FileBackend { } return $opts; } + + /** + * Get the content type to use in HEAD/GET requests for a file + * + * @param string $storagePath + * @param string|null $content File data + * @param string|null $fsPath File system path + * @return MIME type + */ + protected function getContentType( $storagePath, $content, $fsPath ) { + return call_user_func_array( $this->mimeCallback, func_get_args() ); + } } /** @@ -1786,26 +1644,20 @@ abstract class FileBackendStoreOpHandle { * * @ingroup FileBackend */ -abstract class FileBackendStoreShardListIterator implements Iterator { +abstract class FileBackendStoreShardListIterator extends FilterIterator { /** @var FileBackendStore */ protected $backend; /** @var Array */ protected $params; - /** @var Array */ - protected $shardSuffixes; + protected $container; // string; full container name protected $directory; // string; resolved relative path - /** @var Traversable */ - protected $iter; - protected $curShard = 0; // integer - protected $pos = 0; // integer - /** @var Array */ protected $multiShardPaths = array(); // (rel path => 1) /** - * @param $backend FileBackendStore + * @param FileBackendStore $backend * @param string $container Full storage container name * @param string $dir Storage directory relative to container * @param array $suffixes List of container shard suffixes @@ -1817,142 +1669,56 @@ abstract class FileBackendStoreShardListIterator implements Iterator { $this->backend = $backend; $this->container = $container; $this->directory = $dir; - $this->shardSuffixes = $suffixes; $this->params = $params; - } - - /** - * @see Iterator::key() - * @return integer - */ - public function key() { - return $this->pos; - } - /** - * @see Iterator::valid() - * @return bool - */ - public function valid() { - if ( $this->iter instanceof Iterator ) { - return $this->iter->valid(); - } elseif ( is_array( $this->iter ) ) { - return ( current( $this->iter ) !== false ); // no paths can have this value + $iter = new AppendIterator(); + foreach ( $suffixes as $suffix ) { + $iter->append( $this->listFromShard( $this->container . $suffix ) ); } - return false; // some failure? - } - - /** - * @see Iterator::current() - * @return string|bool String or false - */ - public function current() { - return ( $this->iter instanceof Iterator ) - ? $this->iter->current() - : current( $this->iter ); - } - /** - * @see Iterator::next() - * @return void - */ - public function next() { - ++$this->pos; - ( $this->iter instanceof Iterator ) ? $this->iter->next() : next( $this->iter ); - do { - $continue = false; // keep scanning shards? - $this->filterViaNext(); // filter out duplicates - // Find the next non-empty shard if no elements are left - if ( !$this->valid() ) { - $this->nextShardIteratorIfNotValid(); - $continue = $this->valid(); // re-filter unless we ran out of shards - } - } while ( $continue ); - } - - /** - * @see Iterator::rewind() - * @return void - */ - public function rewind() { - $this->pos = 0; - $this->curShard = 0; - $this->setIteratorFromCurrentShard(); - do { - $continue = false; // keep scanning shards? - $this->filterViaNext(); // filter out duplicates - // Find the next non-empty shard if no elements are left - if ( !$this->valid() ) { - $this->nextShardIteratorIfNotValid(); - $continue = $this->valid(); // re-filter unless we ran out of shards - } - } while ( $continue ); - } - - /** - * Filter out duplicate items by advancing to the next ones - */ - protected function filterViaNext() { - while ( $this->valid() ) { - $rel = $this->iter->current(); // path relative to given directory - $path = $this->params['dir'] . "/{$rel}"; // full storage path - if ( $this->backend->isSingleShardPathInternal( $path ) ) { - break; // path is only on one shard; no issue with duplicates - } elseif ( isset( $this->multiShardPaths[$rel] ) ) { - // Don't keep listing paths that are on multiple shards - ( $this->iter instanceof Iterator ) ? $this->iter->next() : next( $this->iter ); - } else { - $this->multiShardPaths[$rel] = 1; - break; - } - } + parent::__construct( $iter ); } - /** - * If the list iterator for this container shard is out of items, - * then move on to the next container that has items. - * If there are none, then it advances to the last container. - */ - protected function nextShardIteratorIfNotValid() { - while ( !$this->valid() && ++$this->curShard < count( $this->shardSuffixes ) ) { - $this->setIteratorFromCurrentShard(); + public function accept() { + $rel = $this->getInnerIterator()->current(); // path relative to given directory + $path = $this->params['dir'] . "/{$rel}"; // full storage path + if ( $this->backend->isSingleShardPathInternal( $path ) ) { + return true; // path is only on one shard; no issue with duplicates + } elseif ( isset( $this->multiShardPaths[$rel] ) ) { + // Don't keep listing paths that are on multiple shards + return false; + } else { + $this->multiShardPaths[$rel] = 1; + return true; } } - /** - * Set the list iterator to that of the current container shard - */ - protected function setIteratorFromCurrentShard() { - $this->iter = $this->listFromShard( - $this->container . $this->shardSuffixes[$this->curShard], - $this->directory, $this->params ); - // Start loading results so that current() works - if ( $this->iter ) { - ( $this->iter instanceof Iterator ) ? $this->iter->rewind() : reset( $this->iter ); - } + public function rewind() { + parent::rewind(); + $this->multiShardPaths = array(); } /** * Get the list for a given container shard * * @param string $container Resolved container name - * @param string $dir Resolved path relative to container - * @param array $params - * @return Traversable|Array|null + * @return Iterator */ - abstract protected function listFromShard( $container, $dir, array $params ); + abstract protected function listFromShard( $container ); } /** * Iterator for listing directories */ class FileBackendStoreShardDirIterator extends FileBackendStoreShardListIterator { - /** - * @see FileBackendStoreShardListIterator::listFromShard() - * @return Array|null|Traversable - */ - protected function listFromShard( $container, $dir, array $params ) { - return $this->backend->getDirectoryListInternal( $container, $dir, $params ); + protected function listFromShard( $container ) { + $list = $this->backend->getDirectoryListInternal( + $container, $this->directory, $this->params ); + if ( $list === null ) { + return new ArrayIterator( array() ); + } else { + return is_array( $list ) ? new ArrayIterator( $list ) : $list; + } } } @@ -1960,11 +1726,13 @@ class FileBackendStoreShardDirIterator extends FileBackendStoreShardListIterator * Iterator for listing regular files */ class FileBackendStoreShardFileIterator extends FileBackendStoreShardListIterator { - /** - * @see FileBackendStoreShardListIterator::listFromShard() - * @return Array|null|Traversable - */ - protected function listFromShard( $container, $dir, array $params ) { - return $this->backend->getFileListInternal( $container, $dir, $params ); + protected function listFromShard( $container ) { + $list = $this->backend->getFileListInternal( + $container, $this->directory, $this->params ); + if ( $list === null ) { + return new ArrayIterator( array() ); + } else { + return is_array( $list ) ? new ArrayIterator( $list ) : $list; + } } } diff --git a/includes/filebackend/FileOp.php b/includes/filebackend/FileOp.php index bb0ab578..fe833084 100644 --- a/includes/filebackend/FileOp.php +++ b/includes/filebackend/FileOp.php @@ -46,7 +46,7 @@ abstract class FileOp { protected $doOperation = true; // boolean; operation is not a no-op protected $sourceSha1; // string - protected $destSameAsSource; // boolean + protected $overwriteSameCase; // boolean protected $destExists; // boolean /* Object life-cycle */ @@ -55,17 +55,19 @@ abstract class FileOp { const STATE_ATTEMPTED = 3; /** - * Build a new file operation transaction + * Build a new batch file operation transaction * - * @param $backend FileBackendStore - * @param $params Array + * @param FileBackendStore $backend + * @param Array $params * @throws MWException */ final public function __construct( FileBackendStore $backend, array $params ) { $this->backend = $backend; list( $required, $optional ) = $this->allowedParams(); + // @todo normalizeAnyStoragePaths() calls are overzealous, use a parameter list foreach ( $required as $name ) { if ( isset( $params[$name] ) ) { + // Normalize paths so the paths to the same file have the same string $this->params[$name] = self::normalizeAnyStoragePaths( $params[$name] ); } else { throw new MWException( "File operation missing parameter '$name'." ); @@ -73,6 +75,7 @@ abstract class FileOp { } foreach ( $optional as $name ) { if ( isset( $params[$name] ) ) { + // Normalize paths so the paths to the same file have the same string $this->params[$name] = self::normalizeAnyStoragePaths( $params[$name] ); } } @@ -82,7 +85,7 @@ abstract class FileOp { /** * Normalize $item or anything in $item that is a valid storage path * - * @param $item string|array + * @param string $item|array * @return string|Array */ protected function normalizeAnyStoragePaths( $item ) { @@ -102,7 +105,7 @@ abstract class FileOp { /** * Normalize a string if it is a valid storage path * - * @param $path string + * @param string $path * @return string */ protected static function normalizeIfValidStoragePath( $path ) { @@ -116,7 +119,7 @@ abstract class FileOp { /** * Set the batch UUID this operation belongs to * - * @param $batchId string + * @param string $batchId * @return void */ final public function setBatchId( $batchId ) { @@ -126,7 +129,7 @@ abstract class FileOp { /** * Get the value of the parameter with the given name * - * @param $name string + * @param string $name * @return mixed Returns null if the parameter is not set */ final public function getParam( $name ) { @@ -209,22 +212,22 @@ abstract class FileOp { $pathsUsed = array_merge( $this->storagePathsRead(), $this->storagePathsChanged() ); foreach ( array_unique( $pathsUsed ) as $path ) { $nullEntries[] = array( // assertion for recovery - 'op' => 'null', - 'path' => $path, + 'op' => 'null', + 'path' => $path, 'newSha1' => $this->fileSha1( $path, $oPredicates ) ); } foreach ( $this->storagePathsChanged() as $path ) { if ( $nPredicates['sha1'][$path] === false ) { // deleted $deleteEntries[] = array( - 'op' => 'delete', - 'path' => $path, + 'op' => 'delete', + 'path' => $path, 'newSha1' => '' ); } else { // created/updated $updateEntries[] = array( - 'op' => $this->fileExists( $path, $oPredicates ) ? 'update' : 'create', - 'path' => $path, + 'op' => $this->fileExists( $path, $oPredicates ) ? 'update' : 'create', + 'path' => $path, 'newSha1' => $nPredicates['sha1'][$path] ); } @@ -237,7 +240,7 @@ abstract class FileOp { * This must update $predicates for each path that the op can change * except when a failing status object is returned. * - * @param $predicates Array + * @param Array $predicates * @return Status */ final public function precheck( array &$predicates ) { @@ -314,7 +317,7 @@ abstract class FileOp { /** * Adjust params to FileBackendStore internal file calls * - * @param $params Array + * @param Array $params * @return Array (required params list, optional params list) */ protected function setFlags( array $params ) { @@ -341,10 +344,10 @@ abstract class FileOp { /** * Check for errors with regards to the destination file already existing. - * Also set the destExists, destSameAsSource and sourceSha1 member variables. + * Also set the destExists, overwriteSameCase and sourceSha1 member variables. * A bad status will be returned if there is no chance it can be overwritten. * - * @param $predicates Array + * @param Array $predicates * @return Status */ protected function precheckDestExistence( array $predicates ) { @@ -354,7 +357,7 @@ abstract class FileOp { if ( $this->sourceSha1 === null ) { // file in storage? $this->sourceSha1 = $this->fileSha1( $this->params['src'], $predicates ); } - $this->destSameAsSource = false; + $this->overwriteSameCase = false; $this->destExists = $this->fileExists( $this->params['dst'], $predicates ); if ( $this->destExists ) { if ( $this->getParam( 'overwrite' ) ) { @@ -368,7 +371,7 @@ abstract class FileOp { // Give an error if the files are not identical $status->fatal( 'backend-fail-notsame', $this->params['dst'] ); } else { - $this->destSameAsSource = true; // OK + $this->overwriteSameCase = true; // OK } return $status; // do nothing; either OK or bad status } else { @@ -381,7 +384,7 @@ abstract class FileOp { /** * precheckDestExistence() helper function to get the source file SHA-1. - * Subclasses should overwride this iff the source is not in storage. + * Subclasses should overwride this if the source is not in storage. * * @return string|bool Returns false on failure */ @@ -393,7 +396,7 @@ abstract class FileOp { * Check if a file will exist in storage when this operation is attempted * * @param string $source Storage path - * @param $predicates Array + * @param Array $predicates * @return bool */ final protected function fileExists( $source, array $predicates ) { @@ -409,7 +412,7 @@ abstract class FileOp { * Get the SHA-1 of a file in storage when this operation is attempted * * @param string $source Storage path - * @param $predicates Array + * @param Array $predicates * @return string|bool False on failure */ final protected function fileSha1( $source, array $predicates ) { @@ -435,7 +438,7 @@ abstract class FileOp { /** * Log a file operation failure and preserve any temp files * - * @param $action string + * @param string $action * @return void */ final public function logFailure( $action ) { @@ -457,7 +460,7 @@ abstract class FileOp { class CreateFileOp extends FileOp { protected function allowedParams() { return array( array( 'content', 'dst' ), - array( 'overwrite', 'overwriteSame', 'disposition', 'headers' ) ); + array( 'overwrite', 'overwriteSame', 'headers' ) ); } protected function doPrecheck( array &$predicates ) { @@ -485,27 +488,18 @@ class CreateFileOp extends FileOp { return $status; // safe to call attempt() } - /** - * @return Status - */ protected function doAttempt() { - if ( !$this->destSameAsSource ) { + if ( !$this->overwriteSameCase ) { // Create the file at the destination return $this->backend->createInternal( $this->setFlags( $this->params ) ); } return Status::newGood(); } - /** - * @return bool|String - */ protected function getSourceSha1Base36() { return wfBaseConvert( sha1( $this->params['content'] ), 16, 36, 31 ); } - /** - * @return array - */ public function storagePathsChanged() { return array( $this->params['dst'] ); } @@ -516,18 +510,11 @@ class CreateFileOp extends FileOp { * Parameters for this operation are outlined in FileBackend::doOperations(). */ class StoreFileOp extends FileOp { - /** - * @return array - */ protected function allowedParams() { return array( array( 'src', 'dst' ), - array( 'overwrite', 'overwriteSame', 'disposition', 'headers' ) ); + array( 'overwrite', 'overwriteSame', 'headers' ) ); } - /** - * @param $predicates array - * @return Status - */ protected function doPrecheck( array &$predicates ) { $status = Status::newGood(); // Check if the source file exists on the file system @@ -557,20 +544,14 @@ class StoreFileOp extends FileOp { return $status; // safe to call attempt() } - /** - * @return Status - */ protected function doAttempt() { - // Store the file at the destination - if ( !$this->destSameAsSource ) { + if ( !$this->overwriteSameCase ) { + // Store the file at the destination return $this->backend->storeInternal( $this->setFlags( $this->params ) ); } return Status::newGood(); } - /** - * @return bool|string - */ protected function getSourceSha1Base36() { wfSuppressWarnings(); $hash = sha1_file( $this->params['src'] ); @@ -591,18 +572,11 @@ class StoreFileOp extends FileOp { * Parameters for this operation are outlined in FileBackend::doOperations(). */ class CopyFileOp extends FileOp { - /** - * @return array - */ protected function allowedParams() { return array( array( 'src', 'dst' ), - array( 'overwrite', 'overwriteSame', 'ignoreMissingSource', 'disposition' ) ); + array( 'overwrite', 'overwriteSame', 'ignoreMissingSource', 'headers' ) ); } - /** - * @param $predicates array - * @return Status - */ protected function doPrecheck( array &$predicates ) { $status = Status::newGood(); // Check if the source file exists @@ -634,30 +608,26 @@ class CopyFileOp extends FileOp { return $status; // safe to call attempt() } - /** - * @return Status - */ protected function doAttempt() { - // Do nothing if the src/dst paths are the same - if ( $this->params['src'] !== $this->params['dst'] ) { - // Copy the file into the destination - if ( !$this->destSameAsSource ) { - return $this->backend->copyInternal( $this->setFlags( $this->params ) ); - } + if ( $this->overwriteSameCase ) { + $status = Status::newGood(); // nothing to do + } elseif ( $this->params['src'] === $this->params['dst'] ) { + // Just update the destination file headers + $headers = $this->getParam( 'headers' ) ?: array(); + $status = $this->backend->describeInternal( $this->setFlags( array( + 'src' => $this->params['dst'], 'headers' => $headers + ) ) ); + } else { + // Copy the file to the destination + $status = $this->backend->copyInternal( $this->setFlags( $this->params ) ); } - return Status::newGood(); + return $status; } - /** - * @return array - */ public function storagePathsRead() { return array( $this->params['src'] ); } - /** - * @return array - */ public function storagePathsChanged() { return array( $this->params['dst'] ); } @@ -668,18 +638,11 @@ class CopyFileOp extends FileOp { * Parameters for this operation are outlined in FileBackend::doOperations(). */ class MoveFileOp extends FileOp { - /** - * @return array - */ protected function allowedParams() { return array( array( 'src', 'dst' ), - array( 'overwrite', 'overwriteSame', 'ignoreMissingSource', 'disposition' ) ); + array( 'overwrite', 'overwriteSame', 'ignoreMissingSource', 'headers' ) ); } - /** - * @param $predicates array - * @return Status - */ protected function doPrecheck( array &$predicates ) { $status = Status::newGood(); // Check if the source file exists @@ -713,34 +676,34 @@ class MoveFileOp extends FileOp { return $status; // safe to call attempt() } - /** - * @return Status - */ protected function doAttempt() { - // Do nothing if the src/dst paths are the same - if ( $this->params['src'] !== $this->params['dst'] ) { - if ( !$this->destSameAsSource ) { - // Move the file into the destination - return $this->backend->moveInternal( $this->setFlags( $this->params ) ); + if ( $this->overwriteSameCase ) { + if ( $this->params['src'] === $this->params['dst'] ) { + // Do nothing to the destination (which is also the source) + $status = Status::newGood(); } else { - // Just delete source as the destination needs no changes - $params = array( 'src' => $this->params['src'] ); - return $this->backend->deleteInternal( $this->setFlags( $params ) ); + // Just delete the source as the destination file needs no changes + $status = $this->backend->deleteInternal( $this->setFlags( + array( 'src' => $this->params['src'] ) + ) ); } + } elseif ( $this->params['src'] === $this->params['dst'] ) { + // Just update the destination file headers + $headers = $this->getParam( 'headers' ) ?: array(); + $status = $this->backend->describeInternal( $this->setFlags( + array( 'src' => $this->params['dst'], 'headers' => $headers ) + ) ); + } else { + // Move the file to the destination + $status = $this->backend->moveInternal( $this->setFlags( $this->params ) ); } - return Status::newGood(); + return $status; } - /** - * @return array - */ public function storagePathsRead() { return array( $this->params['src'] ); } - /** - * @return array - */ public function storagePathsChanged() { return array( $this->params['src'], $this->params['dst'] ); } @@ -751,17 +714,10 @@ class MoveFileOp extends FileOp { * Parameters for this operation are outlined in FileBackend::doOperations(). */ class DeleteFileOp extends FileOp { - /** - * @return array - */ protected function allowedParams() { return array( array( 'src' ), array( 'ignoreMissingSource' ) ); } - /** - * @param $predicates array - * @return Status - */ protected function doPrecheck( array &$predicates ) { $status = Status::newGood(); // Check if the source file exists @@ -788,17 +744,11 @@ class DeleteFileOp extends FileOp { return $status; // safe to call attempt() } - /** - * @return Status - */ protected function doAttempt() { // Delete the source file return $this->backend->deleteInternal( $this->setFlags( $this->params ) ); } - /** - * @return array - */ public function storagePathsChanged() { return array( $this->params['src'] ); } @@ -809,17 +759,10 @@ class DeleteFileOp extends FileOp { * Parameters for this operation are outlined in FileBackend::doOperations(). */ class DescribeFileOp extends FileOp { - /** - * @return array - */ protected function allowedParams() { - return array( array( 'src' ), array( 'disposition', 'headers' ) ); + return array( array( 'src' ), array( 'headers' ) ); } - /** - * @param $predicates array - * @return Status - */ protected function doPrecheck( array &$predicates ) { $status = Status::newGood(); // Check if the source file exists @@ -840,17 +783,11 @@ class DescribeFileOp extends FileOp { return $status; // safe to call attempt() } - /** - * @return Status - */ protected function doAttempt() { // Update the source file's metadata return $this->backend->describeInternal( $this->setFlags( $this->params ) ); } - /** - * @return array - */ public function storagePathsChanged() { return array( $this->params['src'] ); } diff --git a/includes/filebackend/FileOpBatch.php b/includes/filebackend/FileOpBatch.php index fc51d78a..785c0bc9 100644 --- a/includes/filebackend/FileOpBatch.php +++ b/includes/filebackend/FileOpBatch.php @@ -51,7 +51,7 @@ class FileOpBatch { * * @param array $performOps List of FileOp operations * @param array $opts Batch operation options - * @param $journal FileJournal Journal to log operations to + * @param FileJournal $journal Journal to log operations to * @return Status */ public static function attempt( array $performOps, array $opts, FileJournal $journal ) { @@ -145,8 +145,8 @@ class FileOpBatch { * within any given sub-batch do not depend on each other. * This will abort remaining ops on failure. * - * @param $pPerformOps Array - * @param $status Status + * @param Array $pPerformOps + * @param Status $status * @return bool Success */ protected static function runParallelBatches( array $pPerformOps, Status $status ) { diff --git a/includes/filebackend/README b/includes/filebackend/README index 6ab54810..569f3376 100644 --- a/includes/filebackend/README +++ b/includes/filebackend/README @@ -47,7 +47,7 @@ directories. See FileBackend.php for full documentation for each function. The following basic operations are supported for reading from a backend: On files: -* state a file for basic information (timestamp, size) +* stat a file for basic information (timestamp, size) * read a file into a string or several files into a map of path names to strings * download a file or set of files to a temporary file (on a mounted file system) * get the SHA1 hash of a file diff --git a/includes/filebackend/SwiftFileBackend.php b/includes/filebackend/SwiftFileBackend.php index 0f3d97a3..db090a98 100644 --- a/includes/filebackend/SwiftFileBackend.php +++ b/includes/filebackend/SwiftFileBackend.php @@ -24,7 +24,7 @@ */ /** - * @brief Class for an OpenStack Swift based file backend. + * @brief Class for an OpenStack Swift (or Ceph RGW) based file backend. * * This requires the SwiftCloudFiles MediaWiki extension, which includes * the php-cloudfiles library (https://github.com/rackspace/php-cloudfiles). @@ -104,7 +104,7 @@ class SwiftFileBackend extends FileBackendStore { */ public function __construct( array $config ) { parent::__construct( $config ); - if ( !MWInit::classExists( 'CF_Constants' ) ) { + if ( !class_exists( 'CF_Constants' ) ) { throw new MWException( 'SwiftCloudFiles extension not installed.' ); } // Required settings @@ -132,7 +132,7 @@ class SwiftFileBackend extends FileBackendStore { : false; $this->swiftCDNExpiry = isset( $config['swiftCDNExpiry'] ) ? $config['swiftCDNExpiry'] - : 12*3600; // 12 hours is safe (tokens last 24 hours per http://docs.openstack.org) + : 12 * 3600; // 12 hours is safe (tokens last 24 hours per http://docs.openstack.org) $this->swiftCDNPurgable = isset( $config['swiftCDNPurgable'] ) ? $config['swiftCDNPurgable'] : true; @@ -172,10 +172,6 @@ class SwiftFileBackend extends FileBackendStore { return $relStoragePath; } - /** - * @see FileBackendStore::isPathUsableInternal() - * @return bool - */ public function isPathUsableInternal( $storagePath ) { list( $container, $rel ) = $this->resolveStoragePathReal( $storagePath ); if ( $rel === null ) { @@ -193,6 +189,18 @@ class SwiftFileBackend extends FileBackendStore { return false; } + /** + * @param array $headers + * @return array + */ + protected function sanitizeHdrs( array $headers ) { + // By default, Swift has annoyingly low maximum header value limits + if ( isset( $headers['Content-Disposition'] ) ) { + $headers['Content-Disposition'] = $this->truncDisp( $headers['Content-Disposition'] ); + } + return $headers; + } + /** * @param string $disposition Content-Disposition header value * @return string Truncated Content-Disposition header value to meet Swift limits @@ -211,10 +219,6 @@ class SwiftFileBackend extends FileBackendStore { return $res; } - /** - * @see FileBackendStore::doCreateInternal() - * @return Status - */ protected function doCreateInternal( array $params ) { $status = Status::newGood(); @@ -248,17 +252,10 @@ class SwiftFileBackend extends FileBackendStore { // The MD5 here will be checked within Swift against its own MD5. $obj->set_etag( md5( $params['content'] ) ); // Use the same content type as StreamFile for security - $obj->content_type = StreamFile::contentTypeFromPath( $params['dst'] ); - if ( !strlen( $obj->content_type ) ) { // special case - $obj->content_type = 'unknown/unknown'; - } - // Set the Content-Disposition header if requested - if ( isset( $params['disposition'] ) ) { - $obj->headers['Content-Disposition'] = $this->truncDisp( $params['disposition'] ); - } + $obj->content_type = $this->getContentType( $params['dst'], $params['content'], null ); // Set any other custom headers if requested if ( isset( $params['headers'] ) ) { - $obj->headers += $params['headers']; + $obj->headers += $this->sanitizeHdrs( $params['headers'] ); } if ( !empty( $params['async'] ) ) { // deferred $op = $obj->write_async( $params['content'] ); @@ -290,10 +287,6 @@ class SwiftFileBackend extends FileBackendStore { } } - /** - * @see FileBackendStore::doStoreInternal() - * @return Status - */ protected function doStoreInternal( array $params ) { $status = Status::newGood(); @@ -333,17 +326,10 @@ class SwiftFileBackend extends FileBackendStore { // The MD5 here will be checked within Swift against its own MD5. $obj->set_etag( md5_file( $params['src'] ) ); // Use the same content type as StreamFile for security - $obj->content_type = StreamFile::contentTypeFromPath( $params['dst'] ); - if ( !strlen( $obj->content_type ) ) { // special case - $obj->content_type = 'unknown/unknown'; - } - // Set the Content-Disposition header if requested - if ( isset( $params['disposition'] ) ) { - $obj->headers['Content-Disposition'] = $this->truncDisp( $params['disposition'] ); - } + $obj->content_type = $this->getContentType( $params['dst'], null, $params['src'] ); // Set any other custom headers if requested if ( isset( $params['headers'] ) ) { - $obj->headers += $params['headers']; + $obj->headers += $this->sanitizeHdrs( $params['headers'] ); } if ( !empty( $params['async'] ) ) { // deferred wfSuppressWarnings(); @@ -387,10 +373,6 @@ class SwiftFileBackend extends FileBackendStore { } } - /** - * @see FileBackendStore::doCopyInternal() - * @return Status - */ protected function doCopyInternal( array $params ) { $status = Status::newGood(); @@ -424,8 +406,9 @@ class SwiftFileBackend extends FileBackendStore { try { $dstObj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD $hdrs = array(); // source file headers to override with new values - if ( isset( $params['disposition'] ) ) { - $hdrs['Content-Disposition'] = $this->truncDisp( $params['disposition'] ); + // Set any other custom headers if requested + if ( isset( $params['headers'] ) ) { + $hdrs += $this->sanitizeHdrs( $params['headers'] ); } if ( !empty( $params['async'] ) ) { // deferred $op = $sContObj->copy_object_to_async( $srcRel, $dContObj, $dstRel, null, $hdrs ); @@ -459,10 +442,6 @@ class SwiftFileBackend extends FileBackendStore { } } - /** - * @see FileBackendStore::doMoveInternal() - * @return Status - */ protected function doMoveInternal( array $params ) { $status = Status::newGood(); @@ -497,8 +476,9 @@ class SwiftFileBackend extends FileBackendStore { $srcObj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD $dstObj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD $hdrs = array(); // source file headers to override with new values - if ( isset( $params['disposition'] ) ) { - $hdrs['Content-Disposition'] = $this->truncDisp( $params['disposition'] ); + // Set any other custom headers if requested + if ( isset( $params['headers'] ) ) { + $hdrs += $this->sanitizeHdrs( $params['headers'] ); } if ( !empty( $params['async'] ) ) { // deferred $op = $sContObj->move_object_to_async( $srcRel, $dContObj, $dstRel, null, $hdrs ); @@ -534,10 +514,6 @@ class SwiftFileBackend extends FileBackendStore { } } - /** - * @see FileBackendStore::doDeleteInternal() - * @return Status - */ protected function doDeleteInternal( array $params ) { $status = Status::newGood(); @@ -590,10 +566,6 @@ class SwiftFileBackend extends FileBackendStore { } } - /** - * @see FileBackendStore::doDescribeInternal() - * @return Status - */ protected function doDescribeInternal( array $params ) { $status = Status::newGood(); @@ -603,19 +575,15 @@ class SwiftFileBackend extends FileBackendStore { return $status; } - $hdrs = isset( $params['headers'] ) ? $params['headers'] : array(); - // Set the Content-Disposition header if requested - if ( isset( $params['disposition'] ) ) { - $hdrs['Content-Disposition'] = $this->truncDisp( $params['disposition'] ); - } - try { $sContObj = $this->getContainer( $srcCont ); // Get the latest version of the current metadata $srcObj = $sContObj->get_object( $srcRel, $this->headersFromParams( array( 'latest' => true ) ) ); // Merge in the metadata updates... - $srcObj->headers = $hdrs + $srcObj->headers; + if ( isset( $params['headers'] ) ) { + $srcObj->headers = $this->sanitizeHdrs( $params['headers'] ) + $srcObj->headers; + } $srcObj->sync_metadata(); // save to Swift $this->purgeCDNCache( array( $srcObj ) ); } catch ( CDNNotEnabledException $e ) { @@ -631,10 +599,6 @@ class SwiftFileBackend extends FileBackendStore { return $status; } - /** - * @see FileBackendStore::doPrepareInternal() - * @return Status - */ protected function doPrepareInternal( $fullCont, $dir, array $params ) { $status = Status::newGood(); @@ -748,10 +712,6 @@ class SwiftFileBackend extends FileBackendStore { return $status; } - /** - * @see FileBackendStore::doCleanInternal() - * @return Status - */ protected function doCleanInternal( $fullCont, $dir, array $params ) { $status = Status::newGood(); @@ -787,10 +747,6 @@ class SwiftFileBackend extends FileBackendStore { return $status; } - /** - * @see FileBackendStore::doFileExists() - * @return array|bool|null - */ protected function doGetFileStat( array $params ) { list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] ); if ( $srcRel === null ) { @@ -805,8 +761,8 @@ class SwiftFileBackend extends FileBackendStore { $stat = array( // Convert dates like "Tue, 03 Jan 2012 22:01:04 GMT" to TS_MW 'mtime' => wfTimestamp( TS_MW, $srcObj->last_modified ), - 'size' => (int)$srcObj->content_length, - 'sha1' => $srcObj->getMetadataValue( 'Sha1base36' ) + 'size' => (int)$srcObj->content_length, + 'sha1' => $srcObj->getMetadataValue( 'Sha1base36' ) ); } catch ( NoSuchContainerException $e ) { } catch ( NoSuchObjectException $e ) { @@ -821,7 +777,7 @@ class SwiftFileBackend extends FileBackendStore { /** * Fill in any missing object metadata and save it to Swift * - * @param $obj CF_Object + * @param CF_Object $obj * @param string $path Storage path to object * @return bool Success * @throws Exception cloudfiles exceptions @@ -852,10 +808,6 @@ class SwiftFileBackend extends FileBackendStore { return false; // failed } - /** - * @see FileBackendStore::doGetFileContentsMulti() - * @return Array - */ protected function doGetFileContentsMulti( array $params ) { $contents = array(); @@ -965,24 +917,25 @@ class SwiftFileBackend extends FileBackendStore { * @param string $fullCont Resolved container name * @param string $dir Resolved storage directory with no trailing slash * @param string|null $after Storage path of file to list items after - * @param $limit integer Max number of items to list - * @param array $params Includes flag for 'topOnly' - * @return Array List of relative paths of dirs directly under $dir + * @param integer $limit Max number of items to list + * @param array $params Parameters for getDirectoryList() + * @return Array List of resolved paths of directories directly under $dir + * @throws FileBackendError */ public function getDirListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) { $dirs = array(); if ( $after === INF ) { return $dirs; // nothing more } - wfProfileIn( __METHOD__ . '-' . $this->name ); + $section = new ProfileSection( __METHOD__ . '-' . $this->name ); try { $container = $this->getContainer( $fullCont ); $prefix = ( $dir == '' ) ? null : "{$dir}/"; // Non-recursive: only list dirs right under $dir if ( !empty( $params['topOnly'] ) ) { $objects = $container->list_objects( $limit, $after, $prefix, null, '/' ); - foreach ( $objects as $object ) { // files and dirs + foreach ( $objects as $object ) { // files and directories if ( substr( $object, -1 ) === '/' ) { $dirs[] = $object; // directories end in '/' } @@ -1013,6 +966,7 @@ class SwiftFileBackend extends FileBackendStore { } } } + // Page on the unfiltered directory listing (what is returned may be filtered) if ( count( $objects ) < $limit ) { $after = INF; // avoid a second RTT } else { @@ -1022,9 +976,9 @@ class SwiftFileBackend extends FileBackendStore { } catch ( CloudFilesException $e ) { // some other exception? $this->handleException( $e, null, __METHOD__, array( 'cont' => $fullCont, 'dir' => $dir ) ); + throw new FileBackendError( "Got " . get_class( $e ) . " exception." ); } - wfProfileOut( __METHOD__ . '-' . $this->name ); return $dirs; } @@ -1038,33 +992,49 @@ class SwiftFileBackend extends FileBackendStore { * @param string $fullCont Resolved container name * @param string $dir Resolved storage directory with no trailing slash * @param string|null $after Storage path of file to list items after - * @param $limit integer Max number of items to list - * @param array $params Includes flag for 'topOnly' - * @return Array List of relative paths of files under $dir + * @param integer $limit Max number of items to list + * @param array $params Parameters for getDirectoryList() + * @return Array List of resolved paths of files under $dir + * @throws FileBackendError */ public function getFileListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) { $files = array(); if ( $after === INF ) { return $files; // nothing more } - wfProfileIn( __METHOD__ . '-' . $this->name ); + $section = new ProfileSection( __METHOD__ . '-' . $this->name ); try { $container = $this->getContainer( $fullCont ); $prefix = ( $dir == '' ) ? null : "{$dir}/"; // Non-recursive: only list files right under $dir if ( !empty( $params['topOnly'] ) ) { // files and dirs - $objects = $container->list_objects( $limit, $after, $prefix, null, '/' ); - foreach ( $objects as $object ) { - if ( substr( $object, -1 ) !== '/' ) { - $files[] = $object; // directories end in '/' + if ( !empty( $params['adviseStat'] ) ) { + $limit = min( $limit, self::CACHE_CHEAP_SIZE ); + // Note: get_objects() does not include directories + $objects = $this->loadObjectListing( $params, $dir, + $container->get_objects( $limit, $after, $prefix, null, '/' ) ); + $files = $objects; + } else { + $objects = $container->list_objects( $limit, $after, $prefix, null, '/' ); + foreach ( $objects as $object ) { // files and directories + if ( substr( $object, -1 ) !== '/' ) { + $files[] = $object; // directories end in '/' + } } } // Recursive: list all files under $dir and its subdirs } else { // files - $objects = $container->list_objects( $limit, $after, $prefix ); + if ( !empty( $params['adviseStat'] ) ) { + $limit = min( $limit, self::CACHE_CHEAP_SIZE ); + $objects = $this->loadObjectListing( $params, $dir, + $container->get_objects( $limit, $after, $prefix ) ); + } else { + $objects = $container->list_objects( $limit, $after, $prefix ); + } $files = $objects; } + // Page on the unfiltered object listing (what is returned may be filtered) if ( count( $objects ) < $limit ) { $after = INF; // avoid a second RTT } else { @@ -1074,29 +1044,57 @@ class SwiftFileBackend extends FileBackendStore { } catch ( CloudFilesException $e ) { // some other exception? $this->handleException( $e, null, __METHOD__, array( 'cont' => $fullCont, 'dir' => $dir ) ); + throw new FileBackendError( "Got " . get_class( $e ) . " exception." ); } - wfProfileOut( __METHOD__ . '-' . $this->name ); return $files; } /** - * @see FileBackendStore::doGetFileSha1base36() - * @return bool + * Load a list of objects that belong under $dir into stat cache + * and return a list of the names of the objects in the same order. + * + * @param array $params Parameters for getDirectoryList() + * @param string $dir Resolved container directory path + * @param array $cfObjects List of CF_Object items + * @return array List of object names */ + private function loadObjectListing( array $params, $dir, array $cfObjects ) { + $names = array(); + $storageDir = rtrim( $params['dir'], '/' ); + $suffixStart = ( $dir === '' ) ? 0 : strlen( $dir ) + 1; // size of "path/to/dir/" + // Iterate over the list *backwards* as this primes the stat cache, which is LRU. + // If this fills the cache and the caller stats an uncached file before stating + // the ones on the listing, there would be zero cache hits if this went forwards. + for ( end( $cfObjects ); key( $cfObjects ) !== null; prev( $cfObjects ) ) { + $object = current( $cfObjects ); + $path = "{$storageDir}/" . substr( $object->name, $suffixStart ); + $val = array( + // Convert dates like "Tue, 03 Jan 2012 22:01:04 GMT" to TS_MW + 'mtime' => wfTimestamp( TS_MW, $object->last_modified ), + 'size' => (int)$object->content_length, + 'latest' => false // eventually consistent + ); + $this->cheapCache->set( $path, 'stat', $val ); + $names[] = $object->name; + } + return array_reverse( $names ); // keep the paths in original order + } + protected function doGetFileSha1base36( array $params ) { $stat = $this->getFileStat( $params ); if ( $stat ) { + if ( !isset( $stat['sha1'] ) ) { + // Stat entries filled by file listings don't include SHA1 + $this->clearCache( array( $params['src'] ) ); + $stat = $this->getFileStat( $params ); + } return $stat['sha1']; } else { return false; } } - /** - * @see FileBackendStore::doStreamFile() - * @return Status - */ protected function doStreamFile( array $params ) { $status = Status::newGood(); @@ -1128,10 +1126,6 @@ class SwiftFileBackend extends FileBackendStore { return $status; } - /** - * @see FileBackendStore::doGetLocalCopyMulti() - * @return null|TempFSFile - */ protected function doGetLocalCopyMulti( array $params ) { $tmpFiles = array(); @@ -1201,10 +1195,6 @@ class SwiftFileBackend extends FileBackendStore { return $tmpFiles; } - /** - * @see FileBackendStore::getFileHttpUrl() - * @return string|null - */ public function getFileHttpUrl( array $params ) { if ( $this->swiftTempUrlKey != '' || ( $this->rgwS3AccessKey != '' && $this->rgwS3SecretKey != '' ) ) @@ -1237,8 +1227,8 @@ class SwiftFileBackend extends FileBackendStore { str_replace( '/swift/v1', '', // S3 API is the rgw default $sContObj->cfs_http->getStorageUrl() . $spath ), array( - 'Signature' => $signature, - 'Expires' => $expires, + 'Signature' => $signature, + 'Expires' => $expires, 'AWSAccessKeyId' => $this->rgwS3AccessKey ) ); } @@ -1250,10 +1240,6 @@ class SwiftFileBackend extends FileBackendStore { return null; } - /** - * @see FileBackendStore::directoriesAreVirtual() - * @return bool - */ protected function directoriesAreVirtual() { return true; } @@ -1274,10 +1260,6 @@ class SwiftFileBackend extends FileBackendStore { return $hdrs; } - /** - * @see FileBackendStore::doExecuteOpHandlesInternal() - * @return Array List of corresponding Status objects - */ protected function doExecuteOpHandlesInternal( array $fileOpHandles ) { $statuses = array(); @@ -1314,7 +1296,7 @@ class SwiftFileBackend extends FileBackendStore { * matches the expression and the request is not for a listing. * Setting this to '*' effectively makes a container public. * -".rlistings:" : Grants access if the request is from a referrer host that - * matches the expression and the request for a listing. + * matches the expression and the request is for a listing. * * $writeGrps is a list of the possible criteria for a request to have * access to write to a container. Each item is of the following format: @@ -1325,7 +1307,7 @@ class SwiftFileBackend extends FileBackendStore { * In general, we don't allow listings to end-users. It's not useful, isn't well-defined * (lists are truncated to 10000 item with no way to page), and is just a performance risk. * - * @param $contObj CF_Container Swift container + * @param CF_Container $contObj Swift container * @param array $readGrps List of read access routes * @param array $writeGrps List of write access routes * @return Status @@ -1395,12 +1377,12 @@ class SwiftFileBackend extends FileBackendStore { if ( is_array( $creds ) ) { // cache hit $this->auth->load_cached_credentials( $creds['auth_token'], $creds['storage_url'], $creds['cdnm_url'] ); - $this->sessionStarted = time() - ceil( $this->authTTL/2 ); // skew for worst case + $this->sessionStarted = time() - ceil( $this->authTTL / 2 ); // skew for worst case } else { // cache miss try { $this->auth->authenticate(); $creds = $this->auth->export_credentials(); - $this->srvCache->add( $cacheKey, $creds, ceil( $this->authTTL/2 ) ); // cache + $this->srvCache->add( $cacheKey, $creds, ceil( $this->authTTL / 2 ) ); // cache $this->sessionStarted = time(); } catch ( CloudFilesException $e ) { $this->connException = $e; // don't keep re-trying @@ -1433,7 +1415,7 @@ class SwiftFileBackend extends FileBackendStore { /** * Get the cache key for a container * - * @param $username string + * @param string $username * @return string */ private function getCredsCacheKey( $username ) { @@ -1496,10 +1478,6 @@ class SwiftFileBackend extends FileBackendStore { $conn->delete_container( $container ); } - /** - * @see FileBackendStore::doPrimeContainerCache() - * @return void - */ protected function doPrimeContainerCache( array $containerInfo ) { try { $conn = $this->getConnection(); // Swift proxy connection @@ -1517,9 +1495,9 @@ class SwiftFileBackend extends FileBackendStore { * Log an unexpected exception for this backend. * This also sets the Status object to have a fatal error. * - * @param $e Exception - * @param $status Status|null - * @param $func string + * @param Exception $e + * @param Status $status|null + * @param string $func * @param array $params * @return void */ @@ -1554,7 +1532,15 @@ class SwiftFileOpHandle extends FileBackendStoreOpHandle { /** @var Array */ public $affectedObjects = array(); - public function __construct( $backend, array $params, $call, CF_Async_Op $cfOp ) { + /** + * @param SwiftFileBackend $backend + * @param array $params + * @param string $call + * @param CF_Async_Op $cfOp + */ + public function __construct( + SwiftFileBackend $backend, array $params, $call, CF_Async_Op $cfOp + ) { $this->backend = $backend; $this->params = $params; $this->call = $call; @@ -1586,7 +1572,7 @@ abstract class SwiftFileBackendList implements Iterator { const PAGE_SIZE = 9000; // file listing buffer size /** - * @param $backend SwiftFileBackend + * @param SwiftFileBackend $backend * @param string $fullCont Resolved container name * @param string $dir Resolved directory relative to container * @param array $params @@ -1660,10 +1646,10 @@ abstract class SwiftFileBackendList implements Iterator { * * @param string $container Resolved container name * @param string $dir Resolved path relative to container - * @param $after string|null - * @param $limit integer + * @param string $after|null + * @param integer $limit * @param array $params - * @return Traversable|Array|null Returns null on failure + * @return Traversable|Array */ abstract protected function pageFromList( $container, $dir, &$after, $limit, array $params ); } @@ -1682,7 +1668,7 @@ class SwiftFileBackendDirList extends SwiftFileBackendList { /** * @see SwiftFileBackendList::pageFromList() - * @return Array|null + * @return Array */ protected function pageFromList( $container, $dir, &$after, $limit, array $params ) { return $this->backend->getDirListPageInternal( $container, $dir, $after, $limit, $params ); @@ -1703,7 +1689,7 @@ class SwiftFileBackendFileList extends SwiftFileBackendList { /** * @see SwiftFileBackendList::pageFromList() - * @return Array|null + * @return Array */ protected function pageFromList( $container, $dir, &$after, $limit, array $params ) { return $this->backend->getFileListPageInternal( $container, $dir, $after, $limit, $params ); diff --git a/includes/filebackend/TempFSFile.php b/includes/filebackend/TempFSFile.php index 11e125c1..8266e420 100644 --- a/includes/filebackend/TempFSFile.php +++ b/includes/filebackend/TempFSFile.php @@ -37,8 +37,8 @@ class TempFSFile extends FSFile { * Make a new temporary file on the file system. * Temporary files may be purged when the file object falls out of scope. * - * @param $prefix string - * @param $extension string + * @param string $prefix + * @param string $extension * @return TempFSFile|null */ public static function factory( $prefix, $extension = '' ) { @@ -81,7 +81,7 @@ class TempFSFile extends FSFile { /** * Clean up the temporary file only after an object goes out of scope * - * @param $object Object + * @param Object $object * @return TempFSFile This object */ public function bind( $object ) { diff --git a/includes/filebackend/filejournal/DBFileJournal.php b/includes/filebackend/filejournal/DBFileJournal.php index 73f29a95..9250aa5e 100644 --- a/includes/filebackend/filejournal/DBFileJournal.php +++ b/includes/filebackend/filejournal/DBFileJournal.php @@ -65,11 +65,11 @@ class DBFileJournal extends FileJournal { foreach ( $entries as $entry ) { $data[] = array( 'fj_batch_uuid' => $batchId, - 'fj_backend' => $this->backend, - 'fj_op' => $entry['op'], - 'fj_path' => $entry['path'], - 'fj_new_sha1' => $entry['newSha1'], - 'fj_timestamp' => $dbw->timestamp( $now ) + 'fj_backend' => $this->backend, + 'fj_op' => $entry['op'], + 'fj_path' => $entry['path'], + 'fj_new_sha1' => $entry['newSha1'], + 'fj_timestamp' => $dbw->timestamp( $now ) ); } diff --git a/includes/filebackend/lockmanager/DBLockManager.php b/includes/filebackend/lockmanager/DBLockManager.php index f02387dc..3e934ba5 100644 --- a/includes/filebackend/lockmanager/DBLockManager.php +++ b/includes/filebackend/lockmanager/DBLockManager.php @@ -110,6 +110,19 @@ abstract class DBLockManager extends QuorumLockManager { $this->session = wfRandomString( 31 ); } + // @TODO: change this code to work in one batch + protected function getLocksOnServer( $lockSrv, array $pathsByType ) { + $status = Status::newGood(); + foreach ( $pathsByType as $type => $paths ) { + $status->merge( $this->doGetLocksOnServer( $lockSrv, $paths, $type ) ); + } + return $status; + } + + protected function freeLocksOnServer( $lockSrv, array $pathsByType ) { + return Status::newGood(); + } + /** * @see QuorumLockManager::isServerUp() * @return bool @@ -252,7 +265,7 @@ class MySqlLockManager extends DBLockManager { * @see DBLockManager::getLocksOnServer() * @return Status */ - protected function getLocksOnServer( $lockSrv, array $paths, $type ) { + protected function doGetLocksOnServer( $lockSrv, array $paths, $type ) { $status = Status::newGood(); $db = $this->getConnection( $lockSrv ); // checked in isServerUp() @@ -318,14 +331,6 @@ class MySqlLockManager extends DBLockManager { return $status; } - /** - * @see QuorumLockManager::freeLocksOnServer() - * @return Status - */ - protected function freeLocksOnServer( $lockSrv, array $paths, $type ) { - return Status::newGood(); // not supported - } - /** * @see QuorumLockManager::releaseAllLocks() * @return Status @@ -361,7 +366,7 @@ class PostgreSqlLockManager extends DBLockManager { self::LOCK_EX => self::LOCK_EX ); - protected function getLocksOnServer( $lockSrv, array $paths, $type ) { + protected function doGetLocksOnServer( $lockSrv, array $paths, $type ) { $status = Status::newGood(); if ( !count( $paths ) ) { return $status; // nothing to lock @@ -369,7 +374,9 @@ class PostgreSqlLockManager extends DBLockManager { $db = $this->getConnection( $lockSrv ); // checked in isServerUp() $bigints = array_unique( array_map( - function( $key ) { return wfBaseConvert( substr( $key, 0, 15 ), 16, 10 ); }, + function( $key ) { + return wfBaseConvert( substr( $key, 0, 15 ), 16, 10 ); + }, array_map( array( $this, 'sha1Base16Absolute' ), $paths ) ) ); @@ -405,14 +412,6 @@ class PostgreSqlLockManager extends DBLockManager { return $status; } - /** - * @see QuorumLockManager::freeLocksOnServer() - * @return Status - */ - protected function freeLocksOnServer( $lockSrv, array $paths, $type ) { - return Status::newGood(); // not supported - } - /** * @see QuorumLockManager::releaseAllLocks() * @return Status diff --git a/includes/filebackend/lockmanager/LockManager.php b/includes/filebackend/lockmanager/LockManager.php index 0512a01b..dad8a624 100644 --- a/includes/filebackend/lockmanager/LockManager.php +++ b/includes/filebackend/lockmanager/LockManager.php @@ -56,7 +56,7 @@ abstract class LockManager { protected $domain; // string; domain (usually wiki ID) protected $lockTTL; // integer; maximum time locks can be held - /* Lock types; stronger locks have higher values */ + /** Lock types; stronger locks have higher values */ const LOCK_SH = 1; // shared lock (for reads) const LOCK_UW = 2; // shared lock (for reads used to write elsewhere) const LOCK_EX = 3; // exclusive lock (for writes) @@ -76,10 +76,10 @@ abstract class LockManager { if ( isset( $config['lockTTL'] ) ) { $this->lockTTL = max( 1, $config['lockTTL'] ); } elseif ( PHP_SAPI === 'cli' ) { - $this->lockTTL = 2*3600; + $this->lockTTL = 2 * 3600; } else { $met = ini_get( 'max_execution_time' ); // this is 0 in CLI mode - $this->lockTTL = max( 5*60, 2*(int)$met ); + $this->lockTTL = max( 5 * 60, 2 * (int)$met ); } } @@ -88,11 +88,36 @@ abstract class LockManager { * * @param array $paths List of resource names * @param $type integer LockManager::LOCK_* constant + * @param integer $timeout Timeout in seconds (0 means non-blocking) (since 1.21) * @return Status */ - final public function lock( array $paths, $type = self::LOCK_EX ) { + final public function lock( array $paths, $type = self::LOCK_EX, $timeout = 0 ) { + return $this->lockByType( array( $type => $paths ), $timeout ); + } + + /** + * Lock the resources at the given abstract paths + * + * @param array $pathsByType Map of LockManager::LOCK_* constants to lists of paths + * @param integer $timeout Timeout in seconds (0 means non-blocking) (since 1.21) + * @return Status + * @since 1.22 + */ + final public function lockByType( array $pathsByType, $timeout = 0 ) { wfProfileIn( __METHOD__ ); - $status = $this->doLock( array_unique( $paths ), $this->lockTypeMap[$type] ); + $status = Status::newGood(); + $pathsByType = $this->normalizePathsByType( $pathsByType ); + $msleep = array( 0, 50, 100, 300, 500 ); // retry backoff times + $start = microtime( true ); + do { + $status = $this->doLockByType( $pathsByType ); + $elapsed = microtime( true ) - $start; + if ( $status->isOK() || $elapsed >= $timeout || $elapsed < 0 ) { + break; // success, timeout, or clock set back + } + usleep( 1e3 * ( next( $msleep ) ?: 1000 ) ); // use 1 sec after enough times + $elapsed = microtime( true ) - $start; + } while ( $elapsed < $timeout && $elapsed >= 0 ); wfProfileOut( __METHOD__ ); return $status; } @@ -100,13 +125,25 @@ abstract class LockManager { /** * Unlock the resources at the given abstract paths * - * @param array $paths List of storage paths + * @param array $paths List of paths * @param $type integer LockManager::LOCK_* constant * @return Status */ final public function unlock( array $paths, $type = self::LOCK_EX ) { + return $this->unlockByType( array( $type => $paths ) ); + } + + /** + * Unlock the resources at the given abstract paths + * + * @param array $pathsByType Map of LockManager::LOCK_* constants to lists of paths + * @return Status + * @since 1.22 + */ + final public function unlockByType( array $pathsByType ) { wfProfileIn( __METHOD__ ); - $status = $this->doUnlock( array_unique( $paths ), $this->lockTypeMap[$type] ); + $pathsByType = $this->normalizePathsByType( $pathsByType ); + $status = $this->doUnlockByType( $pathsByType ); wfProfileOut( __METHOD__ ); return $status; } @@ -135,21 +172,75 @@ abstract class LockManager { return sha1( "{$this->domain}:{$path}" ); } + /** + * Normalize the $paths array by converting LOCK_UW locks into the + * appropriate type and removing any duplicated paths for each lock type. + * + * @param array $paths Map of LockManager::LOCK_* constants to lists of paths + * @return Array + * @since 1.22 + */ + final protected function normalizePathsByType( array $pathsByType ) { + $res = array(); + foreach ( $pathsByType as $type => $paths ) { + $res[$this->lockTypeMap[$type]] = array_unique( $paths ); + } + return $res; + } + + /** + * @see LockManager::lockByType() + * @param array $paths Map of LockManager::LOCK_* constants to lists of paths + * @return Status + * @since 1.22 + */ + protected function doLockByType( array $pathsByType ) { + $status = Status::newGood(); + $lockedByType = array(); // map of (type => paths) + foreach ( $pathsByType as $type => $paths ) { + $status->merge( $this->doLock( $paths, $type ) ); + if ( $status->isOK() ) { + $lockedByType[$type] = $paths; + } else { + // Release the subset of locks that were acquired + foreach ( $lockedByType as $type => $paths ) { + $status->merge( $this->doUnlock( $paths, $type ) ); + } + break; + } + } + return $status; + } + /** * Lock resources with the given keys and lock type * - * @param array $paths List of storage paths + * @param array $paths List of paths * @param $type integer LockManager::LOCK_* constant - * @return string + * @return Status */ abstract protected function doLock( array $paths, $type ); + /** + * @see LockManager::unlockByType() + * @param array $paths Map of LockManager::LOCK_* constants to lists of paths + * @return Status + * @since 1.22 + */ + protected function doUnlockByType( array $pathsByType ) { + $status = Status::newGood(); + foreach ( $pathsByType as $type => $paths ) { + $status->merge( $this->doUnlock( $paths, $type ) ); + } + return $status; + } + /** * Unlock resources with the given keys and lock type * - * @param array $paths List of storage paths + * @param array $paths List of paths * @param $type integer LockManager::LOCK_* constant - * @return string + * @return Status */ abstract protected function doUnlock( array $paths, $type ); } @@ -159,22 +250,10 @@ abstract class LockManager { * @since 1.19 */ class NullLockManager extends LockManager { - /** - * @see LockManager::doLock() - * @param $paths array - * @param $type int - * @return Status - */ protected function doLock( array $paths, $type ) { return Status::newGood(); } - /** - * @see LockManager::doUnlock() - * @param $paths array - * @param $type int - * @return Status - */ protected function doUnlock( array $paths, $type ) { return Status::newGood(); } diff --git a/includes/filebackend/lockmanager/LockManagerGroup.php b/includes/filebackend/lockmanager/LockManagerGroup.php index ac0bd49b..9aff2415 100644 --- a/includes/filebackend/lockmanager/LockManagerGroup.php +++ b/includes/filebackend/lockmanager/LockManagerGroup.php @@ -97,8 +97,8 @@ class LockManagerGroup { $class = $config['class']; unset( $config['class'] ); // lock manager won't need this $this->managers[$name] = array( - 'class' => $class, - 'config' => $config, + 'class' => $class, + 'config' => $config, 'instance' => null ); } diff --git a/includes/filebackend/lockmanager/MemcLockManager.php b/includes/filebackend/lockmanager/MemcLockManager.php index fafc588a..5eab03ee 100644 --- a/includes/filebackend/lockmanager/MemcLockManager.php +++ b/includes/filebackend/lockmanager/MemcLockManager.php @@ -88,11 +88,44 @@ class MemcLockManager extends QuorumLockManager { $this->session = wfRandomString( 32 ); } + // @TODO: change this code to work in one batch + protected function getLocksOnServer( $lockSrv, array $pathsByType ) { + $status = Status::newGood(); + + $lockedPaths = array(); + foreach ( $pathsByType as $type => $paths ) { + $status->merge( $this->doGetLocksOnServer( $lockSrv, $paths, $type ) ); + if ( $status->isOK() ) { + $lockedPaths[$type] = isset( $lockedPaths[$type] ) + ? array_merge( $lockedPaths[$type], $paths ) + : $paths; + } else { + foreach ( $lockedPaths as $type => $paths ) { + $status->merge( $this->doFreeLocksOnServer( $lockSrv, $paths, $type ) ); + } + break; + } + } + + return $status; + } + + // @TODO: change this code to work in one batch + protected function freeLocksOnServer( $lockSrv, array $pathsByType ) { + $status = Status::newGood(); + + foreach ( $pathsByType as $type => $paths ) { + $status->merge( $this->doFreeLocksOnServer( $lockSrv, $paths, $type ) ); + } + + return $status; + } + /** * @see QuorumLockManager::getLocksOnServer() * @return Status */ - protected function getLocksOnServer( $lockSrv, array $paths, $type ) { + protected function doGetLocksOnServer( $lockSrv, array $paths, $type ) { $status = Status::newGood(); $memc = $this->getCache( $lockSrv ); @@ -145,7 +178,7 @@ class MemcLockManager extends QuorumLockManager { foreach ( $paths as $path ) { $locksKey = $this->recordKeyForPath( $path ); $locksHeld = $lockRecords[$locksKey]; - $ok = $memc->set( $locksKey, $locksHeld, 7*86400 ); + $ok = $memc->set( $locksKey, $locksHeld, 7 * 86400 ); if ( !$ok ) { $status->fatal( 'lockmanager-fail-acquirelock', $path ); } else { @@ -164,7 +197,7 @@ class MemcLockManager extends QuorumLockManager { * @see QuorumLockManager::freeLocksOnServer() * @return Status */ - protected function freeLocksOnServer( $lockSrv, array $paths, $type ) { + protected function doFreeLocksOnServer( $lockSrv, array $paths, $type ) { $status = Status::newGood(); $memc = $this->getCache( $lockSrv ); @@ -297,7 +330,7 @@ class MemcLockManager extends QuorumLockManager { $start = microtime( true ); do { if ( ( ++$rounds % 4 ) == 0 ) { - usleep( 1000*50 ); // 50 ms + usleep( 1000 * 50 ); // 50 ms } foreach ( array_diff( $keys, $lockedKeys ) as $key ) { if ( $memc->add( "$key:mutex", 1, 180 ) ) { // lock record diff --git a/includes/filebackend/lockmanager/QuorumLockManager.php b/includes/filebackend/lockmanager/QuorumLockManager.php index b331b540..8356d32a 100644 --- a/includes/filebackend/lockmanager/QuorumLockManager.php +++ b/includes/filebackend/lockmanager/QuorumLockManager.php @@ -31,81 +31,86 @@ abstract class QuorumLockManager extends LockManager { /** @var Array Map of bucket indexes to peer server lists */ protected $srvsByBucket = array(); // (bucket index => (lsrv1, lsrv2, ...)) + /** @var Array Map of degraded buckets */ + protected $degradedBuckets = array(); // (buckey index => UNIX timestamp) - /** - * @see LockManager::doLock() - * @param $paths array - * @param $type int - * @return Status - */ final protected function doLock( array $paths, $type ) { + return $this->doLockByType( array( $type => $paths ) ); + } + + final protected function doUnlock( array $paths, $type ) { + return $this->doUnlockByType( array( $type => $paths ) ); + } + + protected function doLockByType( array $pathsByType ) { $status = Status::newGood(); - $pathsToLock = array(); // (bucket => paths) + $pathsToLock = array(); // (bucket => type => paths) // Get locks that need to be acquired (buckets => locks)... - foreach ( $paths as $path ) { - if ( isset( $this->locksHeld[$path][$type] ) ) { - ++$this->locksHeld[$path][$type]; - } else { - $bucket = $this->getBucketFromPath( $path ); - $pathsToLock[$bucket][] = $path; + foreach ( $pathsByType as $type => $paths ) { + foreach ( $paths as $path ) { + if ( isset( $this->locksHeld[$path][$type] ) ) { + ++$this->locksHeld[$path][$type]; + } else { + $bucket = $this->getBucketFromPath( $path ); + $pathsToLock[$bucket][$type][] = $path; + } } } - $lockedPaths = array(); // files locked in this attempt + $lockedPaths = array(); // files locked in this attempt (type => paths) // Attempt to acquire these locks... - foreach ( $pathsToLock as $bucket => $paths ) { + foreach ( $pathsToLock as $bucket => $pathsToLockByType ) { // Try to acquire the locks for this bucket - $status->merge( $this->doLockingRequestBucket( $bucket, $paths, $type ) ); + $status->merge( $this->doLockingRequestBucket( $bucket, $pathsToLockByType ) ); if ( !$status->isOK() ) { - $status->merge( $this->doUnlock( $lockedPaths, $type ) ); + $status->merge( $this->doUnlockByType( $lockedPaths ) ); return $status; } // Record these locks as active - foreach ( $paths as $path ) { - $this->locksHeld[$path][$type] = 1; // locked + foreach ( $pathsToLockByType as $type => $paths ) { + foreach ( $paths as $path ) { + $this->locksHeld[$path][$type] = 1; // locked + // Keep track of what locks were made in this attempt + $lockedPaths[$type][] = $path; + } } - // Keep track of what locks were made in this attempt - $lockedPaths = array_merge( $lockedPaths, $paths ); } return $status; } - /** - * @see LockManager::doUnlock() - * @param $paths array - * @param $type int - * @return Status - */ - final protected function doUnlock( array $paths, $type ) { + protected function doUnlockByType( array $pathsByType ) { $status = Status::newGood(); - $pathsToUnlock = array(); - foreach ( $paths as $path ) { - if ( !isset( $this->locksHeld[$path][$type] ) ) { - $status->warning( 'lockmanager-notlocked', $path ); - } else { - --$this->locksHeld[$path][$type]; - // Reference count the locks held and release locks when zero - if ( $this->locksHeld[$path][$type] <= 0 ) { - unset( $this->locksHeld[$path][$type] ); - $bucket = $this->getBucketFromPath( $path ); - $pathsToUnlock[$bucket][] = $path; - } - if ( !count( $this->locksHeld[$path] ) ) { - unset( $this->locksHeld[$path] ); // no SH or EX locks left for key + $pathsToUnlock = array(); // (bucket => type => paths) + foreach ( $pathsByType as $type => $paths ) { + foreach ( $paths as $path ) { + if ( !isset( $this->locksHeld[$path][$type] ) ) { + $status->warning( 'lockmanager-notlocked', $path ); + } else { + --$this->locksHeld[$path][$type]; + // Reference count the locks held and release locks when zero + if ( $this->locksHeld[$path][$type] <= 0 ) { + unset( $this->locksHeld[$path][$type] ); + $bucket = $this->getBucketFromPath( $path ); + $pathsToUnlock[$bucket][$type][] = $path; + } + if ( !count( $this->locksHeld[$path] ) ) { + unset( $this->locksHeld[$path] ); // no SH or EX locks left for key + } } } } // Remove these specific locks if possible, or at least release // all locks once this process is currently not holding any locks. - foreach ( $pathsToUnlock as $bucket => $paths ) { - $status->merge( $this->doUnlockingRequestBucket( $bucket, $paths, $type ) ); + foreach ( $pathsToUnlock as $bucket => $pathsToUnlockByType ) { + $status->merge( $this->doUnlockingRequestBucket( $bucket, $pathsToUnlockByType ) ); } if ( !count( $this->locksHeld ) ) { $status->merge( $this->releaseAllLocks() ); + $this->degradedBuckets = array(); // safe to retry the normal quorum } return $status; @@ -116,25 +121,25 @@ abstract class QuorumLockManager extends LockManager { * This is all or nothing; if any key is locked then this totally fails. * * @param $bucket integer - * @param array $paths List of resource keys to lock - * @param $type integer LockManager::LOCK_EX or LockManager::LOCK_SH + * @param array $pathsByType Map of LockManager::LOCK_* constants to lists of paths * @return Status */ - final protected function doLockingRequestBucket( $bucket, array $paths, $type ) { + final protected function doLockingRequestBucket( $bucket, array $pathsByType ) { $status = Status::newGood(); $yesVotes = 0; // locks made on trustable servers $votesLeft = count( $this->srvsByBucket[$bucket] ); // remaining peers - $quorum = floor( $votesLeft/2 + 1 ); // simple majority + $quorum = floor( $votesLeft / 2 + 1 ); // simple majority // Get votes for each peer, in order, until we have enough... foreach ( $this->srvsByBucket[$bucket] as $lockSrv ) { if ( !$this->isServerUp( $lockSrv ) ) { --$votesLeft; $status->warning( 'lockmanager-fail-svr-acquire', $lockSrv ); + $this->degradedBuckets[$bucket] = time(); continue; // server down? } // Attempt to acquire the lock on this peer - $status->merge( $this->getLocksOnServer( $lockSrv, $paths, $type ) ); + $status->merge( $this->getLocksOnServer( $lockSrv, $pathsByType ) ); if ( !$status->isOK() ) { return $status; // vetoed; resource locked } @@ -158,21 +163,33 @@ abstract class QuorumLockManager extends LockManager { * Attempt to release locks with the peers for a bucket * * @param $bucket integer - * @param array $paths List of resource keys to lock - * @param $type integer LockManager::LOCK_EX or LockManager::LOCK_SH + * @param array $pathsByType Map of LockManager::LOCK_* constants to lists of paths * @return Status */ - final protected function doUnlockingRequestBucket( $bucket, array $paths, $type ) { + final protected function doUnlockingRequestBucket( $bucket, array $pathsByType ) { $status = Status::newGood(); + $yesVotes = 0; // locks freed on trustable servers + $votesLeft = count( $this->srvsByBucket[$bucket] ); // remaining peers + $quorum = floor( $votesLeft / 2 + 1 ); // simple majority + $isDegraded = isset( $this->degradedBuckets[$bucket] ); // not the normal quorum? foreach ( $this->srvsByBucket[$bucket] as $lockSrv ) { if ( !$this->isServerUp( $lockSrv ) ) { - $status->fatal( 'lockmanager-fail-svr-release', $lockSrv ); + $status->warning( 'lockmanager-fail-svr-release', $lockSrv ); // Attempt to release the lock on this peer } else { - $status->merge( $this->freeLocksOnServer( $lockSrv, $paths, $type ) ); + $status->merge( $this->freeLocksOnServer( $lockSrv, $pathsByType ) ); + ++$yesVotes; // success for this peer + // Normally the first peers form the quorum, and the others are ignored. + // Ignore them in this case, but not when an alternative quorum was used. + if ( $yesVotes >= $quorum && !$isDegraded ) { + break; // lock released + } } } + // Set a bad status if the quorum was not met. + // Assumes the same "up" servers as during the acquire step. + $status->setResult( $yesVotes >= $quorum ); return $status; } @@ -190,7 +207,8 @@ abstract class QuorumLockManager extends LockManager { } /** - * Check if a lock server is up + * Check if a lock server is up. + * This should process cache results to reduce RTT. * * @param $lockSrv string * @return bool @@ -198,14 +216,13 @@ abstract class QuorumLockManager extends LockManager { abstract protected function isServerUp( $lockSrv ); /** - * Get a connection to a lock server and acquire locks on $paths + * Get a connection to a lock server and acquire locks * * @param $lockSrv string - * @param $paths array - * @param $type integer + * @param array $pathsByType Map of LockManager::LOCK_* constants to lists of paths * @return Status */ - abstract protected function getLocksOnServer( $lockSrv, array $paths, $type ); + abstract protected function getLocksOnServer( $lockSrv, array $pathsByType ); /** * Get a connection to a lock server and release locks on $paths. @@ -213,11 +230,10 @@ abstract class QuorumLockManager extends LockManager { * Subclasses must effectively implement this or releaseAllLocks(). * * @param $lockSrv string - * @param $paths array - * @param $type integer + * @param array $pathsByType Map of LockManager::LOCK_* constants to lists of paths * @return Status */ - abstract protected function freeLocksOnServer( $lockSrv, array $paths, $type ); + abstract protected function freeLocksOnServer( $lockSrv, array $pathsByType ); /** * Release all locks that this session is holding. diff --git a/includes/filebackend/lockmanager/RedisLockManager.php b/includes/filebackend/lockmanager/RedisLockManager.php new file mode 100644 index 00000000..43b0198a --- /dev/null +++ b/includes/filebackend/lockmanager/RedisLockManager.php @@ -0,0 +1,288 @@ + self::LOCK_SH, + self::LOCK_UW => self::LOCK_SH, + self::LOCK_EX => self::LOCK_EX + ); + + /** @var RedisConnectionPool */ + protected $redisPool; + /** @var Array Map server names to hostname/IP and port numbers */ + protected $lockServers = array(); + + protected $session = ''; // string; random UUID + + /** + * Construct a new instance from configuration. + * + * $config paramaters include: + * - lockServers : Associative array of server names to ":" strings. + * - srvsByBucket : Array of 1-16 consecutive integer keys, starting from 0, + * each having an odd-numbered list of server names (peers) as values. + * - redisConfig : Configuration for RedisConnectionPool::__construct(). + * + * @param Array $config + * @throws MWException + */ + public function __construct( array $config ) { + parent::__construct( $config ); + + $this->lockServers = $config['lockServers']; + // Sanitize srvsByBucket config to prevent PHP errors + $this->srvsByBucket = array_filter( $config['srvsByBucket'], 'is_array' ); + $this->srvsByBucket = array_values( $this->srvsByBucket ); // consecutive + + $config['redisConfig']['serializer'] = 'none'; + $this->redisPool = RedisConnectionPool::singleton( $config['redisConfig'] ); + + $this->session = wfRandomString( 32 ); + } + + // @TODO: change this code to work in one batch + protected function getLocksOnServer( $lockSrv, array $pathsByType ) { + $status = Status::newGood(); + + $lockedPaths = array(); + foreach ( $pathsByType as $type => $paths ) { + $status->merge( $this->doGetLocksOnServer( $lockSrv, $paths, $type ) ); + if ( $status->isOK() ) { + $lockedPaths[$type] = isset( $lockedPaths[$type] ) + ? array_merge( $lockedPaths[$type], $paths ) + : $paths; + } else { + foreach ( $lockedPaths as $type => $paths ) { + $status->merge( $this->doFreeLocksOnServer( $lockSrv, $paths, $type ) ); + } + break; + } + } + + return $status; + } + + // @TODO: change this code to work in one batch + protected function freeLocksOnServer( $lockSrv, array $pathsByType ) { + $status = Status::newGood(); + + foreach ( $pathsByType as $type => $paths ) { + $status->merge( $this->doFreeLocksOnServer( $lockSrv, $paths, $type ) ); + } + + return $status; + } + + protected function doGetLocksOnServer( $lockSrv, array $paths, $type ) { + $status = Status::newGood(); + + $server = $this->lockServers[$lockSrv]; + $conn = $this->redisPool->getConnection( $server ); + if ( !$conn ) { + foreach ( $paths as $path ) { + $status->fatal( 'lockmanager-fail-acquirelock', $path ); + } + return $status; + } + + $keys = array_map( array( $this, 'recordKeyForPath' ), $paths ); // lock records + + try { + static $script = +<<luaEval( $script, + array_merge( + $keys, // KEYS[0], KEYS[1],...KEYS[N] + array( + $type === self::LOCK_SH ? 'SH' : 'EX', // ARGV[1] + $this->session, // ARGV[2] + $this->lockTTL, // ARGV[3] + time() // ARGV[4] + ) + ), + count( $keys ) # number of first argument(s) that are keys + ); + } catch ( RedisException $e ) { + $res = false; + $this->redisPool->handleException( $server, $conn, $e ); + } + + if ( $res === false ) { + foreach ( $paths as $path ) { + $status->fatal( 'lockmanager-fail-acquirelock', $path ); + } + } else { + $pathsByKey = array_combine( $keys, $paths ); + foreach ( $res as $key ) { + $status->fatal( 'lockmanager-fail-acquirelock', $pathsByKey[$key] ); + } + } + + return $status; + } + + protected function doFreeLocksOnServer( $lockSrv, array $paths, $type ) { + $status = Status::newGood(); + + $server = $this->lockServers[$lockSrv]; + $conn = $this->redisPool->getConnection( $server ); + if ( !$conn ) { + foreach ( $paths as $path ) { + $status->fatal( 'lockmanager-fail-releaselock', $path ); + } + return $status; + } + + $keys = array_map( array( $this, 'recordKeyForPath' ), $paths ); // lock records + + try { + static $script = +<< 0 then + -- Remove the whole structure if it is now empty + if redis.call('hLen',resourceKey) == 0 then + redis.call('del',resourceKey) + end + else + failed[#failed+1] = resourceKey + end + end + return failed +LUA; + $res = $conn->luaEval( $script, + array_merge( + $keys, // KEYS[0], KEYS[1],...KEYS[N] + array( + $type === self::LOCK_SH ? 'SH' : 'EX', // ARGV[1] + $this->session // ARGV[2] + ) + ), + count( $keys ) # number of first argument(s) that are keys + ); + } catch ( RedisException $e ) { + $res = false; + $this->redisPool->handleException( $server, $conn, $e ); + } + + if ( $res === false ) { + foreach ( $paths as $path ) { + $status->fatal( 'lockmanager-fail-releaselock', $path ); + } + } else { + $pathsByKey = array_combine( $keys, $paths ); + foreach ( $res as $key ) { + $status->fatal( 'lockmanager-fail-releaselock', $pathsByKey[$key] ); + } + } + + return $status; + } + + protected function releaseAllLocks() { + return Status::newGood(); // not supported + } + + protected function isServerUp( $lockSrv ) { + return (bool)$this->redisPool->getConnection( $this->lockServers[$lockSrv] ); + } + + /** + * @param $path string + * @return string + */ + protected function recordKeyForPath( $path ) { + return implode( ':', array( __CLASS__, 'locks', $this->sha1Base36Absolute( $path ) ) ); + } + + /** + * Make sure remaining locks get cleared for sanity + */ + function __destruct() { + while ( count( $this->locksHeld ) ) { + foreach ( $this->locksHeld as $path => $locks ) { + $this->doUnlock( array( $path ), self::LOCK_EX ); + $this->doUnlock( array( $path ), self::LOCK_SH ); + } + } + } +} diff --git a/includes/filebackend/lockmanager/ScopedLock.php b/includes/filebackend/lockmanager/ScopedLock.php index edcb1d65..5faad4a6 100644 --- a/includes/filebackend/lockmanager/ScopedLock.php +++ b/includes/filebackend/lockmanager/ScopedLock.php @@ -36,24 +36,18 @@ class ScopedLock { protected $manager; /** @var Status */ protected $status; - /** @var Array List of resource paths*/ - protected $paths; - - protected $type; // integer lock type + /** @var Array Map of lock types to resource paths */ + protected $pathsByType; /** - * @param $manager LockManager - * @param array $paths List of storage paths - * @param $type integer LockManager::LOCK_* constant - * @param $status Status + * @param LockManager $manager + * @param array $pathsByType Map of lock types to path lists + * @param Status $status */ - protected function __construct( - LockManager $manager, array $paths, $type, Status $status - ) { + protected function __construct( LockManager $manager, array $pathsByType, Status $status ) { $this->manager = $manager; - $this->paths = $paths; + $this->pathsByType = $pathsByType; $this->status = $status; - $this->type = $type; } /** @@ -61,19 +55,24 @@ class ScopedLock { * Any locks are released once this object goes out of scope. * The status object is updated with any errors or warnings. * - * @param $manager LockManager - * @param array $paths List of storage paths - * @param $type integer LockManager::LOCK_* constant - * @param $status Status + * $type can be "mixed" and $paths can be a map of types to paths (since 1.22). + * Otherwise $type should be an integer and $paths should be a list of paths. + * + * @param LockManager $manager + * @param array $paths List of storage paths or map of lock types to path lists + * @param integer|string $type LockManager::LOCK_* constant or "mixed" + * @param Status $status + * @param integer $timeout Timeout in seconds (0 means non-blocking) (since 1.22) * @return ScopedLock|null Returns null on failure */ public static function factory( - LockManager $manager, array $paths, $type, Status $status + LockManager $manager, array $paths, $type, Status $status, $timeout = 0 ) { - $lockStatus = $manager->lock( $paths, $type ); + $pathsByType = is_integer( $type ) ? array( $type => $paths ) : $paths; + $lockStatus = $manager->lockByType( $pathsByType, $timeout ); $status->merge( $lockStatus ); if ( $lockStatus->isOK() ) { - return new self( $manager, $paths, $type, $status ); + return new self( $manager, $pathsByType, $status ); } return null; } @@ -91,9 +90,12 @@ class ScopedLock { $lock = null; } + /** + * Release the locks when this goes out of scope + */ function __destruct() { $wasOk = $this->status->isOK(); - $this->status->merge( $this->manager->unlock( $this->paths, $this->type ) ); + $this->status->merge( $this->manager->unlockByType( $this->pathsByType ) ); if ( $wasOk ) { // Make sure status is OK, despite any unlockFiles() fatals $this->status->setResult( true, $this->status->value ); -- cgit v1.2.2