From 4ac9fa081a7c045f6a9f1cfc529d82423f485b2e Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 8 Dec 2013 09:55:49 +0100 Subject: Update to MediaWiki 1.22.0 --- includes/filebackend/FileBackendStore.php | 576 +++++++++--------------------- 1 file changed, 172 insertions(+), 404 deletions(-) (limited to 'includes/filebackend/FileBackendStore.php') diff --git a/includes/filebackend/FileBackendStore.php b/includes/filebackend/FileBackendStore.php index 3f1d1857..0921e99f 100644 --- a/includes/filebackend/FileBackendStore.php +++ b/includes/filebackend/FileBackendStore.php @@ -38,28 +38,43 @@ abstract class FileBackendStore extends FileBackend { /** @var BagOStuff */ protected $memCache; - /** @var ProcessCacheLRU */ - protected $cheapCache; // Map of paths to small (RAM/disk) cache items - /** @var ProcessCacheLRU */ - protected $expensiveCache; // Map of paths to large (RAM/disk) cache items + /** @var ProcessCacheLRU Map of paths to small (RAM/disk) cache items */ + protected $cheapCache; + /** @var ProcessCacheLRU Map of paths to large (RAM/disk) cache items */ + protected $expensiveCache; - /** @var Array Map of container names to sharding settings */ - protected $shardViaHashLevels = array(); // (container name => config array) + /** @var Array Map of container names to sharding config */ + protected $shardViaHashLevels = array(); + + /** @var callback Method to get the MIME type of files */ + protected $mimeCallback; protected $maxFileSize = 4294967296; // integer bytes (4GiB) const CACHE_TTL = 10; // integer; TTL in seconds for process cache entries + const CACHE_CHEAP_SIZE = 300; // integer; max entries in "cheap cache" + const CACHE_EXPENSIVE_SIZE = 5; // integer; max entries in "expensive cache" /** * @see FileBackend::__construct() + * Additional $config params include: + * - mimeCallback : Callback that takes (storage path, content, file system path) and + * returns the MIME type of the file or 'unknown/unknown'. The file + * system path parameter should be used if the content one is null. * - * @param $config Array + * @param array $config */ public function __construct( array $config ) { parent::__construct( $config ); + $this->mimeCallback = isset( $config['mimeCallback'] ) + ? $config['mimeCallback'] + : function( $storagePath, $content, $fsPath ) { + // @TODO: handle the case of extension-less files using the contents + return StreamFile::contentTypeFromPath( $storagePath ) ?: 'unknown/unknown'; + }; $this->memCache = new EmptyBagOStuff(); // disabled by default - $this->cheapCache = new ProcessCacheLRU( 300 ); - $this->expensiveCache = new ProcessCacheLRU( 5 ); + $this->cheapCache = new ProcessCacheLRU( self::CACHE_CHEAP_SIZE ); + $this->expensiveCache = new ProcessCacheLRU( self::CACHE_EXPENSIVE_SIZE ); } /** @@ -79,7 +94,7 @@ abstract class FileBackendStore extends FileBackend { * written under it, and that any file already there is writable. * Backends using key/value stores should check if the container exists. * - * @param $storagePath string + * @param string $storagePath * @return bool */ abstract public function isPathUsableInternal( $storagePath ); @@ -92,7 +107,6 @@ abstract class FileBackendStore extends FileBackend { * $params include: * - content : the raw file contents * - dst : destination storage path - * - disposition : Content-Disposition header value for the destination * - headers : HTTP header name/value map * - async : Status will be returned immediately if supported. * If the status is OK, then its value field will be @@ -104,8 +118,7 @@ abstract class FileBackendStore extends FileBackend { * @return Status */ final public function createInternal( array $params ) { - wfProfileIn( __METHOD__ ); - wfProfileIn( __METHOD__ . '-' . $this->name ); + $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); if ( strlen( $params['content'] ) > $this->maxFileSizeInternal() ) { $status = Status::newFatal( 'backend-fail-maxsize', $params['dst'], $this->maxFileSizeInternal() ); @@ -116,8 +129,6 @@ abstract class FileBackendStore extends FileBackend { $this->deleteFileCache( $params['dst'] ); // persistent cache } } - wfProfileOut( __METHOD__ . '-' . $this->name ); - wfProfileOut( __METHOD__ ); return $status; } @@ -135,7 +146,6 @@ abstract class FileBackendStore extends FileBackend { * $params include: * - src : source path on disk * - dst : destination storage path - * - disposition : Content-Disposition header value for the destination * - headers : HTTP header name/value map * - async : Status will be returned immediately if supported. * If the status is OK, then its value field will be @@ -147,8 +157,7 @@ abstract class FileBackendStore extends FileBackend { * @return Status */ final public function storeInternal( array $params ) { - wfProfileIn( __METHOD__ ); - wfProfileIn( __METHOD__ . '-' . $this->name ); + $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); if ( filesize( $params['src'] ) > $this->maxFileSizeInternal() ) { $status = Status::newFatal( 'backend-fail-maxsize', $params['dst'], $this->maxFileSizeInternal() ); @@ -159,8 +168,6 @@ abstract class FileBackendStore extends FileBackend { $this->deleteFileCache( $params['dst'] ); // persistent cache } } - wfProfileOut( __METHOD__ . '-' . $this->name ); - wfProfileOut( __METHOD__ ); return $status; } @@ -179,7 +186,7 @@ abstract class FileBackendStore extends FileBackend { * - src : source storage path * - dst : destination storage path * - ignoreMissingSource : do nothing if the source file does not exist - * - disposition : Content-Disposition header value for the destination + * - headers : HTTP header name/value map * - async : Status will be returned immediately if supported. * If the status is OK, then its value field will be * set to a FileBackendStoreOpHandle object. @@ -190,15 +197,12 @@ abstract class FileBackendStore extends FileBackend { * @return Status */ final public function copyInternal( array $params ) { - wfProfileIn( __METHOD__ ); - wfProfileIn( __METHOD__ . '-' . $this->name ); + $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); $status = $this->doCopyInternal( $params ); $this->clearCache( array( $params['dst'] ) ); if ( !isset( $params['dstExists'] ) || $params['dstExists'] ) { $this->deleteFileCache( $params['dst'] ); // persistent cache } - wfProfileOut( __METHOD__ . '-' . $this->name ); - wfProfileOut( __METHOD__ ); return $status; } @@ -223,13 +227,10 @@ abstract class FileBackendStore extends FileBackend { * @return Status */ final public function deleteInternal( array $params ) { - wfProfileIn( __METHOD__ ); - wfProfileIn( __METHOD__ . '-' . $this->name ); + $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); $status = $this->doDeleteInternal( $params ); $this->clearCache( array( $params['src'] ) ); $this->deleteFileCache( $params['src'] ); // persistent cache - wfProfileOut( __METHOD__ . '-' . $this->name ); - wfProfileOut( __METHOD__ ); return $status; } @@ -248,7 +249,7 @@ abstract class FileBackendStore extends FileBackend { * - src : source storage path * - dst : destination storage path * - ignoreMissingSource : do nothing if the source file does not exist - * - disposition : Content-Disposition header value for the destination + * - headers : HTTP header name/value map * - async : Status will be returned immediately if supported. * If the status is OK, then its value field will be * set to a FileBackendStoreOpHandle object. @@ -259,16 +260,13 @@ abstract class FileBackendStore extends FileBackend { * @return Status */ final public function moveInternal( array $params ) { - wfProfileIn( __METHOD__ ); - wfProfileIn( __METHOD__ . '-' . $this->name ); + $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); $status = $this->doMoveInternal( $params ); $this->clearCache( array( $params['src'], $params['dst'] ) ); $this->deleteFileCache( $params['src'] ); // persistent cache if ( !isset( $params['dstExists'] ) || $params['dstExists'] ) { $this->deleteFileCache( $params['dst'] ); // persistent cache } - wfProfileOut( __METHOD__ . '-' . $this->name ); - wfProfileOut( __METHOD__ ); return $status; } @@ -278,10 +276,12 @@ abstract class FileBackendStore extends FileBackend { */ protected function doMoveInternal( array $params ) { unset( $params['async'] ); // two steps, won't work here :) + $nsrc = FileBackend::normalizeStoragePath( $params['src'] ); + $ndst = FileBackend::normalizeStoragePath( $params['dst'] ); // Copy source to dest $status = $this->copyInternal( $params ); - if ( $status->isOK() ) { - // Delete source (only fails due to races or medium going down) + if ( $nsrc !== $ndst && $status->isOK() ) { + // Delete source (only fails due to races or network problems) $status->merge( $this->deleteInternal( array( 'src' => $params['src'] ) ) ); $status->setResult( true, $status->value ); // ignore delete() errors } @@ -294,7 +294,6 @@ abstract class FileBackendStore extends FileBackend { * * $params include: * - src : source storage path - * - disposition : Content-Disposition header value for the destination * - headers : HTTP header name/value map * - async : Status will be returned immediately if supported. * If the status is OK, then its value field will be @@ -304,13 +303,14 @@ abstract class FileBackendStore extends FileBackend { * @return Status */ final public function describeInternal( array $params ) { - wfProfileIn( __METHOD__ ); - wfProfileIn( __METHOD__ . '-' . $this->name ); - $status = $this->doDescribeInternal( $params ); - $this->clearCache( array( $params['src'] ) ); - $this->deleteFileCache( $params['src'] ); // persistent cache - wfProfileOut( __METHOD__ . '-' . $this->name ); - wfProfileOut( __METHOD__ ); + $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); + if ( count( $params['headers'] ) ) { + $status = $this->doDescribeInternal( $params ); + $this->clearCache( array( $params['src'] ) ); + $this->deleteFileCache( $params['src'] ); // persistent cache + } else { + $status = Status::newGood(); // nothing to do + } return $status; } @@ -333,13 +333,8 @@ abstract class FileBackendStore extends FileBackend { return Status::newGood(); } - /** - * @see FileBackend::concatenate() - * @return Status - */ final public function concatenate( array $params ) { - wfProfileIn( __METHOD__ ); - wfProfileIn( __METHOD__ . '-' . $this->name ); + $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); $status = Status::newGood(); // Try to lock the source files for the scope of this function @@ -355,8 +350,6 @@ abstract class FileBackendStore extends FileBackend { } } - wfProfileOut( __METHOD__ . '-' . $this->name ); - wfProfileOut( __METHOD__ ); return $status; } @@ -426,20 +419,13 @@ abstract class FileBackendStore extends FileBackend { return $status; } - /** - * @see FileBackend::doPrepare() - * @return Status - */ final protected function doPrepare( array $params ) { - wfProfileIn( __METHOD__ ); - wfProfileIn( __METHOD__ . '-' . $this->name ); - + $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); $status = Status::newGood(); + list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] ); if ( $dir === null ) { $status->fatal( 'backend-fail-invalidpath', $params['dir'] ); - wfProfileOut( __METHOD__ . '-' . $this->name ); - wfProfileOut( __METHOD__ ); return $status; // invalid storage path } @@ -453,8 +439,6 @@ abstract class FileBackendStore extends FileBackend { } } - wfProfileOut( __METHOD__ . '-' . $this->name ); - wfProfileOut( __METHOD__ ); return $status; } @@ -466,20 +450,13 @@ abstract class FileBackendStore extends FileBackend { return Status::newGood(); } - /** - * @see FileBackend::doSecure() - * @return Status - */ final protected function doSecure( array $params ) { - wfProfileIn( __METHOD__ ); - wfProfileIn( __METHOD__ . '-' . $this->name ); + $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); $status = Status::newGood(); list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] ); if ( $dir === null ) { $status->fatal( 'backend-fail-invalidpath', $params['dir'] ); - wfProfileOut( __METHOD__ . '-' . $this->name ); - wfProfileOut( __METHOD__ ); return $status; // invalid storage path } @@ -493,8 +470,6 @@ abstract class FileBackendStore extends FileBackend { } } - wfProfileOut( __METHOD__ . '-' . $this->name ); - wfProfileOut( __METHOD__ ); return $status; } @@ -506,20 +481,13 @@ abstract class FileBackendStore extends FileBackend { return Status::newGood(); } - /** - * @see FileBackend::doPublish() - * @return Status - */ final protected function doPublish( array $params ) { - wfProfileIn( __METHOD__ ); - wfProfileIn( __METHOD__ . '-' . $this->name ); + $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); $status = Status::newGood(); list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] ); if ( $dir === null ) { $status->fatal( 'backend-fail-invalidpath', $params['dir'] ); - wfProfileOut( __METHOD__ . '-' . $this->name ); - wfProfileOut( __METHOD__ ); return $status; // invalid storage path } @@ -533,8 +501,6 @@ abstract class FileBackendStore extends FileBackend { } } - wfProfileOut( __METHOD__ . '-' . $this->name ); - wfProfileOut( __METHOD__ ); return $status; } @@ -546,13 +512,8 @@ abstract class FileBackendStore extends FileBackend { return Status::newGood(); } - /** - * @see FileBackend::doClean() - * @return Status - */ final protected function doClean( array $params ) { - wfProfileIn( __METHOD__ ); - wfProfileIn( __METHOD__ . '-' . $this->name ); + $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); $status = Status::newGood(); // Recursive: first delete all empty subdirs recursively @@ -570,8 +531,6 @@ abstract class FileBackendStore extends FileBackend { list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] ); if ( $dir === null ) { $status->fatal( 'backend-fail-invalidpath', $params['dir'] ); - wfProfileOut( __METHOD__ . '-' . $this->name ); - wfProfileOut( __METHOD__ ); return $status; // invalid storage path } @@ -579,8 +538,6 @@ abstract class FileBackendStore extends FileBackend { $filesLockEx = array( $params['dir'] ); $scopedLockE = $this->getScopedFileLocks( $filesLockEx, LockManager::LOCK_EX, $status ); if ( !$status->isOK() ) { - wfProfileOut( __METHOD__ . '-' . $this->name ); - wfProfileOut( __METHOD__ ); return $status; // abort } @@ -596,8 +553,6 @@ abstract class FileBackendStore extends FileBackend { } } - wfProfileOut( __METHOD__ . '-' . $this->name ); - wfProfileOut( __METHOD__ ); return $status; } @@ -609,56 +564,30 @@ abstract class FileBackendStore extends FileBackend { return Status::newGood(); } - /** - * @see FileBackend::fileExists() - * @return bool|null - */ final public function fileExists( array $params ) { - wfProfileIn( __METHOD__ ); - wfProfileIn( __METHOD__ . '-' . $this->name ); + $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); $stat = $this->getFileStat( $params ); - wfProfileOut( __METHOD__ . '-' . $this->name ); - wfProfileOut( __METHOD__ ); return ( $stat === null ) ? null : (bool)$stat; // null => failure } - /** - * @see FileBackend::getFileTimestamp() - * @return bool - */ final public function getFileTimestamp( array $params ) { - wfProfileIn( __METHOD__ ); - wfProfileIn( __METHOD__ . '-' . $this->name ); + $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); $stat = $this->getFileStat( $params ); - wfProfileOut( __METHOD__ . '-' . $this->name ); - wfProfileOut( __METHOD__ ); return $stat ? $stat['mtime'] : false; } - /** - * @see FileBackend::getFileSize() - * @return bool - */ final public function getFileSize( array $params ) { - wfProfileIn( __METHOD__ ); - wfProfileIn( __METHOD__ . '-' . $this->name ); + $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); $stat = $this->getFileStat( $params ); - wfProfileOut( __METHOD__ . '-' . $this->name ); - wfProfileOut( __METHOD__ ); return $stat ? $stat['size'] : false; } - /** - * @see FileBackend::getFileStat() - * @return bool - */ final public function getFileStat( array $params ) { $path = self::normalizeStoragePath( $params['src'] ); if ( $path === null ) { return false; // invalid storage path } - wfProfileIn( __METHOD__ ); - wfProfileIn( __METHOD__ . '-' . $this->name ); + $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); $latest = !empty( $params['latest'] ); // use latest data? if ( !$this->cheapCache->has( $path, 'stat', self::CACHE_TTL ) ) { $this->primeFileCache( array( $path ) ); // check persistent cache @@ -669,14 +598,10 @@ abstract class FileBackendStore extends FileBackend { // value was in fact fetched with the latest available data. if ( is_array( $stat ) ) { if ( !$latest || $stat['latest'] ) { - wfProfileOut( __METHOD__ . '-' . $this->name ); - wfProfileOut( __METHOD__ ); return $stat; } } elseif ( in_array( $stat, array( 'NOT_EXIST', 'NOT_EXIST_LATEST' ) ) ) { if ( !$latest || $stat === 'NOT_EXIST_LATEST' ) { - wfProfileOut( __METHOD__ . '-' . $this->name ); - wfProfileOut( __METHOD__ ); return false; } } @@ -696,12 +621,12 @@ abstract class FileBackendStore extends FileBackend { } } elseif ( $stat === false ) { // file does not exist $this->cheapCache->set( $path, 'stat', $latest ? 'NOT_EXIST_LATEST' : 'NOT_EXIST' ); + $this->cheapCache->set( $path, 'sha1', // the SHA-1 must be false too + array( 'hash' => false, 'latest' => $latest ) ); wfDebug( __METHOD__ . ": File $path does not exist.\n" ); } else { // an error occurred wfDebug( __METHOD__ . ": Could not stat file $path.\n" ); } - wfProfileOut( __METHOD__ . '-' . $this->name ); - wfProfileOut( __METHOD__ ); return $stat; } @@ -710,19 +635,12 @@ abstract class FileBackendStore extends FileBackend { */ abstract protected function doGetFileStat( array $params ); - /** - * @see FileBackend::getFileContentsMulti() - * @return Array - */ public function getFileContentsMulti( array $params ) { - wfProfileIn( __METHOD__ ); - wfProfileIn( __METHOD__ . '-' . $this->name ); + $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); $params = $this->setConcurrencyFlags( $params ); $contents = $this->doGetFileContentsMulti( $params ); - wfProfileOut( __METHOD__ . '-' . $this->name ); - wfProfileOut( __METHOD__ ); return $contents; } @@ -740,25 +658,18 @@ abstract class FileBackendStore extends FileBackend { return $contents; } - /** - * @see FileBackend::getFileSha1Base36() - * @return bool|string - */ final public function getFileSha1Base36( array $params ) { $path = self::normalizeStoragePath( $params['src'] ); if ( $path === null ) { return false; // invalid storage path } - wfProfileIn( __METHOD__ ); - wfProfileIn( __METHOD__ . '-' . $this->name ); + $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); $latest = !empty( $params['latest'] ); // use latest data? if ( $this->cheapCache->has( $path, 'sha1', self::CACHE_TTL ) ) { $stat = $this->cheapCache->get( $path, 'sha1' ); // If we want the latest data, check that this cached // value was in fact fetched with the latest available data. if ( !$latest || $stat['latest'] ) { - wfProfileOut( __METHOD__ . '-' . $this->name ); - wfProfileOut( __METHOD__ ); return $stat['hash']; } } @@ -768,8 +679,6 @@ abstract class FileBackendStore extends FileBackend { wfProfileOut( __METHOD__ . '-miss-' . $this->name ); wfProfileOut( __METHOD__ . '-miss' ); $this->cheapCache->set( $path, 'sha1', array( 'hash' => $hash, 'latest' => $latest ) ); - wfProfileOut( __METHOD__ . '-' . $this->name ); - wfProfileOut( __METHOD__ ); return $hash; } @@ -786,27 +695,15 @@ abstract class FileBackendStore extends FileBackend { } } - /** - * @see FileBackend::getFileProps() - * @return Array - */ final public function getFileProps( array $params ) { - wfProfileIn( __METHOD__ ); - wfProfileIn( __METHOD__ . '-' . $this->name ); + $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); $fsFile = $this->getLocalReference( $params ); $props = $fsFile ? $fsFile->getProps() : FSFile::placeholderProps(); - wfProfileOut( __METHOD__ . '-' . $this->name ); - wfProfileOut( __METHOD__ ); return $props; } - /** - * @see FileBackend::getLocalReferenceMulti() - * @return Array - */ final public function getLocalReferenceMulti( array $params ) { - wfProfileIn( __METHOD__ ); - wfProfileIn( __METHOD__ . '-' . $this->name ); + $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); $params = $this->setConcurrencyFlags( $params ); @@ -836,8 +733,6 @@ abstract class FileBackendStore extends FileBackend { } } - wfProfileOut( __METHOD__ . '-' . $this->name ); - wfProfileOut( __METHOD__ ); return $fsFiles; } @@ -849,19 +744,12 @@ abstract class FileBackendStore extends FileBackend { return $this->doGetLocalCopyMulti( $params ); } - /** - * @see FileBackend::getLocalCopyMulti() - * @return Array - */ final public function getLocalCopyMulti( array $params ) { - wfProfileIn( __METHOD__ ); - wfProfileIn( __METHOD__ . '-' . $this->name ); + $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); $params = $this->setConcurrencyFlags( $params ); $tmpFiles = $this->doGetLocalCopyMulti( $params ); - wfProfileOut( __METHOD__ . '-' . $this->name ); - wfProfileOut( __METHOD__ ); return $tmpFiles; } @@ -879,13 +767,8 @@ abstract class FileBackendStore extends FileBackend { return null; // not supported } - /** - * @see FileBackend::streamFile() - * @return Status - */ final public function streamFile( array $params ) { - wfProfileIn( __METHOD__ ); - wfProfileIn( __METHOD__ . '-' . $this->name ); + $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); $status = Status::newGood(); $info = $this->getFileStat( $params ); @@ -916,8 +799,6 @@ abstract class FileBackendStore extends FileBackend { $status->fatal( 'backend-fail-stream', $params['src'] ); } - wfProfileOut( __METHOD__ . '-' . $this->name ); - wfProfileOut( __METHOD__ ); return $status; } @@ -938,10 +819,6 @@ abstract class FileBackendStore extends FileBackend { return $status; } - /** - * @see FileBackend::directoryExists() - * @return bool|null - */ final public function directoryExists( array $params ) { list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] ); if ( $dir === null ) { @@ -976,10 +853,6 @@ abstract class FileBackendStore extends FileBackend { */ abstract protected function doDirectoryExists( $container, $dir, array $params ); - /** - * @see FileBackend::getDirectoryList() - * @return Traversable|Array|null Returns null on failure - */ final public function getDirectoryList( array $params ) { list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] ); if ( $dir === null ) { // invalid storage path @@ -1009,10 +882,6 @@ abstract class FileBackendStore extends FileBackend { */ abstract public function getDirectoryListInternal( $container, $dir, array $params ); - /** - * @see FileBackend::getFileList() - * @return Traversable|Array|null Returns null on failure - */ final public function getFileList( array $params ) { list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] ); if ( $dir === null ) { // invalid storage path @@ -1055,13 +924,13 @@ abstract class FileBackendStore extends FileBackend { */ final public function getOperationsInternal( array $ops ) { $supportedOps = array( - 'store' => 'StoreFileOp', - 'copy' => 'CopyFileOp', - 'move' => 'MoveFileOp', - 'delete' => 'DeleteFileOp', - 'create' => 'CreateFileOp', + 'store' => 'StoreFileOp', + 'copy' => 'CopyFileOp', + 'move' => 'MoveFileOp', + 'delete' => 'DeleteFileOp', + 'create' => 'CreateFileOp', 'describe' => 'DescribeFileOp', - 'null' => 'NullFileOp' + 'null' => 'NullFileOp' ); $performOps = array(); // array of FileOp objects @@ -1084,12 +953,13 @@ abstract class FileBackendStore extends FileBackend { /** * Get a list of storage paths to lock for a list of operations - * Returns an array with 'sh' (shared) and 'ex' (exclusive) keys, - * each corresponding to a list of storage paths to be locked. - * All returned paths are normalized. + * Returns an array with LockManager::LOCK_UW (shared locks) and + * LockManager::LOCK_EX (exclusive locks) keys, each corresponding + * to a list of storage paths to be locked. All returned paths are + * normalized. * * @param array $performOps List of FileOp objects - * @return Array ('sh' => list of paths, 'ex' => list of paths) + * @return Array (LockManager::LOCK_UW => path list, LockManager::LOCK_EX => path list) */ final public function getPathsToLockForOpsInternal( array $performOps ) { // Build up a list of files to lock... @@ -1103,28 +973,19 @@ abstract class FileBackendStore extends FileBackend { // Get a shared lock on the parent directory of each path changed $paths['sh'] = array_merge( $paths['sh'], array_map( 'dirname', $paths['ex'] ) ); - return $paths; + return array( + LockManager::LOCK_UW => $paths['sh'], + LockManager::LOCK_EX => $paths['ex'] + ); } - /** - * @see FileBackend::getScopedLocksForOps() - * @return Array - */ public function getScopedLocksForOps( array $ops, Status $status ) { $paths = $this->getPathsToLockForOpsInternal( $this->getOperationsInternal( $ops ) ); - return array( - $this->getScopedFileLocks( $paths['sh'], LockManager::LOCK_UW, $status ), - $this->getScopedFileLocks( $paths['ex'], LockManager::LOCK_EX, $status ) - ); + return array( $this->getScopedFileLocks( $paths, 'mixed', $status ) ); } - /** - * @see FileBackend::doOperationsInternal() - * @return Status - */ final protected function doOperationsInternal( array $ops, array $opts ) { - wfProfileIn( __METHOD__ ); - wfProfileIn( __METHOD__ . '-' . $this->name ); + $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); $status = Status::newGood(); // Fix up custom header name/value pairs... @@ -1138,11 +999,8 @@ abstract class FileBackendStore extends FileBackend { // Build up a list of files to lock... $paths = $this->getPathsToLockForOpsInternal( $performOps ); // Try to lock those files for the scope of this function... - $scopeLockS = $this->getScopedFileLocks( $paths['sh'], LockManager::LOCK_UW, $status ); - $scopeLockE = $this->getScopedFileLocks( $paths['ex'], LockManager::LOCK_EX, $status ); + $scopeLock = $this->getScopedFileLocks( $paths, 'mixed', $status ); if ( !$status->isOK() ) { - wfProfileOut( __METHOD__ . '-' . $this->name ); - wfProfileOut( __METHOD__ ); return $status; // abort } } @@ -1164,19 +1022,11 @@ abstract class FileBackendStore extends FileBackend { $status->merge( $subStatus ); $status->success = $subStatus->success; // not done in merge() - wfProfileOut( __METHOD__ . '-' . $this->name ); - wfProfileOut( __METHOD__ ); return $status; } - /** - * @see FileBackend::doQuickOperationsInternal() - * @return Status - * @throws MWException - */ final protected function doQuickOperationsInternal( array $ops ) { - wfProfileIn( __METHOD__ ); - wfProfileIn( __METHOD__ . '-' . $this->name ); + $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); $status = Status::newGood(); // Fix up custom header name/value pairs... @@ -1186,7 +1036,7 @@ abstract class FileBackendStore extends FileBackend { $this->clearCache(); $supportedOps = array( 'create', 'store', 'copy', 'move', 'delete', 'null' ); - $async = ( $this->parallelize === 'implicit' ); + $async = ( $this->parallelize === 'implicit' && count( $ops ) > 1 ); $maxConcurrency = $this->concurrency; // throttle $statuses = array(); // array of (index => Status) @@ -1195,8 +1045,6 @@ abstract class FileBackendStore extends FileBackend { // Perform the sync-only ops and build up op handles for the async ops... foreach ( $ops as $index => $params ) { if ( !in_array( $params['op'], $supportedOps ) ) { - wfProfileOut( __METHOD__ . '-' . $this->name ); - wfProfileOut( __METHOD__ ); throw new MWException( "Operation '{$params['op']}' is not supported." ); } $method = $params['op'] . 'Internal'; // e.g. "storeInternal" @@ -1230,8 +1078,6 @@ abstract class FileBackendStore extends FileBackend { } } - wfProfileOut( __METHOD__ . '-' . $this->name ); - wfProfileOut( __METHOD__ ); return $status; } @@ -1245,8 +1091,7 @@ abstract class FileBackendStore extends FileBackend { * @throws MWException */ final public function executeOpHandlesInternal( array $fileOpHandles ) { - wfProfileIn( __METHOD__ ); - wfProfileIn( __METHOD__ . '-' . $this->name ); + $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); foreach ( $fileOpHandles as $fileOpHandle ) { if ( !( $fileOpHandle instanceof FileBackendStoreOpHandle ) ) { throw new MWException( "Given a non-FileBackendStoreOpHandle object." ); @@ -1258,8 +1103,6 @@ abstract class FileBackendStore extends FileBackend { foreach ( $fileOpHandles as $fileOpHandle ) { $fileOpHandle->closeResources(); } - wfProfileOut( __METHOD__ . '-' . $this->name ); - wfProfileOut( __METHOD__ ); return $res; } @@ -1277,15 +1120,20 @@ abstract class FileBackendStore extends FileBackend { } /** - * Strip long HTTP headers from a file operation + * Strip long HTTP headers from a file operation. + * Most headers are just numbers, but some are allowed to be long. + * This function is useful for cleaning up headers and avoiding backend + * specific errors, especially in the middle of batch file operations. * * @param array $op Same format as doOperation() * @return Array */ protected function stripInvalidHeadersFromOp( array $op ) { - if ( isset( $op['headers'] ) ) { + static $longs = array( 'Content-Disposition' ); + if ( isset( $op['headers'] ) ) { // op sets HTTP headers foreach ( $op['headers'] as $name => $value ) { - if ( strlen( $name ) > 255 || strlen( $value ) > 255 ) { + $maxHVLen = in_array( $name, $longs ) ? INF : 255; + if ( strlen( $name ) > 255 || strlen( $value ) > $maxHVLen ) { trigger_error( "Header '$name: $value' is too long." ); unset( $op['headers'][$name] ); } elseif ( !strlen( $value ) ) { @@ -1296,9 +1144,6 @@ abstract class FileBackendStore extends FileBackend { return $op; } - /** - * @see FileBackend::preloadCache() - */ final public function preloadCache( array $paths ) { $fullConts = array(); // full container names foreach ( $paths as $path ) { @@ -1310,9 +1155,6 @@ abstract class FileBackendStore extends FileBackend { $this->primeFileCache( $paths ); } - /** - * @see FileBackend::clearCache() - */ final public function clearCache( array $paths = null ) { if ( is_array( $paths ) ) { $paths = array_map( 'FileBackend::normalizeStoragePath', $paths ); @@ -1353,7 +1195,7 @@ abstract class FileBackendStore extends FileBackend { * Check if a container name is valid. * This checks for for length and illegal characters. * - * @param $container string + * @param string $container * @return bool */ final protected static function isValidContainerName( $container ) { @@ -1375,7 +1217,7 @@ abstract class FileBackendStore extends FileBackend { * this means that the path can only refer to a directory and can only * be scanned by looking in all the container shards. * - * @param $storagePath string + * @param string $storagePath * @return Array (container, path, container suffix) or (null, null, null) if invalid */ final protected function resolveStoragePath( $storagePath ) { @@ -1405,16 +1247,22 @@ abstract class FileBackendStore extends FileBackend { /** * Like resolveStoragePath() except null values are returned if - * the container is sharded and the shard could not be determined. + * the container is sharded and the shard could not be determined + * or if the path ends with '/'. The later case is illegal for FS + * backends and can confuse listings for object store backends. + * + * This function is used when resolving paths that must be valid + * locations for files. Directory and listing functions should + * generally just use resolveStoragePath() instead. * * @see FileBackendStore::resolveStoragePath() * - * @param $storagePath string + * @param string $storagePath * @return Array (container, path) or (null, null) if invalid */ final protected function resolveStoragePathReal( $storagePath ) { list( $container, $relPath, $cShard ) = $this->resolveStoragePath( $storagePath ); - if ( $cShard !== null ) { + if ( $cShard !== null && substr( $relPath, -1 ) !== '/' ) { return array( $container, $relPath ); } return array( null, null ); @@ -1474,7 +1322,7 @@ abstract class FileBackendStore extends FileBackend { * If greater than 0, then all file storage paths within * the container are required to be hashed accordingly. * - * @param $container string + * @param string $container * @return Array (integer levels, integer base, repeat flag) or (0, 0, false) */ final protected function getContainerHashLevels( $container ) { @@ -1494,7 +1342,7 @@ abstract class FileBackendStore extends FileBackend { /** * Get a list of full container shard suffixes for a container * - * @param $container string + * @param string $container * @return Array */ final protected function getContainerSuffixes( $container ) { @@ -1512,7 +1360,7 @@ abstract class FileBackendStore extends FileBackend { /** * Get the full container name, including the wiki ID prefix * - * @param $container string + * @param string $container * @return string */ final protected function fullContainerName( $container ) { @@ -1528,7 +1376,7 @@ abstract class FileBackendStore extends FileBackend { * This is intended for internal use, such as encoding illegal chars. * Subclasses can override this to be more restrictive. * - * @param $container string + * @param string $container * @return string|null */ protected function resolveContainerName( $container ) { @@ -1563,10 +1411,11 @@ abstract class FileBackendStore extends FileBackend { * Set the cached info for a container * * @param string $container Resolved container name - * @param $val mixed Information to cache + * @param array $val Information to cache + * @return void */ - final protected function setContainerCache( $container, $val ) { - $this->memCache->add( $this->containerCacheKey( $container ), $val, 14*86400 ); + final protected function setContainerCache( $container, array $val ) { + $this->memCache->add( $this->containerCacheKey( $container ), $val, 14 * 86400 ); } /** @@ -1574,6 +1423,7 @@ abstract class FileBackendStore extends FileBackend { * The cache key is salted for a while to prevent race conditions. * * @param string $container Resolved container name + * @return void */ final protected function deleteContainerCache( $container ) { if ( !$this->memCache->set( $this->containerCacheKey( $container ), 'PURGED', 300 ) ) { @@ -1586,12 +1436,11 @@ abstract class FileBackendStore extends FileBackend { * used in a list of container names, storage paths, or FileOp objects. * This loads the persistent cache values into the process cache. * - * @param $items Array + * @param Array $items * @return void */ final protected function primeContainerCache( array $items ) { - wfProfileIn( __METHOD__ ); - wfProfileIn( __METHOD__ . '-' . $this->name ); + $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); $paths = array(); // list of storage paths $contNames = array(); // (cache key => resolved container name) @@ -1623,9 +1472,6 @@ abstract class FileBackendStore extends FileBackend { // Populate the container process cache for the backend... $this->doPrimeContainerCache( array_filter( $contInfo, 'is_array' ) ); - - wfProfileOut( __METHOD__ . '-' . $this->name ); - wfProfileOut( __METHOD__ ); } /** @@ -1654,14 +1500,17 @@ abstract class FileBackendStore extends FileBackend { * salting for the case when a file is created at a path were there was none before. * * @param string $path Storage path - * @param $val mixed Information to cache + * @param array $val Stat information to cache + * @return void */ - final protected function setFileCache( $path, $val ) { + final protected function setFileCache( $path, array $val ) { $path = FileBackend::normalizeStoragePath( $path ); if ( $path === null ) { return; // invalid storage path } - $this->memCache->add( $this->fileCacheKey( $path ), $val, 7*86400 ); + $age = time() - wfTimestamp( TS_UNIX, $val['mtime'] ); + $ttl = min( 7 * 86400, max( 300, floor( .1 * $age ) ) ); + $this->memCache->add( $this->fileCacheKey( $path ), $val, $ttl ); } /** @@ -1671,6 +1520,7 @@ abstract class FileBackendStore extends FileBackend { * a file is created at a path were there was none before. * * @param string $path Storage path + * @return void */ final protected function deleteFileCache( $path ) { $path = FileBackend::normalizeStoragePath( $path ); @@ -1691,8 +1541,7 @@ abstract class FileBackendStore extends FileBackend { * @return void */ final protected function primeFileCache( array $items ) { - wfProfileIn( __METHOD__ ); - wfProfileIn( __METHOD__ . '-' . $this->name ); + $section = new ProfileSection( __METHOD__ . "-{$this->name}" ); $paths = array(); // list of storage paths $pathNames = array(); // (cache key => storage path) @@ -1726,9 +1575,6 @@ abstract class FileBackendStore extends FileBackend { } } } - - wfProfileOut( __METHOD__ . '-' . $this->name ); - wfProfileOut( __METHOD__ ); } /** @@ -1750,6 +1596,18 @@ abstract class FileBackendStore extends FileBackend { } return $opts; } + + /** + * Get the content type to use in HEAD/GET requests for a file + * + * @param string $storagePath + * @param string|null $content File data + * @param string|null $fsPath File system path + * @return MIME type + */ + protected function getContentType( $storagePath, $content, $fsPath ) { + return call_user_func_array( $this->mimeCallback, func_get_args() ); + } } /** @@ -1786,26 +1644,20 @@ abstract class FileBackendStoreOpHandle { * * @ingroup FileBackend */ -abstract class FileBackendStoreShardListIterator implements Iterator { +abstract class FileBackendStoreShardListIterator extends FilterIterator { /** @var FileBackendStore */ protected $backend; /** @var Array */ protected $params; - /** @var Array */ - protected $shardSuffixes; + protected $container; // string; full container name protected $directory; // string; resolved relative path - /** @var Traversable */ - protected $iter; - protected $curShard = 0; // integer - protected $pos = 0; // integer - /** @var Array */ protected $multiShardPaths = array(); // (rel path => 1) /** - * @param $backend FileBackendStore + * @param FileBackendStore $backend * @param string $container Full storage container name * @param string $dir Storage directory relative to container * @param array $suffixes List of container shard suffixes @@ -1817,142 +1669,56 @@ abstract class FileBackendStoreShardListIterator implements Iterator { $this->backend = $backend; $this->container = $container; $this->directory = $dir; - $this->shardSuffixes = $suffixes; $this->params = $params; - } - - /** - * @see Iterator::key() - * @return integer - */ - public function key() { - return $this->pos; - } - /** - * @see Iterator::valid() - * @return bool - */ - public function valid() { - if ( $this->iter instanceof Iterator ) { - return $this->iter->valid(); - } elseif ( is_array( $this->iter ) ) { - return ( current( $this->iter ) !== false ); // no paths can have this value + $iter = new AppendIterator(); + foreach ( $suffixes as $suffix ) { + $iter->append( $this->listFromShard( $this->container . $suffix ) ); } - return false; // some failure? - } - - /** - * @see Iterator::current() - * @return string|bool String or false - */ - public function current() { - return ( $this->iter instanceof Iterator ) - ? $this->iter->current() - : current( $this->iter ); - } - /** - * @see Iterator::next() - * @return void - */ - public function next() { - ++$this->pos; - ( $this->iter instanceof Iterator ) ? $this->iter->next() : next( $this->iter ); - do { - $continue = false; // keep scanning shards? - $this->filterViaNext(); // filter out duplicates - // Find the next non-empty shard if no elements are left - if ( !$this->valid() ) { - $this->nextShardIteratorIfNotValid(); - $continue = $this->valid(); // re-filter unless we ran out of shards - } - } while ( $continue ); - } - - /** - * @see Iterator::rewind() - * @return void - */ - public function rewind() { - $this->pos = 0; - $this->curShard = 0; - $this->setIteratorFromCurrentShard(); - do { - $continue = false; // keep scanning shards? - $this->filterViaNext(); // filter out duplicates - // Find the next non-empty shard if no elements are left - if ( !$this->valid() ) { - $this->nextShardIteratorIfNotValid(); - $continue = $this->valid(); // re-filter unless we ran out of shards - } - } while ( $continue ); - } - - /** - * Filter out duplicate items by advancing to the next ones - */ - protected function filterViaNext() { - while ( $this->valid() ) { - $rel = $this->iter->current(); // path relative to given directory - $path = $this->params['dir'] . "/{$rel}"; // full storage path - if ( $this->backend->isSingleShardPathInternal( $path ) ) { - break; // path is only on one shard; no issue with duplicates - } elseif ( isset( $this->multiShardPaths[$rel] ) ) { - // Don't keep listing paths that are on multiple shards - ( $this->iter instanceof Iterator ) ? $this->iter->next() : next( $this->iter ); - } else { - $this->multiShardPaths[$rel] = 1; - break; - } - } + parent::__construct( $iter ); } - /** - * If the list iterator for this container shard is out of items, - * then move on to the next container that has items. - * If there are none, then it advances to the last container. - */ - protected function nextShardIteratorIfNotValid() { - while ( !$this->valid() && ++$this->curShard < count( $this->shardSuffixes ) ) { - $this->setIteratorFromCurrentShard(); + public function accept() { + $rel = $this->getInnerIterator()->current(); // path relative to given directory + $path = $this->params['dir'] . "/{$rel}"; // full storage path + if ( $this->backend->isSingleShardPathInternal( $path ) ) { + return true; // path is only on one shard; no issue with duplicates + } elseif ( isset( $this->multiShardPaths[$rel] ) ) { + // Don't keep listing paths that are on multiple shards + return false; + } else { + $this->multiShardPaths[$rel] = 1; + return true; } } - /** - * Set the list iterator to that of the current container shard - */ - protected function setIteratorFromCurrentShard() { - $this->iter = $this->listFromShard( - $this->container . $this->shardSuffixes[$this->curShard], - $this->directory, $this->params ); - // Start loading results so that current() works - if ( $this->iter ) { - ( $this->iter instanceof Iterator ) ? $this->iter->rewind() : reset( $this->iter ); - } + public function rewind() { + parent::rewind(); + $this->multiShardPaths = array(); } /** * Get the list for a given container shard * * @param string $container Resolved container name - * @param string $dir Resolved path relative to container - * @param array $params - * @return Traversable|Array|null + * @return Iterator */ - abstract protected function listFromShard( $container, $dir, array $params ); + abstract protected function listFromShard( $container ); } /** * Iterator for listing directories */ class FileBackendStoreShardDirIterator extends FileBackendStoreShardListIterator { - /** - * @see FileBackendStoreShardListIterator::listFromShard() - * @return Array|null|Traversable - */ - protected function listFromShard( $container, $dir, array $params ) { - return $this->backend->getDirectoryListInternal( $container, $dir, $params ); + protected function listFromShard( $container ) { + $list = $this->backend->getDirectoryListInternal( + $container, $this->directory, $this->params ); + if ( $list === null ) { + return new ArrayIterator( array() ); + } else { + return is_array( $list ) ? new ArrayIterator( $list ) : $list; + } } } @@ -1960,11 +1726,13 @@ class FileBackendStoreShardDirIterator extends FileBackendStoreShardListIterator * Iterator for listing regular files */ class FileBackendStoreShardFileIterator extends FileBackendStoreShardListIterator { - /** - * @see FileBackendStoreShardListIterator::listFromShard() - * @return Array|null|Traversable - */ - protected function listFromShard( $container, $dir, array $params ) { - return $this->backend->getFileListInternal( $container, $dir, $params ); + protected function listFromShard( $container ) { + $list = $this->backend->getFileListInternal( + $container, $this->directory, $this->params ); + if ( $list === null ) { + return new ArrayIterator( array() ); + } else { + return is_array( $list ) ? new ArrayIterator( $list ) : $list; + } } } -- cgit v1.2.2