summaryrefslogtreecommitdiff
path: root/includes/filebackend
diff options
context:
space:
mode:
Diffstat (limited to 'includes/filebackend')
-rw-r--r--includes/filebackend/FSFile.php38
-rw-r--r--includes/filebackend/FSFileBackend.php201
-rw-r--r--includes/filebackend/FileBackend.php300
-rw-r--r--includes/filebackend/FileBackendGroup.php46
-rw-r--r--includes/filebackend/FileBackendMultiWrite.php102
-rw-r--r--includes/filebackend/FileBackendStore.php383
-rw-r--r--includes/filebackend/FileOp.php183
-rw-r--r--includes/filebackend/FileOpBatch.php21
-rw-r--r--includes/filebackend/MemoryFileBackend.php274
-rw-r--r--includes/filebackend/README2
-rw-r--r--includes/filebackend/SwiftFileBackend.php2108
-rw-r--r--includes/filebackend/TempFSFile.php50
-rw-r--r--includes/filebackend/filejournal/DBFileJournal.php22
-rw-r--r--includes/filebackend/filejournal/FileJournal.php68
-rw-r--r--includes/filebackend/lockmanager/DBLockManager.php44
-rw-r--r--includes/filebackend/lockmanager/FSLockManager.php35
-rw-r--r--includes/filebackend/lockmanager/LSLockManager.php218
-rw-r--r--includes/filebackend/lockmanager/LockManager.php41
-rw-r--r--includes/filebackend/lockmanager/LockManagerGroup.php22
-rw-r--r--includes/filebackend/lockmanager/MemcLockManager.php50
-rw-r--r--includes/filebackend/lockmanager/QuorumLockManager.php22
-rw-r--r--includes/filebackend/lockmanager/RedisLockManager.php150
-rw-r--r--includes/filebackend/lockmanager/ScopedLock.php15
23 files changed, 2561 insertions, 1834 deletions
diff --git a/includes/filebackend/FSFile.php b/includes/filebackend/FSFile.php
index 8f0a1334..1659c62a 100644
--- a/includes/filebackend/FSFile.php
+++ b/includes/filebackend/FSFile.php
@@ -27,26 +27,25 @@
* @ingroup FileBackend
*/
class FSFile {
- protected $path; // path to file
- protected $sha1Base36; // file SHA-1 in base 36
+ /** @var string Path to file */
+ protected $path;
+
+ /** @var string File SHA-1 in base 36 */
+ protected $sha1Base36;
/**
* Sets up the file object
*
* @param string $path Path to temporary file on local disk
- * @throws MWException
*/
public function __construct( $path ) {
- if ( FileBackend::isStoragePath( $path ) ) {
- throw new MWException( __METHOD__ . " given storage path `$path`." );
- }
$this->path = $path;
}
/**
* Returns the file system path
*
- * @return String
+ * @return string
*/
public function getPath() {
return $this->path;
@@ -82,6 +81,7 @@ class FSFile {
if ( $timestamp !== false ) {
$timestamp = wfTimestamp( TS_MW, $timestamp );
}
+
return $timestamp;
}
@@ -98,7 +98,7 @@ class FSFile {
* Get an associative array containing information about
* a file with the given storage path.
*
- * @param Mixed $ext: the file extension, or true to extract it from the filename.
+ * @param string|bool $ext The file extension, or true to extract it from the filename.
* Set it to false to ignore the extension.
*
* @return array
@@ -118,9 +118,9 @@ class FSFile {
$ext = self::extensionFromPath( $this->path );
}
- # mime type according to file contents
+ # MIME type according to file contents
$info['file-mime'] = $this->getMimeType();
- # logical mime type
+ # logical MIME type
$info['mime'] = $magic->improveTypeFromExtension( $info['file-mime'], $ext );
list( $info['major_mime'], $info['minor_mime'] ) = File::splitMime( $info['mime'] );
@@ -147,13 +147,14 @@ class FSFile {
}
wfProfileOut( __METHOD__ );
+
return $info;
}
/**
* Placeholder file properties to use for files that don't exist
*
- * @return Array
+ * @return array
*/
public static function placeholderProps() {
$info = array();
@@ -165,6 +166,7 @@ class FSFile {
$info['width'] = 0;
$info['height'] = 0;
$info['bits'] = 0;
+
return $info;
}
@@ -172,7 +174,7 @@ class FSFile {
* Exract image size information
*
* @param array $gis
- * @return Array
+ * @return array
*/
protected function extractImageSizeInfo( array $gis ) {
$info = array();
@@ -184,6 +186,7 @@ class FSFile {
} else {
$info['bits'] = 0;
}
+
return $info;
}
@@ -202,6 +205,7 @@ class FSFile {
if ( $this->sha1Base36 !== null && !$recache ) {
wfProfileOut( __METHOD__ );
+
return $this->sha1Base36;
}
@@ -214,6 +218,7 @@ class FSFile {
}
wfProfileOut( __METHOD__ );
+
return $this->sha1Base36;
}
@@ -225,19 +230,21 @@ class FSFile {
*/
public static function extensionFromPath( $path ) {
$i = strrpos( $path, '.' );
+
return strtolower( $i ? substr( $path, $i + 1 ) : '' );
}
/**
* Get an associative array containing information about a file in the local filesystem.
*
- * @param string $path absolute local filesystem path
- * @param Mixed $ext: the file extension, or true to extract it from the filename.
- * Set it to false to ignore the extension.
+ * @param string $path Absolute local filesystem path
+ * @param string|bool $ext The file extension, or true to extract it from the filename.
+ * Set it to false to ignore the extension.
* @return array
*/
public static function getPropsFromPath( $path, $ext = true ) {
$fsFile = new self( $path );
+
return $fsFile->getProps( $ext );
}
@@ -253,6 +260,7 @@ class FSFile {
*/
public static function getSha1Base36FromPath( $path ) {
$fsFile = new self( $path );
+
return $fsFile->getSha1Base36();
}
}
diff --git a/includes/filebackend/FSFileBackend.php b/includes/filebackend/FSFileBackend.php
index 6d642162..b99ffb62 100644
--- a/includes/filebackend/FSFileBackend.php
+++ b/includes/filebackend/FSFileBackend.php
@@ -39,14 +39,22 @@
* @since 1.19
*/
class FSFileBackend extends FileBackendStore {
- protected $basePath; // string; directory holding the container directories
- /** @var Array Map of container names to root paths */
- protected $containerPaths = array(); // for custom container paths
- protected $fileMode; // integer; file permission mode
- protected $fileOwner; // string; required OS username to own files
- protected $currentUser; // string; OS username running this script
-
- /** @var Array */
+ /** @var string Directory holding the container directories */
+ protected $basePath;
+
+ /** @var array Map of container names to root paths for custom container paths */
+ protected $containerPaths = array();
+
+ /** @var int File permission mode */
+ protected $fileMode;
+
+ /** @var string Required OS username to own files */
+ protected $fileOwner;
+
+ /** @var string OS username running this script */
+ protected $currentUser;
+
+ /** @var array */
protected $hadWarningErrors = array();
/**
@@ -82,6 +90,10 @@ class FSFileBackend extends FileBackendStore {
}
}
+ public function getFeatures() {
+ return !wfIsWindows() ? FileBackend::ATTR_UNICODE_PATHS : 0;
+ }
+
protected function resolveContainerPath( $container, $relStoragePath ) {
// Check that container has a root directory
if ( isset( $this->containerPaths[$container] ) || isset( $this->basePath ) ) {
@@ -90,6 +102,7 @@ class FSFileBackend extends FileBackendStore {
return $relStoragePath;
}
}
+
return null;
}
@@ -125,6 +138,7 @@ class FSFileBackend extends FileBackendStore {
} elseif ( isset( $this->basePath ) ) {
return "{$this->basePath}/{$fullCont}";
}
+
return null; // no container base path defined
}
@@ -144,6 +158,7 @@ class FSFileBackend extends FileBackendStore {
if ( $relPath != '' ) {
$fsPath .= "/{$relPath}";
}
+
return $fsPath;
}
@@ -174,6 +189,7 @@ class FSFileBackend extends FileBackendStore {
$dest = $this->resolveToFSPath( $params['dst'] );
if ( $dest === null ) {
$status->fatal( 'backend-fail-invalidpath', $params['dst'] );
+
return $status;
}
@@ -181,6 +197,7 @@ class FSFileBackend extends FileBackendStore {
$tempFile = TempFSFile::factory( 'create_', 'tmp' );
if ( !$tempFile ) {
$status->fatal( 'backend-fail-create', $params['dst'] );
+
return $status;
}
$this->trapWarnings();
@@ -188,6 +205,7 @@ class FSFileBackend extends FileBackendStore {
$this->untrapWarnings();
if ( $bytes === false ) {
$status->fatal( 'backend-fail-create', $params['dst'] );
+
return $status;
}
$cmd = implode( ' ', array(
@@ -195,7 +213,13 @@ class FSFileBackend extends FileBackendStore {
wfEscapeShellArg( $this->cleanPathSlashes( $tempFile->getPath() ) ),
wfEscapeShellArg( $this->cleanPathSlashes( $dest ) )
) );
- $status->value = new FSFileOpHandle( $this, $params, 'Create', $cmd, $dest );
+ $handler = function ( $errors, Status $status, array $params, $cmd ) {
+ if ( $errors !== '' && !( wfIsWindows() && $errors[0] === " " ) ) {
+ $status->fatal( 'backend-fail-create', $params['dst'] );
+ trigger_error( "$cmd\n$errors", E_USER_WARNING ); // command output
+ }
+ };
+ $status->value = new FSFileOpHandle( $this, $params, $handler, $cmd, $dest );
$tempFile->bind( $status->value );
} else { // immediate write
$this->trapWarnings();
@@ -203,6 +227,7 @@ class FSFileBackend extends FileBackendStore {
$this->untrapWarnings();
if ( $bytes === false ) {
$status->fatal( 'backend-fail-create', $params['dst'] );
+
return $status;
}
$this->chmod( $dest );
@@ -211,22 +236,13 @@ class FSFileBackend extends FileBackendStore {
return $status;
}
- /**
- * @see FSFileBackend::doExecuteOpHandlesInternal()
- */
- protected function _getResponseCreate( $errors, Status $status, array $params, $cmd ) {
- if ( $errors !== '' && !( wfIsWindows() && $errors[0] === " " ) ) {
- $status->fatal( 'backend-fail-create', $params['dst'] );
- trigger_error( "$cmd\n$errors", E_USER_WARNING ); // command output
- }
- }
-
protected function doStoreInternal( array $params ) {
$status = Status::newGood();
$dest = $this->resolveToFSPath( $params['dst'] );
if ( $dest === null ) {
$status->fatal( 'backend-fail-invalidpath', $params['dst'] );
+
return $status;
}
@@ -236,7 +252,13 @@ class FSFileBackend extends FileBackendStore {
wfEscapeShellArg( $this->cleanPathSlashes( $params['src'] ) ),
wfEscapeShellArg( $this->cleanPathSlashes( $dest ) )
) );
- $status->value = new FSFileOpHandle( $this, $params, 'Store', $cmd, $dest );
+ $handler = function ( $errors, Status $status, array $params, $cmd ) {
+ if ( $errors !== '' && !( wfIsWindows() && $errors[0] === " " ) ) {
+ $status->fatal( 'backend-fail-store', $params['src'], $params['dst'] );
+ trigger_error( "$cmd\n$errors", E_USER_WARNING ); // command output
+ }
+ };
+ $status->value = new FSFileOpHandle( $this, $params, $handler, $cmd, $dest );
} else { // immediate write
$this->trapWarnings();
$ok = copy( $params['src'], $dest );
@@ -248,6 +270,7 @@ class FSFileBackend extends FileBackendStore {
trigger_error( __METHOD__ . ": copy() failed but returned true." );
}
$status->fatal( 'backend-fail-store', $params['src'], $params['dst'] );
+
return $status;
}
$this->chmod( $dest );
@@ -256,28 +279,20 @@ class FSFileBackend extends FileBackendStore {
return $status;
}
- /**
- * @see FSFileBackend::doExecuteOpHandlesInternal()
- */
- protected function _getResponseStore( $errors, Status $status, array $params, $cmd ) {
- if ( $errors !== '' && !( wfIsWindows() && $errors[0] === " " ) ) {
- $status->fatal( 'backend-fail-store', $params['src'], $params['dst'] );
- trigger_error( "$cmd\n$errors", E_USER_WARNING ); // command output
- }
- }
-
protected function doCopyInternal( array $params ) {
$status = Status::newGood();
$source = $this->resolveToFSPath( $params['src'] );
if ( $source === null ) {
$status->fatal( 'backend-fail-invalidpath', $params['src'] );
+
return $status;
}
$dest = $this->resolveToFSPath( $params['dst'] );
if ( $dest === null ) {
$status->fatal( 'backend-fail-invalidpath', $params['dst'] );
+
return $status;
}
@@ -285,6 +300,7 @@ class FSFileBackend extends FileBackendStore {
if ( empty( $params['ignoreMissingSource'] ) ) {
$status->fatal( 'backend-fail-copy', $params['src'] );
}
+
return $status; // do nothing; either OK or bad status
}
@@ -294,7 +310,13 @@ class FSFileBackend extends FileBackendStore {
wfEscapeShellArg( $this->cleanPathSlashes( $source ) ),
wfEscapeShellArg( $this->cleanPathSlashes( $dest ) )
) );
- $status->value = new FSFileOpHandle( $this, $params, 'Copy', $cmd, $dest );
+ $handler = function ( $errors, Status $status, array $params, $cmd ) {
+ if ( $errors !== '' && !( wfIsWindows() && $errors[0] === " " ) ) {
+ $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
+ trigger_error( "$cmd\n$errors", E_USER_WARNING ); // command output
+ }
+ };
+ $status->value = new FSFileOpHandle( $this, $params, $handler, $cmd, $dest );
} else { // immediate write
$this->trapWarnings();
$ok = ( $source === $dest ) ? true : copy( $source, $dest );
@@ -308,6 +330,7 @@ class FSFileBackend extends FileBackendStore {
trigger_error( __METHOD__ . ": copy() failed but returned true." );
}
$status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
+
return $status;
}
$this->chmod( $dest );
@@ -316,28 +339,20 @@ class FSFileBackend extends FileBackendStore {
return $status;
}
- /**
- * @see FSFileBackend::doExecuteOpHandlesInternal()
- */
- protected function _getResponseCopy( $errors, Status $status, array $params, $cmd ) {
- if ( $errors !== '' && !( wfIsWindows() && $errors[0] === " " ) ) {
- $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
- trigger_error( "$cmd\n$errors", E_USER_WARNING ); // command output
- }
- }
-
protected function doMoveInternal( array $params ) {
$status = Status::newGood();
$source = $this->resolveToFSPath( $params['src'] );
if ( $source === null ) {
$status->fatal( 'backend-fail-invalidpath', $params['src'] );
+
return $status;
}
$dest = $this->resolveToFSPath( $params['dst'] );
if ( $dest === null ) {
$status->fatal( 'backend-fail-invalidpath', $params['dst'] );
+
return $status;
}
@@ -345,6 +360,7 @@ class FSFileBackend extends FileBackendStore {
if ( empty( $params['ignoreMissingSource'] ) ) {
$status->fatal( 'backend-fail-move', $params['src'] );
}
+
return $status; // do nothing; either OK or bad status
}
@@ -354,7 +370,13 @@ class FSFileBackend extends FileBackendStore {
wfEscapeShellArg( $this->cleanPathSlashes( $source ) ),
wfEscapeShellArg( $this->cleanPathSlashes( $dest ) )
) );
- $status->value = new FSFileOpHandle( $this, $params, 'Move', $cmd );
+ $handler = function ( $errors, Status $status, array $params, $cmd ) {
+ if ( $errors !== '' && !( wfIsWindows() && $errors[0] === " " ) ) {
+ $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
+ trigger_error( "$cmd\n$errors", E_USER_WARNING ); // command output
+ }
+ };
+ $status->value = new FSFileOpHandle( $this, $params, $handler, $cmd );
} else { // immediate write
$this->trapWarnings();
$ok = ( $source === $dest ) ? true : rename( $source, $dest );
@@ -362,6 +384,7 @@ class FSFileBackend extends FileBackendStore {
clearstatcache(); // file no longer at source
if ( !$ok ) {
$status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
+
return $status;
}
}
@@ -369,22 +392,13 @@ class FSFileBackend extends FileBackendStore {
return $status;
}
- /**
- * @see FSFileBackend::doExecuteOpHandlesInternal()
- */
- protected function _getResponseMove( $errors, Status $status, array $params, $cmd ) {
- if ( $errors !== '' && !( wfIsWindows() && $errors[0] === " " ) ) {
- $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
- trigger_error( "$cmd\n$errors", E_USER_WARNING ); // command output
- }
- }
-
protected function doDeleteInternal( array $params ) {
$status = Status::newGood();
$source = $this->resolveToFSPath( $params['src'] );
if ( $source === null ) {
$status->fatal( 'backend-fail-invalidpath', $params['src'] );
+
return $status;
}
@@ -392,6 +406,7 @@ class FSFileBackend extends FileBackendStore {
if ( empty( $params['ignoreMissingSource'] ) ) {
$status->fatal( 'backend-fail-delete', $params['src'] );
}
+
return $status; // do nothing; either OK or bad status
}
@@ -400,13 +415,20 @@ class FSFileBackend extends FileBackendStore {
wfIsWindows() ? 'DEL' : 'unlink',
wfEscapeShellArg( $this->cleanPathSlashes( $source ) )
) );
- $status->value = new FSFileOpHandle( $this, $params, 'Copy', $cmd );
+ $handler = function ( $errors, Status $status, array $params, $cmd ) {
+ if ( $errors !== '' && !( wfIsWindows() && $errors[0] === " " ) ) {
+ $status->fatal( 'backend-fail-delete', $params['src'] );
+ trigger_error( "$cmd\n$errors", E_USER_WARNING ); // command output
+ }
+ };
+ $status->value = new FSFileOpHandle( $this, $params, $handler, $cmd );
} else { // immediate write
$this->trapWarnings();
$ok = unlink( $source );
$this->untrapWarnings();
if ( !$ok ) {
$status->fatal( 'backend-fail-delete', $params['src'] );
+
return $status;
}
}
@@ -415,15 +437,11 @@ class FSFileBackend extends FileBackendStore {
}
/**
- * @see FSFileBackend::doExecuteOpHandlesInternal()
+ * @param string $fullCont
+ * @param string $dirRel
+ * @param array $params
+ * @return Status
*/
- protected function _getResponseDelete( $errors, Status $status, array $params, $cmd ) {
- if ( $errors !== '' && !( wfIsWindows() && $errors[0] === " " ) ) {
- $status->fatal( 'backend-fail-delete', $params['src'] );
- trigger_error( "$cmd\n$errors", E_USER_WARNING ); // command output
- }
- }
-
protected function doPrepareInternal( $fullCont, $dirRel, array $params ) {
$status = Status::newGood();
list( , $shortCont, ) = FileBackend::splitStoragePath( $params['dir'] );
@@ -444,6 +462,7 @@ class FSFileBackend extends FileBackendStore {
if ( is_dir( $dir ) && !$existed ) {
$status->merge( $this->doSecureInternal( $fullCont, $dirRel, $params ) );
}
+
return $status;
}
@@ -471,6 +490,7 @@ class FSFileBackend extends FileBackendStore {
$status->fatal( 'backend-fail-create', "{$storeDir}/.htaccess" );
}
}
+
return $status;
}
@@ -498,6 +518,7 @@ class FSFileBackend extends FileBackendStore {
}
$this->untrapWarnings();
}
+
return $status;
}
@@ -511,6 +532,7 @@ class FSFileBackend extends FileBackendStore {
rmdir( $dir ); // remove directory if empty
}
$this->untrapWarnings();
+
return $status;
}
@@ -557,7 +579,10 @@ class FSFileBackend extends FileBackendStore {
/**
* @see FileBackendStore::getDirectoryListInternal()
- * @return Array|null
+ * @param string $fullCont
+ * @param string $dirRel
+ * @param array $params
+ * @return array|null
*/
public function getDirectoryListInternal( $fullCont, $dirRel, array $params ) {
list( , $shortCont, ) = FileBackend::splitStoragePath( $params['dir'] );
@@ -566,17 +591,23 @@ class FSFileBackend extends FileBackendStore {
$exists = is_dir( $dir );
if ( !$exists ) {
wfDebug( __METHOD__ . "() given directory does not exist: '$dir'\n" );
+
return array(); // nothing under this dir
} elseif ( !is_readable( $dir ) ) {
wfDebug( __METHOD__ . "() given directory is unreadable: '$dir'\n" );
+
return null; // bad permissions?
}
+
return new FSFileBackendDirList( $dir, $params );
}
/**
* @see FileBackendStore::getFileListInternal()
- * @return Array|FSFileBackendFileList|null
+ * @param string $fullCont
+ * @param string $dirRel
+ * @param array $params
+ * @return array|FSFileBackendFileList|null
*/
public function getFileListInternal( $fullCont, $dirRel, array $params ) {
list( , $shortCont, ) = FileBackend::splitStoragePath( $params['dir'] );
@@ -585,11 +616,14 @@ class FSFileBackend extends FileBackendStore {
$exists = is_dir( $dir );
if ( !$exists ) {
wfDebug( __METHOD__ . "() given directory does not exist: '$dir'\n" );
+
return array(); // nothing under this dir
} elseif ( !is_readable( $dir ) ) {
wfDebug( __METHOD__ . "() given directory is unreadable: '$dir'\n" );
+
return null; // bad permissions?
}
+
return new FSFileBackendFileList( $dir, $params );
}
@@ -662,8 +696,8 @@ class FSFileBackend extends FileBackendStore {
foreach ( $fileOpHandles as $index => $fileOpHandle ) {
$status = Status::newGood();
- $function = '_getResponse' . $fileOpHandle->call;
- $this->$function( $errs[$index], $status, $fileOpHandle->params, $fileOpHandle->cmd );
+ $function = $fileOpHandle->call;
+ $function( $errs[$index], $status, $fileOpHandle->params, $fileOpHandle->cmd );
$statuses[$index] = $status;
if ( $status->isOK() && $fileOpHandle->chmodPath ) {
$this->chmod( $fileOpHandle->chmodPath );
@@ -718,8 +752,6 @@ class FSFileBackend extends FileBackendStore {
/**
* Listen for E_WARNING errors and track whether any happen
- *
- * @return void
*/
protected function trapWarnings() {
$this->hadWarningErrors[] = false; // push to stack
@@ -737,7 +769,7 @@ class FSFileBackend extends FileBackendStore {
}
/**
- * @param integer $errno
+ * @param int $errno
* @param string $errstr
* @return bool
* @access private
@@ -745,6 +777,7 @@ class FSFileBackend extends FileBackendStore {
public function handleWarning( $errno, $errstr ) {
wfDebugLog( 'FSFileBackend', $errstr ); // more detailed error logging
$this->hadWarningErrors[count( $this->hadWarningErrors ) - 1] = true;
+
return true; // suppress from PHP handler
}
}
@@ -759,9 +792,9 @@ class FSFileOpHandle extends FileBackendStoreOpHandle {
/**
* @param FSFileBackend $backend
* @param array $params
- * @param string $call
+ * @param callable $call
* @param string $cmd
- * @param integer|null $chmodPath
+ * @param int|null $chmodPath
*/
public function __construct(
FSFileBackend $backend, array $params, $call, $cmd, $chmodPath = null
@@ -784,13 +817,18 @@ class FSFileOpHandle extends FileBackendStoreOpHandle {
abstract class FSFileBackendList implements Iterator {
/** @var Iterator */
protected $iter;
- protected $suffixStart; // integer
- protected $pos = 0; // integer
- /** @var Array */
+
+ /** @var int */
+ protected $suffixStart;
+
+ /** @var int */
+ protected $pos = 0;
+
+ /** @var array */
protected $params = array();
/**
- * @param string $dir file system directory
+ * @param string $dir File system directory
* @param array $params
*/
public function __construct( $dir, array $params ) {
@@ -811,7 +849,7 @@ abstract class FSFileBackendList implements Iterator {
/**
* Return an appropriate iterator object to wrap
*
- * @param string $dir file system directory
+ * @param string $dir File system directory
* @return Iterator
*/
protected function initIterator( $dir ) {
@@ -823,6 +861,7 @@ abstract class FSFileBackendList implements Iterator {
# RecursiveDirectoryIterator extends FilesystemIterator.
# FilesystemIterator::SKIP_DOTS default is inconsistent in PHP 5.3.x.
$flags = FilesystemIterator::CURRENT_AS_SELF | FilesystemIterator::SKIP_DOTS;
+
return new RecursiveIteratorIterator(
new RecursiveDirectoryIterator( $dir, $flags ),
RecursiveIteratorIterator::CHILD_FIRST // include dirs
@@ -832,7 +871,7 @@ abstract class FSFileBackendList implements Iterator {
/**
* @see Iterator::key()
- * @return integer
+ * @return int
*/
public function key() {
return $this->pos;
@@ -848,7 +887,7 @@ abstract class FSFileBackendList implements Iterator {
/**
* @see Iterator::next()
- * @return void
+ * @throws FileBackendError
*/
public function next() {
try {
@@ -862,7 +901,7 @@ abstract class FSFileBackendList implements Iterator {
/**
* @see Iterator::rewind()
- * @return void
+ * @throws FileBackendError
*/
public function rewind() {
$this->pos = 0;
@@ -885,13 +924,14 @@ abstract class FSFileBackendList implements Iterator {
/**
* Filter out items by advancing to the next ones
*/
- protected function filterViaNext() {}
+ protected function filterViaNext() {
+ }
/**
* Return only the relative path and normalize slashes to FileBackend-style.
* Uses the "real path" since the suffix is based upon that.
*
- * @param string $path
+ * @param string $dir
* @return string
*/
protected function getRelPath( $dir ) {
@@ -899,6 +939,7 @@ abstract class FSFileBackendList implements Iterator {
if ( $path === false ) {
$path = $dir;
}
+
return strtr( substr( $path, $this->suffixStart ), '\\', '/' );
}
}
diff --git a/includes/filebackend/FileBackend.php b/includes/filebackend/FileBackend.php
index f586578b..8c0a61a1 100644
--- a/includes/filebackend/FileBackend.php
+++ b/includes/filebackend/FileBackend.php
@@ -47,12 +47,35 @@
* For legacy reasons, the FSFileBackend class allows manually setting the paths of
* containers to ones that do not respect the "wiki ID".
*
- * In key/value stores, the container is the only hierarchy (the rest is emulated).
+ * In key/value (object) stores, containers are the only hierarchy (the rest is emulated).
* FS-based backends are somewhat more restrictive due to the existence of real
* directory files; a regular file cannot have the same name as a directory. Other
* backends with virtual directories may not have this limitation. Callers should
* store files in such a way that no files and directories are under the same path.
*
+ * In general, this class allows for callers to access storage through the same
+ * interface, without regard to the underlying storage system. However, calling code
+ * must follow certain patterns and be aware of certain things to ensure compatibility:
+ * - a) Always call prepare() on the parent directory before trying to put a file there;
+ * key/value stores only need the container to exist first, but filesystems need
+ * all the parent directories to exist first (prepare() is aware of all this)
+ * - b) Always call clean() on a directory when it might become empty to avoid empty
+ * directory buildup on filesystems; key/value stores never have empty directories,
+ * so doing this helps preserve consistency in both cases
+ * - c) Likewise, do not rely on the existence of empty directories for anything;
+ * calling directoryExists() on a path that prepare() was previously called on
+ * will return false for key/value stores if there are no files under that path
+ * - d) Never alter the resulting FSFile returned from getLocalReference(), as it could
+ * either be a copy of the source file in /tmp or the original source file itself
+ * - e) Use a file layout that results in never attempting to store files over directories
+ * or directories over files; key/value stores allow this but filesystems do not
+ * - f) Use ASCII file names (e.g. base32, IDs, hashes) to avoid Unicode issues in Windows
+ * - g) Do not assume that move operations are atomic (difficult with key/value stores)
+ * - h) Do not assume that file stat or read operations always have immediate consistency;
+ * various methods have a "latest" flag that should always be used if up-to-date
+ * information is required (this trades performance for correctness as needed)
+ * - i) Do not assume that directory listings have immediate consistency
+ *
* Methods of subclasses should avoid throwing exceptions at all costs.
* As a corollary, external dependencies should be kept to a minimum.
*
@@ -60,58 +83,69 @@
* @since 1.19
*/
abstract class FileBackend {
- protected $name; // string; unique backend name
- protected $wikiId; // string; unique wiki name
- protected $readOnly; // string; read-only explanation message
- protected $parallelize; // string; when to do operations in parallel
- protected $concurrency; // integer; how many operations can be done in parallel
+ /** @var string Unique backend name */
+ protected $name;
+
+ /** @var string Unique wiki name */
+ protected $wikiId;
+
+ /** @var string Read-only explanation message */
+ protected $readOnly;
+
+ /** @var string When to do operations in parallel */
+ protected $parallelize;
+
+ /** @var int How many operations can be done in parallel */
+ protected $concurrency;
/** @var LockManager */
protected $lockManager;
+
/** @var FileJournal */
protected $fileJournal;
+ /** Bitfield flags for supported features */
+ const ATTR_HEADERS = 1; // files can be tagged with standard HTTP headers
+ const ATTR_METADATA = 2; // files can be stored with metadata key/values
+ const ATTR_UNICODE_PATHS = 4; // files can have Unicode paths (not just ASCII)
+
/**
* Create a new backend instance from configuration.
* This should only be called from within FileBackendGroup.
*
- * $config includes:
+ * @param array $config Parameters include:
* - name : The unique name of this backend.
* This should consist of alphanumberic, '-', and '_' characters.
* This name should not be changed after use (e.g. with journaling).
* Note that the name is *not* used in actual container names.
* - wikiId : Prefix to container names that is unique to this backend.
- * If not provided, this defaults to the current wiki ID.
* It should only consist of alphanumberic, '-', and '_' characters.
* This ID is what avoids collisions if multiple logical backends
* use the same storage system, so this should be set carefully.
- * - lockManager : Registered name of a file lock manager to use.
- * - fileJournal : File journal configuration; see FileJournal::factory().
- * Journals simply log changes to files stored in the backend.
+ * - lockManager : LockManager object to use for any file locking.
+ * If not provided, then no file locking will be enforced.
+ * - fileJournal : FileJournal object to use for logging changes to files.
+ * If not provided, then change journaling will be disabled.
* - readOnly : Write operations are disallowed if this is a non-empty string.
* It should be an explanation for the backend being read-only.
* - parallelize : When to do file operations in parallel (when possible).
* Allowed values are "implicit", "explicit" and "off".
* - concurrency : How many file operations can be done in parallel.
- *
- * @param array $config
- * @throws MWException
+ * @throws FileBackendException
*/
public function __construct( array $config ) {
$this->name = $config['name'];
+ $this->wikiId = $config['wikiId']; // e.g. "my_wiki-en_"
if ( !preg_match( '!^[a-zA-Z0-9-_]{1,255}$!', $this->name ) ) {
- throw new MWException( "Backend name `{$this->name}` is invalid." );
+ throw new FileBackendException( "Backend name '{$this->name}' is invalid." );
+ } elseif ( !is_string( $this->wikiId ) ) {
+ throw new FileBackendException( "Backend wiki ID not provided for '{$this->name}'." );
}
- $this->wikiId = isset( $config['wikiId'] )
- ? $config['wikiId']
- : wfWikiID(); // e.g. "my_wiki-en_"
- $this->lockManager = ( $config['lockManager'] instanceof LockManager )
+ $this->lockManager = isset( $config['lockManager'] )
? $config['lockManager']
- : LockManagerGroup::singleton( $this->wikiId )->get( $config['lockManager'] );
+ : new NullLockManager( array() );
$this->fileJournal = isset( $config['fileJournal'] )
- ? ( ( $config['fileJournal'] instanceof FileJournal )
- ? $config['fileJournal']
- : FileJournal::factory( $config['fileJournal'], $this->name ) )
+ ? $config['fileJournal']
: FileJournal::factory( array( 'class' => 'NullFileJournal' ), $this->name );
$this->readOnly = isset( $config['readOnly'] )
? (string)$config['readOnly']
@@ -165,6 +199,27 @@ abstract class FileBackend {
}
/**
+ * Get the a bitfield of extra features supported by the backend medium
+ *
+ * @return int Bitfield of FileBackend::ATTR_* flags
+ * @since 1.23
+ */
+ public function getFeatures() {
+ return self::ATTR_UNICODE_PATHS;
+ }
+
+ /**
+ * Check if the backend medium supports a field of extra features
+ *
+ * @param int $bitfield Bitfield of FileBackend::ATTR_* flags
+ * @return bool
+ * @since 1.23
+ */
+ final public function hasFeatures( $bitfield ) {
+ return ( $this->getFeatures() & $bitfield ) === $bitfield;
+ }
+
+ /**
* This is the main entry point into the backend for write operations.
* Callers supply an ordered list of operations to perform as a transaction.
* Files will be locked, the stat cache cleared, and then the operations attempted.
@@ -671,8 +726,7 @@ abstract class FileBackend {
* otherwise safe from modification from other processes. Normally,
* the file will be a new temp file, which should be adequate.
*
- * @param array $params Operation parameters
- * $params include:
+ * @param array $params Operation parameters, include:
* - srcs : ordered source storage paths (e.g. chunk1, chunk2, ...)
* - dst : file system path to 0-byte temp file
* - parallelize : try to do operations in parallel when possible
@@ -691,8 +745,7 @@ abstract class FileBackend {
* However, setting them is not guaranteed to actually do anything.
* Additional server configuration may be needed to achieve the desired effect.
*
- * @param array $params
- * $params include:
+ * @param array $params Parameters include:
* - dir : storage directory
* - noAccess : try to deny file access (since 1.20)
* - noListing : try to deny file listing (since 1.20)
@@ -721,8 +774,7 @@ abstract class FileBackend {
* This is not guaranteed to actually make files or listings publically hidden.
* Additional server configuration may be needed to achieve the desired effect.
*
- * @param array $params
- * $params include:
+ * @param array $params Parameters include:
* - dir : storage directory
* - noAccess : try to deny file access
* - noListing : try to deny file listing
@@ -752,8 +804,7 @@ abstract class FileBackend {
* This is not guaranteed to actually make files or listings publically viewable.
* Additional server configuration may be needed to achieve the desired effect.
*
- * @param array $params
- * $params include:
+ * @param array $params Parameters include:
* - dir : storage directory
* - access : try to allow file access
* - listing : try to allow file listing
@@ -779,8 +830,7 @@ abstract class FileBackend {
* Backends using key/value stores may do nothing unless the directory
* is that of an empty container, in which case it will be deleted.
*
- * @param array $params
- * $params include:
+ * @param array $params Parameters include:
* - dir : storage directory
* - recursive : recursively delete empty subdirectories first (since 1.20)
* - bypassReadOnly : allow writes in read-only mode (since 1.20)
@@ -807,12 +857,13 @@ abstract class FileBackend {
* @return ScopedCallback|null
*/
final protected function getScopedPHPBehaviorForOps() {
- if ( php_sapi_name() != 'cli' ) { // http://bugs.php.net/bug.php?id=47540
+ if ( PHP_SAPI != 'cli' ) { // http://bugs.php.net/bug.php?id=47540
$old = ignore_user_abort( true ); // avoid half-finished operations
- return new ScopedCallback( function() use ( $old ) {
+ return new ScopedCallback( function () use ( $old ) {
ignore_user_abort( $old );
} );
}
+
return null;
}
@@ -820,8 +871,7 @@ abstract class FileBackend {
* Check if a file exists at a storage path in the backend.
* This returns false if only a directory exists at the path.
*
- * @param array $params
- * $params include:
+ * @param array $params Parameters include:
* - src : source storage path
* - latest : use the latest available data
* @return bool|null Returns null on failure
@@ -831,8 +881,7 @@ abstract class FileBackend {
/**
* Get the last-modified timestamp of the file at a storage path.
*
- * @param array $params
- * $params include:
+ * @param array $params Parameters include:
* - src : source storage path
* - latest : use the latest available data
* @return string|bool TS_MW timestamp or false on failure
@@ -843,8 +892,7 @@ abstract class FileBackend {
* Get the contents of a file at a storage path in the backend.
* This should be avoided for potentially large files.
*
- * @param array $params
- * $params include:
+ * @param array $params Parameters include:
* - src : source storage path
* - latest : use the latest available data
* @return string|bool Returns false on failure
@@ -863,24 +911,42 @@ abstract class FileBackend {
*
* @see FileBackend::getFileContents()
*
- * @param array $params
- * $params include:
+ * @param array $params Parameters include:
* - srcs : list of source storage paths
* - latest : use the latest available data
* - parallelize : try to do operations in parallel when possible
- * @return Array Map of (path name => string or false on failure)
+ * @return array Map of (path name => string or false on failure)
* @since 1.20
*/
abstract public function getFileContentsMulti( array $params );
/**
- * Get the size (bytes) of a file at a storage path in the backend.
+ * Get metadata about a file at a storage path in the backend.
+ * If the file does not exist, then this returns false.
+ * Otherwise, the result is an associative array that includes:
+ * - headers : map of HTTP headers used for GET/HEAD requests (name => value)
+ * - metadata : map of file metadata (name => value)
+ * Metadata keys and headers names will be returned in all lower-case.
+ * Additional values may be included for internal use only.
+ *
+ * Use FileBackend::hasFeatures() to check how well this is supported.
*
* @param array $params
* $params include:
* - src : source storage path
* - latest : use the latest available data
- * @return integer|bool Returns false on failure
+ * @return array|bool Returns false on failure
+ * @since 1.23
+ */
+ abstract public function getFileXAttributes( array $params );
+
+ /**
+ * Get the size (bytes) of a file at a storage path in the backend.
+ *
+ * @param array $params Parameters include:
+ * - src : source storage path
+ * - latest : use the latest available data
+ * @return int|bool Returns false on failure
*/
abstract public function getFileSize( array $params );
@@ -892,19 +958,17 @@ abstract class FileBackend {
* - size : the file size (bytes)
* Additional values may be included for internal use only.
*
- * @param array $params
- * $params include:
+ * @param array $params Parameters include:
* - src : source storage path
* - latest : use the latest available data
- * @return Array|bool|null Returns null on failure
+ * @return array|bool|null Returns null on failure
*/
abstract public function getFileStat( array $params );
/**
* Get a SHA-1 hash of the file at a storage path in the backend.
*
- * @param array $params
- * $params include:
+ * @param array $params Parameters include:
* - src : source storage path
* - latest : use the latest available data
* @return string|bool Hash string or false on failure
@@ -915,11 +979,10 @@ abstract class FileBackend {
* Get the properties of the file at a storage path in the backend.
* This gives the result of FSFile::getProps() on a local copy of the file.
*
- * @param array $params
- * $params include:
+ * @param array $params Parameters include:
* - src : source storage path
* - latest : use the latest available data
- * @return Array Returns FSFile::placeholderProps() on failure
+ * @return array Returns FSFile::placeholderProps() on failure
*/
abstract public function getFileProps( array $params );
@@ -930,8 +993,7 @@ abstract class FileBackend {
* will be sent if streaming began, while none will be sent otherwise.
* Implementations should flush the output buffer before sending data.
*
- * @param array $params
- * $params include:
+ * @param array $params Parameters include:
* - src : source storage path
* - headers : list of additional HTTP headers to send on success
* - latest : use the latest available data
@@ -952,8 +1014,7 @@ abstract class FileBackend {
* In that later case, there are copies of the file that must stay in sync.
* Additionally, further calls to this function may return the same file.
*
- * @param array $params
- * $params include:
+ * @param array $params Parameters include:
* - src : source storage path
* - latest : use the latest available data
* @return FSFile|null Returns null on failure
@@ -972,12 +1033,11 @@ abstract class FileBackend {
*
* @see FileBackend::getLocalReference()
*
- * @param array $params
- * $params include:
+ * @param array $params Parameters include:
* - srcs : list of source storage paths
* - latest : use the latest available data
* - parallelize : try to do operations in parallel when possible
- * @return Array Map of (path name => FSFile or null on failure)
+ * @return array Map of (path name => FSFile or null on failure)
* @since 1.20
*/
abstract public function getLocalReferenceMulti( array $params );
@@ -987,8 +1047,7 @@ abstract class FileBackend {
* The temporary copy will have the same file extension as the source.
* Temporary files may be purged when the file object falls out of scope.
*
- * @param array $params
- * $params include:
+ * @param array $params Parameters include:
* - src : source storage path
* - latest : use the latest available data
* @return TempFSFile|null Returns null on failure
@@ -1007,12 +1066,11 @@ abstract class FileBackend {
*
* @see FileBackend::getLocalCopy()
*
- * @param array $params
- * $params include:
+ * @param array $params Parameters include:
* - srcs : list of source storage paths
* - latest : use the latest available data
* - parallelize : try to do operations in parallel when possible
- * @return Array Map of (path name => TempFSFile or null on failure)
+ * @return array Map of (path name => TempFSFile or null on failure)
* @since 1.20
*/
abstract public function getLocalCopyMulti( array $params );
@@ -1027,8 +1085,7 @@ abstract class FileBackend {
* Otherwise, one would need to use getLocalReference(), which involves loading
* the entire file on to local disk.
*
- * @param array $params
- * $params include:
+ * @param array $params Parameters include:
* - src : source storage path
* - ttl : lifetime (seconds) if pre-authenticated; default is 1 day
* @return string|null
@@ -1043,8 +1100,7 @@ abstract class FileBackend {
*
* Storage backends with eventual consistency might return stale data.
*
- * @param array $params
- * $params include:
+ * @param array $params Parameters include:
* - dir : storage directory
* @return bool|null Returns null on failure
* @since 1.20
@@ -1063,11 +1119,10 @@ abstract class FileBackend {
*
* Failures during iteration can result in FileBackendError exceptions (since 1.22).
*
- * @param array $params
- * $params include:
+ * @param array $params Parameters include:
* - dir : storage directory
* - topOnly : only return direct child dirs of the directory
- * @return Traversable|Array|null Returns null on failure
+ * @return Traversable|array|null Returns null on failure
* @since 1.20
*/
abstract public function getDirectoryList( array $params );
@@ -1080,10 +1135,9 @@ abstract class FileBackend {
*
* Failures during iteration can result in FileBackendError exceptions (since 1.22).
*
- * @param array $params
- * $params include:
+ * @param array $params Parameters include:
* - dir : storage directory
- * @return Traversable|Array|null Returns null on failure
+ * @return Traversable|array|null Returns null on failure
* @since 1.20
*/
final public function getTopDirectoryList( array $params ) {
@@ -1102,12 +1156,11 @@ abstract class FileBackend {
*
* Failures during iteration can result in FileBackendError exceptions (since 1.22).
*
- * @param array $params
- * $params include:
+ * @param array $params Parameters include:
* - dir : storage directory
* - topOnly : only return direct child files of the directory (since 1.20)
* - adviseStat : set to true if stat requests will be made on the files (since 1.22)
- * @return Traversable|Array|null Returns null on failure
+ * @return Traversable|array|null Returns null on failure
*/
abstract public function getFileList( array $params );
@@ -1119,11 +1172,10 @@ abstract class FileBackend {
*
* Failures during iteration can result in FileBackendError exceptions (since 1.22).
*
- * @param array $params
- * $params include:
+ * @param array $params Parameters include:
* - dir : storage directory
* - adviseStat : set to true if stat requests will be made on the files (since 1.22)
- * @return Traversable|Array|null Returns null on failure
+ * @return Traversable|array|null Returns null on failure
* @since 1.20
*/
final public function getTopFileList( array $params ) {
@@ -1131,22 +1183,38 @@ abstract class FileBackend {
}
/**
- * Preload persistent file stat and property cache into in-process cache.
+ * Preload persistent file stat cache and property cache into in-process cache.
* This should be used when stat calls will be made on a known list of a many files.
*
+ * @see FileBackend::getFileStat()
+ *
* @param array $paths Storage paths
- * @return void
*/
- public function preloadCache( array $paths ) {}
+ abstract public function preloadCache( array $paths );
/**
* Invalidate any in-process file stat and property cache.
* If $paths is given, then only the cache for those files will be cleared.
*
+ * @see FileBackend::getFileStat()
+ *
* @param array $paths Storage paths (optional)
- * @return void
*/
- public function clearCache( array $paths = null ) {}
+ abstract public function clearCache( array $paths = null );
+
+ /**
+ * Preload file stat information (concurrently if possible) into in-process cache.
+ * This should be used when stat calls will be made on a known list of a many files.
+ *
+ * @see FileBackend::getFileStat()
+ *
+ * @param array $params Parameters include:
+ * - srcs : list of source storage paths
+ * - latest : use the latest available data
+ * @return bool All requests proceeded without I/O errors (since 1.24)
+ * @since 1.23
+ */
+ abstract public function preloadFileStat( array $params );
/**
* Lock the files at the given storage paths in the backend.
@@ -1155,23 +1223,26 @@ abstract class FileBackend {
* Callers should consider using getScopedFileLocks() instead.
*
* @param array $paths Storage paths
- * @param integer $type LockManager::LOCK_* constant
+ * @param int $type LockManager::LOCK_* constant
+ * @param int $timeout Timeout in seconds (0 means non-blocking) (since 1.24)
* @return Status
*/
- final public function lockFiles( array $paths, $type ) {
+ final public function lockFiles( array $paths, $type, $timeout = 0 ) {
$paths = array_map( 'FileBackend::normalizeStoragePath', $paths );
- return $this->lockManager->lock( $paths, $type );
+
+ return $this->lockManager->lock( $paths, $type, $timeout );
}
/**
* Unlock the files at the given storage paths in the backend.
*
* @param array $paths Storage paths
- * @param integer $type LockManager::LOCK_* constant
+ * @param int $type LockManager::LOCK_* constant
* @return Status
*/
final public function unlockFiles( array $paths, $type ) {
$paths = array_map( 'FileBackend::normalizeStoragePath', $paths );
+
return $this->lockManager->unlock( $paths, $type );
}
@@ -1186,11 +1257,12 @@ abstract class FileBackend {
* @see ScopedLock::factory()
*
* @param array $paths List of storage paths or map of lock types to path lists
- * @param integer|string $type LockManager::LOCK_* constant or "mixed"
+ * @param int|string $type LockManager::LOCK_* constant or "mixed"
* @param Status $status Status to update on lock/unlock
+ * @param int $timeout Timeout in seconds (0 means non-blocking) (since 1.24)
* @return ScopedLock|null Returns null on failure
*/
- final public function getScopedFileLocks( array $paths, $type, Status $status ) {
+ final public function getScopedFileLocks( array $paths, $type, Status $status, $timeout = 0 ) {
if ( $type === 'mixed' ) {
foreach ( $paths as &$typePaths ) {
$typePaths = array_map( 'FileBackend::normalizeStoragePath', $typePaths );
@@ -1198,7 +1270,8 @@ abstract class FileBackend {
} else {
$paths = array_map( 'FileBackend::normalizeStoragePath', $paths );
}
- return ScopedLock::factory( $this->lockManager, $paths, $type, $status );
+
+ return ScopedLock::factory( $this->lockManager, $paths, $type, $status, $timeout );
}
/**
@@ -1214,7 +1287,7 @@ abstract class FileBackend {
*
* @param array $ops List of file operations to FileBackend::doOperations()
* @param Status $status Status to update on lock/unlock
- * @return Array List of ScopedFileLocks or null values
+ * @return array List of ScopedFileLocks or null values
* @since 1.20
*/
abstract public function getScopedLocksForOps( array $ops, Status $status );
@@ -1267,7 +1340,7 @@ abstract class FileBackend {
* This does not do any path normalization or traversal checks.
*
* @param string $storagePath
- * @return Array (backend, container, rel object) or (null, null, null)
+ * @return array (backend, container, rel object) or (null, null, null)
*/
final public static function splitStoragePath( $storagePath ) {
if ( self::isStoragePath( $storagePath ) ) {
@@ -1281,6 +1354,7 @@ abstract class FileBackend {
}
}
}
+
return array( null, null, null );
}
@@ -1301,6 +1375,7 @@ abstract class FileBackend {
: "mwstore://{$backend}/{$container}";
}
}
+
return null;
}
@@ -1315,6 +1390,7 @@ abstract class FileBackend {
final public static function parentStoragePath( $storagePath ) {
$storagePath = dirname( $storagePath );
list( , , $rel ) = self::splitStoragePath( $storagePath );
+
return ( $rel === null ) ? null : $storagePath;
}
@@ -1322,11 +1398,20 @@ abstract class FileBackend {
* Get the final extension from a storage or FS path
*
* @param string $path
+ * @param string $case One of (rawcase, uppercase, lowercase) (since 1.24)
* @return string
*/
- final public static function extensionFromPath( $path ) {
+ final public static function extensionFromPath( $path, $case = 'lowercase' ) {
$i = strrpos( $path, '.' );
- return strtolower( $i ? substr( $path, $i + 1 ) : '' );
+ $ext = $i ? substr( $path, $i + 1 ) : '';
+
+ if ( $case === 'lowercase' ) {
+ $ext = strtolower( $ext );
+ } elseif ( $case === 'uppercase' ) {
+ $ext = strtoupper( $ext );
+ }
+
+ return $ext;
}
/**
@@ -1345,7 +1430,7 @@ abstract class FileBackend {
*
* @param string $type One of (attachment, inline)
* @param string $filename Suggested file name (should not contain slashes)
- * @throws MWException
+ * @throws FileBackendError
* @return string
* @since 1.20
*/
@@ -1354,7 +1439,7 @@ abstract class FileBackend {
$type = strtolower( $type );
if ( !in_array( $type, array( 'inline', 'attachment' ) ) ) {
- throw new MWException( "Invalid Content-Disposition type '$type'." );
+ throw new FileBackendError( "Invalid Content-Disposition type '$type'." );
}
$parts[] = $type;
@@ -1395,12 +1480,25 @@ abstract class FileBackend {
return null;
}
}
+
return $path;
}
}
/**
+ * Generic file backend exception for checked and unexpected (e.g. config) exceptions
+ *
+ * @ingroup FileBackend
+ * @since 1.23
+ */
+class FileBackendException extends MWException {
+}
+
+/**
+ * File backend exception for checked exceptions (e.g. I/O errors)
+ *
* @ingroup FileBackend
* @since 1.22
*/
-class FileBackendError extends MWException {}
+class FileBackendError extends FileBackendException {
+}
diff --git a/includes/filebackend/FileBackendGroup.php b/includes/filebackend/FileBackendGroup.php
index be8a2076..1b88db7e 100644
--- a/includes/filebackend/FileBackendGroup.php
+++ b/includes/filebackend/FileBackendGroup.php
@@ -29,15 +29,14 @@
* @since 1.19
*/
class FileBackendGroup {
- /**
- * @var FileBackendGroup
- */
+ /** @var FileBackendGroup */
protected static $instance = null;
- /** @var Array (name => ('class' => string, 'config' => array, 'instance' => object)) */
+ /** @var array (name => ('class' => string, 'config' => array, 'instance' => object)) */
protected $backends = array();
- protected function __construct() {}
+ protected function __construct() {
+ }
/**
* @return FileBackendGroup
@@ -47,13 +46,12 @@ class FileBackendGroup {
self::$instance = new self();
self::$instance->initFromGlobals();
}
+
return self::$instance;
}
/**
* Destroy the singleton instance
- *
- * @return void
*/
public static function destroySingleton() {
self::$instance = null;
@@ -61,8 +59,6 @@ class FileBackendGroup {
/**
* Register file backends from the global variables
- *
- * @return void
*/
protected function initFromGlobals() {
global $wgLocalFileRepo, $wgForeignFileRepos, $wgFileBackends;
@@ -116,20 +112,19 @@ class FileBackendGroup {
/**
* Register an array of file backend configurations
*
- * @param Array $configs
- * @return void
- * @throws MWException
+ * @param array $configs
+ * @throws FileBackendException
*/
protected function register( array $configs ) {
foreach ( $configs as $config ) {
if ( !isset( $config['name'] ) ) {
- throw new MWException( "Cannot register a backend with no name." );
+ throw new FileBackendException( "Cannot register a backend with no name." );
}
$name = $config['name'];
if ( isset( $this->backends[$name] ) ) {
- throw new MWException( "Backend with name `{$name}` already registered." );
+ throw new FileBackendException( "Backend with name `{$name}` already registered." );
} elseif ( !isset( $config['class'] ) ) {
- throw new MWException( "Cannot register backend `{$name}` with no class." );
+ throw new FileBackendException( "Backend with name `{$name}` has no class." );
}
$class = $config['class'];
@@ -147,18 +142,27 @@ class FileBackendGroup {
*
* @param string $name
* @return FileBackend
- * @throws MWException
+ * @throws FileBackendException
*/
public function get( $name ) {
if ( !isset( $this->backends[$name] ) ) {
- throw new MWException( "No backend defined with the name `$name`." );
+ throw new FileBackendException( "No backend defined with the name `$name`." );
}
// Lazy-load the actual backend instance
if ( !isset( $this->backends[$name]['instance'] ) ) {
$class = $this->backends[$name]['class'];
$config = $this->backends[$name]['config'];
+ $config['wikiId'] = isset( $config['wikiId'] )
+ ? $config['wikiId']
+ : wfWikiID(); // e.g. "my_wiki-en_"
+ $config['lockManager'] =
+ LockManagerGroup::singleton( $config['wikiId'] )->get( $config['lockManager'] );
+ $config['fileJournal'] = isset( $config['fileJournal'] )
+ ? FileJournal::factory( $config['fileJournal'], $name )
+ : FileJournal::factory( array( 'class' => 'NullFileJournal' ), $name );
$this->backends[$name]['instance'] = new $class( $config );
}
+
return $this->backends[$name]['instance'];
}
@@ -166,14 +170,15 @@ class FileBackendGroup {
* Get the config array for a backend object with a given name
*
* @param string $name
- * @return Array
- * @throws MWException
+ * @return array
+ * @throws FileBackendException
*/
public function config( $name ) {
if ( !isset( $this->backends[$name] ) ) {
- throw new MWException( "No backend defined with the name `$name`." );
+ throw new FileBackendException( "No backend defined with the name `$name`." );
}
$class = $this->backends[$name]['class'];
+
return array( 'class' => $class ) + $this->backends[$name]['config'];
}
@@ -188,6 +193,7 @@ class FileBackendGroup {
if ( $backend !== null && isset( $this->backends[$backend] ) ) {
return $this->get( $backend );
}
+
return null;
}
}
diff --git a/includes/filebackend/FileBackendMultiWrite.php b/includes/filebackend/FileBackendMultiWrite.php
index 97584a71..bfffcc0f 100644
--- a/includes/filebackend/FileBackendMultiWrite.php
+++ b/includes/filebackend/FileBackendMultiWrite.php
@@ -40,15 +40,25 @@
* @since 1.19
*/
class FileBackendMultiWrite extends FileBackend {
- /** @var Array Prioritized list of FileBackendStore objects */
- protected $backends = array(); // array of (backend index => backends)
- protected $masterIndex = -1; // integer; index of master backend
- protected $syncChecks = 0; // integer; bitfield
- protected $autoResync = false; // boolean
+ /** @var array Prioritized list of FileBackendStore objects.
+ * array of (backend index => backends)
+ */
+ protected $backends = array();
+
+ /** @var int Index of master backend */
+ protected $masterIndex = -1;
+
+ /** @var int Bitfield */
+ protected $syncChecks = 0;
- /** @var Array */
+ /** @var string|bool */
+ protected $autoResync = false;
+
+ /** @var array */
protected $noPushDirConts = array();
- protected $noPushQuickOps = false; // boolean
+
+ /** @var bool */
+ protected $noPushQuickOps = false;
/* Possible internal backend consistency checks */
const CHECK_SIZE = 1;
@@ -81,8 +91,8 @@ class FileBackendMultiWrite extends FileBackend {
* - noPushQuickOps : (hack) Only apply doQuickOperations() to the master backend.
* - noPushDirConts : (hack) Only apply directory functions to the master backend.
*
- * @param Array $config
- * @throws MWException
+ * @param array $config
+ * @throws FileBackendError
*/
public function __construct( array $config ) {
parent::__construct( $config );
@@ -109,30 +119,30 @@ class FileBackendMultiWrite extends FileBackend {
}
$name = $config['name'];
if ( isset( $namesUsed[$name] ) ) { // don't break FileOp predicates
- throw new MWException( "Two or more backends defined with the name $name." );
+ throw new FileBackendError( "Two or more backends defined with the name $name." );
}
$namesUsed[$name] = 1;
// Alter certain sub-backend settings for sanity
unset( $config['readOnly'] ); // use proxy backend setting
unset( $config['fileJournal'] ); // use proxy backend journal
+ unset( $config['lockManager'] ); // lock under proxy backend
$config['wikiId'] = $this->wikiId; // use the proxy backend wiki ID
- $config['lockManager'] = 'nullLockManager'; // lock under proxy backend
if ( !empty( $config['isMultiMaster'] ) ) {
if ( $this->masterIndex >= 0 ) {
- throw new MWException( 'More than one master backend defined.' );
+ throw new FileBackendError( 'More than one master backend defined.' );
}
$this->masterIndex = $index; // this is the "master"
$config['fileJournal'] = $this->fileJournal; // log under proxy backend
}
// Create sub-backend object
if ( !isset( $config['class'] ) ) {
- throw new MWException( 'No class given for a backend config.' );
+ throw new FileBackendError( 'No class given for a backend config.' );
}
$class = $config['class'];
$this->backends[$index] = new $class( $config );
}
if ( $this->masterIndex < 0 ) { // need backends and must have a master
- throw new MWException( 'No master backend defined.' );
+ throw new FileBackendError( 'No master backend defined.' );
}
}
@@ -167,6 +177,7 @@ class FileBackendMultiWrite extends FileBackend {
// Try to resync the clone backends to the master on the spot...
if ( !$this->autoResync || !$this->resyncFiles( $relevantPaths )->isOK() ) {
$status->merge( $syncStatus );
+
return $status; // abort
}
}
@@ -321,8 +332,8 @@ class FileBackendMultiWrite extends FileBackend {
// already synced; nothing to do
} elseif ( $mSha1 !== false ) { // file is in master
if ( $this->autoResync === 'conservative'
- && $cStat && $cStat['mtime'] > $mStat['mtime'] )
- {
+ && $cStat && $cStat['mtime'] > $mStat['mtime']
+ ) {
$status->fatal( 'backend-fail-synced', $path );
continue; // don't rollback data
}
@@ -348,7 +359,7 @@ class FileBackendMultiWrite extends FileBackend {
* Get a list of file storage paths to read or write for a list of operations
*
* @param array $ops Same format as doOperations()
- * @return Array List of storage paths to files (does not include directories)
+ * @return array List of storage paths to files (does not include directories)
*/
protected function fileStoragePathsForOps( array $ops ) {
$paths = array();
@@ -357,8 +368,8 @@ class FileBackendMultiWrite extends FileBackend {
// For things like copy/move/delete with "ignoreMissingSource" and there
// is no source file, nothing should happen and there should be no errors.
if ( empty( $op['ignoreMissingSource'] )
- || $this->fileExists( array( 'src' => $op['src'] ) ) )
- {
+ || $this->fileExists( array( 'src' => $op['src'] ) )
+ ) {
$paths[] = $op['src'];
}
}
@@ -369,6 +380,7 @@ class FileBackendMultiWrite extends FileBackend {
$paths[] = $op['dst'];
}
}
+
return array_values( array_unique( array_filter( $paths, 'FileBackend::isStoragePath' ) ) );
}
@@ -378,7 +390,7 @@ class FileBackendMultiWrite extends FileBackend {
*
* @param array $ops List of file operation arrays
* @param FileBackendStore $backend
- * @return Array
+ * @return array
*/
protected function substOpBatchPaths( array $ops, FileBackendStore $backend ) {
$newOps = array(); // operations
@@ -391,6 +403,7 @@ class FileBackendMultiWrite extends FileBackend {
}
$newOps[] = $newOp;
}
+
return $newOps;
}
@@ -399,10 +412,11 @@ class FileBackendMultiWrite extends FileBackend {
*
* @param array $ops File operation array
* @param FileBackendStore $backend
- * @return Array
+ * @return array
*/
protected function substOpPaths( array $ops, FileBackendStore $backend ) {
$newOps = $this->substOpBatchPaths( array( $ops ), $backend );
+
return $newOps[0];
}
@@ -411,7 +425,7 @@ class FileBackendMultiWrite extends FileBackend {
*
* @param array|string $paths List of paths or single string path
* @param FileBackendStore $backend
- * @return Array|string
+ * @return array|string
*/
protected function substPaths( $paths, FileBackendStore $backend ) {
return preg_replace(
@@ -425,7 +439,7 @@ class FileBackendMultiWrite extends FileBackend {
* Substitute the backend of internal storage paths with the proxy backend's name
*
* @param array|string $paths List of paths or single string path
- * @return Array|string
+ * @return array|string
*/
protected function unsubstPaths( $paths ) {
return preg_replace(
@@ -456,6 +470,7 @@ class FileBackendMultiWrite extends FileBackend {
$status->success = $masterStatus->success;
$status->successCount = $masterStatus->successCount;
$status->failCount = $masterStatus->failCount;
+
return $status;
}
@@ -465,6 +480,7 @@ class FileBackendMultiWrite extends FileBackend {
*/
protected function replicateContainerDirChanges( $path ) {
list( , $shortCont, ) = self::splitStoragePath( $path );
+
return !in_array( $shortCont, $this->noPushDirConts );
}
@@ -477,6 +493,7 @@ class FileBackendMultiWrite extends FileBackend {
$status->merge( $backend->doPrepare( $realParams ) );
}
}
+
return $status;
}
@@ -489,6 +506,7 @@ class FileBackendMultiWrite extends FileBackend {
$status->merge( $backend->doSecure( $realParams ) );
}
}
+
return $status;
}
@@ -501,6 +519,7 @@ class FileBackendMultiWrite extends FileBackend {
$status->merge( $backend->doPublish( $realParams ) );
}
}
+
return $status;
}
@@ -513,35 +532,47 @@ class FileBackendMultiWrite extends FileBackend {
$status->merge( $backend->doClean( $realParams ) );
}
}
+
return $status;
}
public function concatenate( array $params ) {
// We are writing to an FS file, so we don't need to do this per-backend
$realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
+
return $this->backends[$this->masterIndex]->concatenate( $realParams );
}
public function fileExists( array $params ) {
$realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
+
return $this->backends[$this->masterIndex]->fileExists( $realParams );
}
public function getFileTimestamp( array $params ) {
$realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
+
return $this->backends[$this->masterIndex]->getFileTimestamp( $realParams );
}
public function getFileSize( array $params ) {
$realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
+
return $this->backends[$this->masterIndex]->getFileSize( $realParams );
}
public function getFileStat( array $params ) {
$realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
+
return $this->backends[$this->masterIndex]->getFileStat( $realParams );
}
+ public function getFileXAttributes( array $params ) {
+ $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
+
+ return $this->backends[$this->masterIndex]->getFileXAttributes( $realParams );
+ }
+
public function getFileContentsMulti( array $params ) {
$realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
$contentsM = $this->backends[$this->masterIndex]->getFileContentsMulti( $realParams );
@@ -550,21 +581,25 @@ class FileBackendMultiWrite extends FileBackend {
foreach ( $contentsM as $path => $data ) {
$contents[$this->unsubstPaths( $path )] = $data;
}
+
return $contents;
}
public function getFileSha1Base36( array $params ) {
$realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
+
return $this->backends[$this->masterIndex]->getFileSha1Base36( $realParams );
}
public function getFileProps( array $params ) {
$realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
+
return $this->backends[$this->masterIndex]->getFileProps( $realParams );
}
public function streamFile( array $params ) {
$realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
+
return $this->backends[$this->masterIndex]->streamFile( $realParams );
}
@@ -576,6 +611,7 @@ class FileBackendMultiWrite extends FileBackend {
foreach ( $fsFilesM as $path => $fsFile ) {
$fsFiles[$this->unsubstPaths( $path )] = $fsFile;
}
+
return $fsFiles;
}
@@ -587,29 +623,38 @@ class FileBackendMultiWrite extends FileBackend {
foreach ( $tempFilesM as $path => $tempFile ) {
$tempFiles[$this->unsubstPaths( $path )] = $tempFile;
}
+
return $tempFiles;
}
public function getFileHttpUrl( array $params ) {
$realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
+
return $this->backends[$this->masterIndex]->getFileHttpUrl( $realParams );
}
public function directoryExists( array $params ) {
$realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
+
return $this->backends[$this->masterIndex]->directoryExists( $realParams );
}
public function getDirectoryList( array $params ) {
$realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
+
return $this->backends[$this->masterIndex]->getDirectoryList( $realParams );
}
public function getFileList( array $params ) {
$realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
+
return $this->backends[$this->masterIndex]->getFileList( $realParams );
}
+ public function getFeatures() {
+ return $this->backends[$this->masterIndex]->getFeatures();
+ }
+
public function clearCache( array $paths = null ) {
foreach ( $this->backends as $backend ) {
$realPaths = is_array( $paths ) ? $this->substPaths( $paths, $backend ) : null;
@@ -617,6 +662,16 @@ class FileBackendMultiWrite extends FileBackend {
}
}
+ public function preloadCache( array $paths ) {
+ $realPaths = $this->substPaths( $paths, $this->backends[$this->masterIndex] );
+ $this->backends[$this->masterIndex]->preloadCache( $realPaths );
+ }
+
+ public function preloadFileStat( array $params ) {
+ $realParams = $this->substOpPaths( $params, $this->backends[$this->masterIndex] );
+ return $this->backends[$this->masterIndex]->preloadFileStat( $realParams );
+ }
+
public function getScopedLocksForOps( array $ops, Status $status ) {
$realOps = $this->substOpBatchPaths( $ops, $this->backends[$this->masterIndex] );
$fileOps = $this->backends[$this->masterIndex]->getOperationsInternal( $realOps );
@@ -627,6 +682,7 @@ class FileBackendMultiWrite extends FileBackend {
LockManager::LOCK_UW => $this->unsubstPaths( $paths[LockManager::LOCK_UW] ),
LockManager::LOCK_EX => $this->unsubstPaths( $paths[LockManager::LOCK_EX] )
);
+
// Actually acquire the locks
return array( $this->getScopedFileLocks( $pbPaths, 'mixed', $status ) );
}
diff --git a/includes/filebackend/FileBackendStore.php b/includes/filebackend/FileBackendStore.php
index 0921e99f..495ac3c0 100644
--- a/includes/filebackend/FileBackendStore.php
+++ b/includes/filebackend/FileBackendStore.php
@@ -43,16 +43,16 @@ abstract class FileBackendStore extends FileBackend {
/** @var ProcessCacheLRU Map of paths to large (RAM/disk) cache items */
protected $expensiveCache;
- /** @var Array Map of container names to sharding config */
+ /** @var array Map of container names to sharding config */
protected $shardViaHashLevels = array();
- /** @var callback Method to get the MIME type of files */
+ /** @var callable Method to get the MIME type of files */
protected $mimeCallback;
protected $maxFileSize = 4294967296; // integer bytes (4GiB)
const CACHE_TTL = 10; // integer; TTL in seconds for process cache entries
- const CACHE_CHEAP_SIZE = 300; // integer; max entries in "cheap cache"
+ const CACHE_CHEAP_SIZE = 500; // integer; max entries in "cheap cache"
const CACHE_EXPENSIVE_SIZE = 5; // integer; max entries in "expensive cache"
/**
@@ -68,8 +68,8 @@ abstract class FileBackendStore extends FileBackend {
parent::__construct( $config );
$this->mimeCallback = isset( $config['mimeCallback'] )
? $config['mimeCallback']
- : function( $storagePath, $content, $fsPath ) {
- // @TODO: handle the case of extension-less files using the contents
+ : function ( $storagePath, $content, $fsPath ) {
+ // @todo handle the case of extension-less files using the contents
return StreamFile::contentTypeFromPath( $storagePath ) ?: 'unknown/unknown';
};
$this->memCache = new EmptyBagOStuff(); // disabled by default
@@ -82,7 +82,7 @@ abstract class FileBackendStore extends FileBackend {
* medium restrictions and basic performance constraints.
* Do not call this function from places outside FileBackend and FileOp.
*
- * @return integer Bytes
+ * @return int Bytes
*/
final public function maxFileSizeInternal() {
return $this->maxFileSize;
@@ -129,11 +129,13 @@ abstract class FileBackendStore extends FileBackend {
$this->deleteFileCache( $params['dst'] ); // persistent cache
}
}
+
return $status;
}
/**
* @see FileBackendStore::createInternal()
+ * @param array $params
* @return Status
*/
abstract protected function doCreateInternal( array $params );
@@ -168,11 +170,13 @@ abstract class FileBackendStore extends FileBackend {
$this->deleteFileCache( $params['dst'] ); // persistent cache
}
}
+
return $status;
}
/**
* @see FileBackendStore::storeInternal()
+ * @param array $params
* @return Status
*/
abstract protected function doStoreInternal( array $params );
@@ -203,11 +207,13 @@ abstract class FileBackendStore extends FileBackend {
if ( !isset( $params['dstExists'] ) || $params['dstExists'] ) {
$this->deleteFileCache( $params['dst'] ); // persistent cache
}
+
return $status;
}
/**
* @see FileBackendStore::copyInternal()
+ * @param array $params
* @return Status
*/
abstract protected function doCopyInternal( array $params );
@@ -236,6 +242,7 @@ abstract class FileBackendStore extends FileBackend {
/**
* @see FileBackendStore::deleteInternal()
+ * @param array $params
* @return Status
*/
abstract protected function doDeleteInternal( array $params );
@@ -267,11 +274,13 @@ abstract class FileBackendStore extends FileBackend {
if ( !isset( $params['dstExists'] ) || $params['dstExists'] ) {
$this->deleteFileCache( $params['dst'] ); // persistent cache
}
+
return $status;
}
/**
* @see FileBackendStore::moveInternal()
+ * @param array $params
* @return Status
*/
protected function doMoveInternal( array $params ) {
@@ -285,6 +294,7 @@ abstract class FileBackendStore extends FileBackend {
$status->merge( $this->deleteInternal( array( 'src' => $params['src'] ) ) );
$status->setResult( true, $status->value ); // ignore delete() errors
}
+
return $status;
}
@@ -311,11 +321,13 @@ abstract class FileBackendStore extends FileBackend {
} else {
$status = Status::newGood(); // nothing to do
}
+
return $status;
}
/**
* @see FileBackendStore::describeInternal()
+ * @param array $params
* @return Status
*/
protected function doDescribeInternal( array $params ) {
@@ -345,8 +357,8 @@ abstract class FileBackendStore extends FileBackend {
$status->merge( $this->doConcatenate( $params ) );
$sec = microtime( true ) - $start_time;
if ( !$status->isOK() ) {
- wfDebugLog( 'FileOperation', get_class( $this ) . " failed to concatenate " .
- count( $params['srcs'] ) . " file(s) [$sec sec]" );
+ wfDebugLog( 'FileOperation', get_class( $this ) . "-{$this->name}" .
+ " failed to concatenate " . count( $params['srcs'] ) . " file(s) [$sec sec]" );
}
}
@@ -355,6 +367,7 @@ abstract class FileBackendStore extends FileBackend {
/**
* @see FileBackendStore::concatenate()
+ * @param array $params
* @return Status
*/
protected function doConcatenate( array $params ) {
@@ -368,6 +381,7 @@ abstract class FileBackendStore extends FileBackend {
wfRestoreWarnings();
if ( !$ok ) { // not present or not empty
$status->fatal( 'backend-fail-opentemp', $tmpPath );
+
return $status;
}
@@ -378,6 +392,7 @@ abstract class FileBackendStore extends FileBackend {
$fsFile = $this->getLocalReference( array( 'src' => $path ) );
if ( !$fsFile ) { // retry failed?
$status->fatal( 'backend-fail-read', $path );
+
return $status;
}
}
@@ -388,6 +403,7 @@ abstract class FileBackendStore extends FileBackend {
$tmpHandle = fopen( $tmpPath, 'ab' );
if ( $tmpHandle === false ) {
$status->fatal( 'backend-fail-opentemp', $tmpPath );
+
return $status;
}
@@ -398,6 +414,7 @@ abstract class FileBackendStore extends FileBackend {
if ( $sourceHandle === false ) {
fclose( $tmpHandle );
$status->fatal( 'backend-fail-read', $virtualSource );
+
return $status;
}
// Append chunk to file (pass chunk size to avoid magic quotes)
@@ -405,12 +422,14 @@ abstract class FileBackendStore extends FileBackend {
fclose( $sourceHandle );
fclose( $tmpHandle );
$status->fatal( 'backend-fail-writetemp', $tmpPath );
+
return $status;
}
fclose( $sourceHandle );
}
if ( !fclose( $tmpHandle ) ) {
$status->fatal( 'backend-fail-closetemp', $tmpPath );
+
return $status;
}
@@ -426,6 +445,7 @@ abstract class FileBackendStore extends FileBackend {
list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
if ( $dir === null ) {
$status->fatal( 'backend-fail-invalidpath', $params['dir'] );
+
return $status; // invalid storage path
}
@@ -444,6 +464,9 @@ abstract class FileBackendStore extends FileBackend {
/**
* @see FileBackendStore::doPrepare()
+ * @param string $container
+ * @param string $dir
+ * @param array $params
* @return Status
*/
protected function doPrepareInternal( $container, $dir, array $params ) {
@@ -457,6 +480,7 @@ abstract class FileBackendStore extends FileBackend {
list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
if ( $dir === null ) {
$status->fatal( 'backend-fail-invalidpath', $params['dir'] );
+
return $status; // invalid storage path
}
@@ -475,6 +499,9 @@ abstract class FileBackendStore extends FileBackend {
/**
* @see FileBackendStore::doSecure()
+ * @param string $container
+ * @param string $dir
+ * @param array $params
* @return Status
*/
protected function doSecureInternal( $container, $dir, array $params ) {
@@ -488,6 +515,7 @@ abstract class FileBackendStore extends FileBackend {
list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
if ( $dir === null ) {
$status->fatal( 'backend-fail-invalidpath', $params['dir'] );
+
return $status; // invalid storage path
}
@@ -506,6 +534,9 @@ abstract class FileBackendStore extends FileBackend {
/**
* @see FileBackendStore::doPublish()
+ * @param string $container
+ * @param string $dir
+ * @param array $params
* @return Status
*/
protected function doPublishInternal( $container, $dir, array $params ) {
@@ -531,6 +562,7 @@ abstract class FileBackendStore extends FileBackend {
list( $fullCont, $dir, $shard ) = $this->resolveStoragePath( $params['dir'] );
if ( $dir === null ) {
$status->fatal( 'backend-fail-invalidpath', $params['dir'] );
+
return $status; // invalid storage path
}
@@ -558,6 +590,9 @@ abstract class FileBackendStore extends FileBackend {
/**
* @see FileBackendStore::doClean()
+ * @param string $container
+ * @param string $dir
+ * @param array $params
* @return Status
*/
protected function doCleanInternal( $container, $dir, array $params ) {
@@ -567,18 +602,21 @@ abstract class FileBackendStore extends FileBackend {
final public function fileExists( array $params ) {
$section = new ProfileSection( __METHOD__ . "-{$this->name}" );
$stat = $this->getFileStat( $params );
+
return ( $stat === null ) ? null : (bool)$stat; // null => failure
}
final public function getFileTimestamp( array $params ) {
$section = new ProfileSection( __METHOD__ . "-{$this->name}" );
$stat = $this->getFileStat( $params );
+
return $stat ? $stat['mtime'] : false;
}
final public function getFileSize( array $params ) {
$section = new ProfileSection( __METHOD__ . "-{$this->name}" );
$stat = $this->getFileStat( $params );
+
return $stat ? $stat['size'] : false;
}
@@ -606,27 +644,32 @@ abstract class FileBackendStore extends FileBackend {
}
}
}
- wfProfileIn( __METHOD__ . '-miss' );
wfProfileIn( __METHOD__ . '-miss-' . $this->name );
$stat = $this->doGetFileStat( $params );
wfProfileOut( __METHOD__ . '-miss-' . $this->name );
- wfProfileOut( __METHOD__ . '-miss' );
if ( is_array( $stat ) ) { // file exists
- $stat['latest'] = $latest;
+ // Strongly consistent backends can automatically set "latest"
+ $stat['latest'] = isset( $stat['latest'] ) ? $stat['latest'] : $latest;
$this->cheapCache->set( $path, 'stat', $stat );
$this->setFileCache( $path, $stat ); // update persistent cache
if ( isset( $stat['sha1'] ) ) { // some backends store SHA-1 as metadata
$this->cheapCache->set( $path, 'sha1',
array( 'hash' => $stat['sha1'], 'latest' => $latest ) );
}
+ if ( isset( $stat['xattr'] ) ) { // some backends store headers/metadata
+ $stat['xattr'] = self::normalizeXAttributes( $stat['xattr'] );
+ $this->cheapCache->set( $path, 'xattr',
+ array( 'map' => $stat['xattr'], 'latest' => $latest ) );
+ }
} elseif ( $stat === false ) { // file does not exist
$this->cheapCache->set( $path, 'stat', $latest ? 'NOT_EXIST_LATEST' : 'NOT_EXIST' );
- $this->cheapCache->set( $path, 'sha1', // the SHA-1 must be false too
- array( 'hash' => false, 'latest' => $latest ) );
+ $this->cheapCache->set( $path, 'xattr', array( 'map' => false, 'latest' => $latest ) );
+ $this->cheapCache->set( $path, 'sha1', array( 'hash' => false, 'latest' => $latest ) );
wfDebug( __METHOD__ . ": File $path does not exist.\n" );
} else { // an error occurred
wfDebug( __METHOD__ . ": Could not stat file $path.\n" );
}
+
return $stat;
}
@@ -646,7 +689,8 @@ abstract class FileBackendStore extends FileBackend {
/**
* @see FileBackendStore::getFileContentsMulti()
- * @return Array
+ * @param array $params
+ * @return array
*/
protected function doGetFileContentsMulti( array $params ) {
$contents = array();
@@ -655,9 +699,44 @@ abstract class FileBackendStore extends FileBackend {
$contents[$path] = $fsFile ? file_get_contents( $fsFile->getPath() ) : false;
wfRestoreWarnings();
}
+
return $contents;
}
+ final public function getFileXAttributes( array $params ) {
+ $path = self::normalizeStoragePath( $params['src'] );
+ if ( $path === null ) {
+ return false; // invalid storage path
+ }
+ $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
+ $latest = !empty( $params['latest'] ); // use latest data?
+ if ( $this->cheapCache->has( $path, 'xattr', self::CACHE_TTL ) ) {
+ $stat = $this->cheapCache->get( $path, 'xattr' );
+ // If we want the latest data, check that this cached
+ // value was in fact fetched with the latest available data.
+ if ( !$latest || $stat['latest'] ) {
+ return $stat['map'];
+ }
+ }
+ wfProfileIn( __METHOD__ . '-miss' );
+ wfProfileIn( __METHOD__ . '-miss-' . $this->name );
+ $fields = $this->doGetFileXAttributes( $params );
+ $fields = is_array( $fields ) ? self::normalizeXAttributes( $fields ) : false;
+ wfProfileOut( __METHOD__ . '-miss-' . $this->name );
+ wfProfileOut( __METHOD__ . '-miss' );
+ $this->cheapCache->set( $path, 'xattr', array( 'map' => $fields, 'latest' => $latest ) );
+
+ return $fields;
+ }
+
+ /**
+ * @see FileBackendStore::getFileXAttributes()
+ * @return bool|string
+ */
+ protected function doGetFileXAttributes( array $params ) {
+ return array( 'headers' => array(), 'metadata' => array() ); // not supported
+ }
+
final public function getFileSha1Base36( array $params ) {
$path = self::normalizeStoragePath( $params['src'] );
if ( $path === null ) {
@@ -673,17 +752,17 @@ abstract class FileBackendStore extends FileBackend {
return $stat['hash'];
}
}
- wfProfileIn( __METHOD__ . '-miss' );
wfProfileIn( __METHOD__ . '-miss-' . $this->name );
$hash = $this->doGetFileSha1Base36( $params );
wfProfileOut( __METHOD__ . '-miss-' . $this->name );
- wfProfileOut( __METHOD__ . '-miss' );
$this->cheapCache->set( $path, 'sha1', array( 'hash' => $hash, 'latest' => $latest ) );
+
return $hash;
}
/**
* @see FileBackendStore::getFileSha1Base36()
+ * @param array $params
* @return bool|string
*/
protected function doGetFileSha1Base36( array $params ) {
@@ -699,6 +778,7 @@ abstract class FileBackendStore extends FileBackend {
$section = new ProfileSection( __METHOD__ . "-{$this->name}" );
$fsFile = $this->getLocalReference( $params );
$props = $fsFile ? $fsFile->getProps() : FSFile::placeholderProps();
+
return $props;
}
@@ -738,7 +818,8 @@ abstract class FileBackendStore extends FileBackend {
/**
* @see FileBackendStore::getLocalReferenceMulti()
- * @return Array
+ * @param array $params
+ * @return array
*/
protected function doGetLocalReferenceMulti( array $params ) {
return $this->doGetLocalCopyMulti( $params );
@@ -755,12 +836,14 @@ abstract class FileBackendStore extends FileBackend {
/**
* @see FileBackendStore::getLocalCopyMulti()
- * @return Array
+ * @param array $params
+ * @return array
*/
abstract protected function doGetLocalCopyMulti( array $params );
/**
* @see FileBackend::getFileHttpUrl()
+ * @param array $params
* @return string|null
*/
public function getFileHttpUrl( array $params ) {
@@ -782,11 +865,9 @@ abstract class FileBackendStore extends FileBackend {
if ( $res == StreamFile::NOT_MODIFIED ) {
// do nothing; client cache is up to date
} elseif ( $res == StreamFile::READY_STREAM ) {
- wfProfileIn( __METHOD__ . '-send' );
wfProfileIn( __METHOD__ . '-send-' . $this->name );
$status = $this->doStreamFile( $params );
wfProfileOut( __METHOD__ . '-send-' . $this->name );
- wfProfileOut( __METHOD__ . '-send' );
if ( !$status->isOK() ) {
// Per bug 41113, nasty things can happen if bad cache entries get
// stuck in cache. It's also possible that this error can come up
@@ -804,6 +885,7 @@ abstract class FileBackendStore extends FileBackend {
/**
* @see FileBackendStore::streamFile()
+ * @param array $params
* @return Status
*/
protected function doStreamFile( array $params ) {
@@ -839,6 +921,7 @@ abstract class FileBackendStore extends FileBackend {
$res = null; // if we don't find anything, it is indeterminate
}
}
+
return $res;
}
}
@@ -865,6 +948,7 @@ abstract class FileBackendStore extends FileBackend {
wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
// File listing spans multiple containers/shards
list( , $shortCont, ) = self::splitStoragePath( $params['dir'] );
+
return new FileBackendStoreShardDirIterator( $this,
$fullCont, $dir, $this->getContainerSuffixes( $shortCont ), $params );
}
@@ -878,7 +962,7 @@ abstract class FileBackendStore extends FileBackend {
* @param string $container Resolved container name
* @param string $dir Resolved path relative to container
* @param array $params
- * @return Traversable|Array|null Returns null on failure
+ * @return Traversable|array|null Returns null on failure
*/
abstract public function getDirectoryListInternal( $container, $dir, array $params );
@@ -894,6 +978,7 @@ abstract class FileBackendStore extends FileBackend {
wfDebug( __METHOD__ . ": iterating over all container shards.\n" );
// File listing spans multiple containers/shards
list( , $shortCont, ) = self::splitStoragePath( $params['dir'] );
+
return new FileBackendStoreShardFileIterator( $this,
$fullCont, $dir, $this->getContainerSuffixes( $shortCont ), $params );
}
@@ -907,7 +992,7 @@ abstract class FileBackendStore extends FileBackend {
* @param string $container Resolved container name
* @param string $dir Resolved path relative to container
* @param array $params
- * @return Traversable|Array|null Returns null on failure
+ * @return Traversable|array|null Returns null on failure
*/
abstract public function getFileListInternal( $container, $dir, array $params );
@@ -919,8 +1004,8 @@ abstract class FileBackendStore extends FileBackend {
* An exception is thrown if an unsupported operation is requested.
*
* @param array $ops Same format as doOperations()
- * @return Array List of FileOp objects
- * @throws MWException
+ * @return array List of FileOp objects
+ * @throws FileBackendError
*/
final public function getOperationsInternal( array $ops ) {
$supportedOps = array(
@@ -944,7 +1029,7 @@ abstract class FileBackendStore extends FileBackend {
// Append the FileOp class
$performOps[] = new $class( $this, $params );
} else {
- throw new MWException( "Operation '$opName' is not supported." );
+ throw new FileBackendError( "Operation '$opName' is not supported." );
}
}
@@ -959,7 +1044,7 @@ abstract class FileBackendStore extends FileBackend {
* normalized.
*
* @param array $performOps List of FileOp objects
- * @return Array (LockManager::LOCK_UW => path list, LockManager::LOCK_EX => path list)
+ * @return array (LockManager::LOCK_UW => path list, LockManager::LOCK_EX => path list)
*/
final public function getPathsToLockForOpsInternal( array $performOps ) {
// Build up a list of files to lock...
@@ -981,6 +1066,7 @@ abstract class FileBackendStore extends FileBackend {
public function getScopedLocksForOps( array $ops, Status $status ) {
$paths = $this->getPathsToLockForOpsInternal( $this->getOperationsInternal( $ops ) );
+
return array( $this->getScopedFileLocks( $paths, 'mixed', $status ) );
}
@@ -1010,18 +1096,43 @@ abstract class FileBackendStore extends FileBackend {
$this->clearCache();
}
- // Load from the persistent file and container caches
- $this->primeFileCache( $performOps );
- $this->primeContainerCache( $performOps );
+ // Build the list of paths involved
+ $paths = array();
+ foreach ( $performOps as $op ) {
+ $paths = array_merge( $paths, $op->storagePathsRead() );
+ $paths = array_merge( $paths, $op->storagePathsChanged() );
+ }
- // Actually attempt the operation batch...
- $opts = $this->setConcurrencyFlags( $opts );
- $subStatus = FileOpBatch::attempt( $performOps, $opts, $this->fileJournal );
+ // Enlarge the cache to fit the stat entries of these files
+ $this->cheapCache->resize( max( 2 * count( $paths ), self::CACHE_CHEAP_SIZE ) );
+
+ // Load from the persistent container caches
+ $this->primeContainerCache( $paths );
+ // Get the latest stat info for all the files (having locked them)
+ $ok = $this->preloadFileStat( array( 'srcs' => $paths, 'latest' => true ) );
+
+ if ( $ok ) {
+ // Actually attempt the operation batch...
+ $opts = $this->setConcurrencyFlags( $opts );
+ $subStatus = FileOpBatch::attempt( $performOps, $opts, $this->fileJournal );
+ } else {
+ // If we could not even stat some files, then bail out...
+ $subStatus = Status::newFatal( 'backend-fail-internal', $this->name );
+ foreach ( $ops as $i => $op ) { // mark each op as failed
+ $subStatus->success[$i] = false;
+ ++$subStatus->failCount;
+ }
+ wfDebugLog( 'FileOperation', get_class( $this ) . "-{$this->name} " .
+ " stat failure; aborted operations: " . FormatJson::encode( $ops ) );
+ }
// Merge errors into status fields
$status->merge( $subStatus );
$status->success = $subStatus->success; // not done in merge()
+ // Shrink the stat cache back to normal size
+ $this->cheapCache->resize( self::CACHE_CHEAP_SIZE );
+
return $status;
}
@@ -1035,7 +1146,8 @@ abstract class FileBackendStore extends FileBackend {
// Clear any file cache entries
$this->clearCache();
- $supportedOps = array( 'create', 'store', 'copy', 'move', 'delete', 'null' );
+ $supportedOps = array( 'create', 'store', 'copy', 'move', 'delete', 'describe', 'null' );
+ // Parallel ops may be disabled in config due to dependencies (e.g. needing popen())
$async = ( $this->parallelize === 'implicit' && count( $ops ) > 1 );
$maxConcurrency = $this->concurrency; // throttle
@@ -1045,7 +1157,7 @@ abstract class FileBackendStore extends FileBackend {
// Perform the sync-only ops and build up op handles for the async ops...
foreach ( $ops as $index => $params ) {
if ( !in_array( $params['op'], $supportedOps ) ) {
- throw new MWException( "Operation '{$params['op']}' is not supported." );
+ throw new FileBackendError( "Operation '{$params['op']}' is not supported." );
}
$method = $params['op'] . 'Internal'; // e.g. "storeInternal"
$subStatus = $this->$method( array( 'async' => $async ) + $params );
@@ -1086,36 +1198,40 @@ abstract class FileBackendStore extends FileBackend {
* The resulting Status object fields will correspond
* to the order in which the handles where given.
*
- * @param array $handles List of FileBackendStoreOpHandle objects
- * @return Array Map of Status objects
- * @throws MWException
+ * @param array $fileOpHandles
+ * @throws FileBackendError
+ * @internal param array $handles List of FileBackendStoreOpHandle objects
+ * @return array Map of Status objects
*/
final public function executeOpHandlesInternal( array $fileOpHandles ) {
$section = new ProfileSection( __METHOD__ . "-{$this->name}" );
+
foreach ( $fileOpHandles as $fileOpHandle ) {
if ( !( $fileOpHandle instanceof FileBackendStoreOpHandle ) ) {
- throw new MWException( "Given a non-FileBackendStoreOpHandle object." );
+ throw new FileBackendError( "Given a non-FileBackendStoreOpHandle object." );
} elseif ( $fileOpHandle->backend->getName() !== $this->getName() ) {
- throw new MWException( "Given a FileBackendStoreOpHandle for the wrong backend." );
+ throw new FileBackendError( "Given a FileBackendStoreOpHandle for the wrong backend." );
}
}
$res = $this->doExecuteOpHandlesInternal( $fileOpHandles );
foreach ( $fileOpHandles as $fileOpHandle ) {
$fileOpHandle->closeResources();
}
+
return $res;
}
/**
* @see FileBackendStore::executeOpHandlesInternal()
* @param array $fileOpHandles
- * @throws MWException
- * @return Array List of corresponding Status objects
+ * @throws FileBackendError
+ * @return array List of corresponding Status objects
*/
protected function doExecuteOpHandlesInternal( array $fileOpHandles ) {
- foreach ( $fileOpHandles as $fileOpHandle ) { // OK if empty
- throw new MWException( "This backend supports no asynchronous operations." );
+ if ( count( $fileOpHandles ) ) {
+ throw new FileBackendError( "This backend supports no asynchronous operations." );
}
+
return array();
}
@@ -1126,7 +1242,7 @@ abstract class FileBackendStore extends FileBackend {
* specific errors, especially in the middle of batch file operations.
*
* @param array $op Same format as doOperation()
- * @return Array
+ * @return array
*/
protected function stripInvalidHeadersFromOp( array $op ) {
static $longs = array( 'Content-Disposition' );
@@ -1141,6 +1257,7 @@ abstract class FileBackendStore extends FileBackend {
}
}
}
+
return $op;
}
@@ -1178,9 +1295,71 @@ abstract class FileBackendStore extends FileBackend {
* @see FileBackend::clearCache()
*
* @param array $paths Storage paths (optional)
- * @return void
*/
- protected function doClearCache( array $paths = null ) {}
+ protected function doClearCache( array $paths = null ) {
+ }
+
+ final public function preloadFileStat( array $params ) {
+ $section = new ProfileSection( __METHOD__ . "-{$this->name}" );
+ $success = true; // no network errors
+
+ $params['concurrency'] = ( $this->parallelize !== 'off' ) ? $this->concurrency : 1;
+ $stats = $this->doGetFileStatMulti( $params );
+ if ( $stats === null ) {
+ return true; // not supported
+ }
+
+ $latest = !empty( $params['latest'] ); // use latest data?
+ foreach ( $stats as $path => $stat ) {
+ $path = FileBackend::normalizeStoragePath( $path );
+ if ( $path === null ) {
+ continue; // this shouldn't happen
+ }
+ if ( is_array( $stat ) ) { // file exists
+ // Strongly consistent backends can automatically set "latest"
+ $stat['latest'] = isset( $stat['latest'] ) ? $stat['latest'] : $latest;
+ $this->cheapCache->set( $path, 'stat', $stat );
+ $this->setFileCache( $path, $stat ); // update persistent cache
+ if ( isset( $stat['sha1'] ) ) { // some backends store SHA-1 as metadata
+ $this->cheapCache->set( $path, 'sha1',
+ array( 'hash' => $stat['sha1'], 'latest' => $latest ) );
+ }
+ if ( isset( $stat['xattr'] ) ) { // some backends store headers/metadata
+ $stat['xattr'] = self::normalizeXAttributes( $stat['xattr'] );
+ $this->cheapCache->set( $path, 'xattr',
+ array( 'map' => $stat['xattr'], 'latest' => $latest ) );
+ }
+ } elseif ( $stat === false ) { // file does not exist
+ $this->cheapCache->set( $path, 'stat',
+ $latest ? 'NOT_EXIST_LATEST' : 'NOT_EXIST' );
+ $this->cheapCache->set( $path, 'xattr',
+ array( 'map' => false, 'latest' => $latest ) );
+ $this->cheapCache->set( $path, 'sha1',
+ array( 'hash' => false, 'latest' => $latest ) );
+ wfDebug( __METHOD__ . ": File $path does not exist.\n" );
+ } else { // an error occurred
+ $success = false;
+ wfDebug( __METHOD__ . ": Could not stat file $path.\n" );
+ }
+ }
+
+ return $success;
+ }
+
+ /**
+ * Get file stat information (concurrently if possible) for several files
+ *
+ * @see FileBackend::getFileStat()
+ *
+ * @param array $params Parameters include:
+ * - srcs : list of source storage paths
+ * - latest : use the latest available data
+ * @return array|null Map of storage paths to array|bool|null (returns null if not supported)
+ * @since 1.23
+ */
+ protected function doGetFileStatMulti( array $params ) {
+ return null; // not supported
+ }
/**
* Is this a key/value store where directories are just virtual?
@@ -1218,7 +1397,7 @@ abstract class FileBackendStore extends FileBackend {
* be scanned by looking in all the container shards.
*
* @param string $storagePath
- * @return Array (container, path, container suffix) or (null, null, null) if invalid
+ * @return array (container, path, container suffix) or (null, null, null) if invalid
*/
final protected function resolveStoragePath( $storagePath ) {
list( $backend, $container, $relPath ) = self::splitStoragePath( $storagePath );
@@ -1242,6 +1421,7 @@ abstract class FileBackendStore extends FileBackend {
}
}
}
+
return array( null, null, null );
}
@@ -1258,13 +1438,14 @@ abstract class FileBackendStore extends FileBackend {
* @see FileBackendStore::resolveStoragePath()
*
* @param string $storagePath
- * @return Array (container, path) or (null, null) if invalid
+ * @return array (container, path) or (null, null) if invalid
*/
final protected function resolveStoragePathReal( $storagePath ) {
list( $container, $relPath, $cShard ) = $this->resolveStoragePath( $storagePath );
if ( $cShard !== null && substr( $relPath, -1 ) !== '/' ) {
return array( $container, $relPath );
}
+
return array( null, null );
}
@@ -1299,8 +1480,10 @@ abstract class FileBackendStore extends FileBackend {
if ( preg_match( "!^(?:[^/]{2,}/)*$hashDirRegex(?:/|$)!", $relPath, $m ) ) {
return '.' . implode( '', array_slice( $m, 1 ) );
}
+
return null; // failed to match
}
+
return ''; // no sharding
}
@@ -1314,6 +1497,7 @@ abstract class FileBackendStore extends FileBackend {
*/
final public function isSingleShardPathInternal( $storagePath ) {
list( , , $shard ) = $this->resolveStoragePath( $storagePath );
+
return ( $shard !== null );
}
@@ -1323,7 +1507,7 @@ abstract class FileBackendStore extends FileBackend {
* the container are required to be hashed accordingly.
*
* @param string $container
- * @return Array (integer levels, integer base, repeat flag) or (0, 0, false)
+ * @return array (integer levels, integer base, repeat flag) or (0, 0, false)
*/
final protected function getContainerHashLevels( $container ) {
if ( isset( $this->shardViaHashLevels[$container] ) ) {
@@ -1336,6 +1520,7 @@ abstract class FileBackendStore extends FileBackend {
}
}
}
+
return array( 0, 0, false ); // no sharding
}
@@ -1343,7 +1528,7 @@ abstract class FileBackendStore extends FileBackend {
* Get a list of full container shard suffixes for a container
*
* @param string $container
- * @return Array
+ * @return array
*/
final protected function getContainerSuffixes( $container ) {
$shards = array();
@@ -1354,6 +1539,7 @@ abstract class FileBackendStore extends FileBackend {
$shards[] = '.' . wfBaseConvert( $index, 10, $base, $digits );
}
}
+
return $shards;
}
@@ -1404,7 +1590,7 @@ abstract class FileBackendStore extends FileBackend {
* @return string
*/
private function containerCacheKey( $container ) {
- return wfMemcKey( 'backend', $this->getName(), 'container', $container );
+ return "filebackend:{$this->name}:{$this->wikiId}:container:{$container}";
}
/**
@@ -1412,7 +1598,6 @@ abstract class FileBackendStore extends FileBackend {
*
* @param string $container Resolved container name
* @param array $val Information to cache
- * @return void
*/
final protected function setContainerCache( $container, array $val ) {
$this->memCache->add( $this->containerCacheKey( $container ), $val, 14 * 86400 );
@@ -1423,7 +1608,6 @@ abstract class FileBackendStore extends FileBackend {
* The cache key is salted for a while to prevent race conditions.
*
* @param string $container Resolved container name
- * @return void
*/
final protected function deleteContainerCache( $container ) {
if ( !$this->memCache->set( $this->containerCacheKey( $container ), 'PURGED', 300 ) ) {
@@ -1433,11 +1617,10 @@ abstract class FileBackendStore extends FileBackend {
/**
* Do a batch lookup from cache for container stats for all containers
- * used in a list of container names, storage paths, or FileOp objects.
+ * used in a list of container names or storage paths objects.
* This loads the persistent cache values into the process cache.
*
- * @param Array $items
- * @return void
+ * @param array $items
*/
final protected function primeContainerCache( array $items ) {
$section = new ProfileSection( __METHOD__ . "-{$this->name}" );
@@ -1446,10 +1629,7 @@ abstract class FileBackendStore extends FileBackend {
$contNames = array(); // (cache key => resolved container name)
// Get all the paths/containers from the items...
foreach ( $items as $item ) {
- if ( $item instanceof FileOp ) {
- $paths = array_merge( $paths, $item->storagePathsRead() );
- $paths = array_merge( $paths, $item->storagePathsChanged() );
- } elseif ( self::isStoragePath( $item ) ) {
+ if ( self::isStoragePath( $item ) ) {
$paths[] = $item;
} elseif ( is_string( $item ) ) { // full container name
$contNames[$this->containerCacheKey( $item )] = $item;
@@ -1480,9 +1660,9 @@ abstract class FileBackendStore extends FileBackend {
* Only containers that actually exist should appear in the map.
*
* @param array $containerInfo Map of resolved container names to cached info
- * @return void
*/
- protected function doPrimeContainerCache( array $containerInfo ) {}
+ protected function doPrimeContainerCache( array $containerInfo ) {
+ }
/**
* Get the cache key for a file path
@@ -1491,7 +1671,7 @@ abstract class FileBackendStore extends FileBackend {
* @return string
*/
private function fileCacheKey( $path ) {
- return wfMemcKey( 'backend', $this->getName(), 'file', sha1( $path ) );
+ return "filebackend:{$this->name}:{$this->wikiId}:file:" . sha1( $path );
}
/**
@@ -1501,7 +1681,6 @@ abstract class FileBackendStore extends FileBackend {
*
* @param string $path Storage path
* @param array $val Stat information to cache
- * @return void
*/
final protected function setFileCache( $path, array $val ) {
$path = FileBackend::normalizeStoragePath( $path );
@@ -1510,7 +1689,22 @@ abstract class FileBackendStore extends FileBackend {
}
$age = time() - wfTimestamp( TS_UNIX, $val['mtime'] );
$ttl = min( 7 * 86400, max( 300, floor( .1 * $age ) ) );
- $this->memCache->add( $this->fileCacheKey( $path ), $val, $ttl );
+ $key = $this->fileCacheKey( $path );
+ // Set the cache unless it is currently salted with the value "PURGED".
+ // Using add() handles this except it also is a no-op in that case where
+ // the current value is not "latest" but $val is, so use CAS in that case.
+ if ( !$this->memCache->add( $key, $val, $ttl ) && !empty( $val['latest'] ) ) {
+ $this->memCache->merge(
+ $key,
+ function ( BagOStuff $cache, $key, $cValue ) use ( $val ) {
+ return ( is_array( $cValue ) && empty( $cValue['latest'] ) )
+ ? $val // update the stat cache with the lastest info
+ : false; // do nothing (cache is salted or some error happened)
+ },
+ $ttl,
+ 1
+ );
+ }
}
/**
@@ -1520,7 +1714,6 @@ abstract class FileBackendStore extends FileBackend {
* a file is created at a path were there was none before.
*
* @param string $path Storage path
- * @return void
*/
final protected function deleteFileCache( $path ) {
$path = FileBackend::normalizeStoragePath( $path );
@@ -1537,8 +1730,7 @@ abstract class FileBackendStore extends FileBackend {
* used in a list of storage paths or FileOp objects.
* This loads the persistent cache values into the process cache.
*
- * @param array $items List of storage paths or FileOps
- * @return void
+ * @param array $items List of storage paths
*/
final protected function primeFileCache( array $items ) {
$section = new ProfileSection( __METHOD__ . "-{$this->name}" );
@@ -1547,10 +1739,7 @@ abstract class FileBackendStore extends FileBackend {
$pathNames = array(); // (cache key => storage path)
// Get all the paths/containers from the items...
foreach ( $items as $item ) {
- if ( $item instanceof FileOp ) {
- $paths = array_merge( $paths, $item->storagePathsRead() );
- $paths = array_merge( $paths, $item->storagePathsChanged() );
- } elseif ( self::isStoragePath( $item ) ) {
+ if ( self::isStoragePath( $item ) ) {
$paths[] = FileBackend::normalizeStoragePath( $item );
}
}
@@ -1573,15 +1762,41 @@ abstract class FileBackendStore extends FileBackend {
$this->cheapCache->set( $path, 'sha1',
array( 'hash' => $val['sha1'], 'latest' => $val['latest'] ) );
}
+ if ( isset( $val['xattr'] ) ) { // some backends store headers/metadata
+ $val['xattr'] = self::normalizeXAttributes( $val['xattr'] );
+ $this->cheapCache->set( $path, 'xattr',
+ array( 'map' => $val['xattr'], 'latest' => $val['latest'] ) );
+ }
}
}
}
/**
+ * Normalize file headers/metadata to the FileBackend::getFileXAttributes() format
+ *
+ * @param array $xattr
+ * @return array
+ * @since 1.22
+ */
+ final protected static function normalizeXAttributes( array $xattr ) {
+ $newXAttr = array( 'headers' => array(), 'metadata' => array() );
+
+ foreach ( $xattr['headers'] as $name => $value ) {
+ $newXAttr['headers'][strtolower( $name )] = $value;
+ }
+
+ foreach ( $xattr['metadata'] as $name => $value ) {
+ $newXAttr['metadata'][strtolower( $name )] = $value;
+ }
+
+ return $newXAttr;
+ }
+
+ /**
* Set the 'concurrency' option from a list of operation options
*
* @param array $opts Map of operation options
- * @return Array
+ * @return array
*/
final protected function setConcurrencyFlags( array $opts ) {
$opts['concurrency'] = 1; // off
@@ -1594,6 +1809,7 @@ abstract class FileBackendStore extends FileBackend {
$opts['concurrency'] = $this->concurrency;
}
}
+
return $opts;
}
@@ -1603,7 +1819,7 @@ abstract class FileBackendStore extends FileBackend {
* @param string $storagePath
* @param string|null $content File data
* @param string|null $fsPath File system path
- * @return MIME type
+ * @return string MIME type
*/
protected function getContentType( $storagePath, $content, $fsPath ) {
return call_user_func_array( $this->mimeCallback, func_get_args() );
@@ -1619,19 +1835,17 @@ abstract class FileBackendStore extends FileBackend {
* passed to FileBackendStore::executeOpHandlesInternal().
*/
abstract class FileBackendStoreOpHandle {
- /** @var Array */
+ /** @var array */
public $params = array(); // params to caller functions
/** @var FileBackendStore */
public $backend;
- /** @var Array */
+ /** @var array */
public $resourcesToClose = array();
public $call; // string; name that identifies the function called
/**
* Close all open file handles
- *
- * @return void
*/
public function closeResources() {
array_map( 'fclose', $this->resourcesToClose );
@@ -1647,13 +1861,17 @@ abstract class FileBackendStoreOpHandle {
abstract class FileBackendStoreShardListIterator extends FilterIterator {
/** @var FileBackendStore */
protected $backend;
- /** @var Array */
+
+ /** @var array */
protected $params;
- protected $container; // string; full container name
- protected $directory; // string; resolved relative path
+ /** @var string Full container name */
+ protected $container;
- /** @var Array */
+ /** @var string Resolved relative path */
+ protected $directory;
+
+ /** @var array */
protected $multiShardPaths = array(); // (rel path => 1)
/**
@@ -1689,6 +1907,7 @@ abstract class FileBackendStoreShardListIterator extends FilterIterator {
return false;
} else {
$this->multiShardPaths[$rel] = 1;
+
return true;
}
}
diff --git a/includes/filebackend/FileOp.php b/includes/filebackend/FileOp.php
index fe833084..66d87943 100644
--- a/includes/filebackend/FileOp.php
+++ b/includes/filebackend/FileOp.php
@@ -34,20 +34,35 @@
* @since 1.19
*/
abstract class FileOp {
- /** @var Array */
+ /** @var array */
protected $params = array();
+
/** @var FileBackendStore */
protected $backend;
- protected $state = self::STATE_NEW; // integer
- protected $failed = false; // boolean
- protected $async = false; // boolean
- protected $batchId; // string
+ /** @var int */
+ protected $state = self::STATE_NEW;
+
+ /** @var bool */
+ protected $failed = false;
+
+ /** @var bool */
+ protected $async = false;
+
+ /** @var string */
+ protected $batchId;
- protected $doOperation = true; // boolean; operation is not a no-op
- protected $sourceSha1; // string
- protected $overwriteSameCase; // boolean
- protected $destExists; // boolean
+ /** @var bool Operation is not a no-op */
+ protected $doOperation = true;
+
+ /** @var string */
+ protected $sourceSha1;
+
+ /** @var bool */
+ protected $overwriteSameCase;
+
+ /** @var bool */
+ protected $destExists;
/* Object life-cycle */
const STATE_NEW = 1;
@@ -58,47 +73,29 @@ abstract class FileOp {
* Build a new batch file operation transaction
*
* @param FileBackendStore $backend
- * @param Array $params
- * @throws MWException
+ * @param array $params
+ * @throws FileBackendError
*/
final public function __construct( FileBackendStore $backend, array $params ) {
$this->backend = $backend;
- list( $required, $optional ) = $this->allowedParams();
- // @todo normalizeAnyStoragePaths() calls are overzealous, use a parameter list
+ list( $required, $optional, $paths ) = $this->allowedParams();
foreach ( $required as $name ) {
if ( isset( $params[$name] ) ) {
- // Normalize paths so the paths to the same file have the same string
- $this->params[$name] = self::normalizeAnyStoragePaths( $params[$name] );
+ $this->params[$name] = $params[$name];
} else {
- throw new MWException( "File operation missing parameter '$name'." );
+ throw new FileBackendError( "File operation missing parameter '$name'." );
}
}
foreach ( $optional as $name ) {
if ( isset( $params[$name] ) ) {
- // Normalize paths so the paths to the same file have the same string
- $this->params[$name] = self::normalizeAnyStoragePaths( $params[$name] );
+ $this->params[$name] = $params[$name];
}
}
- $this->params = $params;
- }
-
- /**
- * Normalize $item or anything in $item that is a valid storage path
- *
- * @param string $item|array
- * @return string|Array
- */
- protected function normalizeAnyStoragePaths( $item ) {
- if ( is_array( $item ) ) {
- $res = array();
- foreach ( $item as $k => $v ) {
- $k = self::normalizeIfValidStoragePath( $k );
- $v = self::normalizeIfValidStoragePath( $v );
- $res[$k] = $v;
+ foreach ( $paths as $name ) {
+ if ( isset( $this->params[$name] ) ) {
+ // Normalize paths so the paths to the same file have the same string
+ $this->params[$name] = self::normalizeIfValidStoragePath( $this->params[$name] );
}
- return $res;
- } else {
- return self::normalizeIfValidStoragePath( $item );
}
}
@@ -111,8 +108,10 @@ abstract class FileOp {
protected static function normalizeIfValidStoragePath( $path ) {
if ( FileBackend::isStoragePath( $path ) ) {
$res = FileBackend::normalizeStoragePath( $path );
+
return ( $res !== null ) ? $res : $path;
}
+
return $path;
}
@@ -120,7 +119,6 @@ abstract class FileOp {
* Set the batch UUID this operation belongs to
*
* @param string $batchId
- * @return void
*/
final public function setBatchId( $batchId ) {
$this->batchId = $batchId;
@@ -148,7 +146,7 @@ abstract class FileOp {
/**
* Get a new empty predicates array for precheck()
*
- * @return Array
+ * @return array
*/
final public static function newPredicates() {
return array( 'exists' => array(), 'sha1' => array() );
@@ -157,7 +155,7 @@ abstract class FileOp {
/**
* Get a new empty dependency tracking array for paths read/written to
*
- * @return Array
+ * @return array
*/
final public static function newDependencies() {
return array( 'read' => array(), 'write' => array() );
@@ -167,19 +165,20 @@ abstract class FileOp {
* Update a dependency tracking array to account for this operation
*
* @param array $deps Prior path reads/writes; format of FileOp::newPredicates()
- * @return Array
+ * @return array
*/
final public function applyDependencies( array $deps ) {
$deps['read'] += array_fill_keys( $this->storagePathsRead(), 1 );
$deps['write'] += array_fill_keys( $this->storagePathsChanged(), 1 );
+
return $deps;
}
/**
* Check if this operation changes files listed in $paths
*
- * @param array $paths Prior path reads/writes; format of FileOp::newPredicates()
- * @return boolean
+ * @param array $deps Prior path reads/writes; format of FileOp::newPredicates()
+ * @return bool
*/
final public function dependsOn( array $deps ) {
foreach ( $this->storagePathsChanged() as $path ) {
@@ -192,6 +191,7 @@ abstract class FileOp {
return true; // "flow" dependency
}
}
+
return false;
}
@@ -200,7 +200,7 @@ abstract class FileOp {
*
* @param array $oPredicates Pre-op info about files (format of FileOp::newPredicates)
* @param array $nPredicates Post-op info about files (format of FileOp::newPredicates)
- * @return Array
+ * @return array
*/
final public function getJournalEntries( array $oPredicates, array $nPredicates ) {
if ( !$this->doOperation ) {
@@ -232,6 +232,7 @@ abstract class FileOp {
);
}
}
+
return array_merge( $nullEntries, $updateEntries, $deleteEntries );
}
@@ -240,7 +241,7 @@ abstract class FileOp {
* This must update $predicates for each path that the op can change
* except when a failing status object is returned.
*
- * @param Array $predicates
+ * @param array $predicates
* @return Status
*/
final public function precheck( array &$predicates ) {
@@ -252,10 +253,12 @@ abstract class FileOp {
if ( !$status->isOK() ) {
$this->failed = true;
}
+
return $status;
}
/**
+ * @param array $predicates
* @return Status
*/
protected function doPrecheck( array &$predicates ) {
@@ -283,6 +286,7 @@ abstract class FileOp {
} else { // no-op
$status = Status::newGood();
}
+
return $status;
}
@@ -302,23 +306,24 @@ abstract class FileOp {
$this->async = true;
$result = $this->attempt();
$this->async = false;
+
return $result;
}
/**
* Get the file operation parameters
*
- * @return Array (required params list, optional params list)
+ * @return array (required params list, optional params list, list of params that are paths)
*/
protected function allowedParams() {
- return array( array(), array() );
+ return array( array(), array(), array() );
}
/**
* Adjust params to FileBackendStore internal file calls
*
- * @param Array $params
- * @return Array (required params list, optional params list)
+ * @param array $params
+ * @return array (required params list, optional params list)
*/
protected function setFlags( array $params ) {
return array( 'async' => $this->async ) + $params;
@@ -327,7 +332,7 @@ abstract class FileOp {
/**
* Get a list of storage paths read from for this operation
*
- * @return Array
+ * @return array
*/
public function storagePathsRead() {
return array();
@@ -336,7 +341,7 @@ abstract class FileOp {
/**
* Get a list of storage paths written to for this operation
*
- * @return Array
+ * @return array
*/
public function storagePathsChanged() {
return array();
@@ -347,7 +352,7 @@ abstract class FileOp {
* Also set the destExists, overwriteSameCase and sourceSha1 member variables.
* A bad status will be returned if there is no chance it can be overwritten.
*
- * @param Array $predicates
+ * @param array $predicates
* @return Status
*/
protected function precheckDestExistence( array $predicates ) {
@@ -373,12 +378,15 @@ abstract class FileOp {
} else {
$this->overwriteSameCase = true; // OK
}
+
return $status; // do nothing; either OK or bad status
} else {
$status->fatal( 'backend-fail-alreadyexists', $this->params['dst'] );
+
return $status;
}
}
+
return $status;
}
@@ -396,7 +404,7 @@ abstract class FileOp {
* Check if a file will exist in storage when this operation is attempted
*
* @param string $source Storage path
- * @param Array $predicates
+ * @param array $predicates
* @return bool
*/
final protected function fileExists( $source, array $predicates ) {
@@ -404,6 +412,7 @@ abstract class FileOp {
return $predicates['exists'][$source]; // previous op assures this
} else {
$params = array( 'src' => $source, 'latest' => true );
+
return $this->backend->fileExists( $params );
}
}
@@ -412,7 +421,7 @@ abstract class FileOp {
* Get the SHA-1 of a file in storage when this operation is attempted
*
* @param string $source Storage path
- * @param Array $predicates
+ * @param array $predicates
* @return string|bool False on failure
*/
final protected function fileSha1( $source, array $predicates ) {
@@ -422,6 +431,7 @@ abstract class FileOp {
return false; // previous op assures this
} else {
$params = array( 'src' => $source, 'latest' => true );
+
return $this->backend->getFileSha1Base36( $params );
}
}
@@ -439,7 +449,6 @@ abstract class FileOp {
* Log a file operation failure and preserve any temp files
*
* @param string $action
- * @return void
*/
final public function logFailure( $action ) {
$params = $this->params;
@@ -459,8 +468,11 @@ abstract class FileOp {
*/
class CreateFileOp extends FileOp {
protected function allowedParams() {
- return array( array( 'content', 'dst' ),
- array( 'overwrite', 'overwriteSame', 'headers' ) );
+ return array(
+ array( 'content', 'dst' ),
+ array( 'overwrite', 'overwriteSame', 'headers' ),
+ array( 'dst' )
+ );
}
protected function doPrecheck( array &$predicates ) {
@@ -470,11 +482,13 @@ class CreateFileOp extends FileOp {
$status->fatal( 'backend-fail-maxsize',
$this->params['dst'], $this->backend->maxFileSizeInternal() );
$status->fatal( 'backend-fail-create', $this->params['dst'] );
+
return $status;
// Check if a file can be placed/changed at the destination
} elseif ( !$this->backend->isPathUsableInternal( $this->params['dst'] ) ) {
$status->fatal( 'backend-fail-usable', $this->params['dst'] );
$status->fatal( 'backend-fail-create', $this->params['dst'] );
+
return $status;
}
// Check if destination file exists
@@ -485,6 +499,7 @@ class CreateFileOp extends FileOp {
$predicates['exists'][$this->params['dst']] = true;
$predicates['sha1'][$this->params['dst']] = $this->sourceSha1;
}
+
return $status; // safe to call attempt()
}
@@ -493,6 +508,7 @@ class CreateFileOp extends FileOp {
// Create the file at the destination
return $this->backend->createInternal( $this->setFlags( $this->params ) );
}
+
return Status::newGood();
}
@@ -511,8 +527,11 @@ class CreateFileOp extends FileOp {
*/
class StoreFileOp extends FileOp {
protected function allowedParams() {
- return array( array( 'src', 'dst' ),
- array( 'overwrite', 'overwriteSame', 'headers' ) );
+ return array(
+ array( 'src', 'dst' ),
+ array( 'overwrite', 'overwriteSame', 'headers' ),
+ array( 'src', 'dst' )
+ );
}
protected function doPrecheck( array &$predicates ) {
@@ -520,17 +539,20 @@ class StoreFileOp extends FileOp {
// Check if the source file exists on the file system
if ( !is_file( $this->params['src'] ) ) {
$status->fatal( 'backend-fail-notexists', $this->params['src'] );
+
return $status;
// Check if the source file is too big
} elseif ( filesize( $this->params['src'] ) > $this->backend->maxFileSizeInternal() ) {
$status->fatal( 'backend-fail-maxsize',
$this->params['dst'], $this->backend->maxFileSizeInternal() );
$status->fatal( 'backend-fail-store', $this->params['src'], $this->params['dst'] );
+
return $status;
// Check if a file can be placed/changed at the destination
} elseif ( !$this->backend->isPathUsableInternal( $this->params['dst'] ) ) {
$status->fatal( 'backend-fail-usable', $this->params['dst'] );
$status->fatal( 'backend-fail-store', $this->params['src'], $this->params['dst'] );
+
return $status;
}
// Check if destination file exists
@@ -541,6 +563,7 @@ class StoreFileOp extends FileOp {
$predicates['exists'][$this->params['dst']] = true;
$predicates['sha1'][$this->params['dst']] = $this->sourceSha1;
}
+
return $status; // safe to call attempt()
}
@@ -549,6 +572,7 @@ class StoreFileOp extends FileOp {
// Store the file at the destination
return $this->backend->storeInternal( $this->setFlags( $this->params ) );
}
+
return Status::newGood();
}
@@ -559,6 +583,7 @@ class StoreFileOp extends FileOp {
if ( $hash !== false ) {
$hash = wfBaseConvert( $hash, 16, 36, 31 );
}
+
return $hash;
}
@@ -573,8 +598,11 @@ class StoreFileOp extends FileOp {
*/
class CopyFileOp extends FileOp {
protected function allowedParams() {
- return array( array( 'src', 'dst' ),
- array( 'overwrite', 'overwriteSame', 'ignoreMissingSource', 'headers' ) );
+ return array(
+ array( 'src', 'dst' ),
+ array( 'overwrite', 'overwriteSame', 'ignoreMissingSource', 'headers' ),
+ array( 'src', 'dst' )
+ );
}
protected function doPrecheck( array &$predicates ) {
@@ -586,15 +614,18 @@ class CopyFileOp extends FileOp {
// Update file existence predicates (cache 404s)
$predicates['exists'][$this->params['src']] = false;
$predicates['sha1'][$this->params['src']] = false;
+
return $status; // nothing to do
} else {
$status->fatal( 'backend-fail-notexists', $this->params['src'] );
+
return $status;
}
- // Check if a file can be placed/changed at the destination
+ // Check if a file can be placed/changed at the destination
} elseif ( !$this->backend->isPathUsableInternal( $this->params['dst'] ) ) {
$status->fatal( 'backend-fail-usable', $this->params['dst'] );
$status->fatal( 'backend-fail-copy', $this->params['src'], $this->params['dst'] );
+
return $status;
}
// Check if destination file exists
@@ -605,6 +636,7 @@ class CopyFileOp extends FileOp {
$predicates['exists'][$this->params['dst']] = true;
$predicates['sha1'][$this->params['dst']] = $this->sourceSha1;
}
+
return $status; // safe to call attempt()
}
@@ -621,6 +653,7 @@ class CopyFileOp extends FileOp {
// Copy the file to the destination
$status = $this->backend->copyInternal( $this->setFlags( $this->params ) );
}
+
return $status;
}
@@ -639,8 +672,11 @@ class CopyFileOp extends FileOp {
*/
class MoveFileOp extends FileOp {
protected function allowedParams() {
- return array( array( 'src', 'dst' ),
- array( 'overwrite', 'overwriteSame', 'ignoreMissingSource', 'headers' ) );
+ return array(
+ array( 'src', 'dst' ),
+ array( 'overwrite', 'overwriteSame', 'ignoreMissingSource', 'headers' ),
+ array( 'src', 'dst' )
+ );
}
protected function doPrecheck( array &$predicates ) {
@@ -652,15 +688,18 @@ class MoveFileOp extends FileOp {
// Update file existence predicates (cache 404s)
$predicates['exists'][$this->params['src']] = false;
$predicates['sha1'][$this->params['src']] = false;
+
return $status; // nothing to do
} else {
$status->fatal( 'backend-fail-notexists', $this->params['src'] );
+
return $status;
}
// Check if a file can be placed/changed at the destination
} elseif ( !$this->backend->isPathUsableInternal( $this->params['dst'] ) ) {
$status->fatal( 'backend-fail-usable', $this->params['dst'] );
$status->fatal( 'backend-fail-move', $this->params['src'], $this->params['dst'] );
+
return $status;
}
// Check if destination file exists
@@ -673,6 +712,7 @@ class MoveFileOp extends FileOp {
$predicates['exists'][$this->params['dst']] = true;
$predicates['sha1'][$this->params['dst']] = $this->sourceSha1;
}
+
return $status; // safe to call attempt()
}
@@ -697,6 +737,7 @@ class MoveFileOp extends FileOp {
// Move the file to the destination
$status = $this->backend->moveInternal( $this->setFlags( $this->params ) );
}
+
return $status;
}
@@ -715,7 +756,7 @@ class MoveFileOp extends FileOp {
*/
class DeleteFileOp extends FileOp {
protected function allowedParams() {
- return array( array( 'src' ), array( 'ignoreMissingSource' ) );
+ return array( array( 'src' ), array( 'ignoreMissingSource' ), array( 'src' ) );
}
protected function doPrecheck( array &$predicates ) {
@@ -727,20 +768,24 @@ class DeleteFileOp extends FileOp {
// Update file existence predicates (cache 404s)
$predicates['exists'][$this->params['src']] = false;
$predicates['sha1'][$this->params['src']] = false;
+
return $status; // nothing to do
} else {
$status->fatal( 'backend-fail-notexists', $this->params['src'] );
+
return $status;
}
// Check if a file can be placed/changed at the source
} elseif ( !$this->backend->isPathUsableInternal( $this->params['src'] ) ) {
$status->fatal( 'backend-fail-usable', $this->params['src'] );
$status->fatal( 'backend-fail-delete', $this->params['src'] );
+
return $status;
}
// Update file existence predicates
$predicates['exists'][$this->params['src']] = false;
$predicates['sha1'][$this->params['src']] = false;
+
return $status; // safe to call attempt()
}
@@ -760,7 +805,7 @@ class DeleteFileOp extends FileOp {
*/
class DescribeFileOp extends FileOp {
protected function allowedParams() {
- return array( array( 'src' ), array( 'headers' ) );
+ return array( array( 'src' ), array( 'headers' ), array( 'src' ) );
}
protected function doPrecheck( array &$predicates ) {
@@ -768,11 +813,13 @@ class DescribeFileOp extends FileOp {
// Check if the source file exists
if ( !$this->fileExists( $this->params['src'], $predicates ) ) {
$status->fatal( 'backend-fail-notexists', $this->params['src'] );
+
return $status;
// Check if a file can be placed/changed at the source
} elseif ( !$this->backend->isPathUsableInternal( $this->params['src'] ) ) {
$status->fatal( 'backend-fail-usable', $this->params['src'] );
$status->fatal( 'backend-fail-describe', $this->params['src'] );
+
return $status;
}
// Update file existence predicates
@@ -780,6 +827,7 @@ class DescribeFileOp extends FileOp {
$this->fileExists( $this->params['src'], $predicates );
$predicates['sha1'][$this->params['src']] =
$this->fileSha1( $this->params['src'], $predicates );
+
return $status; // safe to call attempt()
}
@@ -796,4 +844,5 @@ class DescribeFileOp extends FileOp {
/**
* Placeholder operation that has no params and does nothing
*/
-class NullFileOp extends FileOp {}
+class NullFileOp extends FileOp {
+}
diff --git a/includes/filebackend/FileOpBatch.php b/includes/filebackend/FileOpBatch.php
index 785c0bc9..b0d83e01 100644
--- a/includes/filebackend/FileOpBatch.php
+++ b/includes/filebackend/FileOpBatch.php
@@ -55,13 +55,13 @@ class FileOpBatch {
* @return Status
*/
public static function attempt( array $performOps, array $opts, FileJournal $journal ) {
- wfProfileIn( __METHOD__ );
+ $section = new ProfileSection( __METHOD__ );
$status = Status::newGood();
$n = count( $performOps );
if ( $n > self::MAX_BATCH_SIZE ) {
$status->fatal( 'backend-fail-batchsize', $n, self::MAX_BATCH_SIZE );
- wfProfileOut( __METHOD__ );
+
return $status;
}
@@ -107,7 +107,6 @@ class FileOpBatch {
$status->success[$index] = false;
++$status->failCount;
if ( !$ignoreErrors ) {
- wfProfileOut( __METHOD__ );
return $status; // abort
}
}
@@ -121,7 +120,6 @@ class FileOpBatch {
if ( count( $entries ) ) {
$subStatus = $journal->logChangeBatch( $entries, $batchId );
if ( !$subStatus->isOK() ) {
- wfProfileOut( __METHOD__ );
return $subStatus; // abort
}
}
@@ -133,7 +131,6 @@ class FileOpBatch {
// Attempt each operation (in parallel if allowed and possible)...
self::runParallelBatches( $pPerformOps, $status );
- wfProfileOut( __METHOD__ );
return $status;
}
@@ -145,9 +142,8 @@ class FileOpBatch {
* within any given sub-batch do not depend on each other.
* This will abort remaining ops on failure.
*
- * @param Array $pPerformOps
+ * @param array $pPerformOps Batches of file ops (batches use original indexes)
* @param Status $status
- * @return bool Success
*/
protected static function runParallelBatches( array $pPerformOps, Status $status ) {
$aborted = false; // set to true on unexpected errors
@@ -156,6 +152,8 @@ class FileOpBatch {
// We can't continue (even with $ignoreErrors) as $predicates is wrong.
// Log the remaining ops as failed for recovery...
foreach ( $performOpsBatch as $i => $fileOp ) {
+ $status->success[$i] = false;
+ ++$status->failCount;
$performOpsBatch[$i]->logFailure( 'attempt_aborted' );
}
continue;
@@ -168,9 +166,9 @@ class FileOpBatch {
// If attemptAsync() returns a Status, it was either due to an error
// or the backend does not support async ops and did it synchronously.
foreach ( $performOpsBatch as $i => $fileOp ) {
- if ( !$fileOp->failed() ) { // failed => already has Status
- // If the batch is just one operation, it's faster to avoid
- // pipelining as that can involve creating new TCP connections.
+ if ( !isset( $status->success[$i] ) ) { // didn't already fail in precheck()
+ // Parallel ops may be disabled in config due to missing dependencies,
+ // (e.g. needing popen()). When they are, $performOpsBatch has size 1.
$subStatus = ( count( $performOpsBatch ) > 1 )
? $fileOp->attemptAsync()
: $fileOp->attempt();
@@ -185,7 +183,7 @@ class FileOpBatch {
$statuses = $statuses + $backend->executeOpHandlesInternal( $opHandles );
// Marshall and merge all the responses (blocking)...
foreach ( $performOpsBatch as $i => $fileOp ) {
- if ( !$fileOp->failed() ) { // failed => already has Status
+ if ( !isset( $status->success[$i] ) ) { // didn't already fail in precheck()
$subStatus = $statuses[$i];
$status->merge( $subStatus );
if ( $subStatus->isOK() ) {
@@ -199,6 +197,5 @@ class FileOpBatch {
}
}
}
- return $status;
}
}
diff --git a/includes/filebackend/MemoryFileBackend.php b/includes/filebackend/MemoryFileBackend.php
new file mode 100644
index 00000000..7c2f8256
--- /dev/null
+++ b/includes/filebackend/MemoryFileBackend.php
@@ -0,0 +1,274 @@
+<?php
+/**
+ * Simulation of a backend storage in memory.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ * http://www.gnu.org/copyleft/gpl.html
+ *
+ * @file
+ * @ingroup FileBackend
+ * @author Aaron Schulz
+ */
+
+/**
+ * Simulation of a backend storage in memory.
+ *
+ * All data in the backend is automatically deleted at the end of PHP execution.
+ * Since the data stored here is volatile, this is only useful for staging or testing.
+ *
+ * @ingroup FileBackend
+ * @since 1.23
+ */
+class MemoryFileBackend extends FileBackendStore {
+ /** @var array Map of (file path => (data,mtime) */
+ protected $files = array();
+
+ public function isPathUsableInternal( $storagePath ) {
+ return true;
+ }
+
+ protected function doCreateInternal( array $params ) {
+ $status = Status::newGood();
+
+ $dst = $this->resolveHashKey( $params['dst'] );
+ if ( $dst === null ) {
+ $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
+
+ return $status;
+ }
+
+ $this->files[$dst] = array(
+ 'data' => $params['content'],
+ 'mtime' => wfTimestamp( TS_MW, time() )
+ );
+
+ return $status;
+ }
+
+ protected function doStoreInternal( array $params ) {
+ $status = Status::newGood();
+
+ $dst = $this->resolveHashKey( $params['dst'] );
+ if ( $dst === null ) {
+ $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
+
+ return $status;
+ }
+
+ wfSuppressWarnings();
+ $data = file_get_contents( $params['src'] );
+ wfRestoreWarnings();
+ if ( $data === false ) { // source doesn't exist?
+ $status->fatal( 'backend-fail-store', $params['src'], $params['dst'] );
+
+ return $status;
+ }
+
+ $this->files[$dst] = array(
+ 'data' => $data,
+ 'mtime' => wfTimestamp( TS_MW, time() )
+ );
+
+ return $status;
+ }
+
+ protected function doCopyInternal( array $params ) {
+ $status = Status::newGood();
+
+ $src = $this->resolveHashKey( $params['src'] );
+ if ( $src === null ) {
+ $status->fatal( 'backend-fail-invalidpath', $params['src'] );
+
+ return $status;
+ }
+
+ $dst = $this->resolveHashKey( $params['dst'] );
+ if ( $dst === null ) {
+ $status->fatal( 'backend-fail-invalidpath', $params['dst'] );
+
+ return $status;
+ }
+
+ if ( !isset( $this->files[$src] ) ) {
+ if ( empty( $params['ignoreMissingSource'] ) ) {
+ $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
+ }
+
+ return $status;
+ }
+
+ $this->files[$dst] = array(
+ 'data' => $this->files[$src]['data'],
+ 'mtime' => wfTimestamp( TS_MW, time() )
+ );
+
+ return $status;
+ }
+
+ protected function doDeleteInternal( array $params ) {
+ $status = Status::newGood();
+
+ $src = $this->resolveHashKey( $params['src'] );
+ if ( $src === null ) {
+ $status->fatal( 'backend-fail-invalidpath', $params['src'] );
+
+ return $status;
+ }
+
+ if ( !isset( $this->files[$src] ) ) {
+ if ( empty( $params['ignoreMissingSource'] ) ) {
+ $status->fatal( 'backend-fail-delete', $params['src'] );
+ }
+
+ return $status;
+ }
+
+ unset( $this->files[$src] );
+
+ return $status;
+ }
+
+ protected function doGetFileStat( array $params ) {
+ $src = $this->resolveHashKey( $params['src'] );
+ if ( $src === null ) {
+ return null;
+ }
+
+ if ( isset( $this->files[$src] ) ) {
+ return array(
+ 'mtime' => $this->files[$src]['mtime'],
+ 'size' => strlen( $this->files[$src]['data'] ),
+ );
+ }
+
+ return false;
+ }
+
+ protected function doGetLocalCopyMulti( array $params ) {
+ $tmpFiles = array(); // (path => TempFSFile)
+ foreach ( $params['srcs'] as $srcPath ) {
+ $src = $this->resolveHashKey( $srcPath );
+ if ( $src === null || !isset( $this->files[$src] ) ) {
+ $fsFile = null;
+ } else {
+ // Create a new temporary file with the same extension...
+ $ext = FileBackend::extensionFromPath( $src );
+ $fsFile = TempFSFile::factory( 'localcopy_', $ext );
+ if ( $fsFile ) {
+ $bytes = file_put_contents( $fsFile->getPath(), $this->files[$src]['data'] );
+ if ( $bytes !== strlen( $this->files[$src]['data'] ) ) {
+ $fsFile = null;
+ }
+ }
+ }
+ $tmpFiles[$srcPath] = $fsFile;
+ }
+
+ return $tmpFiles;
+ }
+
+ protected function doStreamFile( array $params ) {
+ $status = Status::newGood();
+
+ $src = $this->resolveHashKey( $params['src'] );
+ if ( $src === null || !isset( $this->files[$src] ) ) {
+ $status->fatal( 'backend-fail-stream', $params['src'] );
+
+ return $status;
+ }
+
+ print $this->files[$src]['data'];
+
+ return $status;
+ }
+
+ protected function doDirectoryExists( $container, $dir, array $params ) {
+ $prefix = rtrim( "$container/$dir", '/' ) . '/';
+ foreach ( $this->files as $path => $data ) {
+ if ( strpos( $path, $prefix ) === 0 ) {
+ return true;
+ }
+ }
+
+ return false;
+ }
+
+ public function getDirectoryListInternal( $container, $dir, array $params ) {
+ $dirs = array();
+ $prefix = rtrim( "$container/$dir", '/' ) . '/';
+ $prefixLen = strlen( $prefix );
+ foreach ( $this->files as $path => $data ) {
+ if ( strpos( $path, $prefix ) === 0 ) {
+ $relPath = substr( $path, $prefixLen );
+ if ( $relPath === false ) {
+ continue;
+ } elseif ( strpos( $relPath, '/' ) === false ) {
+ continue; // just a file
+ }
+ $parts = array_slice( explode( '/', $relPath ), 0, -1 ); // last part is file name
+ if ( !empty( $params['topOnly'] ) ) {
+ $dirs[$parts[0]] = 1; // top directory
+ } else {
+ $current = '';
+ foreach ( $parts as $part ) { // all directories
+ $dir = ( $current === '' ) ? $part : "$current/$part";
+ $dirs[$dir] = 1;
+ $current = $dir;
+ }
+ }
+ }
+ }
+
+ return array_keys( $dirs );
+ }
+
+ public function getFileListInternal( $container, $dir, array $params ) {
+ $files = array();
+ $prefix = rtrim( "$container/$dir", '/' ) . '/';
+ $prefixLen = strlen( $prefix );
+ foreach ( $this->files as $path => $data ) {
+ if ( strpos( $path, $prefix ) === 0 ) {
+ $relPath = substr( $path, $prefixLen );
+ if ( $relPath === false ) {
+ continue;
+ } elseif ( !empty( $params['topOnly'] ) && strpos( $relPath, '/' ) !== false ) {
+ continue;
+ }
+ $files[] = $relPath;
+ }
+ }
+
+ return $files;
+ }
+
+ protected function directoriesAreVirtual() {
+ return true;
+ }
+
+ /**
+ * Get the absolute file system path for a storage path
+ *
+ * @param string $storagePath Storage path
+ * @return string|null
+ */
+ protected function resolveHashKey( $storagePath ) {
+ list( $fullCont, $relPath ) = $this->resolveStoragePathReal( $storagePath );
+ if ( $relPath === null ) {
+ return null; // invalid
+ }
+
+ return ( $relPath !== '' ) ? "$fullCont/$relPath" : $fullCont;
+ }
+}
diff --git a/includes/filebackend/README b/includes/filebackend/README
index 569f3376..c06f6fc7 100644
--- a/includes/filebackend/README
+++ b/includes/filebackend/README
@@ -51,7 +51,7 @@ On files:
* read a file into a string or several files into a map of path names to strings
* download a file or set of files to a temporary file (on a mounted file system)
* get the SHA1 hash of a file
-* get various properties of a file (stat information, content time, mime information, ...)
+* get various properties of a file (stat information, content time, MIME information, ...)
On directories:
* get a list of files directly under a directory
diff --git a/includes/filebackend/SwiftFileBackend.php b/includes/filebackend/SwiftFileBackend.php
index db090a98..f40ec46e 100644
--- a/includes/filebackend/SwiftFileBackend.php
+++ b/includes/filebackend/SwiftFileBackend.php
@@ -26,10 +26,6 @@
/**
* @brief Class for an OpenStack Swift (or Ceph RGW) based file backend.
*
- * This requires the SwiftCloudFiles MediaWiki extension, which includes
- * the php-cloudfiles library (https://github.com/rackspace/php-cloudfiles).
- * php-cloudfiles requires the curl, fileinfo, and mb_string PHP extensions.
- *
* Status messages should avoid mentioning the Swift account name.
* Likewise, error suppression should be used to avoid path disclosure.
*
@@ -37,32 +33,47 @@
* @since 1.19
*/
class SwiftFileBackend extends FileBackendStore {
- /** @var CF_Authentication */
- protected $auth; // Swift authentication handler
- protected $authTTL; // integer seconds
- protected $swiftTempUrlKey; // string; shared secret value for making temp urls
- protected $swiftAnonUser; // string; username to handle unauthenticated requests
- protected $swiftUseCDN; // boolean; whether CloudFiles CDN is enabled
- protected $swiftCDNExpiry; // integer; how long to cache things in the CDN
- protected $swiftCDNPurgable; // boolean; whether object CDN purging is enabled
-
- // Rados Gateway specific options
- protected $rgwS3AccessKey; // string; S3 access key
- protected $rgwS3SecretKey; // string; S3 authentication key
-
- /** @var CF_Connection */
- protected $conn; // Swift connection handle
- protected $sessionStarted = 0; // integer UNIX timestamp
-
- /** @var CloudFilesException */
- protected $connException;
- protected $connErrorTime = 0; // UNIX timestamp
+ /** @var MultiHttpClient */
+ protected $http;
+
+ /** @var int TTL in seconds */
+ protected $authTTL;
+
+ /** @var string Authentication base URL (without version) */
+ protected $swiftAuthUrl;
+
+ /** @var string Swift user (account:user) to authenticate as */
+ protected $swiftUser;
+
+ /** @var string Secret key for user */
+ protected $swiftKey;
+
+ /** @var string Shared secret value for making temp URLs */
+ protected $swiftTempUrlKey;
+
+ /** @var string S3 access key (RADOS Gateway) */
+ protected $rgwS3AccessKey;
+
+ /** @var string S3 authentication key (RADOS Gateway) */
+ protected $rgwS3SecretKey;
/** @var BagOStuff */
protected $srvCache;
- /** @var ProcessCacheLRU */
- protected $connContainerCache; // container object cache
+ /** @var ProcessCacheLRU Container stat cache */
+ protected $containerStatCache;
+
+ /** @var array */
+ protected $authCreds;
+
+ /** @var int UNIX timestamp */
+ protected $authSessionTimestamp = 0;
+
+ /** @var int UNIX timestamp */
+ protected $authErrorTimestamp = null;
+
+ /** @var bool Whether the server is an Ceph RGW */
+ protected $isRGW = false;
/**
* @see FileBackendStore::__construct()
@@ -73,16 +84,6 @@ class SwiftFileBackend extends FileBackendStore {
* - swiftAuthTTL : Swift authentication TTL (seconds)
* - swiftTempUrlKey : Swift "X-Account-Meta-Temp-URL-Key" value on the account.
* Do not set this until it has been set in the backend.
- * - swiftAnonUser : Swift user used for end-user requests (account:username).
- * If set, then views of public containers are assumed to go
- * through this user. If not set, then public containers are
- * accessible to unauthenticated requests via ".r:*" in the ACL.
- * - swiftUseCDN : Whether a Cloud Files Content Delivery Network is set up
- * - swiftCDNExpiry : How long (in seconds) to store content in the CDN.
- * If files may likely change, this should probably not exceed
- * a few days. For example, deletions may take this long to apply.
- * If object purging is enabled, however, this is not an issue.
- * - swiftCDNPurgable : Whether object purge requests are allowed by the CDN.
* - shardViaHashLevels : Map of container names to sharding config with:
* - base : base of hash characters, 16 or 36
* - levels : the number of hash levels (and digits)
@@ -91,12 +92,12 @@ class SwiftFileBackend extends FileBackendStore {
* - cacheAuthInfo : Whether to cache authentication tokens in APC, XCache, ect.
* If those are not available, then the main cache will be used.
* This is probably insecure in shared hosting environments.
- * - rgwS3AccessKey : Ragos Gateway S3 "access key" value on the account.
+ * - rgwS3AccessKey : Rados Gateway S3 "access key" value on the account.
* Do not set this until it has been set in the backend.
* This is used for generating expiring pre-authenticated URLs.
* Only use this when using rgw and to work around
* http://tracker.newdream.net/issues/3454.
- * - rgwS3SecretKey : Ragos Gateway S3 "secret key" value on the account.
+ * - rgwS3SecretKey : Rados Gateway S3 "secret key" value on the account.
* Do not set this until it has been set in the backend.
* This is used for generating expiring pre-authenticated URLs.
* Only use this when using rgw and to work around
@@ -104,48 +105,32 @@ class SwiftFileBackend extends FileBackendStore {
*/
public function __construct( array $config ) {
parent::__construct( $config );
- if ( !class_exists( 'CF_Constants' ) ) {
- throw new MWException( 'SwiftCloudFiles extension not installed.' );
- }
// Required settings
- $this->auth = new CF_Authentication(
- $config['swiftUser'],
- $config['swiftKey'],
- null, // account; unused
- $config['swiftAuthUrl']
- );
+ $this->swiftAuthUrl = $config['swiftAuthUrl'];
+ $this->swiftUser = $config['swiftUser'];
+ $this->swiftKey = $config['swiftKey'];
// Optional settings
$this->authTTL = isset( $config['swiftAuthTTL'] )
? $config['swiftAuthTTL']
: 5 * 60; // some sane number
- $this->swiftAnonUser = isset( $config['swiftAnonUser'] )
- ? $config['swiftAnonUser']
- : '';
$this->swiftTempUrlKey = isset( $config['swiftTempUrlKey'] )
? $config['swiftTempUrlKey']
: '';
$this->shardViaHashLevels = isset( $config['shardViaHashLevels'] )
? $config['shardViaHashLevels']
: '';
- $this->swiftUseCDN = isset( $config['swiftUseCDN'] )
- ? $config['swiftUseCDN']
- : false;
- $this->swiftCDNExpiry = isset( $config['swiftCDNExpiry'] )
- ? $config['swiftCDNExpiry']
- : 12 * 3600; // 12 hours is safe (tokens last 24 hours per http://docs.openstack.org)
- $this->swiftCDNPurgable = isset( $config['swiftCDNPurgable'] )
- ? $config['swiftCDNPurgable']
- : true;
$this->rgwS3AccessKey = isset( $config['rgwS3AccessKey'] )
? $config['rgwS3AccessKey']
: '';
$this->rgwS3SecretKey = isset( $config['rgwS3SecretKey'] )
? $config['rgwS3SecretKey']
: '';
+ // HTTP helper client
+ $this->http = new MultiHttpClient( array() );
// Cache container information to mask latency
$this->memCache = wfGetMainCache();
// Process cache for container info
- $this->connContainerCache = new ProcessCacheLRU( 300 );
+ $this->containerStatCache = new ProcessCacheLRU( 300 );
// Cache auth token information to avoid RTTs
if ( !empty( $config['cacheAuthInfo'] ) ) {
if ( PHP_SAPI === 'cli' ) {
@@ -153,22 +138,25 @@ class SwiftFileBackend extends FileBackendStore {
} else {
try { // look for APC, XCache, WinCache, ect...
$this->srvCache = ObjectCache::newAccelerator( array() );
- } catch ( Exception $e ) {}
+ } catch ( Exception $e ) {
+ }
}
}
- $this->srvCache = $this->srvCache ? $this->srvCache : new EmptyBagOStuff();
+ $this->srvCache = $this->srvCache ?: new EmptyBagOStuff();
+ }
+
+ public function getFeatures() {
+ return ( FileBackend::ATTR_UNICODE_PATHS |
+ FileBackend::ATTR_HEADERS | FileBackend::ATTR_METADATA );
}
- /**
- * @see FileBackendStore::resolveContainerPath()
- * @return null
- */
protected function resolveContainerPath( $container, $relStoragePath ) {
if ( !mb_check_encoding( $relStoragePath, 'UTF-8' ) ) { // mb_string required by CF
return null; // not UTF-8, makes it hard to use CF and the swift HTTP API
} elseif ( strlen( urlencode( $relStoragePath ) ) > 1024 ) {
return null; // too long for Swift
}
+
return $relStoragePath;
}
@@ -178,45 +166,48 @@ class SwiftFileBackend extends FileBackendStore {
return false; // invalid
}
- try {
- $this->getContainer( $container );
- return true; // container exists
- } catch ( NoSuchContainerException $e ) {
- } catch ( CloudFilesException $e ) { // some other exception?
- $this->handleException( $e, null, __METHOD__, array( 'path' => $storagePath ) );
- }
-
- return false;
+ return is_array( $this->getContainerStat( $container ) );
}
/**
- * @param array $headers
- * @return array
+ * Sanitize and filter the custom headers from a $params array.
+ * We only allow certain Content- and X-Content- headers.
+ *
+ * @param array $params
+ * @return array Sanitized value of 'headers' field in $params
*/
- protected function sanitizeHdrs( array $headers ) {
- // By default, Swift has annoyingly low maximum header value limits
- if ( isset( $headers['Content-Disposition'] ) ) {
- $headers['Content-Disposition'] = $this->truncDisp( $headers['Content-Disposition'] );
+ protected function sanitizeHdrs( array $params ) {
+ $headers = array();
+
+ // Normalize casing, and strip out illegal headers
+ if ( isset( $params['headers'] ) ) {
+ foreach ( $params['headers'] as $name => $value ) {
+ $name = strtolower( $name );
+ if ( preg_match( '/^content-(type|length)$/', $name ) ) {
+ continue; // blacklisted
+ } elseif ( preg_match( '/^(x-)?content-/', $name ) ) {
+ $headers[$name] = $value; // allowed
+ } elseif ( preg_match( '/^content-(disposition)/', $name ) ) {
+ $headers[$name] = $value; // allowed
+ }
+ }
}
- return $headers;
- }
-
- /**
- * @param string $disposition Content-Disposition header value
- * @return string Truncated Content-Disposition header value to meet Swift limits
- */
- protected function truncDisp( $disposition ) {
- $res = '';
- foreach ( explode( ';', $disposition ) as $part ) {
- $part = trim( $part );
- $new = ( $res === '' ) ? $part : "{$res};{$part}";
- if ( strlen( $new ) <= 255 ) {
- $res = $new;
- } else {
- break; // too long; sigh
+ // By default, Swift has annoyingly low maximum header value limits
+ if ( isset( $headers['content-disposition'] ) ) {
+ $disposition = '';
+ foreach ( explode( ';', $headers['content-disposition'] ) as $part ) {
+ $part = trim( $part );
+ $new = ( $disposition === '' ) ? $part : "{$disposition};{$part}";
+ if ( strlen( $new ) <= 255 ) {
+ $disposition = $new;
+ } else {
+ break; // too long; sigh
+ }
}
+ $headers['content-disposition'] = $disposition;
}
- return $res;
+
+ return $headers;
}
protected function doCreateInternal( array $params ) {
@@ -225,152 +216,109 @@ class SwiftFileBackend extends FileBackendStore {
list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
if ( $dstRel === null ) {
$status->fatal( 'backend-fail-invalidpath', $params['dst'] );
- return $status;
- }
- // (a) Check the destination container and object
- try {
- $dContObj = $this->getContainer( $dstCont );
- } catch ( NoSuchContainerException $e ) {
- $status->fatal( 'backend-fail-create', $params['dst'] );
- return $status;
- } catch ( CloudFilesException $e ) { // some other exception?
- $this->handleException( $e, $status, __METHOD__, $params );
return $status;
}
- // (b) Get a SHA-1 hash of the object
$sha1Hash = wfBaseConvert( sha1( $params['content'] ), 16, 36, 31 );
-
- // (c) Actually create the object
- try {
- // Create a fresh CF_Object with no fields preloaded.
- // We don't want to preserve headers, metadata, and such.
- $obj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
- $obj->setMetadataValues( array( 'Sha1base36' => $sha1Hash ) );
- // Manually set the ETag (https://github.com/rackspace/php-cloudfiles/issues/59).
- // The MD5 here will be checked within Swift against its own MD5.
- $obj->set_etag( md5( $params['content'] ) );
- // Use the same content type as StreamFile for security
- $obj->content_type = $this->getContentType( $params['dst'], $params['content'], null );
- // Set any other custom headers if requested
- if ( isset( $params['headers'] ) ) {
- $obj->headers += $this->sanitizeHdrs( $params['headers'] );
- }
- if ( !empty( $params['async'] ) ) { // deferred
- $op = $obj->write_async( $params['content'] );
- $status->value = new SwiftFileOpHandle( $this, $params, 'Create', $op );
- $status->value->affectedObjects[] = $obj;
- } else { // actually write the object in Swift
- $obj->write( $params['content'] );
- $this->purgeCDNCache( array( $obj ) );
+ $contentType = $this->getContentType( $params['dst'], $params['content'], null );
+
+ $reqs = array( array(
+ 'method' => 'PUT',
+ 'url' => array( $dstCont, $dstRel ),
+ 'headers' => array(
+ 'content-length' => strlen( $params['content'] ),
+ 'etag' => md5( $params['content'] ),
+ 'content-type' => $contentType,
+ 'x-object-meta-sha1base36' => $sha1Hash
+ ) + $this->sanitizeHdrs( $params ),
+ 'body' => $params['content']
+ ) );
+
+ $be = $this;
+ $method = __METHOD__;
+ $handler = function ( array $request, Status $status ) use ( $be, $method, $params ) {
+ list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request['response'];
+ if ( $rcode === 201 ) {
+ // good
+ } elseif ( $rcode === 412 ) {
+ $status->fatal( 'backend-fail-contenttype', $params['dst'] );
+ } else {
+ $be->onError( $status, $method, $params, $rerr, $rcode, $rdesc );
}
- } catch ( CDNNotEnabledException $e ) {
- // CDN not enabled; nothing to see here
- } catch ( BadContentTypeException $e ) {
- $status->fatal( 'backend-fail-contenttype', $params['dst'] );
- } catch ( CloudFilesException $e ) { // some other exception?
- $this->handleException( $e, $status, __METHOD__, $params );
+ };
+
+ $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs );
+ if ( !empty( $params['async'] ) ) { // deferred
+ $status->value = $opHandle;
+ } else { // actually write the object in Swift
+ $status->merge( current( $this->doExecuteOpHandlesInternal( array( $opHandle ) ) ) );
}
return $status;
}
- /**
- * @see SwiftFileBackend::doExecuteOpHandlesInternal()
- */
- protected function _getResponseCreate( CF_Async_Op $cfOp, Status $status, array $params ) {
- try {
- $cfOp->getLastResponse();
- } catch ( BadContentTypeException $e ) {
- $status->fatal( 'backend-fail-contenttype', $params['dst'] );
- }
- }
-
protected function doStoreInternal( array $params ) {
$status = Status::newGood();
list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
if ( $dstRel === null ) {
$status->fatal( 'backend-fail-invalidpath', $params['dst'] );
- return $status;
- }
- // (a) Check the destination container and object
- try {
- $dContObj = $this->getContainer( $dstCont );
- } catch ( NoSuchContainerException $e ) {
- $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
- return $status;
- } catch ( CloudFilesException $e ) { // some other exception?
- $this->handleException( $e, $status, __METHOD__, $params );
return $status;
}
- // (b) Get a SHA-1 hash of the object
wfSuppressWarnings();
$sha1Hash = sha1_file( $params['src'] );
wfRestoreWarnings();
if ( $sha1Hash === false ) { // source doesn't exist?
- $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
+ $status->fatal( 'backend-fail-store', $params['src'], $params['dst'] );
+
return $status;
}
$sha1Hash = wfBaseConvert( $sha1Hash, 16, 36, 31 );
+ $contentType = $this->getContentType( $params['dst'], null, $params['src'] );
- // (c) Actually store the object
- try {
- // Create a fresh CF_Object with no fields preloaded.
- // We don't want to preserve headers, metadata, and such.
- $obj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
- $obj->setMetadataValues( array( 'Sha1base36' => $sha1Hash ) );
- // The MD5 here will be checked within Swift against its own MD5.
- $obj->set_etag( md5_file( $params['src'] ) );
- // Use the same content type as StreamFile for security
- $obj->content_type = $this->getContentType( $params['dst'], null, $params['src'] );
- // Set any other custom headers if requested
- if ( isset( $params['headers'] ) ) {
- $obj->headers += $this->sanitizeHdrs( $params['headers'] );
- }
- if ( !empty( $params['async'] ) ) { // deferred
- wfSuppressWarnings();
- $fp = fopen( $params['src'], 'rb' );
- wfRestoreWarnings();
- if ( !$fp ) {
- $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
- } else {
- $op = $obj->write_async( $fp, filesize( $params['src'] ), true );
- $status->value = new SwiftFileOpHandle( $this, $params, 'Store', $op );
- $status->value->resourcesToClose[] = $fp;
- $status->value->affectedObjects[] = $obj;
- }
- } else { // actually write the object in Swift
- $obj->load_from_filename( $params['src'], true ); // calls $obj->write()
- $this->purgeCDNCache( array( $obj ) );
- }
- } catch ( CDNNotEnabledException $e ) {
- // CDN not enabled; nothing to see here
- } catch ( BadContentTypeException $e ) {
- $status->fatal( 'backend-fail-contenttype', $params['dst'] );
- } catch ( IOException $e ) {
- $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
- } catch ( CloudFilesException $e ) { // some other exception?
- $this->handleException( $e, $status, __METHOD__, $params );
+ $handle = fopen( $params['src'], 'rb' );
+ if ( $handle === false ) { // source doesn't exist?
+ $status->fatal( 'backend-fail-store', $params['src'], $params['dst'] );
+
+ return $status;
}
- return $status;
- }
+ $reqs = array( array(
+ 'method' => 'PUT',
+ 'url' => array( $dstCont, $dstRel ),
+ 'headers' => array(
+ 'content-length' => filesize( $params['src'] ),
+ 'etag' => md5_file( $params['src'] ),
+ 'content-type' => $contentType,
+ 'x-object-meta-sha1base36' => $sha1Hash
+ ) + $this->sanitizeHdrs( $params ),
+ 'body' => $handle // resource
+ ) );
+
+ $be = $this;
+ $method = __METHOD__;
+ $handler = function ( array $request, Status $status ) use ( $be, $method, $params ) {
+ list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request['response'];
+ if ( $rcode === 201 ) {
+ // good
+ } elseif ( $rcode === 412 ) {
+ $status->fatal( 'backend-fail-contenttype', $params['dst'] );
+ } else {
+ $be->onError( $status, $method, $params, $rerr, $rcode, $rdesc );
+ }
+ };
- /**
- * @see SwiftFileBackend::doExecuteOpHandlesInternal()
- */
- protected function _getResponseStore( CF_Async_Op $cfOp, Status $status, array $params ) {
- try {
- $cfOp->getLastResponse();
- } catch ( BadContentTypeException $e ) {
- $status->fatal( 'backend-fail-contenttype', $params['dst'] );
- } catch ( IOException $e ) {
- $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
+ $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs );
+ if ( !empty( $params['async'] ) ) { // deferred
+ $status->value = $opHandle;
+ } else { // actually write the object in Swift
+ $status->merge( current( $this->doExecuteOpHandlesInternal( array( $opHandle ) ) ) );
}
+
+ return $status;
}
protected function doCopyInternal( array $params ) {
@@ -379,221 +327,202 @@ class SwiftFileBackend extends FileBackendStore {
list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
if ( $srcRel === null ) {
$status->fatal( 'backend-fail-invalidpath', $params['src'] );
+
return $status;
}
list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
if ( $dstRel === null ) {
$status->fatal( 'backend-fail-invalidpath', $params['dst'] );
- return $status;
- }
- // (a) Check the source/destination containers and destination object
- try {
- $sContObj = $this->getContainer( $srcCont );
- $dContObj = $this->getContainer( $dstCont );
- } catch ( NoSuchContainerException $e ) {
- if ( empty( $params['ignoreMissingSource'] ) || isset( $sContObj ) ) {
- $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
- }
- return $status;
- } catch ( CloudFilesException $e ) { // some other exception?
- $this->handleException( $e, $status, __METHOD__, $params );
return $status;
}
- // (b) Actually copy the file to the destination
- try {
- $dstObj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
- $hdrs = array(); // source file headers to override with new values
- // Set any other custom headers if requested
- if ( isset( $params['headers'] ) ) {
- $hdrs += $this->sanitizeHdrs( $params['headers'] );
- }
- if ( !empty( $params['async'] ) ) { // deferred
- $op = $sContObj->copy_object_to_async( $srcRel, $dContObj, $dstRel, null, $hdrs );
- $status->value = new SwiftFileOpHandle( $this, $params, 'Copy', $op );
- $status->value->affectedObjects[] = $dstObj;
- } else { // actually write the object in Swift
- $sContObj->copy_object_to( $srcRel, $dContObj, $dstRel, null, $hdrs );
- $this->purgeCDNCache( array( $dstObj ) );
- }
- } catch ( CDNNotEnabledException $e ) {
- // CDN not enabled; nothing to see here
- } catch ( NoSuchObjectException $e ) { // source object does not exist
- if ( empty( $params['ignoreMissingSource'] ) ) {
+ $reqs = array( array(
+ 'method' => 'PUT',
+ 'url' => array( $dstCont, $dstRel ),
+ 'headers' => array(
+ 'x-copy-from' => '/' . rawurlencode( $srcCont ) .
+ '/' . str_replace( "%2F", "/", rawurlencode( $srcRel ) )
+ ) + $this->sanitizeHdrs( $params ), // extra headers merged into object
+ ) );
+
+ $be = $this;
+ $method = __METHOD__;
+ $handler = function ( array $request, Status $status ) use ( $be, $method, $params ) {
+ list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request['response'];
+ if ( $rcode === 201 ) {
+ // good
+ } elseif ( $rcode === 404 ) {
$status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
+ } else {
+ $be->onError( $status, $method, $params, $rerr, $rcode, $rdesc );
}
- } catch ( CloudFilesException $e ) { // some other exception?
- $this->handleException( $e, $status, __METHOD__, $params );
+ };
+
+ $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs );
+ if ( !empty( $params['async'] ) ) { // deferred
+ $status->value = $opHandle;
+ } else { // actually write the object in Swift
+ $status->merge( current( $this->doExecuteOpHandlesInternal( array( $opHandle ) ) ) );
}
return $status;
}
- /**
- * @see SwiftFileBackend::doExecuteOpHandlesInternal()
- */
- protected function _getResponseCopy( CF_Async_Op $cfOp, Status $status, array $params ) {
- try {
- $cfOp->getLastResponse();
- } catch ( NoSuchObjectException $e ) { // source object does not exist
- $status->fatal( 'backend-fail-copy', $params['src'], $params['dst'] );
- }
- }
-
protected function doMoveInternal( array $params ) {
$status = Status::newGood();
list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
if ( $srcRel === null ) {
$status->fatal( 'backend-fail-invalidpath', $params['src'] );
+
return $status;
}
list( $dstCont, $dstRel ) = $this->resolveStoragePathReal( $params['dst'] );
if ( $dstRel === null ) {
$status->fatal( 'backend-fail-invalidpath', $params['dst'] );
+
return $status;
}
- // (a) Check the source/destination containers and destination object
- try {
- $sContObj = $this->getContainer( $srcCont );
- $dContObj = $this->getContainer( $dstCont );
- } catch ( NoSuchContainerException $e ) {
- if ( empty( $params['ignoreMissingSource'] ) || isset( $sContObj ) ) {
- $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
- }
- return $status;
- } catch ( CloudFilesException $e ) { // some other exception?
- $this->handleException( $e, $status, __METHOD__, $params );
- return $status;
+ $reqs = array(
+ array(
+ 'method' => 'PUT',
+ 'url' => array( $dstCont, $dstRel ),
+ 'headers' => array(
+ 'x-copy-from' => '/' . rawurlencode( $srcCont ) .
+ '/' . str_replace( "%2F", "/", rawurlencode( $srcRel ) )
+ ) + $this->sanitizeHdrs( $params ) // extra headers merged into object
+ )
+ );
+ if ( "{$srcCont}/{$srcRel}" !== "{$dstCont}/{$dstRel}" ) {
+ $reqs[] = array(
+ 'method' => 'DELETE',
+ 'url' => array( $srcCont, $srcRel ),
+ 'headers' => array()
+ );
}
- // (b) Actually move the file to the destination
- try {
- $srcObj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
- $dstObj = new CF_Object( $dContObj, $dstRel, false, false ); // skip HEAD
- $hdrs = array(); // source file headers to override with new values
- // Set any other custom headers if requested
- if ( isset( $params['headers'] ) ) {
- $hdrs += $this->sanitizeHdrs( $params['headers'] );
- }
- if ( !empty( $params['async'] ) ) { // deferred
- $op = $sContObj->move_object_to_async( $srcRel, $dContObj, $dstRel, null, $hdrs );
- $status->value = new SwiftFileOpHandle( $this, $params, 'Move', $op );
- $status->value->affectedObjects[] = $srcObj;
- $status->value->affectedObjects[] = $dstObj;
- } else { // actually write the object in Swift
- $sContObj->move_object_to( $srcRel, $dContObj, $dstRel, null, $hdrs );
- $this->purgeCDNCache( array( $srcObj ) );
- $this->purgeCDNCache( array( $dstObj ) );
- }
- } catch ( CDNNotEnabledException $e ) {
- // CDN not enabled; nothing to see here
- } catch ( NoSuchObjectException $e ) { // source object does not exist
- if ( empty( $params['ignoreMissingSource'] ) ) {
+ $be = $this;
+ $method = __METHOD__;
+ $handler = function ( array $request, Status $status ) use ( $be, $method, $params ) {
+ list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request['response'];
+ if ( $request['method'] === 'PUT' && $rcode === 201 ) {
+ // good
+ } elseif ( $request['method'] === 'DELETE' && $rcode === 204 ) {
+ // good
+ } elseif ( $rcode === 404 ) {
$status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
+ } else {
+ $be->onError( $status, $method, $params, $rerr, $rcode, $rdesc );
}
- } catch ( CloudFilesException $e ) { // some other exception?
- $this->handleException( $e, $status, __METHOD__, $params );
+ };
+
+ $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs );
+ if ( !empty( $params['async'] ) ) { // deferred
+ $status->value = $opHandle;
+ } else { // actually move the object in Swift
+ $status->merge( current( $this->doExecuteOpHandlesInternal( array( $opHandle ) ) ) );
}
return $status;
}
- /**
- * @see SwiftFileBackend::doExecuteOpHandlesInternal()
- */
- protected function _getResponseMove( CF_Async_Op $cfOp, Status $status, array $params ) {
- try {
- $cfOp->getLastResponse();
- } catch ( NoSuchObjectException $e ) { // source object does not exist
- $status->fatal( 'backend-fail-move', $params['src'], $params['dst'] );
- }
- }
-
protected function doDeleteInternal( array $params ) {
$status = Status::newGood();
list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
if ( $srcRel === null ) {
$status->fatal( 'backend-fail-invalidpath', $params['src'] );
+
return $status;
}
- try {
- $sContObj = $this->getContainer( $srcCont );
- $srcObj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
- if ( !empty( $params['async'] ) ) { // deferred
- $op = $sContObj->delete_object_async( $srcRel );
- $status->value = new SwiftFileOpHandle( $this, $params, 'Delete', $op );
- $status->value->affectedObjects[] = $srcObj;
- } else { // actually write the object in Swift
- $sContObj->delete_object( $srcRel );
- $this->purgeCDNCache( array( $srcObj ) );
- }
- } catch ( CDNNotEnabledException $e ) {
- // CDN not enabled; nothing to see here
- } catch ( NoSuchContainerException $e ) {
- if ( empty( $params['ignoreMissingSource'] ) ) {
- $status->fatal( 'backend-fail-delete', $params['src'] );
- }
- } catch ( NoSuchObjectException $e ) {
- if ( empty( $params['ignoreMissingSource'] ) ) {
- $status->fatal( 'backend-fail-delete', $params['src'] );
+ $reqs = array( array(
+ 'method' => 'DELETE',
+ 'url' => array( $srcCont, $srcRel ),
+ 'headers' => array()
+ ) );
+
+ $be = $this;
+ $method = __METHOD__;
+ $handler = function ( array $request, Status $status ) use ( $be, $method, $params ) {
+ list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request['response'];
+ if ( $rcode === 204 ) {
+ // good
+ } elseif ( $rcode === 404 ) {
+ if ( empty( $params['ignoreMissingSource'] ) ) {
+ $status->fatal( 'backend-fail-delete', $params['src'] );
+ }
+ } else {
+ $be->onError( $status, $method, $params, $rerr, $rcode, $rdesc );
}
- } catch ( CloudFilesException $e ) { // some other exception?
- $this->handleException( $e, $status, __METHOD__, $params );
+ };
+
+ $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs );
+ if ( !empty( $params['async'] ) ) { // deferred
+ $status->value = $opHandle;
+ } else { // actually delete the object in Swift
+ $status->merge( current( $this->doExecuteOpHandlesInternal( array( $opHandle ) ) ) );
}
return $status;
}
- /**
- * @see SwiftFileBackend::doExecuteOpHandlesInternal()
- */
- protected function _getResponseDelete( CF_Async_Op $cfOp, Status $status, array $params ) {
- try {
- $cfOp->getLastResponse();
- } catch ( NoSuchContainerException $e ) {
- $status->fatal( 'backend-fail-delete', $params['src'] );
- } catch ( NoSuchObjectException $e ) {
- if ( empty( $params['ignoreMissingSource'] ) ) {
- $status->fatal( 'backend-fail-delete', $params['src'] );
- }
- }
- }
-
protected function doDescribeInternal( array $params ) {
$status = Status::newGood();
list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
if ( $srcRel === null ) {
$status->fatal( 'backend-fail-invalidpath', $params['src'] );
+
return $status;
}
- try {
- $sContObj = $this->getContainer( $srcCont );
- // Get the latest version of the current metadata
- $srcObj = $sContObj->get_object( $srcRel,
- $this->headersFromParams( array( 'latest' => true ) ) );
- // Merge in the metadata updates...
- if ( isset( $params['headers'] ) ) {
- $srcObj->headers = $this->sanitizeHdrs( $params['headers'] ) + $srcObj->headers;
- }
- $srcObj->sync_metadata(); // save to Swift
- $this->purgeCDNCache( array( $srcObj ) );
- } catch ( CDNNotEnabledException $e ) {
- // CDN not enabled; nothing to see here
- } catch ( NoSuchContainerException $e ) {
- $status->fatal( 'backend-fail-describe', $params['src'] );
- } catch ( NoSuchObjectException $e ) {
+ // Fetch the old object headers/metadata...this should be in stat cache by now
+ $stat = $this->getFileStat( array( 'src' => $params['src'], 'latest' => 1 ) );
+ if ( $stat && !isset( $stat['xattr'] ) ) { // older cache entry
+ $stat = $this->doGetFileStat( array( 'src' => $params['src'], 'latest' => 1 ) );
+ }
+ if ( !$stat ) {
$status->fatal( 'backend-fail-describe', $params['src'] );
- } catch ( CloudFilesException $e ) { // some other exception?
- $this->handleException( $e, $status, __METHOD__, $params );
+
+ return $status;
+ }
+
+ // POST clears prior headers, so we need to merge the changes in to the old ones
+ $metaHdrs = array();
+ foreach ( $stat['xattr']['metadata'] as $name => $value ) {
+ $metaHdrs["x-object-meta-$name"] = $value;
+ }
+ $customHdrs = $this->sanitizeHdrs( $params ) + $stat['xattr']['headers'];
+
+ $reqs = array( array(
+ 'method' => 'POST',
+ 'url' => array( $srcCont, $srcRel ),
+ 'headers' => $metaHdrs + $customHdrs
+ ) );
+
+ $be = $this;
+ $method = __METHOD__;
+ $handler = function ( array $request, Status $status ) use ( $be, $method, $params ) {
+ list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $request['response'];
+ if ( $rcode === 202 ) {
+ // good
+ } elseif ( $rcode === 404 ) {
+ $status->fatal( 'backend-fail-describe', $params['src'] );
+ } else {
+ $be->onError( $status, $method, $params, $rerr, $rcode, $rdesc );
+ }
+ };
+
+ $opHandle = new SwiftFileOpHandle( $this, $handler, $reqs );
+ if ( !empty( $params['async'] ) ) { // deferred
+ $status->value = $opHandle;
+ } else { // actually change the object in Swift
+ $status->merge( current( $this->doExecuteOpHandlesInternal( array( $opHandle ) ) ) );
}
return $status;
@@ -603,110 +532,62 @@ class SwiftFileBackend extends FileBackendStore {
$status = Status::newGood();
// (a) Check if container already exists
- try {
- $this->getContainer( $fullCont );
- // NoSuchContainerException not thrown: container must exist
- return $status; // already exists
- } catch ( NoSuchContainerException $e ) {
- // NoSuchContainerException thrown: container does not exist
- } catch ( CloudFilesException $e ) { // some other exception?
- $this->handleException( $e, $status, __METHOD__, $params );
+ $stat = $this->getContainerStat( $fullCont );
+ if ( is_array( $stat ) ) {
+ return $status; // already there
+ } elseif ( $stat === null ) {
+ $status->fatal( 'backend-fail-internal', $this->name );
+
return $status;
}
- // (b) Create container as needed
- try {
- $contObj = $this->createContainer( $fullCont );
- if ( !empty( $params['noAccess'] ) ) {
- // Make container private to end-users...
- $status->merge( $this->doSecureInternal( $fullCont, $dir, $params ) );
- } else {
- // Make container public to end-users...
- $status->merge( $this->doPublishInternal( $fullCont, $dir, $params ) );
- }
- if ( $this->swiftUseCDN ) { // Rackspace style CDN
- $contObj->make_public( $this->swiftCDNExpiry );
- }
- } catch ( CDNNotEnabledException $e ) {
- // CDN not enabled; nothing to see here
- } catch ( CloudFilesException $e ) { // some other exception?
- $this->handleException( $e, $status, __METHOD__, $params );
- return $status;
+ // (b) Create container as needed with proper ACLs
+ if ( $stat === false ) {
+ $params['op'] = 'prepare';
+ $status->merge( $this->createContainer( $fullCont, $params ) );
}
return $status;
}
- /**
- * @see FileBackendStore::doSecureInternal()
- * @return Status
- */
protected function doSecureInternal( $fullCont, $dir, array $params ) {
$status = Status::newGood();
if ( empty( $params['noAccess'] ) ) {
return $status; // nothing to do
}
- // Restrict container from end-users...
- try {
- // doPrepareInternal() should have been called,
- // so the Swift container should already exist...
- $contObj = $this->getContainer( $fullCont ); // normally a cache hit
- // NoSuchContainerException not thrown: container must exist
-
+ $stat = $this->getContainerStat( $fullCont );
+ if ( is_array( $stat ) ) {
// Make container private to end-users...
$status->merge( $this->setContainerAccess(
- $contObj,
- array( $this->auth->username ), // read
- array( $this->auth->username ) // write
+ $fullCont,
+ array( $this->swiftUser ), // read
+ array( $this->swiftUser ) // write
) );
- if ( $this->swiftUseCDN && $contObj->is_public() ) { // Rackspace style CDN
- $contObj->make_private();
- }
- } catch ( CDNNotEnabledException $e ) {
- // CDN not enabled; nothing to see here
- } catch ( CloudFilesException $e ) { // some other exception?
- $this->handleException( $e, $status, __METHOD__, $params );
+ } elseif ( $stat === false ) {
+ $status->fatal( 'backend-fail-usable', $params['dir'] );
+ } else {
+ $status->fatal( 'backend-fail-internal', $this->name );
}
return $status;
}
- /**
- * @see FileBackendStore::doPublishInternal()
- * @return Status
- */
protected function doPublishInternal( $fullCont, $dir, array $params ) {
$status = Status::newGood();
- // Unrestrict container from end-users...
- try {
- // doPrepareInternal() should have been called,
- // so the Swift container should already exist...
- $contObj = $this->getContainer( $fullCont ); // normally a cache hit
- // NoSuchContainerException not thrown: container must exist
-
+ $stat = $this->getContainerStat( $fullCont );
+ if ( is_array( $stat ) ) {
// Make container public to end-users...
- if ( $this->swiftAnonUser != '' ) {
- $status->merge( $this->setContainerAccess(
- $contObj,
- array( $this->auth->username, $this->swiftAnonUser ), // read
- array( $this->auth->username, $this->swiftAnonUser ) // write
- ) );
- } else {
- $status->merge( $this->setContainerAccess(
- $contObj,
- array( $this->auth->username, '.r:*' ), // read
- array( $this->auth->username ) // write
- ) );
- }
- if ( $this->swiftUseCDN && !$contObj->is_public() ) { // Rackspace style CDN
- $contObj->make_public();
- }
- } catch ( CDNNotEnabledException $e ) {
- // CDN not enabled; nothing to see here
- } catch ( CloudFilesException $e ) { // some other exception?
- $this->handleException( $e, $status, __METHOD__, $params );
+ $status->merge( $this->setContainerAccess(
+ $fullCont,
+ array( $this->swiftUser, '.r:*' ), // read
+ array( $this->swiftUser ) // write
+ ) );
+ } elseif ( $stat === false ) {
+ $status->fatal( 'backend-fail-usable', $params['dir'] );
+ } else {
+ $status->fatal( 'backend-fail-internal', $this->name );
}
return $status;
@@ -721,73 +602,74 @@ class SwiftFileBackend extends FileBackendStore {
}
// (a) Check the container
- try {
- $contObj = $this->getContainer( $fullCont, true );
- } catch ( NoSuchContainerException $e ) {
+ $stat = $this->getContainerStat( $fullCont, true );
+ if ( $stat === false ) {
return $status; // ok, nothing to do
- } catch ( CloudFilesException $e ) { // some other exception?
- $this->handleException( $e, $status, __METHOD__, $params );
+ } elseif ( !is_array( $stat ) ) {
+ $status->fatal( 'backend-fail-internal', $this->name );
+
return $status;
}
// (b) Delete the container if empty
- if ( $contObj->object_count == 0 ) {
- try {
- $this->deleteContainer( $fullCont );
- } catch ( NoSuchContainerException $e ) {
- return $status; // race?
- } catch ( NonEmptyContainerException $e ) {
- return $status; // race? consistency delay?
- } catch ( CloudFilesException $e ) { // some other exception?
- $this->handleException( $e, $status, __METHOD__, $params );
- return $status;
- }
+ if ( $stat['count'] == 0 ) {
+ $params['op'] = 'clean';
+ $status->merge( $this->deleteContainer( $fullCont, $params ) );
}
return $status;
}
protected function doGetFileStat( array $params ) {
- list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
- if ( $srcRel === null ) {
- return false; // invalid storage path
- }
+ $params = array( 'srcs' => array( $params['src'] ), 'concurrency' => 1 ) + $params;
+ unset( $params['src'] );
+ $stats = $this->doGetFileStatMulti( $params );
+
+ return reset( $stats );
+ }
- $stat = false;
+ /**
+ * Convert dates like "Tue, 03 Jan 2012 22:01:04 GMT"/"2013-05-11T07:37:27.678360Z".
+ * Dates might also come in like "2013-05-11T07:37:27.678360" from Swift listings,
+ * missing the timezone suffix (though Ceph RGW does not appear to have this bug).
+ *
+ * @param string $ts
+ * @param int $format Output format (TS_* constant)
+ * @return string
+ * @throws FileBackendError
+ */
+ protected function convertSwiftDate( $ts, $format = TS_MW ) {
try {
- $contObj = $this->getContainer( $srcCont );
- $srcObj = $contObj->get_object( $srcRel, $this->headersFromParams( $params ) );
- $this->addMissingMetadata( $srcObj, $params['src'] );
- $stat = array(
- // Convert dates like "Tue, 03 Jan 2012 22:01:04 GMT" to TS_MW
- 'mtime' => wfTimestamp( TS_MW, $srcObj->last_modified ),
- 'size' => (int)$srcObj->content_length,
- 'sha1' => $srcObj->getMetadataValue( 'Sha1base36' )
- );
- } catch ( NoSuchContainerException $e ) {
- } catch ( NoSuchObjectException $e ) {
- } catch ( CloudFilesException $e ) { // some other exception?
- $stat = null;
- $this->handleException( $e, null, __METHOD__, $params );
- }
+ $timestamp = new MWTimestamp( $ts );
- return $stat;
+ return $timestamp->getTimestamp( $format );
+ } catch ( MWException $e ) {
+ throw new FileBackendError( $e->getMessage() );
+ }
}
/**
* Fill in any missing object metadata and save it to Swift
*
- * @param CF_Object $obj
+ * @param array $objHdrs Object response headers
* @param string $path Storage path to object
- * @return bool Success
- * @throws Exception cloudfiles exceptions
+ * @return array New headers
*/
- protected function addMissingMetadata( CF_Object $obj, $path ) {
- if ( $obj->getMetadataValue( 'Sha1base36' ) !== null ) {
- return true; // nothing to do
+ protected function addMissingMetadata( array $objHdrs, $path ) {
+ if ( isset( $objHdrs['x-object-meta-sha1base36'] ) ) {
+ return $objHdrs; // nothing to do
}
- wfProfileIn( __METHOD__ );
+
+ $section = new ProfileSection( __METHOD__ . '-' . $this->name );
trigger_error( "$path was not stored with SHA-1 metadata.", E_USER_WARNING );
+
+ $auth = $this->getAuthentication();
+ if ( !$auth ) {
+ $objHdrs['x-object-meta-sha1base36'] = false;
+
+ return $objHdrs; // failed
+ }
+
$status = Status::newGood();
$scopeLockS = $this->getScopedFileLocks( array( $path ), LockManager::LOCK_UW, $status );
if ( $status->isOK() ) {
@@ -795,101 +677,79 @@ class SwiftFileBackend extends FileBackendStore {
if ( $tmpFile ) {
$hash = $tmpFile->getSha1Base36();
if ( $hash !== false ) {
- $obj->setMetadataValues( array( 'Sha1base36' => $hash ) );
- $obj->sync_metadata(); // save to Swift
- wfProfileOut( __METHOD__ );
- return true; // success
+ $objHdrs['x-object-meta-sha1base36'] = $hash;
+ list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $path );
+ list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( array(
+ 'method' => 'POST',
+ 'url' => $this->storageUrl( $auth, $srcCont, $srcRel ),
+ 'headers' => $this->authTokenHeaders( $auth ) + $objHdrs
+ ) );
+ if ( $rcode >= 200 && $rcode <= 299 ) {
+ return $objHdrs; // success
+ }
}
}
}
trigger_error( "Unable to set SHA-1 metadata for $path", E_USER_WARNING );
- $obj->setMetadataValues( array( 'Sha1base36' => false ) );
- wfProfileOut( __METHOD__ );
- return false; // failed
+ $objHdrs['x-object-meta-sha1base36'] = false;
+
+ return $objHdrs; // failed
}
protected function doGetFileContentsMulti( array $params ) {
$contents = array();
+ $auth = $this->getAuthentication();
+
$ep = array_diff_key( $params, array( 'srcs' => 1 ) ); // for error logging
// Blindly create tmp files and stream to them, catching any exception if the file does
// not exist. Doing stats here is useless and will loop infinitely in addMissingMetadata().
- foreach ( array_chunk( $params['srcs'], $params['concurrency'] ) as $pathBatch ) {
- $cfOps = array(); // (path => CF_Async_Op)
-
- foreach ( $pathBatch as $path ) { // each path in this concurrent batch
- list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $path );
- if ( $srcRel === null ) {
- $contents[$path] = false;
- continue;
- }
- $data = false;
- try {
- $sContObj = $this->getContainer( $srcCont );
- $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
- // Create a new temporary memory file...
- $handle = fopen( 'php://temp', 'wb' );
- if ( $handle ) {
- $headers = $this->headersFromParams( $params );
- if ( count( $pathBatch ) > 1 ) {
- $cfOps[$path] = $obj->stream_async( $handle, $headers );
- $cfOps[$path]->_file_handle = $handle; // close this later
- } else {
- $obj->stream( $handle, $headers );
- rewind( $handle ); // start from the beginning
- $data = stream_get_contents( $handle );
- fclose( $handle );
- }
- } else {
- $data = false;
- }
- } catch ( NoSuchContainerException $e ) {
- $data = false;
- } catch ( NoSuchObjectException $e ) {
- $data = false;
- } catch ( CloudFilesException $e ) { // some other exception?
- $data = false;
- $this->handleException( $e, null, __METHOD__, array( 'src' => $path ) + $ep );
- }
- $contents[$path] = $data;
- }
+ $reqs = array(); // (path => op)
- $batch = new CF_Async_Op_Batch( $cfOps );
- $cfOps = $batch->execute();
- foreach ( $cfOps as $path => $cfOp ) {
- try {
- $cfOp->getLastResponse();
- rewind( $cfOp->_file_handle ); // start from the beginning
- $contents[$path] = stream_get_contents( $cfOp->_file_handle );
- } catch ( NoSuchContainerException $e ) {
- $contents[$path] = false;
- } catch ( NoSuchObjectException $e ) {
- $contents[$path] = false;
- } catch ( CloudFilesException $e ) { // some other exception?
- $contents[$path] = false;
- $this->handleException( $e, null, __METHOD__, array( 'src' => $path ) + $ep );
- }
- fclose( $cfOp->_file_handle ); // close open handle
+ foreach ( $params['srcs'] as $path ) { // each path in this concurrent batch
+ list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $path );
+ if ( $srcRel === null || !$auth ) {
+ $contents[$path] = false;
+ continue;
}
+ // Create a new temporary memory file...
+ $handle = fopen( 'php://temp', 'wb' );
+ if ( $handle ) {
+ $reqs[$path] = array(
+ 'method' => 'GET',
+ 'url' => $this->storageUrl( $auth, $srcCont, $srcRel ),
+ 'headers' => $this->authTokenHeaders( $auth )
+ + $this->headersFromParams( $params ),
+ 'stream' => $handle,
+ );
+ }
+ $contents[$path] = false;
+ }
+
+ $opts = array( 'maxConnsPerHost' => $params['concurrency'] );
+ $reqs = $this->http->runMulti( $reqs, $opts );
+ foreach ( $reqs as $path => $op ) {
+ list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $op['response'];
+ if ( $rcode >= 200 && $rcode <= 299 ) {
+ rewind( $op['stream'] ); // start from the beginning
+ $contents[$path] = stream_get_contents( $op['stream'] );
+ } elseif ( $rcode === 404 ) {
+ $contents[$path] = false;
+ } else {
+ $this->onError( null, __METHOD__,
+ array( 'src' => $path ) + $ep, $rerr, $rcode, $rdesc );
+ }
+ fclose( $op['stream'] ); // close open handle
}
return $contents;
}
- /**
- * @see FileBackendStore::doDirectoryExists()
- * @return bool|null
- */
protected function doDirectoryExists( $fullCont, $dir, array $params ) {
- try {
- $container = $this->getContainer( $fullCont );
- $prefix = ( $dir == '' ) ? null : "{$dir}/";
- return ( count( $container->list_objects( 1, null, $prefix ) ) > 0 );
- } catch ( NoSuchContainerException $e ) {
- return false;
- } catch ( CloudFilesException $e ) { // some other exception?
- $this->handleException( $e, null, __METHOD__,
- array( 'cont' => $fullCont, 'dir' => $dir ) );
+ $prefix = ( $dir == '' ) ? null : "{$dir}/";
+ $status = $this->objectListing( $fullCont, 'names', 1, null, $prefix );
+ if ( $status->isOk() ) {
+ return ( count( $status->value ) ) > 0;
}
return null; // error
@@ -897,6 +757,9 @@ class SwiftFileBackend extends FileBackendStore {
/**
* @see FileBackendStore::getDirectoryListInternal()
+ * @param string $fullCont
+ * @param string $dir
+ * @param array $params
* @return SwiftFileBackendDirList
*/
public function getDirectoryListInternal( $fullCont, $dir, array $params ) {
@@ -905,6 +768,9 @@ class SwiftFileBackend extends FileBackendStore {
/**
* @see FileBackendStore::getFileListInternal()
+ * @param string $fullCont
+ * @param string $dir
+ * @param array $params
* @return SwiftFileBackendFileList
*/
public function getFileListInternal( $fullCont, $dir, array $params ) {
@@ -916,10 +782,10 @@ class SwiftFileBackend extends FileBackendStore {
*
* @param string $fullCont Resolved container name
* @param string $dir Resolved storage directory with no trailing slash
- * @param string|null $after Storage path of file to list items after
- * @param integer $limit Max number of items to list
+ * @param string|null $after Resolved container relative path to list items after
+ * @param int $limit Max number of items to list
* @param array $params Parameters for getDirectoryList()
- * @return Array List of resolved paths of directories directly under $dir
+ * @return array List of container relative resolved paths of directories directly under $dir
* @throws FileBackendError
*/
public function getDirListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) {
@@ -929,156 +795,181 @@ class SwiftFileBackend extends FileBackendStore {
}
$section = new ProfileSection( __METHOD__ . '-' . $this->name );
- try {
- $container = $this->getContainer( $fullCont );
- $prefix = ( $dir == '' ) ? null : "{$dir}/";
- // Non-recursive: only list dirs right under $dir
- if ( !empty( $params['topOnly'] ) ) {
- $objects = $container->list_objects( $limit, $after, $prefix, null, '/' );
- foreach ( $objects as $object ) { // files and directories
- if ( substr( $object, -1 ) === '/' ) {
- $dirs[] = $object; // directories end in '/'
- }
+
+ $prefix = ( $dir == '' ) ? null : "{$dir}/";
+ // Non-recursive: only list dirs right under $dir
+ if ( !empty( $params['topOnly'] ) ) {
+ $status = $this->objectListing( $fullCont, 'names', $limit, $after, $prefix, '/' );
+ if ( !$status->isOk() ) {
+ return $dirs; // error
+ }
+ $objects = $status->value;
+ foreach ( $objects as $object ) { // files and directories
+ if ( substr( $object, -1 ) === '/' ) {
+ $dirs[] = $object; // directories end in '/'
}
+ }
+ } else {
// Recursive: list all dirs under $dir and its subdirs
- } else {
- // Get directory from last item of prior page
- $lastDir = $this->getParentDir( $after ); // must be first page
- $objects = $container->list_objects( $limit, $after, $prefix );
- foreach ( $objects as $object ) { // files
- $objectDir = $this->getParentDir( $object ); // directory of object
- if ( $objectDir !== false && $objectDir !== $dir ) {
- // Swift stores paths in UTF-8, using binary sorting.
- // See function "create_container_table" in common/db.py.
- // If a directory is not "greater" than the last one,
- // then it was already listed by the calling iterator.
- if ( strcmp( $objectDir, $lastDir ) > 0 ) {
- $pDir = $objectDir;
- do { // add dir and all its parent dirs
- $dirs[] = "{$pDir}/";
- $pDir = $this->getParentDir( $pDir );
- } while ( $pDir !== false // sanity
- && strcmp( $pDir, $lastDir ) > 0 // not done already
- && strlen( $pDir ) > strlen( $dir ) // within $dir
- );
- }
- $lastDir = $objectDir;
+ $getParentDir = function ( $path ) {
+ return ( strpos( $path, '/' ) !== false ) ? dirname( $path ) : false;
+ };
+
+ // Get directory from last item of prior page
+ $lastDir = $getParentDir( $after ); // must be first page
+ $status = $this->objectListing( $fullCont, 'names', $limit, $after, $prefix );
+
+ if ( !$status->isOk() ) {
+ return $dirs; // error
+ }
+
+ $objects = $status->value;
+
+ foreach ( $objects as $object ) { // files
+ $objectDir = $getParentDir( $object ); // directory of object
+
+ if ( $objectDir !== false && $objectDir !== $dir ) {
+ // Swift stores paths in UTF-8, using binary sorting.
+ // See function "create_container_table" in common/db.py.
+ // If a directory is not "greater" than the last one,
+ // then it was already listed by the calling iterator.
+ if ( strcmp( $objectDir, $lastDir ) > 0 ) {
+ $pDir = $objectDir;
+ do { // add dir and all its parent dirs
+ $dirs[] = "{$pDir}/";
+ $pDir = $getParentDir( $pDir );
+ } while ( $pDir !== false // sanity
+ && strcmp( $pDir, $lastDir ) > 0 // not done already
+ && strlen( $pDir ) > strlen( $dir ) // within $dir
+ );
}
+ $lastDir = $objectDir;
}
}
- // Page on the unfiltered directory listing (what is returned may be filtered)
- if ( count( $objects ) < $limit ) {
- $after = INF; // avoid a second RTT
- } else {
- $after = end( $objects ); // update last item
- }
- } catch ( NoSuchContainerException $e ) {
- } catch ( CloudFilesException $e ) { // some other exception?
- $this->handleException( $e, null, __METHOD__,
- array( 'cont' => $fullCont, 'dir' => $dir ) );
- throw new FileBackendError( "Got " . get_class( $e ) . " exception." );
+ }
+ // Page on the unfiltered directory listing (what is returned may be filtered)
+ if ( count( $objects ) < $limit ) {
+ $after = INF; // avoid a second RTT
+ } else {
+ $after = end( $objects ); // update last item
}
return $dirs;
}
- protected function getParentDir( $path ) {
- return ( strpos( $path, '/' ) !== false ) ? dirname( $path ) : false;
- }
-
/**
* Do not call this function outside of SwiftFileBackendFileList
*
* @param string $fullCont Resolved container name
* @param string $dir Resolved storage directory with no trailing slash
- * @param string|null $after Storage path of file to list items after
- * @param integer $limit Max number of items to list
+ * @param string|null $after Resolved container relative path of file to list items after
+ * @param int $limit Max number of items to list
* @param array $params Parameters for getDirectoryList()
- * @return Array List of resolved paths of files under $dir
+ * @return array List of resolved container relative paths of files under $dir
* @throws FileBackendError
*/
public function getFileListPageInternal( $fullCont, $dir, &$after, $limit, array $params ) {
- $files = array();
+ $files = array(); // list of (path, stat array or null) entries
if ( $after === INF ) {
return $files; // nothing more
}
$section = new ProfileSection( __METHOD__ . '-' . $this->name );
- try {
- $container = $this->getContainer( $fullCont );
- $prefix = ( $dir == '' ) ? null : "{$dir}/";
- // Non-recursive: only list files right under $dir
- if ( !empty( $params['topOnly'] ) ) { // files and dirs
- if ( !empty( $params['adviseStat'] ) ) {
- $limit = min( $limit, self::CACHE_CHEAP_SIZE );
- // Note: get_objects() does not include directories
- $objects = $this->loadObjectListing( $params, $dir,
- $container->get_objects( $limit, $after, $prefix, null, '/' ) );
- $files = $objects;
- } else {
- $objects = $container->list_objects( $limit, $after, $prefix, null, '/' );
- foreach ( $objects as $object ) { // files and directories
- if ( substr( $object, -1 ) !== '/' ) {
- $files[] = $object; // directories end in '/'
- }
- }
- }
- // Recursive: list all files under $dir and its subdirs
- } else { // files
- if ( !empty( $params['adviseStat'] ) ) {
- $limit = min( $limit, self::CACHE_CHEAP_SIZE );
- $objects = $this->loadObjectListing( $params, $dir,
- $container->get_objects( $limit, $after, $prefix ) );
- } else {
- $objects = $container->list_objects( $limit, $after, $prefix );
- }
- $files = $objects;
+
+ $prefix = ( $dir == '' ) ? null : "{$dir}/";
+ // $objects will contain a list of unfiltered names or CF_Object items
+ // Non-recursive: only list files right under $dir
+ if ( !empty( $params['topOnly'] ) ) {
+ if ( !empty( $params['adviseStat'] ) ) {
+ $status = $this->objectListing( $fullCont, 'info', $limit, $after, $prefix, '/' );
+ } else {
+ $status = $this->objectListing( $fullCont, 'names', $limit, $after, $prefix, '/' );
}
- // Page on the unfiltered object listing (what is returned may be filtered)
- if ( count( $objects ) < $limit ) {
- $after = INF; // avoid a second RTT
+ } else {
+ // Recursive: list all files under $dir and its subdirs
+ if ( !empty( $params['adviseStat'] ) ) {
+ $status = $this->objectListing( $fullCont, 'info', $limit, $after, $prefix );
} else {
- $after = end( $objects ); // update last item
+ $status = $this->objectListing( $fullCont, 'names', $limit, $after, $prefix );
}
- } catch ( NoSuchContainerException $e ) {
- } catch ( CloudFilesException $e ) { // some other exception?
- $this->handleException( $e, null, __METHOD__,
- array( 'cont' => $fullCont, 'dir' => $dir ) );
- throw new FileBackendError( "Got " . get_class( $e ) . " exception." );
+ }
+
+ // Reformat this list into a list of (name, stat array or null) entries
+ if ( !$status->isOk() ) {
+ return $files; // error
+ }
+
+ $objects = $status->value;
+ $files = $this->buildFileObjectListing( $params, $dir, $objects );
+
+ // Page on the unfiltered object listing (what is returned may be filtered)
+ if ( count( $objects ) < $limit ) {
+ $after = INF; // avoid a second RTT
+ } else {
+ $after = end( $objects ); // update last item
+ $after = is_object( $after ) ? $after->name : $after;
}
return $files;
}
/**
- * Load a list of objects that belong under $dir into stat cache
- * and return a list of the names of the objects in the same order.
+ * Build a list of file objects, filtering out any directories
+ * and extracting any stat info if provided in $objects (for CF_Objects)
*
* @param array $params Parameters for getDirectoryList()
* @param string $dir Resolved container directory path
- * @param array $cfObjects List of CF_Object items
- * @return array List of object names
+ * @param array $objects List of CF_Object items or object names
+ * @return array List of (names,stat array or null) entries
*/
- private function loadObjectListing( array $params, $dir, array $cfObjects ) {
+ private function buildFileObjectListing( array $params, $dir, array $objects ) {
$names = array();
- $storageDir = rtrim( $params['dir'], '/' );
- $suffixStart = ( $dir === '' ) ? 0 : strlen( $dir ) + 1; // size of "path/to/dir/"
- // Iterate over the list *backwards* as this primes the stat cache, which is LRU.
- // If this fills the cache and the caller stats an uncached file before stating
- // the ones on the listing, there would be zero cache hits if this went forwards.
- for ( end( $cfObjects ); key( $cfObjects ) !== null; prev( $cfObjects ) ) {
- $object = current( $cfObjects );
- $path = "{$storageDir}/" . substr( $object->name, $suffixStart );
- $val = array(
- // Convert dates like "Tue, 03 Jan 2012 22:01:04 GMT" to TS_MW
- 'mtime' => wfTimestamp( TS_MW, $object->last_modified ),
- 'size' => (int)$object->content_length,
- 'latest' => false // eventually consistent
- );
- $this->cheapCache->set( $path, 'stat', $val );
- $names[] = $object->name;
+ foreach ( $objects as $object ) {
+ if ( is_object( $object ) ) {
+ if ( isset( $object->subdir ) || !isset( $object->name ) ) {
+ continue; // virtual directory entry; ignore
+ }
+ $stat = array(
+ // Convert various random Swift dates to TS_MW
+ 'mtime' => $this->convertSwiftDate( $object->last_modified, TS_MW ),
+ 'size' => (int)$object->bytes,
+ // Note: manifiest ETags are not an MD5 of the file
+ 'md5' => ctype_xdigit( $object->hash ) ? $object->hash : null,
+ 'latest' => false // eventually consistent
+ );
+ $names[] = array( $object->name, $stat );
+ } elseif ( substr( $object, -1 ) !== '/' ) {
+ // Omit directories, which end in '/' in listings
+ $names[] = array( $object, null );
+ }
+ }
+
+ return $names;
+ }
+
+ /**
+ * Do not call this function outside of SwiftFileBackendFileList
+ *
+ * @param string $path Storage path
+ * @param array $val Stat value
+ */
+ public function loadListingStatInternal( $path, array $val ) {
+ $this->cheapCache->set( $path, 'stat', $val );
+ }
+
+ protected function doGetFileXAttributes( array $params ) {
+ $stat = $this->getFileStat( $params );
+ if ( $stat ) {
+ if ( !isset( $stat['xattr'] ) ) {
+ // Stat entries filled by file listings don't include metadata/headers
+ $this->clearCache( array( $params['src'] ) );
+ $stat = $this->getFileStat( $params );
+ }
+
+ return $stat['xattr'];
+ } else {
+ return false;
}
- return array_reverse( $names ); // keep the paths in original order
}
protected function doGetFileSha1base36( array $params ) {
@@ -1089,6 +980,7 @@ class SwiftFileBackend extends FileBackendStore {
$this->clearCache( array( $params['src'] ) );
$stat = $this->getFileStat( $params );
}
+
return $stat['sha1'];
} else {
return false;
@@ -1103,24 +995,29 @@ class SwiftFileBackend extends FileBackendStore {
$status->fatal( 'backend-fail-invalidpath', $params['src'] );
}
- try {
- $cont = $this->getContainer( $srcCont );
- } catch ( NoSuchContainerException $e ) {
+ $auth = $this->getAuthentication();
+ if ( !$auth || !is_array( $this->getContainerStat( $srcCont ) ) ) {
$status->fatal( 'backend-fail-stream', $params['src'] );
- return $status;
- } catch ( CloudFilesException $e ) { // some other exception?
- $this->handleException( $e, $status, __METHOD__, $params );
+
return $status;
}
- try {
- $output = fopen( 'php://output', 'wb' );
- $obj = new CF_Object( $cont, $srcRel, false, false ); // skip HEAD
- $obj->stream( $output, $this->headersFromParams( $params ) );
- } catch ( NoSuchObjectException $e ) {
+ $handle = fopen( 'php://output', 'wb' );
+
+ list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( array(
+ 'method' => 'GET',
+ 'url' => $this->storageUrl( $auth, $srcCont, $srcRel ),
+ 'headers' => $this->authTokenHeaders( $auth )
+ + $this->headersFromParams( $params ),
+ 'stream' => $handle,
+ ) );
+
+ if ( $rcode >= 200 && $rcode <= 299 ) {
+ // good
+ } elseif ( $rcode === 404 ) {
$status->fatal( 'backend-fail-stream', $params['src'] );
- } catch ( CloudFilesException $e ) { // some other exception?
- $this->handleException( $e, $status, __METHOD__, $params );
+ } else {
+ $this->onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc );
}
return $status;
@@ -1129,66 +1026,60 @@ class SwiftFileBackend extends FileBackendStore {
protected function doGetLocalCopyMulti( array $params ) {
$tmpFiles = array();
+ $auth = $this->getAuthentication();
+
$ep = array_diff_key( $params, array( 'srcs' => 1 ) ); // for error logging
// Blindly create tmp files and stream to them, catching any exception if the file does
// not exist. Doing a stat here is useless causes infinite loops in addMissingMetadata().
- foreach ( array_chunk( $params['srcs'], $params['concurrency'] ) as $pathBatch ) {
- $cfOps = array(); // (path => CF_Async_Op)
+ $reqs = array(); // (path => op)
- foreach ( $pathBatch as $path ) { // each path in this concurrent batch
- list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $path );
- if ( $srcRel === null ) {
- $tmpFiles[$path] = null;
- continue;
- }
- $tmpFile = null;
- try {
- $sContObj = $this->getContainer( $srcCont );
- $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
- // Get source file extension
- $ext = FileBackend::extensionFromPath( $path );
- // Create a new temporary file...
- $tmpFile = TempFSFile::factory( 'localcopy_', $ext );
- if ( $tmpFile ) {
- $handle = fopen( $tmpFile->getPath(), 'wb' );
- if ( $handle ) {
- $headers = $this->headersFromParams( $params );
- if ( count( $pathBatch ) > 1 ) {
- $cfOps[$path] = $obj->stream_async( $handle, $headers );
- $cfOps[$path]->_file_handle = $handle; // close this later
- } else {
- $obj->stream( $handle, $headers );
- fclose( $handle );
- }
- } else {
- $tmpFile = null;
- }
- }
- } catch ( NoSuchContainerException $e ) {
- $tmpFile = null;
- } catch ( NoSuchObjectException $e ) {
- $tmpFile = null;
- } catch ( CloudFilesException $e ) { // some other exception?
+ foreach ( $params['srcs'] as $path ) { // each path in this concurrent batch
+ list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $path );
+ if ( $srcRel === null || !$auth ) {
+ $tmpFiles[$path] = null;
+ continue;
+ }
+ // Get source file extension
+ $ext = FileBackend::extensionFromPath( $path );
+ // Create a new temporary file...
+ $tmpFile = TempFSFile::factory( 'localcopy_', $ext );
+ if ( $tmpFile ) {
+ $handle = fopen( $tmpFile->getPath(), 'wb' );
+ if ( $handle ) {
+ $reqs[$path] = array(
+ 'method' => 'GET',
+ 'url' => $this->storageUrl( $auth, $srcCont, $srcRel ),
+ 'headers' => $this->authTokenHeaders( $auth )
+ + $this->headersFromParams( $params ),
+ 'stream' => $handle,
+ );
+ } else {
$tmpFile = null;
- $this->handleException( $e, null, __METHOD__, array( 'src' => $path ) + $ep );
}
- $tmpFiles[$path] = $tmpFile;
}
-
- $batch = new CF_Async_Op_Batch( $cfOps );
- $cfOps = $batch->execute();
- foreach ( $cfOps as $path => $cfOp ) {
- try {
- $cfOp->getLastResponse();
- } catch ( NoSuchContainerException $e ) {
+ $tmpFiles[$path] = $tmpFile;
+ }
+
+ $opts = array( 'maxConnsPerHost' => $params['concurrency'] );
+ $reqs = $this->http->runMulti( $reqs, $opts );
+ foreach ( $reqs as $path => $op ) {
+ list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $op['response'];
+ fclose( $op['stream'] ); // close open handle
+ if ( $rcode >= 200 && $rcode <= 299 ) {
+ $size = $tmpFiles[$path] ? $tmpFiles[$path]->getSize() : 0;
+ // Double check that the disk is not full/broken
+ if ( $size != $rhdrs['content-length'] ) {
$tmpFiles[$path] = null;
- } catch ( NoSuchObjectException $e ) {
- $tmpFiles[$path] = null;
- } catch ( CloudFilesException $e ) { // some other exception?
- $tmpFiles[$path] = null;
- $this->handleException( $e, null, __METHOD__, array( 'src' => $path ) + $ep );
+ $rerr = "Got {$size}/{$rhdrs['content-length']} bytes";
+ $this->onError( null, __METHOD__,
+ array( 'src' => $path ) + $ep, $rerr, $rcode, $rdesc );
}
- fclose( $cfOp->_file_handle ); // close open handle
+ } elseif ( $rcode === 404 ) {
+ $tmpFiles[$path] = false;
+ } else {
+ $tmpFiles[$path] = null;
+ $this->onError( null, __METHOD__,
+ array( 'src' => $path ) + $ep, $rerr, $rcode, $rdesc );
}
}
@@ -1197,46 +1088,55 @@ class SwiftFileBackend extends FileBackendStore {
public function getFileHttpUrl( array $params ) {
if ( $this->swiftTempUrlKey != '' ||
- ( $this->rgwS3AccessKey != '' && $this->rgwS3SecretKey != '' ) )
- {
+ ( $this->rgwS3AccessKey != '' && $this->rgwS3SecretKey != '' )
+ ) {
list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $params['src'] );
if ( $srcRel === null ) {
return null; // invalid path
}
- try {
- $ttl = isset( $params['ttl'] ) ? $params['ttl'] : 86400;
- $sContObj = $this->getContainer( $srcCont );
- $obj = new CF_Object( $sContObj, $srcRel, false, false ); // skip HEAD
- if ( $this->swiftTempUrlKey != '' ) {
- return $obj->get_temp_url( $this->swiftTempUrlKey, $ttl, "GET" );
- } else { // give S3 API URL for rgw
- $expires = time() + $ttl;
- // Path for signature starts with the bucket
- $spath = '/' . rawurlencode( $srcCont ) . '/' .
- str_replace( '%2F', '/', rawurlencode( $srcRel ) );
- // Calculate the hash
- $signature = base64_encode( hash_hmac(
- 'sha1',
- "GET\n\n\n{$expires}\n{$spath}",
- $this->rgwS3SecretKey,
- true // raw
- ) );
- // See http://s3.amazonaws.com/doc/s3-developer-guide/RESTAuthentication.html.
- // Note: adding a newline for empty CanonicalizedAmzHeaders does not work.
- return wfAppendQuery(
- str_replace( '/swift/v1', '', // S3 API is the rgw default
- $sContObj->cfs_http->getStorageUrl() . $spath ),
- array(
- 'Signature' => $signature,
- 'Expires' => $expires,
- 'AWSAccessKeyId' => $this->rgwS3AccessKey )
- );
- }
- } catch ( NoSuchContainerException $e ) {
- } catch ( CloudFilesException $e ) { // some other exception?
- $this->handleException( $e, null, __METHOD__, $params );
+
+ $auth = $this->getAuthentication();
+ if ( !$auth ) {
+ return null;
+ }
+
+ $ttl = isset( $params['ttl'] ) ? $params['ttl'] : 86400;
+ $expires = time() + $ttl;
+
+ if ( $this->swiftTempUrlKey != '' ) {
+ $url = $this->storageUrl( $auth, $srcCont, $srcRel );
+ // Swift wants the signature based on the unencoded object name
+ $contPath = parse_url( $this->storageUrl( $auth, $srcCont ), PHP_URL_PATH );
+ $signature = hash_hmac( 'sha1',
+ "GET\n{$expires}\n{$contPath}/{$srcRel}",
+ $this->swiftTempUrlKey
+ );
+
+ return "{$url}?temp_url_sig={$signature}&temp_url_expires={$expires}";
+ } else { // give S3 API URL for rgw
+ // Path for signature starts with the bucket
+ $spath = '/' . rawurlencode( $srcCont ) . '/' .
+ str_replace( '%2F', '/', rawurlencode( $srcRel ) );
+ // Calculate the hash
+ $signature = base64_encode( hash_hmac(
+ 'sha1',
+ "GET\n\n\n{$expires}\n{$spath}",
+ $this->rgwS3SecretKey,
+ true // raw
+ ) );
+ // See http://s3.amazonaws.com/doc/s3-developer-guide/RESTAuthentication.html.
+ // Note: adding a newline for empty CanonicalizedAmzHeaders does not work.
+ return wfAppendQuery(
+ str_replace( '/swift/v1', '', // S3 API is the rgw default
+ $this->storageUrl( $auth ) . $spath ),
+ array(
+ 'Signature' => $signature,
+ 'Expires' => $expires,
+ 'AWSAccessKeyId' => $this->rgwS3AccessKey )
+ );
}
}
+
return null;
}
@@ -1250,37 +1150,61 @@ class SwiftFileBackend extends FileBackendStore {
* $params is currently only checked for a 'latest' flag.
*
* @param array $params
- * @return Array
+ * @return array
*/
protected function headersFromParams( array $params ) {
$hdrs = array();
if ( !empty( $params['latest'] ) ) {
- $hdrs[] = 'X-Newest: true';
+ $hdrs['x-newest'] = 'true';
}
+
return $hdrs;
}
protected function doExecuteOpHandlesInternal( array $fileOpHandles ) {
$statuses = array();
- $cfOps = array(); // list of CF_Async_Op objects
+ $auth = $this->getAuthentication();
+ if ( !$auth ) {
+ foreach ( $fileOpHandles as $index => $fileOpHandle ) {
+ $statuses[$index] = Status::newFatal( 'backend-fail-connect', $this->name );
+ }
+
+ return $statuses;
+ }
+
+ // Split the HTTP requests into stages that can be done concurrently
+ $httpReqsByStage = array(); // map of (stage => index => HTTP request)
foreach ( $fileOpHandles as $index => $fileOpHandle ) {
- $cfOps[$index] = $fileOpHandle->cfOp;
- }
- $batch = new CF_Async_Op_Batch( $cfOps );
-
- $cfOps = $batch->execute();
- foreach ( $cfOps as $index => $cfOp ) {
- $status = Status::newGood();
- $function = '_getResponse' . $fileOpHandles[$index]->call;
- try { // catch exceptions; update status
- $this->$function( $cfOp, $status, $fileOpHandles[$index]->params );
- $this->purgeCDNCache( $fileOpHandles[$index]->affectedObjects );
- } catch ( CloudFilesException $e ) { // some other exception?
- $this->handleException( $e, $status,
- __CLASS__ . ":$function", $fileOpHandles[$index]->params );
+ $reqs = $fileOpHandle->httpOp;
+ // Convert the 'url' parameter to an actual URL using $auth
+ foreach ( $reqs as $stage => &$req ) {
+ list( $container, $relPath ) = $req['url'];
+ $req['url'] = $this->storageUrl( $auth, $container, $relPath );
+ $req['headers'] = isset( $req['headers'] ) ? $req['headers'] : array();
+ $req['headers'] = $this->authTokenHeaders( $auth ) + $req['headers'];
+ $httpReqsByStage[$stage][$index] = $req;
+ }
+ $statuses[$index] = Status::newGood();
+ }
+
+ // Run all requests for the first stage, then the next, and so on
+ $reqCount = count( $httpReqsByStage );
+ for ( $stage = 0; $stage < $reqCount; ++$stage ) {
+ $httpReqs = $this->http->runMulti( $httpReqsByStage[$stage] );
+ foreach ( $httpReqs as $index => $httpReq ) {
+ // Run the callback for each request of this operation
+ $callback = $fileOpHandles[$index]->callback;
+ call_user_func_array( $callback, array( $httpReq, $statuses[$index] ) );
+ // On failure, abort all remaining requests for this operation
+ // (e.g. abort the DELETE request if the COPY request fails for a move)
+ if ( !$statuses[$index]->isOK() ) {
+ $stages = count( $fileOpHandles[$index]->httpOp );
+ for ( $s = ( $stage + 1 ); $s < $stages; ++$s ) {
+ unset( $httpReqsByStage[$s][$index] );
+ }
+ }
}
- $statuses[$index] = $status;
}
return $statuses;
@@ -1289,7 +1213,13 @@ class SwiftFileBackend extends FileBackendStore {
/**
* Set read/write permissions for a Swift container.
*
- * $readGrps is a list of the possible criteria for a request to have
+ * @see http://swift.openstack.org/misc.html#acls
+ *
+ * In general, we don't allow listings to end-users. It's not useful, isn't well-defined
+ * (lists are truncated to 10000 item with no way to page), and is just a performance risk.
+ *
+ * @param string $container Resolved Swift container
+ * @param array $readGrps List of the possible criteria for a request to have
* access to read a container. Each item is one of the following formats:
* - account:user : Grants access if the request is by the given user
* - ".r:<regex>" : Grants access if the request is from a referrer host that
@@ -1297,228 +1227,438 @@ class SwiftFileBackend extends FileBackendStore {
* Setting this to '*' effectively makes a container public.
* -".rlistings:<regex>" : Grants access if the request is from a referrer host that
* matches the expression and the request is for a listing.
- *
- * $writeGrps is a list of the possible criteria for a request to have
+ * @param array $writeGrps A list of the possible criteria for a request to have
* access to write to a container. Each item is of the following format:
* - account:user : Grants access if the request is by the given user
- *
- * @see http://swift.openstack.org/misc.html#acls
- *
- * In general, we don't allow listings to end-users. It's not useful, isn't well-defined
- * (lists are truncated to 10000 item with no way to page), and is just a performance risk.
- *
- * @param CF_Container $contObj Swift container
- * @param array $readGrps List of read access routes
- * @param array $writeGrps List of write access routes
* @return Status
*/
- protected function setContainerAccess(
- CF_Container $contObj, array $readGrps, array $writeGrps
- ) {
- $creds = $contObj->cfs_auth->export_credentials();
+ protected function setContainerAccess( $container, array $readGrps, array $writeGrps ) {
+ $status = Status::newGood();
+ $auth = $this->getAuthentication();
- $url = $creds['storage_url'] . '/' . rawurlencode( $contObj->name );
+ if ( !$auth ) {
+ $status->fatal( 'backend-fail-connect', $this->name );
- // Note: 10 second timeout consistent with php-cloudfiles
- $req = MWHttpRequest::factory( $url, array( 'method' => 'POST', 'timeout' => 10 ) );
- $req->setHeader( 'X-Auth-Token', $creds['auth_token'] );
- $req->setHeader( 'X-Container-Read', implode( ',', $readGrps ) );
- $req->setHeader( 'X-Container-Write', implode( ',', $writeGrps ) );
+ return $status;
+ }
+
+ list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( array(
+ 'method' => 'POST',
+ 'url' => $this->storageUrl( $auth, $container ),
+ 'headers' => $this->authTokenHeaders( $auth ) + array(
+ 'x-container-read' => implode( ',', $readGrps ),
+ 'x-container-write' => implode( ',', $writeGrps )
+ )
+ ) );
+
+ if ( $rcode != 204 && $rcode !== 202 ) {
+ $status->fatal( 'backend-fail-internal', $this->name );
+ }
- return $req->execute(); // should return 204
+ return $status;
}
/**
- * Purge the CDN cache of affected objects if CDN caching is enabled.
- * This is for Rackspace/Akamai CDNs.
+ * Get a Swift container stat array, possibly from process cache.
+ * Use $reCache if the file count or byte count is needed.
*
- * @param array $objects List of CF_Object items
- * @return void
+ * @param string $container Container name
+ * @param bool $bypassCache Bypass all caches and load from Swift
+ * @return array|bool|null False on 404, null on failure
*/
- public function purgeCDNCache( array $objects ) {
- if ( $this->swiftUseCDN && $this->swiftCDNPurgable ) {
- foreach ( $objects as $object ) {
- try {
- $object->purge_from_cdn();
- } catch ( CDNNotEnabledException $e ) {
- // CDN not enabled; nothing to see here
- } catch ( CloudFilesException $e ) {
- $this->handleException( $e, null, __METHOD__,
- array( 'cont' => $object->container->name, 'obj' => $object->name ) );
+ protected function getContainerStat( $container, $bypassCache = false ) {
+ $section = new ProfileSection( __METHOD__ . '-' . $this->name );
+
+ if ( $bypassCache ) { // purge cache
+ $this->containerStatCache->clear( $container );
+ } elseif ( !$this->containerStatCache->has( $container, 'stat' ) ) {
+ $this->primeContainerCache( array( $container ) ); // check persistent cache
+ }
+ if ( !$this->containerStatCache->has( $container, 'stat' ) ) {
+ $auth = $this->getAuthentication();
+ if ( !$auth ) {
+ return null;
+ }
+
+ wfProfileIn( __METHOD__ . "-{$this->name}-miss" );
+ list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( array(
+ 'method' => 'HEAD',
+ 'url' => $this->storageUrl( $auth, $container ),
+ 'headers' => $this->authTokenHeaders( $auth )
+ ) );
+ wfProfileOut( __METHOD__ . "-{$this->name}-miss" );
+
+ if ( $rcode === 204 ) {
+ $stat = array(
+ 'count' => $rhdrs['x-container-object-count'],
+ 'bytes' => $rhdrs['x-container-bytes-used']
+ );
+ if ( $bypassCache ) {
+ return $stat;
+ } else {
+ $this->containerStatCache->set( $container, 'stat', $stat ); // cache it
+ $this->setContainerCache( $container, $stat ); // update persistent cache
}
+ } elseif ( $rcode === 404 ) {
+ return false;
+ } else {
+ $this->onError( null, __METHOD__,
+ array( 'cont' => $container ), $rerr, $rcode, $rdesc );
+
+ return null;
}
}
+
+ return $this->containerStatCache->get( $container, 'stat' );
}
/**
- * Get an authenticated connection handle to the Swift proxy
+ * Create a Swift container
*
- * @throws CloudFilesException
- * @throws CloudFilesException|Exception
- * @return CF_Connection|bool False on failure
+ * @param string $container Container name
+ * @param array $params
+ * @return Status
*/
- protected function getConnection() {
- if ( $this->connException instanceof CloudFilesException ) {
- if ( ( time() - $this->connErrorTime ) < 60 ) {
- throw $this->connException; // failed last attempt; don't bother
- } else { // actually retry this time
- $this->connException = null;
- $this->connErrorTime = 0;
- }
+ protected function createContainer( $container, array $params ) {
+ $status = Status::newGood();
+
+ $auth = $this->getAuthentication();
+ if ( !$auth ) {
+ $status->fatal( 'backend-fail-connect', $this->name );
+
+ return $status;
}
- // Session keys expire after a while, so we renew them periodically
- $reAuth = ( ( time() - $this->sessionStarted ) > $this->authTTL );
- // Authenticate with proxy and get a session key...
- if ( !$this->conn || $reAuth ) {
- $this->sessionStarted = 0;
- $this->connContainerCache->clear();
- $cacheKey = $this->getCredsCacheKey( $this->auth->username );
- $creds = $this->srvCache->get( $cacheKey ); // credentials
- if ( is_array( $creds ) ) { // cache hit
- $this->auth->load_cached_credentials(
- $creds['auth_token'], $creds['storage_url'], $creds['cdnm_url'] );
- $this->sessionStarted = time() - ceil( $this->authTTL / 2 ); // skew for worst case
- } else { // cache miss
- try {
- $this->auth->authenticate();
- $creds = $this->auth->export_credentials();
- $this->srvCache->add( $cacheKey, $creds, ceil( $this->authTTL / 2 ) ); // cache
- $this->sessionStarted = time();
- } catch ( CloudFilesException $e ) {
- $this->connException = $e; // don't keep re-trying
- $this->connErrorTime = time();
- throw $e; // throw it back
- }
- }
- if ( $this->conn ) { // re-authorizing?
- $this->conn->close(); // close active cURL handles in CF_Http object
- }
- $this->conn = new CF_Connection( $this->auth );
+
+ // @see SwiftFileBackend::setContainerAccess()
+ if ( empty( $params['noAccess'] ) ) {
+ $readGrps = array( '.r:*', $this->swiftUser ); // public
+ } else {
+ $readGrps = array( $this->swiftUser ); // private
+ }
+ $writeGrps = array( $this->swiftUser ); // sanity
+
+ list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( array(
+ 'method' => 'PUT',
+ 'url' => $this->storageUrl( $auth, $container ),
+ 'headers' => $this->authTokenHeaders( $auth ) + array(
+ 'x-container-read' => implode( ',', $readGrps ),
+ 'x-container-write' => implode( ',', $writeGrps )
+ )
+ ) );
+
+ if ( $rcode === 201 ) { // new
+ // good
+ } elseif ( $rcode === 202 ) { // already there
+ // this shouldn't really happen, but is OK
+ } else {
+ $this->onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc );
}
- return $this->conn;
+
+ return $status;
}
/**
- * Close the connection to the Swift proxy
+ * Delete a Swift container
*
- * @return void
+ * @param string $container Container name
+ * @param array $params
+ * @return Status
*/
- protected function closeConnection() {
- if ( $this->conn ) {
- $this->conn->close(); // close active cURL handles in CF_Http object
- $this->conn = null;
- $this->sessionStarted = 0;
- $this->connContainerCache->clear();
+ protected function deleteContainer( $container, array $params ) {
+ $status = Status::newGood();
+
+ $auth = $this->getAuthentication();
+ if ( !$auth ) {
+ $status->fatal( 'backend-fail-connect', $this->name );
+
+ return $status;
+ }
+
+ list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( array(
+ 'method' => 'DELETE',
+ 'url' => $this->storageUrl( $auth, $container ),
+ 'headers' => $this->authTokenHeaders( $auth )
+ ) );
+
+ if ( $rcode >= 200 && $rcode <= 299 ) { // deleted
+ $this->containerStatCache->clear( $container ); // purge
+ } elseif ( $rcode === 404 ) { // not there
+ // this shouldn't really happen, but is OK
+ } elseif ( $rcode === 409 ) { // not empty
+ $this->onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc ); // race?
+ } else {
+ $this->onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc );
}
+
+ return $status;
}
/**
- * Get the cache key for a container
+ * Get a list of objects under a container.
+ * Either just the names or a list of stdClass objects with details can be returned.
*
- * @param string $username
- * @return string
+ * @param string $fullCont
+ * @param string $type ('info' for a list of object detail maps, 'names' for names only)
+ * @param int $limit
+ * @param string|null $after
+ * @param string|null $prefix
+ * @param string|null $delim
+ * @return Status With the list as value
*/
- private function getCredsCacheKey( $username ) {
- return wfMemcKey( 'backend', $this->getName(), 'usercreds', $username );
+ private function objectListing(
+ $fullCont, $type, $limit, $after = null, $prefix = null, $delim = null
+ ) {
+ $status = Status::newGood();
+
+ $auth = $this->getAuthentication();
+ if ( !$auth ) {
+ $status->fatal( 'backend-fail-connect', $this->name );
+
+ return $status;
+ }
+
+ $query = array( 'limit' => $limit );
+ if ( $type === 'info' ) {
+ $query['format'] = 'json';
+ }
+ if ( $after !== null ) {
+ $query['marker'] = $after;
+ }
+ if ( $prefix !== null ) {
+ $query['prefix'] = $prefix;
+ }
+ if ( $delim !== null ) {
+ $query['delimiter'] = $delim;
+ }
+
+ list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( array(
+ 'method' => 'GET',
+ 'url' => $this->storageUrl( $auth, $fullCont ),
+ 'query' => $query,
+ 'headers' => $this->authTokenHeaders( $auth )
+ ) );
+
+ $params = array( 'cont' => $fullCont, 'prefix' => $prefix, 'delim' => $delim );
+ if ( $rcode === 200 ) { // good
+ if ( $type === 'info' ) {
+ $status->value = FormatJson::decode( trim( $rbody ) );
+ } else {
+ $status->value = explode( "\n", trim( $rbody ) );
+ }
+ } elseif ( $rcode === 204 ) {
+ $status->value = array(); // empty container
+ } elseif ( $rcode === 404 ) {
+ $status->value = array(); // no container
+ } else {
+ $this->onError( $status, __METHOD__, $params, $rerr, $rcode, $rdesc );
+ }
+
+ return $status;
+ }
+
+ protected function doPrimeContainerCache( array $containerInfo ) {
+ foreach ( $containerInfo as $container => $info ) {
+ $this->containerStatCache->set( $container, 'stat', $info );
+ }
+ }
+
+ protected function doGetFileStatMulti( array $params ) {
+ $stats = array();
+
+ $auth = $this->getAuthentication();
+
+ $reqs = array();
+ foreach ( $params['srcs'] as $path ) {
+ list( $srcCont, $srcRel ) = $this->resolveStoragePathReal( $path );
+ if ( $srcRel === null ) {
+ $stats[$path] = false;
+ continue; // invalid storage path
+ } elseif ( !$auth ) {
+ $stats[$path] = null;
+ continue;
+ }
+
+ // (a) Check the container
+ $cstat = $this->getContainerStat( $srcCont );
+ if ( $cstat === false ) {
+ $stats[$path] = false;
+ continue; // ok, nothing to do
+ } elseif ( !is_array( $cstat ) ) {
+ $stats[$path] = null;
+ continue;
+ }
+
+ $reqs[$path] = array(
+ 'method' => 'HEAD',
+ 'url' => $this->storageUrl( $auth, $srcCont, $srcRel ),
+ 'headers' => $this->authTokenHeaders( $auth ) + $this->headersFromParams( $params )
+ );
+ }
+
+ $opts = array( 'maxConnsPerHost' => $params['concurrency'] );
+ $reqs = $this->http->runMulti( $reqs, $opts );
+
+ foreach ( $params['srcs'] as $path ) {
+ if ( array_key_exists( $path, $stats ) ) {
+ continue; // some sort of failure above
+ }
+ // (b) Check the file
+ list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $reqs[$path]['response'];
+ if ( $rcode === 200 || $rcode === 204 ) {
+ // Update the object if it is missing some headers
+ $rhdrs = $this->addMissingMetadata( $rhdrs, $path );
+ // Fetch all of the custom metadata headers
+ $metadata = array();
+ foreach ( $rhdrs as $name => $value ) {
+ if ( strpos( $name, 'x-object-meta-' ) === 0 ) {
+ $metadata[substr( $name, strlen( 'x-object-meta-' ) )] = $value;
+ }
+ }
+ // Fetch all of the custom raw HTTP headers
+ $headers = $this->sanitizeHdrs( array( 'headers' => $rhdrs ) );
+ $stat = array(
+ // Convert various random Swift dates to TS_MW
+ 'mtime' => $this->convertSwiftDate( $rhdrs['last-modified'], TS_MW ),
+ // Empty objects actually return no content-length header in Ceph
+ 'size' => isset( $rhdrs['content-length'] ) ? (int)$rhdrs['content-length'] : 0,
+ 'sha1' => $rhdrs['x-object-meta-sha1base36'],
+ // Note: manifiest ETags are not an MD5 of the file
+ 'md5' => ctype_xdigit( $rhdrs['etag'] ) ? $rhdrs['etag'] : null,
+ 'xattr' => array( 'metadata' => $metadata, 'headers' => $headers )
+ );
+ if ( $this->isRGW ) {
+ $stat['latest'] = true; // strong consistency
+ }
+ } elseif ( $rcode === 404 ) {
+ $stat = false;
+ } else {
+ $stat = null;
+ $this->onError( null, __METHOD__, $params, $rerr, $rcode, $rdesc );
+ }
+ $stats[$path] = $stat;
+ }
+
+ return $stats;
}
/**
- * Get a Swift container object, possibly from process cache.
- * Use $reCache if the file count or byte count is needed.
- *
- * @param string $container Container name
- * @param bool $bypassCache Bypass all caches and load from Swift
- * @return CF_Container
- * @throws CloudFilesException
+ * @return array|null Credential map
*/
- protected function getContainer( $container, $bypassCache = false ) {
- $conn = $this->getConnection(); // Swift proxy connection
- if ( $bypassCache ) { // purge cache
- $this->connContainerCache->clear( $container );
- } elseif ( !$this->connContainerCache->has( $container, 'obj' ) ) {
- $this->primeContainerCache( array( $container ) ); // check persistent cache
+ protected function getAuthentication() {
+ if ( $this->authErrorTimestamp !== null ) {
+ if ( ( time() - $this->authErrorTimestamp ) < 60 ) {
+ return null; // failed last attempt; don't bother
+ } else { // actually retry this time
+ $this->authErrorTimestamp = null;
+ }
}
- if ( !$this->connContainerCache->has( $container, 'obj' ) ) {
- $contObj = $conn->get_container( $container );
- // NoSuchContainerException not thrown: container must exist
- $this->connContainerCache->set( $container, 'obj', $contObj ); // cache it
- if ( !$bypassCache ) {
- $this->setContainerCache( $container, // update persistent cache
- array( 'bytes' => $contObj->bytes_used, 'count' => $contObj->object_count )
- );
+ // Session keys expire after a while, so we renew them periodically
+ $reAuth = ( ( time() - $this->authSessionTimestamp ) > $this->authTTL );
+ // Authenticate with proxy and get a session key...
+ if ( !$this->authCreds || $reAuth ) {
+ $this->authSessionTimestamp = 0;
+ $cacheKey = $this->getCredsCacheKey( $this->swiftUser );
+ $creds = $this->srvCache->get( $cacheKey ); // credentials
+ // Try to use the credential cache
+ if ( isset( $creds['auth_token'] ) && isset( $creds['storage_url'] ) ) {
+ $this->authCreds = $creds;
+ // Skew the timestamp for worst case to avoid using stale credentials
+ $this->authSessionTimestamp = time() - ceil( $this->authTTL / 2 );
+ } else { // cache miss
+ list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( array(
+ 'method' => 'GET',
+ 'url' => "{$this->swiftAuthUrl}/v1.0",
+ 'headers' => array(
+ 'x-auth-user' => $this->swiftUser,
+ 'x-auth-key' => $this->swiftKey
+ )
+ ) );
+
+ if ( $rcode >= 200 && $rcode <= 299 ) { // OK
+ $this->authCreds = array(
+ 'auth_token' => $rhdrs['x-auth-token'],
+ 'storage_url' => $rhdrs['x-storage-url']
+ );
+ $this->srvCache->set( $cacheKey, $this->authCreds, ceil( $this->authTTL / 2 ) );
+ $this->authSessionTimestamp = time();
+ } elseif ( $rcode === 401 ) {
+ $this->onError( null, __METHOD__, array(), "Authentication failed.", $rcode );
+ $this->authErrorTimestamp = time();
+
+ return null;
+ } else {
+ $this->onError( null, __METHOD__, array(), "HTTP return code: $rcode", $rcode );
+ $this->authErrorTimestamp = time();
+
+ return null;
+ }
+ }
+ // Ceph RGW does not use <account> in URLs (OpenStack Swift uses "/v1/<account>")
+ if ( substr( $this->authCreds['storage_url'], -3 ) === '/v1' ) {
+ $this->isRGW = true; // take advantage of strong consistency
}
}
- return $this->connContainerCache->get( $container, 'obj' );
+
+ return $this->authCreds;
}
/**
- * Create a Swift container
- *
- * @param string $container Container name
- * @return CF_Container
- * @throws CloudFilesException
+ * @param array $creds From getAuthentication()
+ * @param string $container
+ * @param string $object
+ * @return array
*/
- protected function createContainer( $container ) {
- $conn = $this->getConnection(); // Swift proxy connection
- $contObj = $conn->create_container( $container );
- $this->connContainerCache->set( $container, 'obj', $contObj ); // cache
- return $contObj;
+ protected function storageUrl( array $creds, $container = null, $object = null ) {
+ $parts = array( $creds['storage_url'] );
+ if ( strlen( $container ) ) {
+ $parts[] = rawurlencode( $container );
+ }
+ if ( strlen( $object ) ) {
+ $parts[] = str_replace( "%2F", "/", rawurlencode( $object ) );
+ }
+
+ return implode( '/', $parts );
}
/**
- * Delete a Swift container
- *
- * @param string $container Container name
- * @return void
- * @throws CloudFilesException
+ * @param array $creds From getAuthentication()
+ * @return array
*/
- protected function deleteContainer( $container ) {
- $conn = $this->getConnection(); // Swift proxy connection
- $this->connContainerCache->clear( $container ); // purge
- $conn->delete_container( $container );
+ protected function authTokenHeaders( array $creds ) {
+ return array( 'x-auth-token' => $creds['auth_token'] );
}
- protected function doPrimeContainerCache( array $containerInfo ) {
- try {
- $conn = $this->getConnection(); // Swift proxy connection
- foreach ( $containerInfo as $container => $info ) {
- $contObj = new CF_Container( $conn->cfs_auth, $conn->cfs_http,
- $container, $info['count'], $info['bytes'] );
- $this->connContainerCache->set( $container, 'obj', $contObj );
- }
- } catch ( CloudFilesException $e ) { // some other exception?
- $this->handleException( $e, null, __METHOD__, array() );
- }
+ /**
+ * Get the cache key for a container
+ *
+ * @param string $username
+ * @return string
+ */
+ private function getCredsCacheKey( $username ) {
+ return 'swiftcredentials:' . md5( $username . ':' . $this->swiftAuthUrl );
}
/**
* Log an unexpected exception for this backend.
* This also sets the Status object to have a fatal error.
*
- * @param Exception $e
- * @param Status $status|null
+ * @param Status|null $status
* @param string $func
* @param array $params
- * @return void
+ * @param string $err Error string
+ * @param int $code HTTP status
+ * @param string $desc HTTP status description
*/
- protected function handleException( Exception $e, $status, $func, array $params ) {
+ public function onError( $status, $func, array $params, $err = '', $code = 0, $desc = '' ) {
if ( $status instanceof Status ) {
- if ( $e instanceof AuthenticationException ) {
- $status->fatal( 'backend-fail-connect', $this->name );
- } else {
- $status->fatal( 'backend-fail-internal', $this->name );
- }
- }
- if ( $e->getMessage() ) {
- trigger_error( "$func: " . $e->getMessage(), E_USER_WARNING );
+ $status->fatal( 'backend-fail-internal', $this->name );
}
- if ( $e instanceof InvalidResponseException ) { // possibly a stale token
- $this->srvCache->delete( $this->getCredsCacheKey( $this->auth->username ) );
- $this->closeConnection(); // force a re-connect and re-auth next time
+ if ( $code == 401 ) { // possibly a stale token
+ $this->srvCache->delete( $this->getCredsCacheKey( $this->swiftUser ) );
}
wfDebugLog( 'SwiftBackend',
- get_class( $e ) . " in '{$func}' (given '" . FormatJson::encode( $params ) . "')" .
- ( $e->getMessage() ? ": {$e->getMessage()}" : "" )
+ "HTTP $code ($desc) in '{$func}' (given '" . FormatJson::encode( $params ) . "')" .
+ ( $err ? ": $err" : "" )
);
}
}
@@ -1527,24 +1667,20 @@ class SwiftFileBackend extends FileBackendStore {
* @see FileBackendStoreOpHandle
*/
class SwiftFileOpHandle extends FileBackendStoreOpHandle {
- /** @var CF_Async_Op */
- public $cfOp;
- /** @var Array */
- public $affectedObjects = array();
+ /** @var array List of Requests for MultiHttpClient */
+ public $httpOp;
+ /** @var Closure */
+ public $callback;
/**
* @param SwiftFileBackend $backend
- * @param array $params
- * @param string $call
- * @param CF_Async_Op $cfOp
+ * @param Closure $callback Function that takes (HTTP request array, status)
+ * @param array $httpOp MultiHttpClient op
*/
- public function __construct(
- SwiftFileBackend $backend, array $params, $call, CF_Async_Op $cfOp
- ) {
+ public function __construct( SwiftFileBackend $backend, Closure $callback, array $httpOp ) {
$this->backend = $backend;
- $this->params = $params;
- $this->call = $call;
- $this->cfOp = $cfOp;
+ $this->callback = $callback;
+ $this->httpOp = $httpOp;
}
}
@@ -1556,18 +1692,29 @@ class SwiftFileOpHandle extends FileBackendStoreOpHandle {
* @ingroup FileBackend
*/
abstract class SwiftFileBackendList implements Iterator {
- /** @var Array */
+ /** @var array List of path or (path,stat array) entries */
protected $bufferIter = array();
- protected $bufferAfter = null; // string; list items *after* this path
- protected $pos = 0; // integer
- /** @var Array */
+
+ /** @var string List items *after* this path */
+ protected $bufferAfter = null;
+
+ /** @var int */
+ protected $pos = 0;
+
+ /** @var array */
protected $params = array();
/** @var SwiftFileBackend */
protected $backend;
- protected $container; // string; container name
- protected $dir; // string; storage directory
- protected $suffixStart; // integer
+
+ /** @var string Container name */
+ protected $container;
+
+ /** @var string Storage directory */
+ protected $dir;
+
+ /** @var int */
+ protected $suffixStart;
const PAGE_SIZE = 9000; // file listing buffer size
@@ -1594,7 +1741,7 @@ abstract class SwiftFileBackendList implements Iterator {
/**
* @see Iterator::key()
- * @return integer
+ * @return int
*/
public function key() {
return $this->pos;
@@ -1602,7 +1749,6 @@ abstract class SwiftFileBackendList implements Iterator {
/**
* @see Iterator::next()
- * @return void
*/
public function next() {
// Advance to the next file in the page
@@ -1619,7 +1765,6 @@ abstract class SwiftFileBackendList implements Iterator {
/**
* @see Iterator::rewind()
- * @return void
*/
public function rewind() {
$this->pos = 0;
@@ -1646,10 +1791,10 @@ abstract class SwiftFileBackendList implements Iterator {
*
* @param string $container Resolved container name
* @param string $dir Resolved path relative to container
- * @param string $after|null
- * @param integer $limit
+ * @param string $after
+ * @param int $limit
* @param array $params
- * @return Traversable|Array
+ * @return Traversable|array
*/
abstract protected function pageFromList( $container, $dir, &$after, $limit, array $params );
}
@@ -1666,10 +1811,6 @@ class SwiftFileBackendDirList extends SwiftFileBackendList {
return substr( current( $this->bufferIter ), $this->suffixStart, -1 );
}
- /**
- * @see SwiftFileBackendList::pageFromList()
- * @return Array
- */
protected function pageFromList( $container, $dir, &$after, $limit, array $params ) {
return $this->backend->getDirListPageInternal( $container, $dir, $after, $limit, $params );
}
@@ -1684,13 +1825,16 @@ class SwiftFileBackendFileList extends SwiftFileBackendList {
* @return string|bool String (relative path) or false
*/
public function current() {
- return substr( current( $this->bufferIter ), $this->suffixStart );
+ list( $path, $stat ) = current( $this->bufferIter );
+ $relPath = substr( $path, $this->suffixStart );
+ if ( is_array( $stat ) ) {
+ $storageDir = rtrim( $this->params['dir'], '/' );
+ $this->backend->loadListingStatInternal( "$storageDir/$relPath", $stat );
+ }
+
+ return $relPath;
}
- /**
- * @see SwiftFileBackendList::pageFromList()
- * @return Array
- */
protected function pageFromList( $container, $dir, &$after, $limit, array $params ) {
return $this->backend->getFileListPageInternal( $container, $dir, $after, $limit, $params );
}
diff --git a/includes/filebackend/TempFSFile.php b/includes/filebackend/TempFSFile.php
index 8266e420..1b68130f 100644
--- a/includes/filebackend/TempFSFile.php
+++ b/includes/filebackend/TempFSFile.php
@@ -28,11 +28,24 @@
* @ingroup FileBackend
*/
class TempFSFile extends FSFile {
- protected $canDelete = false; // bool; garbage collect the temp file
+ /** @var bool Garbage collect the temp file */
+ protected $canDelete = false;
- /** @var Array of active temp files to purge on shutdown */
+ /** @var array Active temp files to purge on shutdown */
protected static $instances = array();
+ /** @var array Map of (path => 1) for paths to delete on shutdown */
+ protected static $pathsCollect = null;
+
+ public function __construct( $path ) {
+ parent::__construct( $path );
+
+ if ( self::$pathsCollect === null ) {
+ self::$pathsCollect = array();
+ register_shutdown_function( array( __CLASS__, 'purgeAllOnShutdown' ) );
+ }
+ }
+
/**
* Make a new temporary file on the file system.
* Temporary files may be purged when the file object falls out of scope.
@@ -56,12 +69,14 @@ class TempFSFile extends FSFile {
}
if ( $attempt >= 5 ) {
wfProfileOut( __METHOD__ );
+
return null; // give up
}
}
$tmpFile = new self( $path );
- $tmpFile->canDelete = true; // safely instantiated
+ $tmpFile->autocollect(); // safely instantiated
wfProfileOut( __METHOD__ );
+
return $tmpFile;
}
@@ -75,13 +90,16 @@ class TempFSFile extends FSFile {
wfSuppressWarnings();
$ok = unlink( $this->path );
wfRestoreWarnings();
+
+ unset( self::$pathsCollect[$this->path] );
+
return $ok;
}
/**
* Clean up the temporary file only after an object goes out of scope
*
- * @param Object $object
+ * @param stdClass $object
* @return TempFSFile This object
*/
public function bind( $object ) {
@@ -92,6 +110,7 @@ class TempFSFile extends FSFile {
}
$object->tempFSFileReferences[] = $this;
}
+
return $this;
}
@@ -102,6 +121,9 @@ class TempFSFile extends FSFile {
*/
public function preserve() {
$this->canDelete = false;
+
+ unset( self::$pathsCollect[$this->path] );
+
return $this;
}
@@ -112,17 +134,31 @@ class TempFSFile extends FSFile {
*/
public function autocollect() {
$this->canDelete = true;
+
+ self::$pathsCollect[$this->path] = 1;
+
return $this;
}
/**
+ * Try to make sure that all files are purged on error
+ *
+ * This method should only be called internally
+ */
+ public static function purgeAllOnShutdown() {
+ foreach ( self::$pathsCollect as $path ) {
+ wfSuppressWarnings();
+ unlink( $path );
+ wfRestoreWarnings();
+ }
+ }
+
+ /**
* Cleans up after the temporary file by deleting it
*/
function __destruct() {
if ( $this->canDelete ) {
- wfSuppressWarnings();
- unlink( $this->path );
- wfRestoreWarnings();
+ $this->purge();
}
}
}
diff --git a/includes/filebackend/filejournal/DBFileJournal.php b/includes/filebackend/filejournal/DBFileJournal.php
index 9250aa5e..4f64f022 100644
--- a/includes/filebackend/filejournal/DBFileJournal.php
+++ b/includes/filebackend/filejournal/DBFileJournal.php
@@ -34,10 +34,9 @@ class DBFileJournal extends FileJournal {
/**
* Construct a new instance from configuration.
- * $config includes:
- * 'wiki' : wiki name to use for LoadBalancer
*
- * @param $config Array
+ * @param array $config Includes:
+ * 'wiki' : wiki name to use for LoadBalancer
*/
protected function __construct( array $config ) {
parent::__construct( $config );
@@ -47,6 +46,8 @@ class DBFileJournal extends FileJournal {
/**
* @see FileJournal::logChangeBatch()
+ * @param array $entries
+ * @param string $batchId
* @return Status
*/
protected function doLogChangeBatch( array $entries, $batchId ) {
@@ -56,6 +57,7 @@ class DBFileJournal extends FileJournal {
$dbw = $this->getMasterDB();
} catch ( DBError $e ) {
$status->fatal( 'filejournal-fail-dbconnect', $this->backend );
+
return $status;
}
@@ -80,6 +82,7 @@ class DBFileJournal extends FileJournal {
}
} catch ( DBError $e ) {
$status->fatal( 'filejournal-fail-dbquery', $this->backend );
+
return $status;
}
@@ -88,7 +91,7 @@ class DBFileJournal extends FileJournal {
/**
* @see FileJournal::doGetCurrentPosition()
- * @return integer|false
+ * @return bool|mixed The value from the field, or false on failure.
*/
protected function doGetCurrentPosition() {
$dbw = $this->getMasterDB();
@@ -101,13 +104,14 @@ class DBFileJournal extends FileJournal {
/**
* @see FileJournal::doGetPositionAtTime()
- * @param $time integer|string timestamp
- * @return integer|false
+ * @param int|string $time Timestamp
+ * @return bool|mixed The value from the field, or false on failure.
*/
protected function doGetPositionAtTime( $time ) {
$dbw = $this->getMasterDB();
$encTimestamp = $dbw->addQuotes( $dbw->timestamp( $time ) );
+
return $dbw->selectField( 'filejournal', 'fj_id',
array( 'fj_backend' => $this->backend, "fj_timestamp <= $encTimestamp" ),
__METHOD__,
@@ -117,8 +121,9 @@ class DBFileJournal extends FileJournal {
/**
* @see FileJournal::doGetChangeEntries()
- * @return Array
- * @throws DBError
+ * @param int $start
+ * @param int $limit
+ * @return array
*/
protected function doGetChangeEntries( $start, $limit ) {
$dbw = $this->getMasterDB();
@@ -179,6 +184,7 @@ class DBFileJournal extends FileJournal {
$this->dbw = $lb->getConnection( DB_MASTER, array(), $this->wiki );
$this->dbw->clearFlag( DBO_TRX );
}
+
return $this->dbw;
}
}
diff --git a/includes/filebackend/filejournal/FileJournal.php b/includes/filebackend/filejournal/FileJournal.php
index a1b7a459..c0651485 100644
--- a/includes/filebackend/filejournal/FileJournal.php
+++ b/includes/filebackend/filejournal/FileJournal.php
@@ -36,15 +36,17 @@
* @since 1.20
*/
abstract class FileJournal {
- protected $backend; // string
- protected $ttlDays; // integer
+ /** @var string */
+ protected $backend;
+
+ /** @var int */
+ protected $ttlDays;
/**
* Construct a new instance from configuration.
- * $config includes:
- * 'ttlDays' : days to keep log entries around (false means "forever")
*
- * @param $config Array
+ * @param array $config Includes:
+ * 'ttlDays' : days to keep log entries around (false means "forever")
*/
protected function __construct( array $config ) {
$this->ttlDays = isset( $config['ttlDays'] ) ? $config['ttlDays'] : false;
@@ -53,7 +55,7 @@ abstract class FileJournal {
/**
* Create an appropriate FileJournal object from config
*
- * @param $config Array
+ * @param array $config
* @param string $backend A registered file backend name
* @throws MWException
* @return FileJournal
@@ -65,6 +67,7 @@ abstract class FileJournal {
throw new MWException( "Class given is not an instance of FileJournal." );
}
$jrn->backend = $backend;
+
return $jrn;
}
@@ -79,18 +82,18 @@ abstract class FileJournal {
$s .= mt_rand( 0, 2147483647 );
}
$s = wfBaseConvert( sha1( $s ), 16, 36, 31 );
+
return substr( wfBaseConvert( wfTimestamp( TS_MW ), 10, 36, 9 ) . $s, 0, 31 );
}
/**
* Log changes made by a batch file operation.
- * $entries is an array of log entries, each of which contains:
+ *
+ * @param array $entries List of file operations (each an array of parameters) which contain:
* op : Basic operation name (create, update, delete)
* path : The storage path of the file
* newSha1 : The final base 36 SHA-1 of the file
- * Note that 'false' should be used as the SHA-1 for non-existing files.
- *
- * @param array $entries List of file operations (each an array of parameters)
+ * Note that 'false' should be used as the SHA-1 for non-existing files.
* @param string $batchId UUID string that identifies the operation batch
* @return Status
*/
@@ -98,6 +101,7 @@ abstract class FileJournal {
if ( !count( $entries ) ) {
return Status::newGood();
}
+
return $this->doLogChangeBatch( $entries, $batchId );
}
@@ -113,7 +117,7 @@ abstract class FileJournal {
/**
* Get the position ID of the latest journal entry
*
- * @return integer|false
+ * @return int|bool
*/
final public function getCurrentPosition() {
return $this->doGetCurrentPosition();
@@ -121,15 +125,15 @@ abstract class FileJournal {
/**
* @see FileJournal::getCurrentPosition()
- * @return integer|false
+ * @return int|bool
*/
abstract protected function doGetCurrentPosition();
/**
* Get the position ID of the latest journal entry at some point in time
*
- * @param $time integer|string timestamp
- * @return integer|false
+ * @param int|string $time Timestamp
+ * @return int|bool
*/
final public function getPositionAtTime( $time ) {
return $this->doGetPositionAtTime( $time );
@@ -137,8 +141,8 @@ abstract class FileJournal {
/**
* @see FileJournal::getPositionAtTime()
- * @param $time integer|string timestamp
- * @return integer|false
+ * @param int|string $time Timestamp
+ * @return int|bool
*/
abstract protected function doGetPositionAtTime( $time );
@@ -146,7 +150,10 @@ abstract class FileJournal {
* Get an array of file change log entries.
* A starting change ID and/or limit can be specified.
*
- * The result as a list of associative arrays, each having:
+ * @param int $start Starting change ID or null
+ * @param int $limit Maximum number of items to return
+ * @param string &$next Updated to the ID of the next entry.
+ * @return array List of associative arrays, each having:
* id : unique, monotonic, ID for this change
* batch_uuid : UUID for an operation batch
* backend : the backend name
@@ -154,13 +161,7 @@ abstract class FileJournal {
* path : affected storage path
* new_sha1 : base 36 sha1 of the new file had the operation succeeded
* timestamp : TS_MW timestamp of the batch change
-
- * Also, $next is updated to the ID of the next entry.
- *
- * @param $start integer Starting change ID or null
- * @param $limit integer Maximum number of items to return
- * @param &$next string
- * @return Array
+ * Also, $next is updated to the ID of the next entry.
*/
final public function getChangeEntries( $start = null, $limit = 0, &$next = null ) {
$entries = $this->doGetChangeEntries( $start, $limit ? $limit + 1 : 0 );
@@ -170,12 +171,15 @@ abstract class FileJournal {
} else {
$next = null; // end of list
}
+
return $entries;
}
/**
* @see FileJournal::getChangeEntries()
- * @return Array
+ * @param int $start
+ * @param int $limit
+ * @return array
*/
abstract protected function doGetChangeEntries( $start, $limit );
@@ -202,8 +206,8 @@ abstract class FileJournal {
class NullFileJournal extends FileJournal {
/**
* @see FileJournal::doLogChangeBatch()
- * @param $entries array
- * @param $batchId string
+ * @param array $entries
+ * @param string $batchId
* @return Status
*/
protected function doLogChangeBatch( array $entries, $batchId ) {
@@ -212,7 +216,7 @@ class NullFileJournal extends FileJournal {
/**
* @see FileJournal::doGetCurrentPosition()
- * @return integer|false
+ * @return int|bool
*/
protected function doGetCurrentPosition() {
return false;
@@ -220,8 +224,8 @@ class NullFileJournal extends FileJournal {
/**
* @see FileJournal::doGetPositionAtTime()
- * @param $time integer|string timestamp
- * @return integer|false
+ * @param int|string $time Timestamp
+ * @return int|bool
*/
protected function doGetPositionAtTime( $time ) {
return false;
@@ -229,7 +233,9 @@ class NullFileJournal extends FileJournal {
/**
* @see FileJournal::doGetChangeEntries()
- * @return Array
+ * @param int $start
+ * @param int $limit
+ * @return array
*/
protected function doGetChangeEntries( $start, $limit ) {
return array();
diff --git a/includes/filebackend/lockmanager/DBLockManager.php b/includes/filebackend/lockmanager/DBLockManager.php
index 3e934ba5..450ccc82 100644
--- a/includes/filebackend/lockmanager/DBLockManager.php
+++ b/includes/filebackend/lockmanager/DBLockManager.php
@@ -37,7 +37,7 @@
* @since 1.19
*/
abstract class DBLockManager extends QuorumLockManager {
- /** @var Array Map of DB names to server config */
+ /** @var array Map of DB names to server config */
protected $dbServers; // (DB name => server config array)
/** @var BagOStuff */
protected $statusCache;
@@ -46,13 +46,13 @@ abstract class DBLockManager extends QuorumLockManager {
protected $safeDelay; // integer number of seconds
protected $session = 0; // random integer
- /** @var Array Map Database connections (DB name => Database) */
+ /** @var array Map Database connections (DB name => Database) */
protected $conns = array();
/**
* Construct a new instance from configuration.
*
- * $config paramaters include:
+ * @param array $config Paramaters include:
* - dbServers : Associative array of DB names to server configuration.
* Configuration is an associative array that includes:
* - host : DB server name
@@ -70,8 +70,6 @@ abstract class DBLockManager extends QuorumLockManager {
* - lockExpiry : Lock timeout (seconds) for dropped connections. [optional]
* This tells the DB server how long to wait before assuming
* connection failure and releasing all the locks for a session.
- *
- * @param array $config
*/
public function __construct( array $config ) {
parent::__construct( $config );
@@ -110,12 +108,13 @@ abstract class DBLockManager extends QuorumLockManager {
$this->session = wfRandomString( 31 );
}
- // @TODO: change this code to work in one batch
+ // @todo change this code to work in one batch
protected function getLocksOnServer( $lockSrv, array $pathsByType ) {
$status = Status::newGood();
foreach ( $pathsByType as $type => $paths ) {
$status->merge( $this->doGetLocksOnServer( $lockSrv, $paths, $type ) );
}
+
return $status;
}
@@ -125,6 +124,7 @@ abstract class DBLockManager extends QuorumLockManager {
/**
* @see QuorumLockManager::isServerUp()
+ * @param string $lockSrv
* @return bool
*/
protected function isServerUp( $lockSrv ) {
@@ -135,15 +135,17 @@ abstract class DBLockManager extends QuorumLockManager {
$this->getConnection( $lockSrv );
} catch ( DBError $e ) {
$this->cacheRecordFailure( $lockSrv );
+
return false; // failed to connect
}
+
return true;
}
/**
* Get (or reuse) a connection to a lock DB
*
- * @param $lockDb string
+ * @param string $lockDb
* @return DatabaseBase
* @throws DBError
*/
@@ -175,24 +177,25 @@ abstract class DBLockManager extends QuorumLockManager {
if ( !$this->conns[$lockDb]->trxLevel() ) {
$this->conns[$lockDb]->begin( __METHOD__ ); // start transaction
}
+
return $this->conns[$lockDb];
}
/**
* Do additional initialization for new lock DB connection
*
- * @param $lockDb string
- * @param $db DatabaseBase
- * @return void
+ * @param string $lockDb
+ * @param DatabaseBase $db
* @throws DBError
*/
- protected function initConnection( $lockDb, DatabaseBase $db ) {}
+ protected function initConnection( $lockDb, DatabaseBase $db ) {
+ }
/**
* Checks if the DB has not recently had connection/query errors.
* This just avoids wasting time on doomed connection attempts.
*
- * @param $lockDb string
+ * @param string $lockDb
* @return bool
*/
protected function cacheCheckFailures( $lockDb ) {
@@ -204,7 +207,7 @@ abstract class DBLockManager extends QuorumLockManager {
/**
* Log a lock request failure to the cache
*
- * @param $lockDb string
+ * @param string $lockDb
* @return bool Success
*/
protected function cacheRecordFailure( $lockDb ) {
@@ -216,7 +219,7 @@ abstract class DBLockManager extends QuorumLockManager {
/**
* Get a cache key for recent query misses for a DB
*
- * @param $lockDb string
+ * @param string $lockDb
* @return string
*/
protected function getMissKey( $lockDb ) {
@@ -242,7 +245,7 @@ abstract class DBLockManager extends QuorumLockManager {
* @ingroup LockManager
*/
class MySqlLockManager extends DBLockManager {
- /** @var Array Mapping of lock types to the type actually used */
+ /** @var array Mapping of lock types to the type actually used */
protected $lockTypeMap = array(
self::LOCK_SH => self::LOCK_SH,
self::LOCK_UW => self::LOCK_SH,
@@ -250,8 +253,8 @@ class MySqlLockManager extends DBLockManager {
);
/**
- * @param $lockDb string
- * @param $db DatabaseBase
+ * @param string $lockDb
+ * @param DatabaseBase $db
*/
protected function initConnection( $lockDb, DatabaseBase $db ) {
# Let this transaction see lock rows from other transactions
@@ -263,6 +266,9 @@ class MySqlLockManager extends DBLockManager {
* This does not use GET_LOCK() per http://bugs.mysql.com/bug.php?id=1118.
*
* @see DBLockManager::getLocksOnServer()
+ * @param string $lockSrv
+ * @param array $paths
+ * @param string $type
* @return Status
*/
protected function doGetLocksOnServer( $lockSrv, array $paths, $type ) {
@@ -359,7 +365,7 @@ class MySqlLockManager extends DBLockManager {
* @ingroup LockManager
*/
class PostgreSqlLockManager extends DBLockManager {
- /** @var Array Mapping of lock types to the type actually used */
+ /** @var array Mapping of lock types to the type actually used */
protected $lockTypeMap = array(
self::LOCK_SH => self::LOCK_SH,
self::LOCK_UW => self::LOCK_SH,
@@ -374,7 +380,7 @@ class PostgreSqlLockManager extends DBLockManager {
$db = $this->getConnection( $lockSrv ); // checked in isServerUp()
$bigints = array_unique( array_map(
- function( $key ) {
+ function ( $key ) {
return wfBaseConvert( substr( $key, 0, 15 ), 16, 10 );
},
array_map( array( $this, 'sha1Base16Absolute' ), $paths )
diff --git a/includes/filebackend/lockmanager/FSLockManager.php b/includes/filebackend/lockmanager/FSLockManager.php
index eacba704..bce6b34c 100644
--- a/includes/filebackend/lockmanager/FSLockManager.php
+++ b/includes/filebackend/lockmanager/FSLockManager.php
@@ -34,7 +34,7 @@
* @since 1.19
*/
class FSLockManager extends LockManager {
- /** @var Array Mapping of lock types to the type actually used */
+ /** @var array Mapping of lock types to the type actually used */
protected $lockTypeMap = array(
self::LOCK_SH => self::LOCK_SH,
self::LOCK_UW => self::LOCK_SH,
@@ -43,16 +43,14 @@ class FSLockManager extends LockManager {
protected $lockDir; // global dir for all servers
- /** @var Array Map of (locked key => lock file handle) */
+ /** @var array Map of (locked key => lock file handle) */
protected $handles = array();
/**
* Construct a new instance from configuration.
*
- * $config includes:
+ * @param array $config Includes:
* - lockDirectory : Directory containing the lock files
- *
- * @param array $config
*/
function __construct( array $config ) {
parent::__construct( $config );
@@ -62,8 +60,8 @@ class FSLockManager extends LockManager {
/**
* @see LockManager::doLock()
- * @param $paths array
- * @param $type int
+ * @param array $paths
+ * @param int $type
* @return Status
*/
protected function doLock( array $paths, $type ) {
@@ -77,6 +75,7 @@ class FSLockManager extends LockManager {
} else {
// Abort and unlock everything
$status->merge( $this->doUnlock( $lockedPaths, $type ) );
+
return $status;
}
}
@@ -86,8 +85,8 @@ class FSLockManager extends LockManager {
/**
* @see LockManager::doUnlock()
- * @param $paths array
- * @param $type int
+ * @param array $paths
+ * @param int $type
* @return Status
*/
protected function doUnlock( array $paths, $type ) {
@@ -103,8 +102,8 @@ class FSLockManager extends LockManager {
/**
* Lock a single resource key
*
- * @param $path string
- * @param $type integer
+ * @param string $path
+ * @param int $type
* @return Status
*/
protected function doSingleLock( $path, $type ) {
@@ -148,8 +147,8 @@ class FSLockManager extends LockManager {
/**
* Unlock a single resource key
*
- * @param $path string
- * @param $type integer
+ * @param string $path
+ * @param int $type
* @return Status
*/
protected function doSingleUnlock( $path, $type ) {
@@ -191,8 +190,8 @@ class FSLockManager extends LockManager {
}
/**
- * @param $path string
- * @param $handlesToClose array
+ * @param string $path
+ * @param array $handlesToClose
* @return Status
*/
private function closeLockHandles( $path, array $handlesToClose ) {
@@ -205,11 +204,12 @@ class FSLockManager extends LockManager {
$status->warning( 'lockmanager-fail-closelock', $path );
}
}
+
return $status;
}
/**
- * @param $path string
+ * @param string $path
* @return Status
*/
private function pruneKeyLockFiles( $path ) {
@@ -221,12 +221,13 @@ class FSLockManager extends LockManager {
}
unset( $this->handles[$path] );
}
+
return $status;
}
/**
* Get the path to the lock file for a key
- * @param $path string
+ * @param string $path
* @return string
*/
protected function getLockPath( $path ) {
diff --git a/includes/filebackend/lockmanager/LSLockManager.php b/includes/filebackend/lockmanager/LSLockManager.php
deleted file mode 100644
index 97de8dca..00000000
--- a/includes/filebackend/lockmanager/LSLockManager.php
+++ /dev/null
@@ -1,218 +0,0 @@
-<?php
-/**
- * Version of LockManager based on using lock daemon servers.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
- * http://www.gnu.org/copyleft/gpl.html
- *
- * @file
- * @ingroup LockManager
- */
-
-/**
- * Manage locks using a lock daemon server.
- *
- * Version of LockManager based on using lock daemon servers.
- * This is meant for multi-wiki systems that may share files.
- * All locks are non-blocking, which avoids deadlocks.
- *
- * All lock requests for a resource, identified by a hash string, will map
- * to one bucket. Each bucket maps to one or several peer servers, each
- * running LockServerDaemon.php, listening on a designated TCP port.
- * A majority of peers must agree for a lock to be acquired.
- *
- * @ingroup LockManager
- * @since 1.19
- */
-class LSLockManager extends QuorumLockManager {
- /** @var Array Mapping of lock types to the type actually used */
- protected $lockTypeMap = array(
- self::LOCK_SH => self::LOCK_SH,
- self::LOCK_UW => self::LOCK_SH,
- self::LOCK_EX => self::LOCK_EX
- );
-
- /** @var Array Map of server names to server config */
- protected $lockServers; // (server name => server config array)
-
- /** @var Array Map Server connections (server name => resource) */
- protected $conns = array();
-
- protected $connTimeout; // float number of seconds
- protected $session = ''; // random SHA-1 string
-
- /**
- * Construct a new instance from configuration.
- *
- * $config paramaters include:
- * - lockServers : Associative array of server names to configuration.
- * Configuration is an associative array that includes:
- * - host : IP address/hostname
- * - port : TCP port
- * - authKey : Secret string the lock server uses
- * - srvsByBucket : Array of 1-16 consecutive integer keys, starting from 0,
- * each having an odd-numbered list of server names (peers) as values.
- * - connTimeout : Lock server connection attempt timeout. [optional]
- *
- * @param array $config
- */
- public function __construct( array $config ) {
- parent::__construct( $config );
-
- $this->lockServers = $config['lockServers'];
- // Sanitize srvsByBucket config to prevent PHP errors
- $this->srvsByBucket = array_filter( $config['srvsByBucket'], 'is_array' );
- $this->srvsByBucket = array_values( $this->srvsByBucket ); // consecutive
-
- if ( isset( $config['connTimeout'] ) ) {
- $this->connTimeout = $config['connTimeout'];
- } else {
- $this->connTimeout = 3; // use some sane amount
- }
-
- $this->session = wfRandomString( 32 ); // 128 bits
- }
-
- /**
- * @see QuorumLockManager::getLocksOnServer()
- * @return Status
- */
- protected function getLocksOnServer( $lockSrv, array $paths, $type ) {
- $status = Status::newGood();
-
- // Send out the command and get the response...
- $type = ( $type == self::LOCK_SH ) ? 'SH' : 'EX';
- $keys = array_unique( array_map( array( $this, 'sha1Base36Absolute' ), $paths ) );
- $response = $this->sendCommand( $lockSrv, 'ACQUIRE', $type, $keys );
-
- if ( $response !== 'ACQUIRED' ) {
- foreach ( $paths as $path ) {
- $status->fatal( 'lockmanager-fail-acquirelock', $path );
- }
- }
-
- return $status;
- }
-
- /**
- * @see QuorumLockManager::freeLocksOnServer()
- * @return Status
- */
- protected function freeLocksOnServer( $lockSrv, array $paths, $type ) {
- $status = Status::newGood();
-
- // Send out the command and get the response...
- $type = ( $type == self::LOCK_SH ) ? 'SH' : 'EX';
- $keys = array_unique( array_map( array( $this, 'sha1Base36Absolute' ), $paths ) );
- $response = $this->sendCommand( $lockSrv, 'RELEASE', $type, $keys );
-
- if ( $response !== 'RELEASED' ) {
- foreach ( $paths as $path ) {
- $status->fatal( 'lockmanager-fail-releaselock', $path );
- }
- }
-
- return $status;
- }
-
- /**
- * @see QuorumLockManager::releaseAllLocks()
- * @return Status
- */
- protected function releaseAllLocks() {
- $status = Status::newGood();
-
- foreach ( $this->conns as $lockSrv => $conn ) {
- $response = $this->sendCommand( $lockSrv, 'RELEASE_ALL', '', array() );
- if ( $response !== 'RELEASED_ALL' ) {
- $status->fatal( 'lockmanager-fail-svr-release', $lockSrv );
- }
- }
-
- return $status;
- }
-
- /**
- * @see QuorumLockManager::isServerUp()
- * @return bool
- */
- protected function isServerUp( $lockSrv ) {
- return (bool)$this->getConnection( $lockSrv );
- }
-
- /**
- * Send a command and get back the response
- *
- * @param $lockSrv string
- * @param $action string
- * @param $type string
- * @param $values Array
- * @return string|bool
- */
- protected function sendCommand( $lockSrv, $action, $type, $values ) {
- $conn = $this->getConnection( $lockSrv );
- if ( !$conn ) {
- return false; // no connection
- }
- $authKey = $this->lockServers[$lockSrv]['authKey'];
- // Build of the command as a flat string...
- $values = implode( '|', $values );
- $key = hash_hmac( 'sha1', "{$this->session}\n{$action}\n{$type}\n{$values}", $authKey );
- // Send out the command...
- if ( fwrite( $conn, "{$this->session}:$key:$action:$type:$values\n" ) === false ) {
- return false;
- }
- // Get the response...
- $response = fgets( $conn );
- if ( $response === false ) {
- return false;
- }
- return trim( $response );
- }
-
- /**
- * Get (or reuse) a connection to a lock server
- *
- * @param $lockSrv string
- * @return resource
- */
- protected function getConnection( $lockSrv ) {
- if ( !isset( $this->conns[$lockSrv] ) ) {
- $cfg = $this->lockServers[$lockSrv];
- wfSuppressWarnings();
- $errno = $errstr = '';
- $conn = fsockopen( $cfg['host'], $cfg['port'], $errno, $errstr, $this->connTimeout );
- wfRestoreWarnings();
- if ( $conn === false ) {
- return null;
- }
- $sec = floor( $this->connTimeout );
- $usec = floor( ( $this->connTimeout - floor( $this->connTimeout ) ) * 1e6 );
- stream_set_timeout( $conn, $sec, $usec );
- $this->conns[$lockSrv] = $conn;
- }
- return $this->conns[$lockSrv];
- }
-
- /**
- * Make sure remaining locks get cleared for sanity
- */
- function __destruct() {
- $this->releaseAllLocks();
- foreach ( $this->conns as $conn ) {
- fclose( $conn );
- }
- }
-}
diff --git a/includes/filebackend/lockmanager/LockManager.php b/includes/filebackend/lockmanager/LockManager.php
index dad8a624..df8d2d4f 100644
--- a/includes/filebackend/lockmanager/LockManager.php
+++ b/includes/filebackend/lockmanager/LockManager.php
@@ -43,14 +43,14 @@
* @since 1.19
*/
abstract class LockManager {
- /** @var Array Mapping of lock types to the type actually used */
+ /** @var array Mapping of lock types to the type actually used */
protected $lockTypeMap = array(
self::LOCK_SH => self::LOCK_SH,
self::LOCK_UW => self::LOCK_EX, // subclasses may use self::LOCK_SH
self::LOCK_EX => self::LOCK_EX
);
- /** @var Array Map of (resource path => lock type => count) */
+ /** @var array Map of (resource path => lock type => count) */
protected $locksHeld = array();
protected $domain; // string; domain (usually wiki ID)
@@ -64,12 +64,10 @@ abstract class LockManager {
/**
* Construct a new instance from configuration
*
- * $config paramaters include:
+ * @param array $config Paramaters include:
* - domain : Domain (usually wiki ID) that all resources are relative to [optional]
* - lockTTL : Age (in seconds) at which resource locks should expire.
* This only applies if locks are not tied to a connection/process.
- *
- * @param $config Array
*/
public function __construct( array $config ) {
$this->domain = isset( $config['domain'] ) ? $config['domain'] : wfWikiID();
@@ -87,8 +85,8 @@ abstract class LockManager {
* Lock the resources at the given abstract paths
*
* @param array $paths List of resource names
- * @param $type integer LockManager::LOCK_* constant
- * @param integer $timeout Timeout in seconds (0 means non-blocking) (since 1.21)
+ * @param int $type LockManager::LOCK_* constant
+ * @param int $timeout Timeout in seconds (0 means non-blocking) (since 1.21)
* @return Status
*/
final public function lock( array $paths, $type = self::LOCK_EX, $timeout = 0 ) {
@@ -99,7 +97,7 @@ abstract class LockManager {
* Lock the resources at the given abstract paths
*
* @param array $pathsByType Map of LockManager::LOCK_* constants to lists of paths
- * @param integer $timeout Timeout in seconds (0 means non-blocking) (since 1.21)
+ * @param int $timeout Timeout in seconds (0 means non-blocking) (since 1.21)
* @return Status
* @since 1.22
*/
@@ -119,6 +117,7 @@ abstract class LockManager {
$elapsed = microtime( true ) - $start;
} while ( $elapsed < $timeout && $elapsed >= 0 );
wfProfileOut( __METHOD__ );
+
return $status;
}
@@ -126,7 +125,7 @@ abstract class LockManager {
* Unlock the resources at the given abstract paths
*
* @param array $paths List of paths
- * @param $type integer LockManager::LOCK_* constant
+ * @param int $type LockManager::LOCK_* constant
* @return Status
*/
final public function unlock( array $paths, $type = self::LOCK_EX ) {
@@ -145,6 +144,7 @@ abstract class LockManager {
$pathsByType = $this->normalizePathsByType( $pathsByType );
$status = $this->doUnlockByType( $pathsByType );
wfProfileOut( __METHOD__ );
+
return $status;
}
@@ -153,7 +153,7 @@ abstract class LockManager {
* Before hashing, the path will be prefixed with the domain ID.
* This should be used interally for lock key or file names.
*
- * @param $path string
+ * @param string $path
* @return string
*/
final protected function sha1Base36Absolute( $path ) {
@@ -165,7 +165,7 @@ abstract class LockManager {
* Before hashing, the path will be prefixed with the domain ID.
* This should be used interally for lock key or file names.
*
- * @param $path string
+ * @param string $path
* @return string
*/
final protected function sha1Base16Absolute( $path ) {
@@ -176,8 +176,8 @@ abstract class LockManager {
* Normalize the $paths array by converting LOCK_UW locks into the
* appropriate type and removing any duplicated paths for each lock type.
*
- * @param array $paths Map of LockManager::LOCK_* constants to lists of paths
- * @return Array
+ * @param array $pathsByType Map of LockManager::LOCK_* constants to lists of paths
+ * @return array
* @since 1.22
*/
final protected function normalizePathsByType( array $pathsByType ) {
@@ -185,12 +185,13 @@ abstract class LockManager {
foreach ( $pathsByType as $type => $paths ) {
$res[$this->lockTypeMap[$type]] = array_unique( $paths );
}
+
return $res;
}
/**
* @see LockManager::lockByType()
- * @param array $paths Map of LockManager::LOCK_* constants to lists of paths
+ * @param array $pathsByType Map of LockManager::LOCK_* constants to lists of paths
* @return Status
* @since 1.22
*/
@@ -203,12 +204,13 @@ abstract class LockManager {
$lockedByType[$type] = $paths;
} else {
// Release the subset of locks that were acquired
- foreach ( $lockedByType as $type => $paths ) {
- $status->merge( $this->doUnlock( $paths, $type ) );
+ foreach ( $lockedByType as $lType => $lPaths ) {
+ $status->merge( $this->doUnlock( $lPaths, $lType ) );
}
break;
}
}
+
return $status;
}
@@ -216,14 +218,14 @@ abstract class LockManager {
* Lock resources with the given keys and lock type
*
* @param array $paths List of paths
- * @param $type integer LockManager::LOCK_* constant
+ * @param int $type LockManager::LOCK_* constant
* @return Status
*/
abstract protected function doLock( array $paths, $type );
/**
* @see LockManager::unlockByType()
- * @param array $paths Map of LockManager::LOCK_* constants to lists of paths
+ * @param array $pathsByType Map of LockManager::LOCK_* constants to lists of paths
* @return Status
* @since 1.22
*/
@@ -232,6 +234,7 @@ abstract class LockManager {
foreach ( $pathsByType as $type => $paths ) {
$status->merge( $this->doUnlock( $paths, $type ) );
}
+
return $status;
}
@@ -239,7 +242,7 @@ abstract class LockManager {
* Unlock resources with the given keys and lock type
*
* @param array $paths List of paths
- * @param $type integer LockManager::LOCK_* constant
+ * @param int $type LockManager::LOCK_* constant
* @return Status
*/
abstract protected function doUnlock( array $paths, $type );
diff --git a/includes/filebackend/lockmanager/LockManagerGroup.php b/includes/filebackend/lockmanager/LockManagerGroup.php
index 9aff2415..19fc4fef 100644
--- a/includes/filebackend/lockmanager/LockManagerGroup.php
+++ b/includes/filebackend/lockmanager/LockManagerGroup.php
@@ -29,12 +29,12 @@
* @since 1.19
*/
class LockManagerGroup {
- /** @var Array (domain => LockManager) */
+ /** @var array (domain => LockManager) */
protected static $instances = array();
protected $domain; // string; domain (usually wiki ID)
- /** @var Array of (name => ('class' => ..., 'config' => ..., 'instance' => ...)) */
+ /** @var array Array of (name => ('class' => ..., 'config' => ..., 'instance' => ...)) */
protected $managers = array();
/**
@@ -45,7 +45,7 @@ class LockManagerGroup {
}
/**
- * @param string $domain Domain (usually wiki ID)
+ * @param bool|string $domain Domain (usually wiki ID). Default: false.
* @return LockManagerGroup
*/
public static function singleton( $domain = false ) {
@@ -54,13 +54,12 @@ class LockManagerGroup {
self::$instances[$domain] = new self( $domain );
self::$instances[$domain]->initFromGlobals();
}
+
return self::$instances[$domain];
}
/**
* Destroy the singleton instances
- *
- * @return void
*/
public static function destroySingletons() {
self::$instances = array();
@@ -68,8 +67,6 @@ class LockManagerGroup {
/**
* Register lock managers from the global variables
- *
- * @return void
*/
protected function initFromGlobals() {
global $wgLockManagers;
@@ -80,8 +77,7 @@ class LockManagerGroup {
/**
* Register an array of file lock manager configurations
*
- * @param $configs Array
- * @return void
+ * @param array $configs
* @throws MWException
*/
protected function register( array $configs ) {
@@ -107,7 +103,7 @@ class LockManagerGroup {
/**
* Get the lock manager object with a given name
*
- * @param $name string
+ * @param string $name
* @return LockManager
* @throws MWException
*/
@@ -121,14 +117,15 @@ class LockManagerGroup {
$config = $this->managers[$name]['config'];
$this->managers[$name]['instance'] = new $class( $config );
}
+
return $this->managers[$name]['instance'];
}
/**
* Get the config array for a lock manager object with a given name
*
- * @param $name string
- * @return Array
+ * @param string $name
+ * @return array
* @throws MWException
*/
public function config( $name ) {
@@ -136,6 +133,7 @@ class LockManagerGroup {
throw new MWException( "No lock manager defined with the name `$name`." );
}
$class = $this->managers[$name]['class'];
+
return array( 'class' => $class ) + $this->managers[$name]['config'];
}
diff --git a/includes/filebackend/lockmanager/MemcLockManager.php b/includes/filebackend/lockmanager/MemcLockManager.php
index 5eab03ee..9bb01c21 100644
--- a/includes/filebackend/lockmanager/MemcLockManager.php
+++ b/includes/filebackend/lockmanager/MemcLockManager.php
@@ -36,31 +36,31 @@
* @since 1.20
*/
class MemcLockManager extends QuorumLockManager {
- /** @var Array Mapping of lock types to the type actually used */
+ /** @var array Mapping of lock types to the type actually used */
protected $lockTypeMap = array(
self::LOCK_SH => self::LOCK_SH,
self::LOCK_UW => self::LOCK_SH,
self::LOCK_EX => self::LOCK_EX
);
- /** @var Array Map server names to MemcachedBagOStuff objects */
+ /** @var array Map server names to MemcachedBagOStuff objects */
protected $bagOStuffs = array();
- /** @var Array */
- protected $serversUp = array(); // (server name => bool)
- protected $session = ''; // string; random UUID
+ /** @var array (server name => bool) */
+ protected $serversUp = array();
+
+ /** @var string Random UUID */
+ protected $session = '';
/**
* Construct a new instance from configuration.
*
- * $config paramaters include:
+ * @param array $config Paramaters include:
* - lockServers : Associative array of server names to "<IP>:<port>" strings.
* - srvsByBucket : Array of 1-16 consecutive integer keys, starting from 0,
* each having an odd-numbered list of server names (peers) as values.
* - memcConfig : Configuration array for ObjectCache::newFromParams. [optional]
* If set, this must use one of the memcached classes.
- *
- * @param array $config
* @throws MWException
*/
public function __construct( array $config ) {
@@ -88,7 +88,7 @@ class MemcLockManager extends QuorumLockManager {
$this->session = wfRandomString( 32 );
}
- // @TODO: change this code to work in one batch
+ // @todo Change this code to work in one batch
protected function getLocksOnServer( $lockSrv, array $pathsByType ) {
$status = Status::newGood();
@@ -100,8 +100,8 @@ class MemcLockManager extends QuorumLockManager {
? array_merge( $lockedPaths[$type], $paths )
: $paths;
} else {
- foreach ( $lockedPaths as $type => $paths ) {
- $status->merge( $this->doFreeLocksOnServer( $lockSrv, $paths, $type ) );
+ foreach ( $lockedPaths as $lType => $lPaths ) {
+ $status->merge( $this->doFreeLocksOnServer( $lockSrv, $lPaths, $lType ) );
}
break;
}
@@ -110,7 +110,7 @@ class MemcLockManager extends QuorumLockManager {
return $status;
}
- // @TODO: change this code to work in one batch
+ // @todo Change this code to work in one batch
protected function freeLocksOnServer( $lockSrv, array $pathsByType ) {
$status = Status::newGood();
@@ -123,6 +123,9 @@ class MemcLockManager extends QuorumLockManager {
/**
* @see QuorumLockManager::getLocksOnServer()
+ * @param string $lockSrv
+ * @param array $paths
+ * @param string $type
* @return Status
*/
protected function doGetLocksOnServer( $lockSrv, array $paths, $type ) {
@@ -136,6 +139,7 @@ class MemcLockManager extends QuorumLockManager {
foreach ( $paths as $path ) {
$status->fatal( 'lockmanager-fail-acquirelock', $path );
}
+
return $status;
}
@@ -195,6 +199,9 @@ class MemcLockManager extends QuorumLockManager {
/**
* @see QuorumLockManager::freeLocksOnServer()
+ * @param string $lockSrv
+ * @param array $paths
+ * @param string $type
* @return Status
*/
protected function doFreeLocksOnServer( $lockSrv, array $paths, $type ) {
@@ -208,7 +215,8 @@ class MemcLockManager extends QuorumLockManager {
foreach ( $paths as $path ) {
$status->fatal( 'lockmanager-fail-releaselock', $path );
}
- return;
+
+ return $status;
}
// Fetch all the existing lock records...
@@ -254,6 +262,7 @@ class MemcLockManager extends QuorumLockManager {
/**
* @see QuorumLockManager::isServerUp()
+ * @param string $lockSrv
* @return bool
*/
protected function isServerUp( $lockSrv ) {
@@ -280,11 +289,12 @@ class MemcLockManager extends QuorumLockManager {
return null; // server appears to be down
}
}
+
return $memc;
}
/**
- * @param $path string
+ * @param string $path
* @return string
*/
protected function recordKeyForPath( $path ) {
@@ -292,27 +302,28 @@ class MemcLockManager extends QuorumLockManager {
}
/**
- * @return Array An empty lock structure for a key
+ * @return array An empty lock structure for a key
*/
protected static function newLockArray() {
return array( self::LOCK_SH => array(), self::LOCK_EX => array() );
}
/**
- * @param $a array
- * @return Array An empty lock structure for a key
+ * @param array $a
+ * @return array An empty lock structure for a key
*/
protected static function sanitizeLockArray( $a ) {
if ( is_array( $a ) && isset( $a[self::LOCK_EX] ) && isset( $a[self::LOCK_SH] ) ) {
return $a;
} else {
trigger_error( __METHOD__ . ": reset invalid lock array.", E_USER_WARNING );
+
return self::newLockArray();
}
}
/**
- * @param $memc MemcachedBagOStuff
+ * @param MemcachedBagOStuff $memc
* @param array $keys List of keys to acquire
* @return bool
*/
@@ -350,9 +361,8 @@ class MemcLockManager extends QuorumLockManager {
}
/**
- * @param $memc MemcachedBagOStuff
+ * @param MemcachedBagOStuff $memc
* @param array $keys List of acquired keys
- * @return void
*/
protected function releaseMutexes( MemcachedBagOStuff $memc, array $keys ) {
foreach ( $keys as $key ) {
diff --git a/includes/filebackend/lockmanager/QuorumLockManager.php b/includes/filebackend/lockmanager/QuorumLockManager.php
index 8356d32a..a692012d 100644
--- a/includes/filebackend/lockmanager/QuorumLockManager.php
+++ b/includes/filebackend/lockmanager/QuorumLockManager.php
@@ -29,9 +29,10 @@
* @since 1.20
*/
abstract class QuorumLockManager extends LockManager {
- /** @var Array Map of bucket indexes to peer server lists */
+ /** @var array Map of bucket indexes to peer server lists */
protected $srvsByBucket = array(); // (bucket index => (lsrv1, lsrv2, ...))
- /** @var Array Map of degraded buckets */
+
+ /** @var array Map of degraded buckets */
protected $degradedBuckets = array(); // (buckey index => UNIX timestamp)
final protected function doLock( array $paths, $type ) {
@@ -65,6 +66,7 @@ abstract class QuorumLockManager extends LockManager {
$status->merge( $this->doLockingRequestBucket( $bucket, $pathsToLockByType ) );
if ( !$status->isOK() ) {
$status->merge( $this->doUnlockByType( $lockedPaths ) );
+
return $status;
}
// Record these locks as active
@@ -120,7 +122,7 @@ abstract class QuorumLockManager extends LockManager {
* Attempt to acquire locks with the peers for a bucket.
* This is all or nothing; if any key is locked then this totally fails.
*
- * @param $bucket integer
+ * @param int $bucket
* @param array $pathsByType Map of LockManager::LOCK_* constants to lists of paths
* @return Status
*/
@@ -162,7 +164,7 @@ abstract class QuorumLockManager extends LockManager {
/**
* Attempt to release locks with the peers for a bucket
*
- * @param $bucket integer
+ * @param int $bucket
* @param array $pathsByType Map of LockManager::LOCK_* constants to lists of paths
* @return Status
*/
@@ -176,8 +178,8 @@ abstract class QuorumLockManager extends LockManager {
foreach ( $this->srvsByBucket[$bucket] as $lockSrv ) {
if ( !$this->isServerUp( $lockSrv ) ) {
$status->warning( 'lockmanager-fail-svr-release', $lockSrv );
- // Attempt to release the lock on this peer
} else {
+ // Attempt to release the lock on this peer
$status->merge( $this->freeLocksOnServer( $lockSrv, $pathsByType ) );
++$yesVotes; // success for this peer
// Normally the first peers form the quorum, and the others are ignored.
@@ -198,8 +200,8 @@ abstract class QuorumLockManager extends LockManager {
* Get the bucket for resource path.
* This should avoid throwing any exceptions.
*
- * @param $path string
- * @return integer
+ * @param string $path
+ * @return int
*/
protected function getBucketFromPath( $path ) {
$prefix = substr( sha1( $path ), 0, 2 ); // first 2 hex chars (8 bits)
@@ -210,7 +212,7 @@ abstract class QuorumLockManager extends LockManager {
* Check if a lock server is up.
* This should process cache results to reduce RTT.
*
- * @param $lockSrv string
+ * @param string $lockSrv
* @return bool
*/
abstract protected function isServerUp( $lockSrv );
@@ -218,7 +220,7 @@ abstract class QuorumLockManager extends LockManager {
/**
* Get a connection to a lock server and acquire locks
*
- * @param $lockSrv string
+ * @param string $lockSrv
* @param array $pathsByType Map of LockManager::LOCK_* constants to lists of paths
* @return Status
*/
@@ -229,7 +231,7 @@ abstract class QuorumLockManager extends LockManager {
*
* Subclasses must effectively implement this or releaseAllLocks().
*
- * @param $lockSrv string
+ * @param string $lockSrv
* @param array $pathsByType Map of LockManager::LOCK_* constants to lists of paths
* @return Status
*/
diff --git a/includes/filebackend/lockmanager/RedisLockManager.php b/includes/filebackend/lockmanager/RedisLockManager.php
index 43b0198a..90e05817 100644
--- a/includes/filebackend/lockmanager/RedisLockManager.php
+++ b/includes/filebackend/lockmanager/RedisLockManager.php
@@ -38,7 +38,7 @@
* @since 1.22
*/
class RedisLockManager extends QuorumLockManager {
- /** @var Array Mapping of lock types to the type actually used */
+ /** @var array Mapping of lock types to the type actually used */
protected $lockTypeMap = array(
self::LOCK_SH => self::LOCK_SH,
self::LOCK_UW => self::LOCK_SH,
@@ -47,21 +47,21 @@ class RedisLockManager extends QuorumLockManager {
/** @var RedisConnectionPool */
protected $redisPool;
- /** @var Array Map server names to hostname/IP and port numbers */
+
+ /** @var array Map server names to hostname/IP and port numbers */
protected $lockServers = array();
- protected $session = ''; // string; random UUID
+ /** @var string Random UUID */
+ protected $session = '';
/**
* Construct a new instance from configuration.
*
- * $config paramaters include:
+ * @param array $config Parameters include:
* - lockServers : Associative array of server names to "<IP>:<port>" strings.
* - srvsByBucket : Array of 1-16 consecutive integer keys, starting from 0,
* each having an odd-numbered list of server names (peers) as values.
* - redisConfig : Configuration for RedisConnectionPool::__construct().
- *
- * @param Array $config
* @throws MWException
*/
public function __construct( array $config ) {
@@ -78,115 +78,89 @@ class RedisLockManager extends QuorumLockManager {
$this->session = wfRandomString( 32 );
}
- // @TODO: change this code to work in one batch
protected function getLocksOnServer( $lockSrv, array $pathsByType ) {
$status = Status::newGood();
- $lockedPaths = array();
- foreach ( $pathsByType as $type => $paths ) {
- $status->merge( $this->doGetLocksOnServer( $lockSrv, $paths, $type ) );
- if ( $status->isOK() ) {
- $lockedPaths[$type] = isset( $lockedPaths[$type] )
- ? array_merge( $lockedPaths[$type], $paths )
- : $paths;
- } else {
- foreach ( $lockedPaths as $type => $paths ) {
- $status->merge( $this->doFreeLocksOnServer( $lockSrv, $paths, $type ) );
- }
- break;
- }
- }
-
- return $status;
- }
-
- // @TODO: change this code to work in one batch
- protected function freeLocksOnServer( $lockSrv, array $pathsByType ) {
- $status = Status::newGood();
-
- foreach ( $pathsByType as $type => $paths ) {
- $status->merge( $this->doFreeLocksOnServer( $lockSrv, $paths, $type ) );
- }
-
- return $status;
- }
-
- protected function doGetLocksOnServer( $lockSrv, array $paths, $type ) {
- $status = Status::newGood();
-
$server = $this->lockServers[$lockSrv];
$conn = $this->redisPool->getConnection( $server );
if ( !$conn ) {
- foreach ( $paths as $path ) {
+ foreach ( array_merge( array_values( $pathsByType ) ) as $path ) {
$status->fatal( 'lockmanager-fail-acquirelock', $path );
}
+
return $status;
}
- $keys = array_map( array( $this, 'recordKeyForPath' ), $paths ); // lock records
+ $pathsByKey = array(); // (type:hash => path) map
+ foreach ( $pathsByType as $type => $paths ) {
+ $typeString = ( $type == LockManager::LOCK_SH ) ? 'SH' : 'EX';
+ foreach ( $paths as $path ) {
+ $pathsByKey[$this->recordKeyForPath( $path, $typeString )] = $path;
+ }
+ }
try {
static $script =
<<<LUA
- if ARGV[1] ~= 'EX' and ARGV[1] ~= 'SH' then
- return redis.error_reply('Unrecognized lock type given (must be EX or SH)')
- end
local failed = {}
+ -- Load input params (e.g. session, ttl, time of request)
+ local rSession, rTTL, rTime = unpack(ARGV)
-- Check that all the locks can be acquired
- for i,resourceKey in ipairs(KEYS) do
+ for i,requestKey in ipairs(KEYS) do
+ local _, _, rType, resourceKey = string.find(requestKey,"(%w+):(%w+)$")
local keyIsFree = true
local currentLocks = redis.call('hKeys',resourceKey)
for i,lockKey in ipairs(currentLocks) do
+ -- Get the type and session of this lock
local _, _, type, session = string.find(lockKey,"(%w+):(%w+)")
-- Check any locks that are not owned by this session
- if session ~= ARGV[2] then
- local lockTimestamp = redis.call('hGet',resourceKey,lockKey)
- if 1*lockTimestamp < ( ARGV[4] - ARGV[3] ) then
+ if session ~= rSession then
+ local lockExpiry = redis.call('hGet',resourceKey,lockKey)
+ if 1*lockExpiry < 1*rTime then
-- Lock is stale, so just prune it out
redis.call('hDel',resourceKey,lockKey)
- elseif ARGV[1] == 'EX' or type == 'EX' then
+ elseif rType == 'EX' or type == 'EX' then
keyIsFree = false
break
end
end
end
if not keyIsFree then
- failed[#failed+1] = resourceKey
+ failed[#failed+1] = requestKey
end
end
-- If all locks could be acquired, then do so
if #failed == 0 then
- for i,resourceKey in ipairs(KEYS) do
- redis.call('hSet',resourceKey,ARGV[1] .. ':' .. ARGV[2],ARGV[4])
+ for i,requestKey in ipairs(KEYS) do
+ local _, _, rType, resourceKey = string.find(requestKey,"(%w+):(%w+)$")
+ redis.call('hSet',resourceKey,rType .. ':' .. rSession,rTime + rTTL)
-- In addition to invalidation logic, be sure to garbage collect
- redis.call('expire',resourceKey,ARGV[3])
+ redis.call('expire',resourceKey,rTTL)
end
end
return failed
LUA;
$res = $conn->luaEval( $script,
array_merge(
- $keys, // KEYS[0], KEYS[1],...KEYS[N]
+ array_keys( $pathsByKey ), // KEYS[0], KEYS[1],...,KEYS[N]
array(
- $type === self::LOCK_SH ? 'SH' : 'EX', // ARGV[1]
- $this->session, // ARGV[2]
- $this->lockTTL, // ARGV[3]
- time() // ARGV[4]
+ $this->session, // ARGV[1]
+ $this->lockTTL, // ARGV[2]
+ time() // ARGV[3]
)
),
- count( $keys ) # number of first argument(s) that are keys
+ count( $pathsByKey ) # number of first argument(s) that are keys
);
} catch ( RedisException $e ) {
$res = false;
- $this->redisPool->handleException( $server, $conn, $e );
+ $this->redisPool->handleError( $conn, $e );
}
if ( $res === false ) {
- foreach ( $paths as $path ) {
+ foreach ( array_merge( array_values( $pathsByType ) ) as $path ) {
$status->fatal( 'lockmanager-fail-acquirelock', $path );
}
} else {
- $pathsByKey = array_combine( $keys, $paths );
foreach ( $res as $key ) {
$status->fatal( 'lockmanager-fail-acquirelock', $pathsByKey[$key] );
}
@@ -195,61 +169,66 @@ LUA;
return $status;
}
- protected function doFreeLocksOnServer( $lockSrv, array $paths, $type ) {
+ protected function freeLocksOnServer( $lockSrv, array $pathsByType ) {
$status = Status::newGood();
$server = $this->lockServers[$lockSrv];
$conn = $this->redisPool->getConnection( $server );
if ( !$conn ) {
- foreach ( $paths as $path ) {
+ foreach ( array_merge( array_values( $pathsByType ) ) as $path ) {
$status->fatal( 'lockmanager-fail-releaselock', $path );
}
+
return $status;
}
- $keys = array_map( array( $this, 'recordKeyForPath' ), $paths ); // lock records
+ $pathsByKey = array(); // (type:hash => path) map
+ foreach ( $pathsByType as $type => $paths ) {
+ $typeString = ( $type == LockManager::LOCK_SH ) ? 'SH' : 'EX';
+ foreach ( $paths as $path ) {
+ $pathsByKey[$this->recordKeyForPath( $path, $typeString )] = $path;
+ }
+ }
try {
static $script =
<<<LUA
- if ARGV[1] ~= 'EX' and ARGV[1] ~= 'SH' then
- return redis.error_reply('Unrecognized lock type given (must be EX or SH)')
- end
local failed = {}
- for i,resourceKey in ipairs(KEYS) do
- local released = redis.call('hDel',resourceKey,ARGV[1] .. ':' .. ARGV[2])
+ -- Load input params (e.g. session)
+ local rSession = unpack(ARGV)
+ for i,requestKey in ipairs(KEYS) do
+ local _, _, rType, resourceKey = string.find(requestKey,"(%w+):(%w+)$")
+ local released = redis.call('hDel',resourceKey,rType .. ':' .. rSession)
if released > 0 then
-- Remove the whole structure if it is now empty
if redis.call('hLen',resourceKey) == 0 then
redis.call('del',resourceKey)
end
else
- failed[#failed+1] = resourceKey
+ failed[#failed+1] = requestKey
end
end
return failed
LUA;
$res = $conn->luaEval( $script,
array_merge(
- $keys, // KEYS[0], KEYS[1],...KEYS[N]
+ array_keys( $pathsByKey ), // KEYS[0], KEYS[1],...,KEYS[N]
array(
- $type === self::LOCK_SH ? 'SH' : 'EX', // ARGV[1]
- $this->session // ARGV[2]
+ $this->session, // ARGV[1]
)
),
- count( $keys ) # number of first argument(s) that are keys
+ count( $pathsByKey ) # number of first argument(s) that are keys
);
} catch ( RedisException $e ) {
$res = false;
- $this->redisPool->handleException( $server, $conn, $e );
+ $this->redisPool->handleError( $conn, $e );
}
if ( $res === false ) {
- foreach ( $paths as $path ) {
+ foreach ( array_merge( array_values( $pathsByType ) ) as $path ) {
$status->fatal( 'lockmanager-fail-releaselock', $path );
}
} else {
- $pathsByKey = array_combine( $keys, $paths );
foreach ( $res as $key ) {
$status->fatal( 'lockmanager-fail-releaselock', $pathsByKey[$key] );
}
@@ -267,11 +246,13 @@ LUA;
}
/**
- * @param $path string
+ * @param string $path
+ * @param string $type One of (EX,SH)
* @return string
*/
- protected function recordKeyForPath( $path ) {
- return implode( ':', array( __CLASS__, 'locks', $this->sha1Base36Absolute( $path ) ) );
+ protected function recordKeyForPath( $path, $type ) {
+ return implode( ':',
+ array( __CLASS__, 'locks', "$type:" . $this->sha1Base36Absolute( $path ) ) );
}
/**
@@ -279,10 +260,13 @@ LUA;
*/
function __destruct() {
while ( count( $this->locksHeld ) ) {
+ $pathsByType = array();
foreach ( $this->locksHeld as $path => $locks ) {
- $this->doUnlock( array( $path ), self::LOCK_EX );
- $this->doUnlock( array( $path ), self::LOCK_SH );
+ foreach ( $locks as $type => $count ) {
+ $pathsByType[$type][] = $path;
+ }
}
+ $this->unlockByType( $pathsByType );
}
}
}
diff --git a/includes/filebackend/lockmanager/ScopedLock.php b/includes/filebackend/lockmanager/ScopedLock.php
index 5faad4a6..2056e101 100644
--- a/includes/filebackend/lockmanager/ScopedLock.php
+++ b/includes/filebackend/lockmanager/ScopedLock.php
@@ -34,9 +34,11 @@
class ScopedLock {
/** @var LockManager */
protected $manager;
+
/** @var Status */
protected $status;
- /** @var Array Map of lock types to resource paths */
+
+ /** @var array Map of lock types to resource paths */
protected $pathsByType;
/**
@@ -55,14 +57,13 @@ class ScopedLock {
* Any locks are released once this object goes out of scope.
* The status object is updated with any errors or warnings.
*
- * $type can be "mixed" and $paths can be a map of types to paths (since 1.22).
- * Otherwise $type should be an integer and $paths should be a list of paths.
- *
* @param LockManager $manager
* @param array $paths List of storage paths or map of lock types to path lists
- * @param integer|string $type LockManager::LOCK_* constant or "mixed"
+ * @param int|string $type LockManager::LOCK_* constant or "mixed" and $paths
+ * can be a map of types to paths (since 1.22). Otherwise $type should be an
+ * integer and $paths should be a list of paths.
* @param Status $status
- * @param integer $timeout Timeout in seconds (0 means non-blocking) (since 1.22)
+ * @param int $timeout Timeout in seconds (0 means non-blocking) (since 1.22)
* @return ScopedLock|null Returns null on failure
*/
public static function factory(
@@ -74,6 +75,7 @@ class ScopedLock {
if ( $lockStatus->isOK() ) {
return new self( $manager, $pathsByType, $status );
}
+
return null;
}
@@ -83,7 +85,6 @@ class ScopedLock {
* This is the same as setting the lock object to null.
*
* @param ScopedLock $lock
- * @return void
* @since 1.21
*/
public static function release( ScopedLock &$lock = null ) {