From f6d65e533c62f6deb21342d4901ece24497b433e Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Thu, 4 Jun 2015 07:31:04 +0200 Subject: Update to MediaWiki 1.25.1 --- includes/filebackend/SwiftFileBackend.php | 81 ++++++++++++++++++++----------- 1 file changed, 53 insertions(+), 28 deletions(-) (limited to 'includes/filebackend/SwiftFileBackend.php') diff --git a/includes/filebackend/SwiftFileBackend.php b/includes/filebackend/SwiftFileBackend.php index f40ec46e..5f406c9b 100644 --- a/includes/filebackend/SwiftFileBackend.php +++ b/includes/filebackend/SwiftFileBackend.php @@ -112,7 +112,7 @@ class SwiftFileBackend extends FileBackendStore { // Optional settings $this->authTTL = isset( $config['swiftAuthTTL'] ) ? $config['swiftAuthTTL'] - : 5 * 60; // some sane number + : 15 * 60; // some sane number $this->swiftTempUrlKey = isset( $config['swiftTempUrlKey'] ) ? $config['swiftTempUrlKey'] : ''; @@ -537,6 +537,7 @@ class SwiftFileBackend extends FileBackendStore { return $status; // already there } elseif ( $stat === null ) { $status->fatal( 'backend-fail-internal', $this->name ); + wfDebugLog( 'SwiftBackend', __METHOD__ . ': cannot get container stat' ); return $status; } @@ -568,6 +569,7 @@ class SwiftFileBackend extends FileBackendStore { $status->fatal( 'backend-fail-usable', $params['dir'] ); } else { $status->fatal( 'backend-fail-internal', $this->name ); + wfDebugLog( 'SwiftBackend', __METHOD__ . ': cannot get container stat' ); } return $status; @@ -588,6 +590,7 @@ class SwiftFileBackend extends FileBackendStore { $status->fatal( 'backend-fail-usable', $params['dir'] ); } else { $status->fatal( 'backend-fail-internal', $this->name ); + wfDebugLog( 'SwiftBackend', __METHOD__ . ': cannot get container stat' ); } return $status; @@ -607,6 +610,7 @@ class SwiftFileBackend extends FileBackendStore { return $status; // ok, nothing to do } elseif ( !is_array( $stat ) ) { $status->fatal( 'backend-fail-internal', $this->name ); + wfDebugLog( 'SwiftBackend', __METHOD__ . ': cannot get container stat' ); return $status; } @@ -643,7 +647,7 @@ class SwiftFileBackend extends FileBackendStore { $timestamp = new MWTimestamp( $ts ); return $timestamp->getTimestamp( $format ); - } catch ( MWException $e ) { + } catch ( Exception $e ) { throw new FileBackendError( $e->getMessage() ); } } @@ -660,7 +664,7 @@ class SwiftFileBackend extends FileBackendStore { return $objHdrs; // nothing to do } - $section = new ProfileSection( __METHOD__ . '-' . $this->name ); + $ps = Profiler::instance()->scopedProfileIn( __METHOD__ . "-{$this->name}" ); trigger_error( "$path was not stored with SHA-1 metadata.", E_USER_WARNING ); $auth = $this->getAuthentication(); @@ -794,7 +798,7 @@ class SwiftFileBackend extends FileBackendStore { return $dirs; // nothing more } - $section = new ProfileSection( __METHOD__ . '-' . $this->name ); + $ps = Profiler::instance()->scopedProfileIn( __METHOD__ . "-{$this->name}" ); $prefix = ( $dir == '' ) ? null : "{$dir}/"; // Non-recursive: only list dirs right under $dir @@ -874,7 +878,7 @@ class SwiftFileBackend extends FileBackendStore { return $files; // nothing more } - $section = new ProfileSection( __METHOD__ . '-' . $this->name ); + $ps = Profiler::instance()->scopedProfileIn( __METHOD__ . "-{$this->name}" ); $prefix = ( $dir == '' ) ? null : "{$dir}/"; // $objects will contain a list of unfiltered names or CF_Object items @@ -933,6 +937,7 @@ class SwiftFileBackend extends FileBackendStore { // Convert various random Swift dates to TS_MW 'mtime' => $this->convertSwiftDate( $object->last_modified, TS_MW ), 'size' => (int)$object->bytes, + 'sha1' => null, // Note: manifiest ETags are not an MD5 of the file 'md5' => ctype_xdigit( $object->hash ) ? $object->hash : null, 'latest' => false // eventually consistent @@ -1060,6 +1065,7 @@ class SwiftFileBackend extends FileBackendStore { $tmpFiles[$path] = $tmpFile; } + $isLatest = ( $this->isRGW || !empty( $params['latest'] ) ); $opts = array( 'maxConnsPerHost' => $params['concurrency'] ); $reqs = $this->http->runMulti( $reqs, $opts ); foreach ( $reqs as $path => $op ) { @@ -1074,6 +1080,10 @@ class SwiftFileBackend extends FileBackendStore { $this->onError( null, __METHOD__, array( 'src' => $path ) + $ep, $rerr, $rcode, $rdesc ); } + // Set the file stat process cache in passing + $stat = $this->getStatFromHeaders( $rhdrs ); + $stat['latest'] = $isLatest; + $this->cheapCache->set( $path, 'stat', $stat ); } elseif ( $rcode === 404 ) { $tmpFiles[$path] = false; } else { @@ -1161,6 +1171,11 @@ class SwiftFileBackend extends FileBackendStore { return $hdrs; } + /** + * @param FileBackendStoreOpHandle[] $fileOpHandles + * + * @return Status[] + */ protected function doExecuteOpHandlesInternal( array $fileOpHandles ) { $statuses = array(); @@ -1253,6 +1268,7 @@ class SwiftFileBackend extends FileBackendStore { if ( $rcode != 204 && $rcode !== 202 ) { $status->fatal( 'backend-fail-internal', $this->name ); + wfDebugLog( 'SwiftBackend', __METHOD__ . ': unexpected rcode value (' . $rcode . ')' ); } return $status; @@ -1267,7 +1283,7 @@ class SwiftFileBackend extends FileBackendStore { * @return array|bool|null False on 404, null on failure */ protected function getContainerStat( $container, $bypassCache = false ) { - $section = new ProfileSection( __METHOD__ . '-' . $this->name ); + $ps = Profiler::instance()->scopedProfileIn( __METHOD__ . "-{$this->name}" ); if ( $bypassCache ) { // purge cache $this->containerStatCache->clear( $container ); @@ -1280,13 +1296,11 @@ class SwiftFileBackend extends FileBackendStore { return null; } - wfProfileIn( __METHOD__ . "-{$this->name}-miss" ); list( $rcode, $rdesc, $rhdrs, $rbody, $rerr ) = $this->http->run( array( 'method' => 'HEAD', 'url' => $this->storageUrl( $auth, $container ), 'headers' => $this->authTokenHeaders( $auth ) ) ); - wfProfileOut( __METHOD__ . "-{$this->name}-miss" ); if ( $rcode === 204 ) { $stat = array( @@ -1507,25 +1521,8 @@ class SwiftFileBackend extends FileBackendStore { if ( $rcode === 200 || $rcode === 204 ) { // Update the object if it is missing some headers $rhdrs = $this->addMissingMetadata( $rhdrs, $path ); - // Fetch all of the custom metadata headers - $metadata = array(); - foreach ( $rhdrs as $name => $value ) { - if ( strpos( $name, 'x-object-meta-' ) === 0 ) { - $metadata[substr( $name, strlen( 'x-object-meta-' ) )] = $value; - } - } - // Fetch all of the custom raw HTTP headers - $headers = $this->sanitizeHdrs( array( 'headers' => $rhdrs ) ); - $stat = array( - // Convert various random Swift dates to TS_MW - 'mtime' => $this->convertSwiftDate( $rhdrs['last-modified'], TS_MW ), - // Empty objects actually return no content-length header in Ceph - 'size' => isset( $rhdrs['content-length'] ) ? (int)$rhdrs['content-length'] : 0, - 'sha1' => $rhdrs['x-object-meta-sha1base36'], - // Note: manifiest ETags are not an MD5 of the file - 'md5' => ctype_xdigit( $rhdrs['etag'] ) ? $rhdrs['etag'] : null, - 'xattr' => array( 'metadata' => $metadata, 'headers' => $headers ) - ); + // Load the stat array from the headers + $stat = $this->getStatFromHeaders( $rhdrs ); if ( $this->isRGW ) { $stat['latest'] = true; // strong consistency } @@ -1541,6 +1538,34 @@ class SwiftFileBackend extends FileBackendStore { return $stats; } + /** + * @param array $rhdrs + * @return array + */ + protected function getStatFromHeaders( array $rhdrs ) { + // Fetch all of the custom metadata headers + $metadata = array(); + foreach ( $rhdrs as $name => $value ) { + if ( strpos( $name, 'x-object-meta-' ) === 0 ) { + $metadata[substr( $name, strlen( 'x-object-meta-' ) )] = $value; + } + } + // Fetch all of the custom raw HTTP headers + $headers = $this->sanitizeHdrs( array( 'headers' => $rhdrs ) ); + return array( + // Convert various random Swift dates to TS_MW + 'mtime' => $this->convertSwiftDate( $rhdrs['last-modified'], TS_MW ), + // Empty objects actually return no content-length header in Ceph + 'size' => isset( $rhdrs['content-length'] ) ? (int)$rhdrs['content-length'] : 0, + 'sha1' => isset( $rhdrs['x-object-meta-sha1base36'] ) + ? $rhdrs['x-object-meta-sha1base36'] + : null, + // Note: manifiest ETags are not an MD5 of the file + 'md5' => ctype_xdigit( $rhdrs['etag'] ) ? $rhdrs['etag'] : null, + 'xattr' => array( 'metadata' => $metadata, 'headers' => $headers ) + ); + } + /** * @return array|null Credential map */ @@ -1595,7 +1620,7 @@ class SwiftFileBackend extends FileBackendStore { } // Ceph RGW does not use in URLs (OpenStack Swift uses "/v1/") if ( substr( $this->authCreds['storage_url'], -3 ) === '/v1' ) { - $this->isRGW = true; // take advantage of strong consistency + $this->isRGW = true; // take advantage of strong consistency in Ceph } } -- cgit v1.2.2