From 4ac9fa081a7c045f6a9f1cfc529d82423f485b2e Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 8 Dec 2013 09:55:49 +0100 Subject: Update to MediaWiki 1.22.0 --- includes/filerepo/FSRepo.php | 14 +- includes/filerepo/FileRepo.php | 108 ++++++++------ includes/filerepo/ForeignAPIRepo.php | 226 ++++++++++++++++++++++------- includes/filerepo/ForeignDBRepo.php | 5 +- includes/filerepo/LocalRepo.php | 42 +++--- includes/filerepo/RepoGroup.php | 12 +- includes/filerepo/file/ArchivedFile.php | 23 +-- includes/filerepo/file/File.php | 37 +++-- includes/filerepo/file/ForeignAPIFile.php | 49 ++++--- includes/filerepo/file/ForeignDBFile.php | 5 +- includes/filerepo/file/LocalFile.php | 234 ++++++++++++++++++++---------- includes/filerepo/file/OldLocalFile.php | 3 +- 12 files changed, 504 insertions(+), 254 deletions(-) (limited to 'includes/filerepo') diff --git a/includes/filerepo/FSRepo.php b/includes/filerepo/FSRepo.php index e49f37d2..42c9c945 100644 --- a/includes/filerepo/FSRepo.php +++ b/includes/filerepo/FSRepo.php @@ -56,16 +56,16 @@ class FSRepo extends FileRepo { $repoName = $info['name']; // Get the FS backend configuration $backend = new FSFileBackend( array( - 'name' => $info['name'] . '-backend', - 'lockManager' => 'fsLockManager', + 'name' => $info['name'] . '-backend', + 'lockManager' => 'fsLockManager', 'containerPaths' => array( - "{$repoName}-public" => "{$directory}", - "{$repoName}-temp" => "{$directory}/temp", - "{$repoName}-thumb" => $thumbDir, - "{$repoName}-transcoded" => $transcodedDir, + "{$repoName}-public" => "{$directory}", + "{$repoName}-temp" => "{$directory}/temp", + "{$repoName}-thumb" => $thumbDir, + "{$repoName}-transcoded" => $transcodedDir, "{$repoName}-deleted" => $deletedDir ), - 'fileMode' => $fileMode, + 'fileMode' => $fileMode, ) ); // Update repo config to use this backend $info['backend'] = $backend; diff --git a/includes/filerepo/FileRepo.php b/includes/filerepo/FileRepo.php index 366dd8a5..1195d5f8 100644 --- a/includes/filerepo/FileRepo.php +++ b/includes/filerepo/FileRepo.php @@ -67,7 +67,7 @@ class FileRepo { */ public function __construct( array $info = null ) { // Verify required settings presence - if( + if ( $info === null || !array_key_exists( 'name', $info ) || !array_key_exists( 'backend', $info ) @@ -259,19 +259,19 @@ class FileRepo { */ public function resolveVirtualUrl( $url ) { if ( substr( $url, 0, 9 ) != 'mwrepo://' ) { - throw new MWException( __METHOD__.': unknown protocol' ); + throw new MWException( __METHOD__ . ': unknown protocol' ); } $bits = explode( '/', substr( $url, 9 ), 3 ); if ( count( $bits ) != 3 ) { - throw new MWException( __METHOD__.": invalid mwrepo URL: $url" ); + throw new MWException( __METHOD__ . ": invalid mwrepo URL: $url" ); } list( $repo, $zone, $rel ) = $bits; if ( $repo !== $this->name ) { - throw new MWException( __METHOD__.": fetching from a foreign repo is not supported" ); + throw new MWException( __METHOD__ . ": fetching from a foreign repo is not supported" ); } $base = $this->getZonePath( $zone ); if ( !$base ) { - throw new MWException( __METHOD__.": invalid zone: $zone" ); + throw new MWException( __METHOD__ . ": invalid zone: $zone" ); } return $base . '/' . rawurldecode( $rel ); } @@ -383,7 +383,7 @@ class FileRepo { return false; } $redir = $this->checkRedirect( $title ); - if ( $redir && $title->getNamespace() == NS_FILE) { + if ( $redir && $title->getNamespace() == NS_FILE ) { $img = $this->newFile( $redir ); if ( !$img ) { return false; @@ -794,10 +794,10 @@ class FileRepo { } } $operations[] = array( - 'op' => $opName, - 'src' => $srcPath, - 'dst' => $dstPath, - 'overwrite' => $flags & self::OVERWRITE, + 'op' => $opName, + 'src' => $srcPath, + 'dst' => $dstPath, + 'overwrite' => $flags & self::OVERWRITE, 'overwriteSame' => $flags & self::OVERWRITE_SAME, ); } @@ -917,9 +917,9 @@ class FileRepo { $src = $this->resolveToStoragePath( $src ); $dst = $this->resolveToStoragePath( $dst ); $operations[] = array( - 'op' => FileBackend::isStoragePath( $src ) ? 'copy' : 'store', - 'src' => $src, - 'dst' => $dst, + 'op' => FileBackend::isStoragePath( $src ) ? 'copy' : 'store', + 'src' => $src, + 'dst' => $dst, 'disposition' => isset( $triple[2] ) ? $triple[2] : null ); $status->merge( $this->initDirectory( dirname( $dst ) ) ); @@ -942,8 +942,8 @@ class FileRepo { $operations = array(); foreach ( $paths as $path ) { $operations[] = array( - 'op' => 'delete', - 'src' => $this->resolveToStoragePath( $path ), + 'op' => 'delete', + 'src' => $this->resolveToStoragePath( $path ), 'ignoreMissingSource' => true ); } @@ -965,7 +965,7 @@ class FileRepo { public function storeTemp( $originalName, $srcPath ) { $this->assertWritableRepo(); // fail out if read-only - $date = gmdate( "YmdHis" ); + $date = MWTimestamp::getInstance()->format( 'YmdHis' ); $hashPath = $this->getHashPath( $originalName ); $dstUrlRel = $hashPath . $date . '!' . rawurlencode( $originalName ); $virtualUrl = $this->getVirtualUrl( 'temp' ) . '/' . $dstUrlRel; @@ -987,7 +987,7 @@ class FileRepo { $temp = $this->getVirtualUrl( 'temp' ); if ( substr( $virtualUrl, 0, strlen( $temp ) ) != $temp ) { - wfDebug( __METHOD__.": Invalid temp virtual URL\n" ); + wfDebug( __METHOD__ . ": Invalid temp virtual URL\n" ); return false; } @@ -1132,9 +1132,9 @@ class FileRepo { // race conditions unless an functioning LockManager is used. // LocalFile also uses SELECT FOR UPDATE for synchronization. $operations[] = array( - 'op' => 'copy', - 'src' => $dstPath, - 'dst' => $archivePath, + 'op' => 'copy', + 'src' => $dstPath, + 'dst' => $archivePath, 'ignoreMissingSource' => true ); @@ -1142,28 +1142,28 @@ class FileRepo { if ( FileBackend::isStoragePath( $srcPath ) ) { if ( $flags & self::DELETE_SOURCE ) { $operations[] = array( - 'op' => 'move', - 'src' => $srcPath, - 'dst' => $dstPath, + 'op' => 'move', + 'src' => $srcPath, + 'dst' => $dstPath, 'overwrite' => true, // replace current - 'headers' => $headers + 'headers' => $headers ); } else { $operations[] = array( - 'op' => 'copy', - 'src' => $srcPath, - 'dst' => $dstPath, + 'op' => 'copy', + 'src' => $srcPath, + 'dst' => $dstPath, 'overwrite' => true, // replace current - 'headers' => $headers + 'headers' => $headers ); } } else { // FS source path $operations[] = array( - 'op' => 'store', - 'src' => $srcPath, - 'dst' => $dstPath, + 'op' => 'store', + 'src' => $srcPath, + 'dst' => $dstPath, 'overwrite' => true, // replace current - 'headers' => $headers + 'headers' => $headers ); if ( $flags & self::DELETE_SOURCE ) { $sourceFSFilesToDelete[] = $srcPath; @@ -1306,9 +1306,9 @@ class FileRepo { foreach ( $sourceDestPairs as $pair ) { list( $srcRel, $archiveRel ) = $pair; if ( !$this->validateFilename( $srcRel ) ) { - throw new MWException( __METHOD__.':Validation error in $srcRel' ); + throw new MWException( __METHOD__ . ':Validation error in $srcRel' ); } elseif ( !$this->validateFilename( $archiveRel ) ) { - throw new MWException( __METHOD__.':Validation error in $archiveRel' ); + throw new MWException( __METHOD__ . ':Validation error in $archiveRel' ); } $publicRoot = $this->getZonePath( 'public' ); @@ -1324,9 +1324,9 @@ class FileRepo { } $operations[] = array( - 'op' => 'move', - 'src' => $srcPath, - 'dst' => $archivePath, + 'op' => 'move', + 'src' => $srcPath, + 'dst' => $archivePath, // We may have 2+ identical files being deleted, // all of which will map to the same destination file 'overwriteSame' => true // also see bug 31792 @@ -1564,7 +1564,7 @@ class FileRepo { public function newFatal( $message /*, parameters...*/ ) { $params = func_get_args(); array_unshift( $params, $this ); - return MWInit::callStaticMethod( 'FileRepoStatus', 'newFatal', $params ); + return call_user_func_array( array( 'FileRepoStatus', 'newFatal' ), $params ); } /** @@ -1671,29 +1671,29 @@ class FileRepo { */ public function getTempRepo() { return new TempFileRepo( array( - 'name' => "{$this->name}-temp", - 'backend' => $this->backend, - 'zones' => array( + 'name' => "{$this->name}-temp", + 'backend' => $this->backend, + 'zones' => array( 'public' => array( 'container' => $this->zones['temp']['container'], 'directory' => $this->zones['temp']['directory'] ), - 'thumb' => array( + 'thumb' => array( 'container' => $this->zones['thumb']['container'], 'directory' => ( $this->zones['thumb']['directory'] == '' ) ? 'temp' : $this->zones['thumb']['directory'] . '/temp' ), - 'transcoded' => array( + 'transcoded' => array( 'container' => $this->zones['transcoded']['container'], 'directory' => ( $this->zones['transcoded']['directory'] == '' ) ? 'temp' : $this->zones['transcoded']['directory'] . '/temp' ) ), - 'url' => $this->getZoneUrl( 'temp' ), - 'thumbUrl' => $this->getZoneUrl( 'thumb' ) . '/temp', - 'transcodedUrl' => $this->getZoneUrl( 'transcoded' ) . '/temp', + 'url' => $this->getZoneUrl( 'temp' ), + 'thumbUrl' => $this->getZoneUrl( 'thumb' ) . '/temp', + 'transcodedUrl' => $this->getZoneUrl( 'transcoded' ) . '/temp', 'hashLevels' => $this->hashLevels // performance ) ); } @@ -1716,6 +1716,22 @@ class FileRepo { * @throws MWException */ protected function assertWritableRepo() {} + + + /** + * Return information about the repository. + * + * @return array + * @since 1.22 + */ + public function getInfo() { + return array( + 'name' => $this->getName(), + 'displayname' => $this->getDisplayName(), + 'rootUrl' => $this->getRootUrl(), + 'local' => $this->isLocal(), + ); + } } /** diff --git a/includes/filerepo/ForeignAPIRepo.php b/includes/filerepo/ForeignAPIRepo.php index ba574da1..5eec9a50 100644 --- a/includes/filerepo/ForeignAPIRepo.php +++ b/includes/filerepo/ForeignAPIRepo.php @@ -61,25 +61,33 @@ class ForeignAPIRepo extends FileRepo { // http://commons.wikimedia.org/w/api.php $this->mApiBase = isset( $info['apibase'] ) ? $info['apibase'] : null; - if( isset( $info['apiThumbCacheExpiry'] ) ) { + if ( isset( $info['apiThumbCacheExpiry'] ) ) { $this->apiThumbCacheExpiry = $info['apiThumbCacheExpiry']; } - if( isset( $info['fileCacheExpiry'] ) ) { + if ( isset( $info['fileCacheExpiry'] ) ) { $this->fileCacheExpiry = $info['fileCacheExpiry']; } - if( !$this->scriptDirUrl ) { + if ( !$this->scriptDirUrl ) { // hack for description fetches $this->scriptDirUrl = dirname( $this->mApiBase ); } // If we can cache thumbs we can guess sane defaults for these - if( $this->canCacheThumbs() && !$this->url ) { + if ( $this->canCacheThumbs() && !$this->url ) { $this->url = $wgLocalFileRepo['url']; } - if( $this->canCacheThumbs() && !$this->thumbUrl ) { + if ( $this->canCacheThumbs() && !$this->thumbUrl ) { $this->thumbUrl = $this->url . '/thumb'; } } + /** + * @return string + * @since 1.22 + */ + function getApiUrl() { + return $this->mApiBase; + } + /** * Per docs in FileRepo, this needs to return false if we don't support versioned * files. Well, we don't. @@ -102,10 +110,10 @@ class ForeignAPIRepo extends FileRepo { function fileExistsBatch( array $files ) { $results = array(); foreach ( $files as $k => $f ) { - if ( isset( $this->mFileExists[$k] ) ) { - $results[$k] = true; + if ( isset( $this->mFileExists[$f] ) ) { + $results[$k] = $this->mFileExists[$f]; unset( $files[$k] ); - } elseif( self::isVirtualUrl( $f ) ) { + } elseif ( self::isVirtualUrl( $f ) ) { # @todo FIXME: We need to be able to handle virtual # URLs better, at least when we know they refer to the # same repo. @@ -120,11 +128,27 @@ class ForeignAPIRepo extends FileRepo { $data = $this->fetchImageQuery( array( 'titles' => implode( $files, '|' ), 'prop' => 'imageinfo' ) ); - if( isset( $data['query']['pages'] ) ) { - $i = 0; - foreach( $files as $key => $file ) { - $results[$key] = $this->mFileExists[$key] = !isset( $data['query']['pages'][$i]['missing'] ); - $i++; + if ( isset( $data['query']['pages'] ) ) { + # First, get results from the query. Note we only care whether the image exists, + # not whether it has a description page. + foreach ( $data['query']['pages'] as $p ) { + $this->mFileExists[$p['title']] = ( $p['imagerepository'] !== '' ); + } + # Second, copy the results to any redirects that were queried + if ( isset( $data['query']['redirects'] ) ) { + foreach ( $data['query']['redirects'] as $r ) { + $this->mFileExists[$r['from']] = $this->mFileExists[$r['to']]; + } + } + # Third, copy the results to any non-normalized titles that were queried + if ( isset( $data['query']['normalized'] ) ) { + foreach ( $data['query']['normalized'] as $n ) { + $this->mFileExists[$n['from']] = $this->mFileExists[$n['to']]; + } + } + # Finally, copy the results to the output + foreach ( $files as $key => $file ) { + $results[$key] = $this->mFileExists[$file]; } } return $results; @@ -143,38 +167,26 @@ class ForeignAPIRepo extends FileRepo { * @return string */ function fetchImageQuery( $query ) { - global $wgMemc; + global $wgMemc, $wgLanguageCode; $query = array_merge( $query, array( - 'format' => 'json', - 'action' => 'query', + 'format' => 'json', + 'action' => 'query', 'redirects' => 'true' ) ); - if ( $this->mApiBase ) { - $url = wfAppendQuery( $this->mApiBase, $query ); - } else { - $url = $this->makeUrl( $query, 'api' ); + + if ( !isset( $query['uselang'] ) ) { // uselang is unset or null + $query['uselang'] = $wgLanguageCode; } - if( !isset( $this->mQueryCache[$url] ) ) { - $key = $this->getLocalCacheKey( 'ForeignAPIRepo', 'Metadata', md5( $url ) ); - $data = $wgMemc->get( $key ); - if( !$data ) { - $data = self::httpGet( $url ); - if ( !$data ) { - return null; - } - $wgMemc->set( $key, $data, 3600 ); - } + $data = $this->httpGetCached( 'Metadata', $query ); - if( count( $this->mQueryCache ) > 100 ) { - // Keep the cache from growing infinitely - $this->mQueryCache = array(); - } - $this->mQueryCache[$url] = $data; + if ( $data ) { + return FormatJson::decode( $data, true ); + } else { + return null; } - return FormatJson::decode( $this->mQueryCache[$url], true ); } /** @@ -182,9 +194,9 @@ class ForeignAPIRepo extends FileRepo { * @return bool|array */ function getImageInfo( $data ) { - if( $data && isset( $data['query']['pages'] ) ) { - foreach( $data['query']['pages'] as $info ) { - if( isset( $info['imageinfo'][0] ) ) { + if ( $data && isset( $data['query']['pages'] ) ) { + foreach ( $data['query']['pages'] as $info ) { + if ( isset( $info['imageinfo'][0] ) ) { return $info['imageinfo'][0]; } } @@ -198,14 +210,15 @@ class ForeignAPIRepo extends FileRepo { */ function findBySha1( $hash ) { $results = $this->fetchImageQuery( array( - 'aisha1base36' => $hash, - 'aiprop' => ForeignAPIFile::getProps(), - 'list' => 'allimages', ) ); + 'aisha1base36' => $hash, + 'aiprop' => ForeignAPIFile::getProps(), + 'list' => 'allimages', + ) ); $ret = array(); if ( isset( $results['query']['allimages'] ) ) { foreach ( $results['query']['allimages'] as $img ) { // 1.14 was broken, doesn't return name attribute - if( !isset( $img['name'] ) ) { + if ( !isset( $img['name'] ) ) { continue; } $ret[] = new ForeignAPIFile( Title::makeTitle( NS_FILE, $img['name'] ), $this, $img ); @@ -228,11 +241,11 @@ class ForeignAPIRepo extends FileRepo { 'iiprop' => 'url|timestamp', 'iiurlwidth' => $width, 'iiurlheight' => $height, - 'iiurlparam' => $otherParams, + 'iiurlparam' => $otherParams, 'prop' => 'imageinfo' ) ); $info = $this->getImageInfo( $data ); - if( $data && $info && isset( $info['thumburl'] ) ) { + if ( $data && $info && isset( $info['thumburl'] ) ) { wfDebug( __METHOD__ . " got remote thumb " . $info['thumburl'] . "\n" ); $result = $info; return $info['thumburl']; @@ -241,6 +254,40 @@ class ForeignAPIRepo extends FileRepo { } } + /** + * @param $name string + * @param $width int + * @param $height int + * @param $otherParams string + * @return bool|MediaTransformError + * @since 1.22 + */ + function getThumbError( $name, $width = -1, $height = -1, $otherParams = '', $lang = null ) { + $data = $this->fetchImageQuery( array( + 'titles' => 'File:' . $name, + 'iiprop' => 'url|timestamp', + 'iiurlwidth' => $width, + 'iiurlheight' => $height, + 'iiurlparam' => $otherParams, + 'prop' => 'imageinfo', + 'uselang' => $lang, + ) ); + $info = $this->getImageInfo( $data ); + + if ( $data && $info && isset( $info['thumberror'] ) ) { + wfDebug( __METHOD__ . " got remote thumb error " . $info['thumberror'] . "\n" ); + return new MediaTransformError( + 'thumbnail_error_remote', + $width, + $height, + $this->getDisplayName(), + $info['thumberror'] // already parsed message from foreign repo + ); + } else { + return false; + } + } + /** * Return the imageurl from cache if possible * @@ -268,11 +315,11 @@ class ForeignAPIRepo extends FileRepo { /* Get the array of urls that we already know */ $knownThumbUrls = $wgMemc->get( $key ); - if( !$knownThumbUrls ) { + if ( !$knownThumbUrls ) { /* No knownThumbUrls for this file */ $knownThumbUrls = array(); } else { - if( isset( $knownThumbUrls[$sizekey] ) ) { + if ( isset( $knownThumbUrls[$sizekey] ) ) { wfDebug( __METHOD__ . ': Got thumburl from local cache: ' . "{$knownThumbUrls[$sizekey]} \n" ); return $knownThumbUrls[$sizekey]; @@ -283,14 +330,14 @@ class ForeignAPIRepo extends FileRepo { $metadata = null; $foreignUrl = $this->getThumbUrl( $name, $width, $height, $metadata, $params ); - if( !$foreignUrl ) { + if ( !$foreignUrl ) { wfDebug( __METHOD__ . " Could not find thumburl\n" ); return false; } // We need the same filename as the remote one :) $fileName = rawurldecode( pathinfo( $foreignUrl, PATHINFO_BASENAME ) ); - if( !$this->validateFilename( $fileName ) ) { + if ( !$this->validateFilename( $fileName ) ) { wfDebug( __METHOD__ . " The deduced filename $fileName is not safe\n" ); return false; } @@ -298,15 +345,14 @@ class ForeignAPIRepo extends FileRepo { $localFilename = $localPath . "/" . $fileName; $localUrl = $this->getZoneUrl( 'thumb' ) . "/" . $this->getHashPath( $name ) . rawurlencode( $name ) . "/" . rawurlencode( $fileName ); - if( $backend->fileExists( array( 'src' => $localFilename ) ) - && isset( $metadata['timestamp'] ) ) - { + if ( $backend->fileExists( array( 'src' => $localFilename ) ) + && isset( $metadata['timestamp'] ) ) { wfDebug( __METHOD__ . " Thumbnail was already downloaded before\n" ); $modified = $backend->getFileTimestamp( array( 'src' => $localFilename ) ); $remoteModified = strtotime( $metadata['timestamp'] ); $current = time(); $diff = abs( $modified - $current ); - if( $remoteModified < $modified && $diff < $this->fileCacheExpiry ) { + if ( $remoteModified < $modified && $diff < $this->fileCacheExpiry ) { /* Use our current and already downloaded thumbnail */ $knownThumbUrls[$sizekey] = $localUrl; $wgMemc->set( $key, $knownThumbUrls, $this->apiThumbCacheExpiry ); @@ -315,7 +361,7 @@ class ForeignAPIRepo extends FileRepo { /* There is a new Commons file, or existing thumbnail older than a month */ } $thumb = self::httpGet( $foreignUrl ); - if( !$thumb ) { + if ( !$thumb ) { wfDebug( __METHOD__ . " Could not download thumb\n" ); return false; } @@ -323,7 +369,7 @@ class ForeignAPIRepo extends FileRepo { # @todo FIXME: Delete old thumbs that aren't being used. Maintenance script? $backend->prepare( array( 'dir' => dirname( $localFilename ) ) ); $params = array( 'dst' => $localFilename, 'content' => $thumb ); - if( !$backend->quickCreate( $params )->isOK() ) { + if ( !$backend->quickCreate( $params )->isOK() ) { wfDebug( __METHOD__ . " could not write to thumb path '$localFilename'\n" ); return $foreignUrl; } @@ -379,6 +425,36 @@ class ForeignAPIRepo extends FileRepo { return Http::userAgent() . " ForeignAPIRepo/" . self::VERSION; } + /** + * Get information about the repo - overrides/extends the parent + * class's information. + * @return array + * @since 1.22 + */ + function getInfo() { + $info = parent::getInfo(); + $info['apiurl'] = $this->getApiUrl(); + + $query = array( + 'format' => 'json', + 'action' => 'query', + 'meta' => 'siteinfo', + 'siprop' => 'general', + ); + + $data = $this->httpGetCached( 'SiteInfo', $query, 7200 ); + + if ( $data ) { + $siteInfo = FormatJson::decode( $data, true ); + $general = $siteInfo['query']['general']; + + $info['articlepath'] = $general['articlepath']; + $info['server'] = $general['server']; + } + + return $info; + } + /** * Like a Http:get request, but with custom User-Agent. * @see Http:get @@ -409,6 +485,46 @@ class ForeignAPIRepo extends FileRepo { } } + /** + * HTTP GET request to a mediawiki API (with caching) + * @param $target string Used in cache key creation, mostly + * @param $query array The query parameters for the API request + * @param $cacheTTL int Time to live for the memcached caching + */ + public function httpGetCached( $target, $query, $cacheTTL = 3600 ) { + if ( $this->mApiBase ) { + $url = wfAppendQuery( $this->mApiBase, $query ); + } else { + $url = $this->makeUrl( $query, 'api' ); + } + + if ( !isset( $this->mQueryCache[$url] ) ) { + global $wgMemc; + + $key = $this->getLocalCacheKey( get_class( $this ), $target, md5( $url ) ); + $data = $wgMemc->get( $key ); + + if ( !$data ) { + $data = self::httpGet( $url ); + + if ( !$data ) { + return null; + } + + $wgMemc->set( $key, $data, $cacheTTL ); + } + + if ( count( $this->mQueryCache ) > 100 ) { + // Keep the cache from growing infinitely + $this->mQueryCache = array(); + } + + $this->mQueryCache[$url] = $data; + } + + return $this->mQueryCache[$url]; + } + /** * @param $callback Array|string * @throws MWException diff --git a/includes/filerepo/ForeignDBRepo.php b/includes/filerepo/ForeignDBRepo.php index 18659852..37c65723 100644 --- a/includes/filerepo/ForeignDBRepo.php +++ b/includes/filerepo/ForeignDBRepo.php @@ -59,11 +59,12 @@ class ForeignDBRepo extends LocalRepo { $this->dbConn = DatabaseBase::factory( $this->dbType, array( 'host' => $this->dbServer, - 'user' => $this->dbUser, + 'user' => $this->dbUser, 'password' => $this->dbPassword, 'dbname' => $this->dbName, 'flags' => $this->dbFlags, - 'tablePrefix' => $this->tablePrefix + 'tablePrefix' => $this->tablePrefix, + 'foreign' => true, ) ); } diff --git a/includes/filerepo/LocalRepo.php b/includes/filerepo/LocalRepo.php index be11b233..9b62243b 100644 --- a/includes/filerepo/LocalRepo.php +++ b/includes/filerepo/LocalRepo.php @@ -47,7 +47,7 @@ class LocalRepo extends FileRepo { } elseif ( isset( $row->oi_name ) ) { return call_user_func( $this->oldFileFromRowFactory, $row, $this ); } else { - throw new MWException( __METHOD__.': invalid row' ); + throw new MWException( __METHOD__ . ': invalid row' ); } } @@ -171,13 +171,13 @@ class LocalRepo extends FileRepo { if ( $cachedValue === ' ' || $cachedValue === '' ) { // Does not exist return false; - } elseif ( strval( $cachedValue ) !== '' ) { + } elseif ( strval( $cachedValue ) !== '' && $cachedValue !== ' PURGED' ) { return Title::newFromText( $cachedValue, NS_FILE ); } // else $cachedValue is false or null: cache miss $id = $this->getArticleID( $title ); - if( !$id ) { - $wgMemc->set( $memcKey, " ", $expiry ); + if ( !$id ) { + $wgMemc->add( $memcKey, " ", $expiry ); return false; } $dbr = $this->getSlaveDB(); @@ -188,12 +188,12 @@ class LocalRepo extends FileRepo { __METHOD__ ); - if( $row && $row->rd_namespace == NS_FILE ) { + if ( $row && $row->rd_namespace == NS_FILE ) { $targetTitle = Title::makeTitle( $row->rd_namespace, $row->rd_title ); - $wgMemc->set( $memcKey, $targetTitle->getDBkey(), $expiry ); + $wgMemc->add( $memcKey, $targetTitle->getDBkey(), $expiry ); return $targetTitle; } else { - $wgMemc->set( $memcKey, '', $expiry ); + $wgMemc->add( $memcKey, '', $expiry ); return false; } } @@ -206,7 +206,7 @@ class LocalRepo extends FileRepo { * @return bool|int|mixed */ protected function getArticleID( $title ) { - if( !$title instanceof Title ) { + if ( !$title instanceof Title ) { return 0; } $dbr = $this->getSlaveDB(); @@ -258,7 +258,7 @@ class LocalRepo extends FileRepo { * @return array An Array of arrays or iterators of file objects and the hash as key */ function findBySha1s( array $hashes ) { - if( !count( $hashes ) ) { + if ( !count( $hashes ) ) { return array(); //empty parameter } @@ -281,17 +281,17 @@ class LocalRepo extends FileRepo { return $result; } - /** - * Return an array of files where the name starts with $prefix. - * - * @param string $prefix The prefix to search for - * @param int $limit The maximum amount of files to return - * @return array - */ + /** + * Return an array of files where the name starts with $prefix. + * + * @param string $prefix The prefix to search for + * @param int $limit The maximum amount of files to return + * @return array + */ public function findFilesByPrefix( $prefix, $limit ) { $selectOptions = array( 'ORDER BY' => 'img_name', 'LIMIT' => intval( $limit ) ); - // Query database + // Query database $dbr = $this->getSlaveDB(); $res = $dbr->select( 'image', @@ -306,7 +306,7 @@ class LocalRepo extends FileRepo { foreach ( $res as $row ) { $files[] = $this->newFileFromRow( $row ); } - return $files; + return $files; } /** @@ -347,7 +347,11 @@ class LocalRepo extends FileRepo { global $wgMemc; $memcKey = $this->getSharedCacheKey( 'image_redirect', md5( $title->getDBkey() ) ); if ( $memcKey ) { - $wgMemc->delete( $memcKey ); + // Set a temporary value for the cache key, to ensure + // that this value stays purged long enough so that + // it isn't refreshed with a stale value due to a + // lagged slave. + $wgMemc->set( $memcKey, ' PURGED', 12 ); } } } diff --git a/includes/filerepo/RepoGroup.php b/includes/filerepo/RepoGroup.php index 02dfdad6..b2b9477a 100644 --- a/includes/filerepo/RepoGroup.php +++ b/includes/filerepo/RepoGroup.php @@ -209,7 +209,7 @@ class RepoGroup { } $redir = $this->localRepo->checkRedirect( $title ); - if( $redir ) { + if ( $redir ) { return $redir; } foreach ( $this->foreignRepos as $repo ) { @@ -238,7 +238,9 @@ class RepoGroup { if ( !$file ) { foreach ( $this->foreignRepos as $repo ) { $file = $repo->findFileFromKey( $hash, $options ); - if ( $file ) break; + if ( $file ) { + break; + } } } return $file; @@ -279,7 +281,7 @@ class RepoGroup { $result = array_merge_recursive( $result, $repo->findBySha1s( $hashes ) ); } //sort the merged (and presorted) sublist of each hash - foreach( $result as $hash => $files ) { + foreach ( $result as $hash => $files ) { usort( $result[$hash], 'File::compare' ); } return $result; @@ -339,9 +341,9 @@ class RepoGroup { * @return bool */ function forEachForeignRepo( $callback, $params = array() ) { - foreach( $this->foreignRepos as $repo ) { + foreach ( $this->foreignRepos as $repo ) { $args = array_merge( array( $repo ), $params ); - if( call_user_func_array( $callback, $args ) ) { + if ( call_user_func_array( $callback, $args ) ) { return true; } } diff --git a/includes/filerepo/file/ArchivedFile.php b/includes/filerepo/file/ArchivedFile.php index 3f786197..749f11a5 100644 --- a/includes/filerepo/file/ArchivedFile.php +++ b/includes/filerepo/file/ArchivedFile.php @@ -90,7 +90,7 @@ class ArchivedFile { $this->exists = false; $this->sha1 = ''; - if( $title instanceof Title ) { + if ( $title instanceof Title ) { $this->title = File::normalizeTitle( $title, 'exception' ); $this->name = $title->getDBkey(); } @@ -119,22 +119,22 @@ class ArchivedFile { } $conds = array(); - if( $this->id > 0 ) { + if ( $this->id > 0 ) { $conds['fa_id'] = $this->id; } - if( $this->key ) { + if ( $this->key ) { $conds['fa_storage_group'] = $this->group; $conds['fa_storage_key'] = $this->key; } - if( $this->title ) { + if ( $this->title ) { $conds['fa_name'] = $this->title->getDBkey(); } - if( !count( $conds ) ) { + if ( !count( $conds ) ) { throw new MWException( "No specific information for retrieving archived file" ); } - if( !$this->title || $this->title->getNamespace() == NS_FILE ) { + if ( !$this->title || $this->title->getNamespace() == NS_FILE ) { $this->dataLoaded = true; // set it here, to have also true on miss $dbr = wfGetDB( DB_SLAVE ); $row = $dbr->selectRow( @@ -196,6 +196,7 @@ class ArchivedFile { 'fa_user_text', 'fa_timestamp', 'fa_deleted', + 'fa_deleted_timestamp', /* Used by LocalFileRestoreBatch */ 'fa_sha1', ); } @@ -224,7 +225,7 @@ class ArchivedFile { $this->user_text = $row->fa_user_text; $this->timestamp = $row->fa_timestamp; $this->deleted = $row->fa_deleted; - if( isset( $row->fa_sha1 ) ) { + if ( isset( $row->fa_sha1 ) ) { $this->sha1 = $row->fa_sha1; } else { // old row, populate from key @@ -409,7 +410,7 @@ class ArchivedFile { */ public function getUser() { $this->load(); - if( $this->isDeleted( File::DELETED_USER ) ) { + if ( $this->isDeleted( File::DELETED_USER ) ) { return 0; } else { return $this->user; @@ -423,7 +424,7 @@ class ArchivedFile { */ public function getUserText() { $this->load(); - if( $this->isDeleted( File::DELETED_USER ) ) { + if ( $this->isDeleted( File::DELETED_USER ) ) { return 0; } else { return $this->user_text; @@ -437,7 +438,7 @@ class ArchivedFile { */ public function getDescription() { $this->load(); - if( $this->isDeleted( File::DELETED_COMMENT ) ) { + if ( $this->isDeleted( File::DELETED_COMMENT ) ) { return 0; } else { return $this->description; @@ -491,7 +492,7 @@ class ArchivedFile { */ public function isDeleted( $field ) { $this->load(); - return ($this->deleted & $field) == $field; + return ( $this->deleted & $field ) == $field; } /** diff --git a/includes/filerepo/file/File.php b/includes/filerepo/file/File.php index cecd0aee..ec5f927b 100644 --- a/includes/filerepo/file/File.php +++ b/includes/filerepo/file/File.php @@ -48,6 +48,7 @@ * @ingroup FileAbstraction */ abstract class File { + // Bitfield values akin to the Revision deletion constants const DELETED_FILE = 1; const DELETED_COMMENT = 2; const DELETED_USER = 4; @@ -201,9 +202,9 @@ abstract class File { 'mpeg' => 'mpg', 'tiff' => 'tif', 'ogv' => 'ogg' ); - if( isset( $squish[$lower] ) ) { + if ( isset( $squish[$lower] ) ) { return $squish[$lower]; - } elseif( preg_match( '/^[0-9a-z]+$/', $lower ) ) { + } elseif ( preg_match( '/^[0-9a-z]+$/', $lower ) ) { return $lower; } else { return ''; @@ -241,7 +242,7 @@ abstract class File { * @return array ("text", "html") etc */ public static function splitMime( $mime ) { - if( strpos( $mime, '/' ) !== false ) { + if ( strpos( $mime, '/' ) !== false ) { return explode( '/', $mime, 2 ); } else { return array( $mime, 'unknown' ); @@ -518,7 +519,7 @@ abstract class File { * @param $version integer version number. * @return Array containing metadata, or what was passed to it on fail (unserializing if not array) */ - public function convertMetadataVersion($metadata, $version) { + public function convertMetadataVersion( $metadata, $version ) { $handler = $this->getHandler(); if ( !is_array( $metadata ) ) { // Just to make the return type consistent @@ -681,7 +682,7 @@ abstract class File { if ( $mime === "unknown/unknown" ) { return false; #unknown type, not trusted } - if ( in_array( $mime, $wgTrustedMediaFormats) ) { + if ( in_array( $mime, $wgTrustedMediaFormats ) ) { return true; } @@ -737,7 +738,7 @@ abstract class File { if ( $this->repo ) { $script = $this->repo->getThumbScriptUrl(); if ( $script ) { - $this->transformScript = "$script?f=" . urlencode( $this->getName() ); + $this->transformScript = wfAppendQuery( $script, array( 'f' => $this->getName() ) ); } } } @@ -841,8 +842,9 @@ abstract class File { protected function transformErrorOutput( $thumbPath, $thumbUrl, $params, $flags ) { global $wgIgnoreImageErrors; - if ( $wgIgnoreImageErrors && !( $flags & self::RENDER_NOW ) ) { - return $this->getHandler()->getTransform( $this, $thumbPath, $thumbUrl, $params ); + $handler = $this->getHandler(); + if ( $handler && $wgIgnoreImageErrors && !( $flags & self::RENDER_NOW ) ) { + return $handler->getTransform( $this, $thumbPath, $thumbUrl, $params ); } else { return new MediaTransformError( 'thumbnail_error', $params['width'], 0, wfMessage( 'thumbnail-dest-create' )->text() ); @@ -910,8 +912,7 @@ abstract class File { // XXX: Pass in the storage path even though we are not rendering anything // and the path is supposed to be an FS path. This is due to getScalerType() // getting called on the path and clobbering $thumb->getUrl() if it's false. - $thumb = $handler->getTransform( - $this, $thumbPath, $thumbUrl, $params ); + $thumb = $handler->getTransform( $this, $thumbPath, $thumbUrl, $params ); $thumb->setStoragePath( $thumbPath ); break; } @@ -1000,7 +1001,7 @@ abstract class File { /** * Get a MediaHandler instance for this file * - * @return MediaHandler + * @return MediaHandler|boolean Registered MediaHandler for file's mime type or false if none found */ function getHandler() { if ( !isset( $this->handler ) ) { @@ -1097,7 +1098,7 @@ abstract class File { * * @return array */ - function getHistory( $limit = null, $start = null, $end = null, $inc=true ) { + function getHistory( $limit = null, $start = null, $end = null, $inc = true ) { return array(); } @@ -1500,7 +1501,7 @@ abstract class File { * Is this file a "deleted" file in a private archive? * STUB * - * @param $field + * @param integer $field one of DELETED_* bitfield constants * * @return bool */ @@ -1656,18 +1657,22 @@ abstract class File { /** * Get the HTML text of the description page, if available * + * @param $lang Language Optional language to fetch description in * @return string */ - function getDescriptionText() { + function getDescriptionText( $lang = false ) { global $wgMemc, $wgLang; if ( !$this->repo || !$this->repo->fetchDescription ) { return false; } - $renderUrl = $this->repo->getDescriptionRenderUrl( $this->getName(), $wgLang->getCode() ); + if ( !$lang ) { + $lang = $wgLang; + } + $renderUrl = $this->repo->getDescriptionRenderUrl( $this->getName(), $lang->getCode() ); if ( $renderUrl ) { if ( $this->repo->descriptionCacheExpiry > 0 ) { wfDebug( "Attempting to get the description from cache..." ); - $key = $this->repo->getLocalCacheKey( 'RemoteFileDescription', 'url', $wgLang->getCode(), + $key = $this->repo->getLocalCacheKey( 'RemoteFileDescription', 'url', $lang->getCode(), $this->getName() ); $obj = $wgMemc->get( $key ); if ( $obj ) { diff --git a/includes/filerepo/file/ForeignAPIFile.php b/includes/filerepo/file/ForeignAPIFile.php index a96c1f3f..ed96d446 100644 --- a/includes/filerepo/file/ForeignAPIFile.php +++ b/includes/filerepo/file/ForeignAPIFile.php @@ -54,22 +54,22 @@ class ForeignAPIFile extends File { */ static function newFromTitle( Title $title, $repo ) { $data = $repo->fetchImageQuery( array( - 'titles' => 'File:' . $title->getDBKey(), - 'iiprop' => self::getProps(), - 'prop' => 'imageinfo', + 'titles' => 'File:' . $title->getDBkey(), + 'iiprop' => self::getProps(), + 'prop' => 'imageinfo', 'iimetadataversion' => MediaHandler::getMetadataVersion() ) ); $info = $repo->getImageInfo( $data ); - if( $info ) { + if ( $info ) { $lastRedirect = isset( $data['query']['redirects'] ) ? count( $data['query']['redirects'] ) - 1 : -1; - if( $lastRedirect >= 0 ) { - $newtitle = Title::newFromText( $data['query']['redirects'][$lastRedirect]['to']); + if ( $lastRedirect >= 0 ) { + $newtitle = Title::newFromText( $data['query']['redirects'][$lastRedirect]['to'] ); $img = new self( $newtitle, $repo, $info, true ); - if( $img ) { + if ( $img ) { $img->redirectedFrom( $title->getDBkey() ); } } else { @@ -86,7 +86,7 @@ class ForeignAPIFile extends File { * @return string */ static function getProps() { - return 'timestamp|user|comment|url|size|sha1|metadata|mime'; + return 'timestamp|user|comment|url|size|sha1|metadata|mime|mediatype'; } // Dummy functions... @@ -111,7 +111,7 @@ class ForeignAPIFile extends File { * @return bool|MediaTransformOutput */ function transform( $params, $flags = 0 ) { - if( !$this->canRender() ) { + if ( !$this->canRender() ) { // show icon return parent::transform( $params, $flags ); } @@ -119,12 +119,25 @@ class ForeignAPIFile extends File { // Note, the this->canRender() check above implies // that we have a handler, and it can do makeParamString. $otherParams = $this->handler->makeParamString( $params ); + $width = isset( $params['width'] ) ? $params['width'] : -1; + $height = isset( $params['height'] ) ? $params['height'] : -1; $thumbUrl = $this->repo->getThumbUrlFromCache( $this->getName(), - isset( $params['width'] ) ? $params['width'] : -1, - isset( $params['height'] ) ? $params['height'] : -1, - $otherParams ); + $width, + $height, + $otherParams + ); + if ( $thumbUrl === false ) { + global $wgLang; + return $this->repo->getThumbError( + $this->getName(), + $width, + $height, + $otherParams, + $wgLang->getCode() + ); + } return $this->handler->getTransform( $this, 'bogus', $thumbUrl, $params ); } @@ -161,12 +174,12 @@ class ForeignAPIFile extends File { * @return array */ public static function parseMetadata( $metadata ) { - if( !is_array( $metadata ) ) { + if ( !is_array( $metadata ) ) { return $metadata; } $ret = array(); - foreach( $metadata as $meta ) { - $ret[ $meta['name'] ] = self::parseMetadata( $meta['value'] ); + foreach ( $metadata as $meta ) { + $ret[$meta['name']] = self::parseMetadata( $meta['value'] ); } return $ret; } @@ -224,7 +237,7 @@ class ForeignAPIFile extends File { * @return string */ function getMimeType() { - if( !isset( $this->mInfo['mime'] ) ) { + if ( !isset( $this->mInfo['mime'] ) ) { $magic = MimeMagic::singleton(); $this->mInfo['mime'] = $magic->guessTypesForExtension( $this->getExtension() ); } @@ -232,10 +245,12 @@ class ForeignAPIFile extends File { } /** - * @todo FIXME: May guess wrong on file types that can be eg audio or video * @return int|string */ function getMediaType() { + if ( isset( $this->mInfo['mediatype'] ) ) { + return $this->mInfo['mediatype']; + } $magic = MimeMagic::singleton(); return $magic->getMediaType( null, $this->getMimeType() ); } diff --git a/includes/filerepo/file/ForeignDBFile.php b/includes/filerepo/file/ForeignDBFile.php index ee5883c4..01d6b0f5 100644 --- a/includes/filerepo/file/ForeignDBFile.php +++ b/includes/filerepo/file/ForeignDBFile.php @@ -120,10 +120,11 @@ class ForeignDBFile extends LocalFile { } /** + * @param $lang Language Optional language to fetch description in. * @return string */ - function getDescriptionText() { + function getDescriptionText( $lang = false ) { // Restore remote behavior - return File::getDescriptionText(); + return File::getDescriptionText( $lang ); } } diff --git a/includes/filerepo/file/LocalFile.php b/includes/filerepo/file/LocalFile.php index 4f50bfaa..fe769be2 100644 --- a/includes/filerepo/file/LocalFile.php +++ b/includes/filerepo/file/LocalFile.php @@ -382,6 +382,7 @@ class LocalFile extends File { $this->$name = $value; } } else { + wfProfileOut( $fname ); throw new MWException( "Could not find data for image '{$this->getName()}'." ); } @@ -531,15 +532,15 @@ class LocalFile extends File { $dbw->update( 'image', array( - 'img_size' => $this->size, // sanity - 'img_width' => $this->width, - 'img_height' => $this->height, - 'img_bits' => $this->bits, + 'img_size' => $this->size, // sanity + 'img_width' => $this->width, + 'img_height' => $this->height, + 'img_bits' => $this->bits, 'img_media_type' => $this->media_type, 'img_major_mime' => $major, 'img_minor_mime' => $minor, - 'img_metadata' => $this->metadata, - 'img_sha1' => $this->sha1, + 'img_metadata' => $dbw->encodeBlob($this->metadata), + 'img_sha1' => $this->sha1, ), array( 'img_name' => $this->getName() ), __METHOD__ @@ -603,17 +604,23 @@ class LocalFile extends File { * Return the width of the image * * @param $page int - * @return bool|int Returns false on error + * @return int */ public function getWidth( $page = 1 ) { $this->load(); if ( $this->isMultipage() ) { - $dim = $this->getHandler()->getPageDimensions( $this, $page ); + $handler = $this->getHandler(); + if ( !$handler ) { + return 0; + } + $dim = $handler->getPageDimensions( $this, $page ); if ( $dim ) { return $dim['width']; } else { - return false; + // For non-paged media, the false goes through an + // intval, turning failure into 0, so do same here. + return 0; } } else { return $this->width; @@ -624,17 +631,23 @@ class LocalFile extends File { * Return the height of the image * * @param $page int - * @return bool|int Returns false on error + * @return int */ public function getHeight( $page = 1 ) { $this->load(); if ( $this->isMultipage() ) { - $dim = $this->getHandler()->getPageDimensions( $this, $page ); + $handler = $this->getHandler(); + if ( !$handler ) { + return 0; + } + $dim = $handler->getPageDimensions( $this, $page ); if ( $dim ) { return $dim['height']; } else { - return false; + // For non-paged media, the false goes through an + // intval, turning failure into 0, so do same here. + return 0; } } else { return $this->height; @@ -775,10 +788,12 @@ class LocalFile extends File { $backend = $this->repo->getBackend(); $files = array( $dir ); - $iterator = $backend->getFileList( array( 'dir' => $dir ) ); - foreach ( $iterator as $file ) { - $files[] = $file; - } + try { + $iterator = $backend->getFileList( array( 'dir' => $dir ) ); + foreach ( $iterator as $file ) { + $files[] = $file; + } + } catch ( FileBackendError $e ) {} // suppress (bug 54674) return $files; } @@ -793,7 +808,9 @@ class LocalFile extends File { } /** - * Purge the shared history (OldLocalFile) cache + * Purge the shared history (OldLocalFile) cache. + * + * @note This used to purge old thumbnails as well. */ function purgeHistory() { global $wgMemc; @@ -801,20 +818,20 @@ class LocalFile extends File { $hashedName = md5( $this->getName() ); $oldKey = $this->repo->getSharedCacheKey( 'oldfile', $hashedName ); - // Must purge thumbnails for old versions too! bug 30192 - foreach( $this->getHistory() as $oldFile ) { - $oldFile->purgeThumbnails(); - } - if ( $oldKey ) { $wgMemc->delete( $oldKey ); } } /** - * Delete all previously generated thumbnails, refresh metadata in memcached and purge the squid + * Delete all previously generated thumbnails, refresh metadata in memcached and purge the squid. + * + * @param Array $options An array potentially with the key forThumbRefresh. + * + * @note This used to purge old thumbnails by default as well, but doesn't anymore. */ function purgeCache( $options = array() ) { + wfProfileIn( __METHOD__ ); // Refresh metadata cache $this->purgeMetadataCache(); @@ -823,6 +840,7 @@ class LocalFile extends File { // Purge squid cache for this file SquidUpdate::purge( array( $this->getURL() ) ); + wfProfileOut( __METHOD__ ); } /** @@ -844,7 +862,7 @@ class LocalFile extends File { // Purge the squid if ( $wgUseSquid ) { $urls = array(); - foreach( $files as $file ) { + foreach ( $files as $file ) { $urls[] = $this->getArchiveThumbUrl( $archiveName, $file ); } SquidUpdate::purge( $urls ); @@ -865,7 +883,7 @@ class LocalFile extends File { // Always purge all files from squid regardless of handler filters if ( $wgUseSquid ) { $urls = array(); - foreach( $files as $file ) { + foreach ( $files as $file ) { $urls[] = $this->getThumbUrl( $file ); } array_shift( $urls ); // don't purge directory @@ -1195,20 +1213,20 @@ class LocalFile extends File { # doesn't deadlock. SELECT FOR UPDATE causes a deadlock for every race condition. $dbw->insert( 'image', array( - 'img_name' => $this->getName(), - 'img_size' => $this->size, - 'img_width' => intval( $this->width ), - 'img_height' => intval( $this->height ), - 'img_bits' => $this->bits, - 'img_media_type' => $this->media_type, - 'img_major_mime' => $this->major_mime, - 'img_minor_mime' => $this->minor_mime, - 'img_timestamp' => $timestamp, + 'img_name' => $this->getName(), + 'img_size' => $this->size, + 'img_width' => intval( $this->width ), + 'img_height' => intval( $this->height ), + 'img_bits' => $this->bits, + 'img_media_type' => $this->media_type, + 'img_major_mime' => $this->major_mime, + 'img_minor_mime' => $this->minor_mime, + 'img_timestamp' => $timestamp, 'img_description' => $comment, - 'img_user' => $user->getId(), - 'img_user_text' => $user->getName(), - 'img_metadata' => $this->metadata, - 'img_sha1' => $this->sha1 + 'img_user' => $user->getId(), + 'img_user_text' => $user->getName(), + 'img_metadata' => $dbw->encodeBlob($this->metadata), + 'img_sha1' => $this->sha1 ), __METHOD__, 'IGNORE' @@ -1258,7 +1276,7 @@ class LocalFile extends File { 'img_description' => $comment, 'img_user' => $user->getId(), 'img_user_text' => $user->getName(), - 'img_metadata' => $this->metadata, + 'img_metadata' => $dbw->encodeBlob($this->metadata), 'img_sha1' => $this->sha1 ), array( 'img_name' => $this->getName() ), @@ -1274,20 +1292,45 @@ class LocalFile extends File { $wikiPage->setFile( $this ); # Add the log entry - $log = new LogPage( 'upload' ); $action = $reupload ? 'overwrite' : 'upload'; - $logId = $log->addEntry( $action, $descTitle, $comment, array(), $user ); - wfProfileIn( __METHOD__ . '-edit' ); + $logEntry = new ManualLogEntry( 'upload', $action ); + $logEntry->setPerformer( $user ); + $logEntry->setComment( $comment ); + $logEntry->setTarget( $descTitle ); + + // Allow people using the api to associate log entries with the upload. + // Log has a timestamp, but sometimes different from upload timestamp. + $logEntry->setParameters( + array( + 'img_sha1' => $this->sha1, + 'img_timestamp' => $timestamp, + ) + ); + // Note we keep $logId around since during new image + // creation, page doesn't exist yet, so log_page = 0 + // but we want it to point to the page we're making, + // so we later modify the log entry. + // For a similar reason, we avoid making an RC entry + // now and wait until the page exists. + $logId = $logEntry->insert(); + $exists = $descTitle->exists(); + if ( $exists ) { + // Page exists, do RC entry now (otherwise we wait for later). + $logEntry->publish( $logId ); + } + wfProfileIn( __METHOD__ . '-edit' ); if ( $exists ) { # Create a null revision $latest = $descTitle->getLatestRevID(); + $editSummary = LogFormatter::newFromEntry( $logEntry )->getPlainActionText(); + $nullRevision = Revision::newNullRevision( $dbw, $descTitle->getArticleID(), - $log->getRcComment(), + $editSummary, false ); if ( !is_null( $nullRevision ) ) { @@ -1315,16 +1358,20 @@ class LocalFile extends File { $content = ContentHandler::makeContent( $pageText, $descTitle ); $status = $wikiPage->doEditContent( $content, $comment, EDIT_NEW | EDIT_SUPPRESS_RC, false, $user ); - if ( isset( $status->value['revision'] ) ) { // XXX; doEdit() uses a transaction - $dbw->begin( __METHOD__ ); + $dbw->begin( __METHOD__ ); // XXX; doEdit() uses a transaction + // Now that the page exists, make an RC entry. + $logEntry->publish( $logId ); + if ( isset( $status->value['revision'] ) ) { $dbw->update( 'logging', array( 'log_page' => $status->value['revision']->getPage() ), array( 'log_id' => $logId ), __METHOD__ ); - $dbw->commit( __METHOD__ ); // commit before anything bad can happen } + $dbw->commit( __METHOD__ ); // commit before anything bad can happen } + + wfProfileOut( __METHOD__ . '-edit' ); # Save to cache and purge the squid @@ -1351,11 +1398,17 @@ class LocalFile extends File { # Invalidate cache for all pages using this file $update = new HTMLCacheUpdate( $this->getTitle(), 'imagelinks' ); $update->doUpdate(); + if ( !$reupload ) { + LinksUpdate::queueRecursiveJobsForTable( $this->getTitle(), 'imagelinks' ); + } # Invalidate cache for all pages that redirects on this page $redirs = $this->getTitle()->getRedirectsHere(); foreach ( $redirs as $redir ) { + if ( !$reupload && $redir->getNamespace() === NS_FILE ) { + LinksUpdate::queueRecursiveJobsForTable( $redir, 'imagelinks' ); + } $update = new HTMLCacheUpdate( $redir, 'imagelinks' ); $update->doUpdate(); } @@ -1405,7 +1458,7 @@ class LocalFile extends File { $this->lock(); // begin - $archiveName = wfTimestamp( TS_MW ) . '!'. $this->getName(); + $archiveName = wfTimestamp( TS_MW ) . '!' . $this->getName(); $archiveRel = 'archive/' . $this->getHashPath() . $archiveName; $flags = $flags & File::DELETE_SOURCE ? LocalRepo::DELETE_SOURCE : 0; $status = $this->repo->publish( $srcPath, $dstRel, $archiveRel, $flags, $options ); @@ -1454,18 +1507,27 @@ class LocalFile extends File { wfDebugLog( 'imagemove', "Finished moving {$this->name}" ); - $this->purgeEverything(); - foreach ( $archiveNames as $archiveName ) { - $this->purgeOldThumbnails( $archiveName ); - } + // Purge the source and target files... + $oldTitleFile = wfLocalFile( $this->title ); + $newTitleFile = wfLocalFile( $target ); + // Hack: the lock()/unlock() pair is nested in a transaction so the locking is not + // tied to BEGIN/COMMIT. To avoid slow purges in the transaction, move them outside. + $this->getRepo()->getMasterDB()->onTransactionIdle( + function() use ( $oldTitleFile, $newTitleFile, $archiveNames ) { + $oldTitleFile->purgeEverything(); + foreach ( $archiveNames as $archiveName ) { + $oldTitleFile->purgeOldThumbnails( $archiveName ); + } + $newTitleFile->purgeEverything(); + } + ); + if ( $status->isOK() ) { // Now switch the object $this->title = $target; // Force regeneration of the name and hashpath unset( $this->name ); unset( $this->hashPath ); - // Purge the new image - $this->purgeEverything(); } return $status; @@ -1484,7 +1546,6 @@ class LocalFile extends File { * @return FileRepoStatus object. */ function delete( $reason, $suppress = false ) { - global $wgUseSquid; if ( $this->getRepo()->getReadOnlyReason() !== false ) { return $this->readOnlyFatalStatus(); } @@ -1502,19 +1563,28 @@ class LocalFile extends File { DeferredUpdates::addUpdate( SiteStatsUpdate::factory( array( 'images' => -1 ) ) ); } - $this->purgeEverything(); - foreach ( $archiveNames as $archiveName ) { - $this->purgeOldThumbnails( $archiveName ); - } + // Hack: the lock()/unlock() pair is nested in a transaction so the locking is not + // tied to BEGIN/COMMIT. To avoid slow purges in the transaction, move them outside. + $file = $this; + $this->getRepo()->getMasterDB()->onTransactionIdle( + function() use ( $file, $archiveNames ) { + global $wgUseSquid; - if ( $wgUseSquid ) { - // Purge the squid - $purgeUrls = array(); - foreach ($archiveNames as $archiveName ) { - $purgeUrls[] = $this->getArchiveUrl( $archiveName ); + $file->purgeEverything(); + foreach ( $archiveNames as $archiveName ) { + $file->purgeOldThumbnails( $archiveName ); + } + + if ( $wgUseSquid ) { + // Purge the squid + $purgeUrls = array(); + foreach ( $archiveNames as $archiveName ) { + $purgeUrls[] = $file->getArchiveUrl( $archiveName ); + } + SquidUpdate::purge( $purgeUrls ); + } } - SquidUpdate::purge( $purgeUrls ); - } + ); return $status; } @@ -1606,21 +1676,27 @@ class LocalFile extends File { * @return String */ function getDescriptionUrl() { - return $this->title->getLocalUrl(); + return $this->title->getLocalURL(); } /** * Get the HTML text of the description page * This is not used by ImagePage for local files, since (among other things) * it skips the parser cache. + * + * @param $lang Language What language to get description in (Optional) * @return bool|mixed */ - function getDescriptionText() { + function getDescriptionText( $lang = null ) { $revision = Revision::newFromTitle( $this->title, false, Revision::READ_NORMAL ); - if ( !$revision ) return false; + if ( !$revision ) { + return false; + } $content = $revision->getContent(); - if ( !$content ) return false; - $pout = $content->getParserOutput( $this->title, null, new ParserOptions() ); + if ( !$content ) { + return false; + } + $pout = $content->getParserOutput( $this->title, null, new ParserOptions( null, $lang ) ); return $pout->getText(); } @@ -1674,11 +1750,13 @@ class LocalFile extends File { } /** - * @return bool + * @return bool Whether to cache in RepoGroup (this avoids OOMs) */ function isCacheable() { $this->load(); - return strlen( $this->metadata ) <= self::CACHE_FIELD_MAX_LEN; // avoid OOMs + // If extra data (metadata) was not loaded then it must have been large + return $this->extraDataLoaded + && strlen( serialize( $this->metadata ) ) <= self::CACHE_FIELD_MAX_LEN; } /** @@ -1695,6 +1773,16 @@ class LocalFile extends File { $this->lockedOwnTrx = true; } $this->locked++; + // Bug 54736: use simple lock to handle when the file does not exist. + // SELECT FOR UPDATE only locks records not the gaps where there are none. + $cache = wfGetMainCache(); + $key = $this->getCacheKey(); + if ( !$cache->lock( $key, 60 ) ) { + throw new MWException( "Could not acquire lock for '{$this->getName()}.'" ); + } + $dbw->onTransactionIdle( function() use ( $cache, $key ) { + $cache->unlock( $key ); // release on commit + } ); } return $dbw->selectField( 'image', '1', @@ -2189,7 +2277,7 @@ class LocalFileRestoreBatch { $deletedRel = $this->file->repo->getDeletedHashPath( $row->fa_storage_key ) . $row->fa_storage_key; $deletedUrl = $this->file->repo->getVirtualUrl() . '/deleted/' . $deletedRel; - if( isset( $row->fa_sha1 ) ) { + if ( isset( $row->fa_sha1 ) ) { $sha1 = $row->fa_sha1; } else { // old row, populate from key diff --git a/includes/filerepo/file/OldLocalFile.php b/includes/filerepo/file/OldLocalFile.php index 5c505928..2c545963 100644 --- a/includes/filerepo/file/OldLocalFile.php +++ b/includes/filerepo/file/OldLocalFile.php @@ -218,6 +218,7 @@ class OldLocalFile extends LocalFile { $this->$name = $value; } } else { + wfProfileOut( __METHOD__ ); throw new MWException( "Could not find data for image '{$this->archive_name}'." ); } @@ -290,7 +291,7 @@ class OldLocalFile extends LocalFile { */ function isDeleted( $field ) { $this->load(); - return ($this->deleted & $field) == $field; + return ( $this->deleted & $field ) == $field; } /** -- cgit v1.2.2