Update to MediaWiki 1.21.1

6 files changed, 211 insertions, 129 deletions

diff --git a/includes/upload/UploadBase.php b/includes/upload/UploadBase.php
index 0848780f..5a823622 100644
--- a/includes/upload/UploadBase.php
+++ b/includes/upload/UploadBase.php
@@ -65,6 +65,8 @@ abstract class UploadBase {
 	const WINDOWS_NONASCII_FILENAME = 13;
 	const FILENAME_TOO_LONG = 14;
 
+	const SESSION_STATUS_KEY = 'wsUploadStatusData';
+
 	/**
 	 * @param $error int
 	 * @return string
@@ -78,7 +80,7 @@ abstract class UploadBase {
 								self::ILLEGAL_FILENAME => 'illegal-filename',
 								self::OVERWRITE_EXISTING_FILE => 'overwrite',
 								self::VERIFICATION_ERROR => 'verification-error',
-								self::HOOK_ABORTED =>  'hookaborted',
+								self::HOOK_ABORTED => 'hookaborted',
 								self::WINDOWS_NONASCII_FILENAME => 'windows-nonascii-filename',
 								self::FILENAME_TOO_LONG => 'filename-toolong',
 		);
@@ -108,7 +110,7 @@ abstract class UploadBase {
 	/**
 	 * Returns true if the user can use this upload module or else a string
 	 * identifying the missing permission.
-	 * Can be overriden by subclasses.
+	 * Can be overridden by subclasses.
 	 *
 	 * @param $user User
 	 * @return bool
@@ -190,10 +192,10 @@ abstract class UploadBase {
 
 	/**
 	 * Initialize the path information
-	 * @param $name string the desired destination name
-	 * @param $tempPath string the temporary path
-	 * @param $fileSize int the file size
-	 * @param $removeTempFile bool (false) remove the temporary file?
+	 * @param string $name the desired destination name
+	 * @param string $tempPath the temporary path
+	 * @param int $fileSize the file size
+	 * @param bool $removeTempFile (false) remove the temporary file?
 	 * @throws MWException
 	 */
 	public function initializePathInfo( $name, $tempPath, $fileSize, $removeTempFile = false ) {
@@ -209,7 +211,7 @@ abstract class UploadBase {
 	/**
 	 * Initialize from a WebRequest. Override this in a subclass.
 	 */
-	public abstract function initializeFromRequest( &$request );
+	abstract public function initializeFromRequest( &$request );
 
 	/**
 	 * Fetch the file. Usually a no-op
@@ -236,7 +238,15 @@ abstract class UploadBase {
 	}
 
 	/**
-	 * @param $srcPath String: the source path
+	 * Get the base 36 SHA1 of the file
+	 * @return string
+	 */
+	public function getTempFileSha1Base36() {
+		return FSFile::getSha1Base36FromPath( $this->mTempPath );
+	}
+
+	/**
+	 * @param string $srcPath the source path
 	 * @return string the real path if it was a virtual URL
 	 */
 	function getRealPath( $srcPath ) {
@@ -244,9 +254,9 @@ abstract class UploadBase {
 		$repo = RepoGroup::singleton()->getLocalRepo();
 		if ( $repo->isVirtualUrl( $srcPath ) ) {
 			// @TODO: just make uploads work with storage paths
-			// UploadFromStash loads files via virtuals URLs
+			// UploadFromStash loads files via virtual URLs
 			$tmpFile = $repo->getLocalCopy( $srcPath );
-			$tmpFile->bind( $this ); // keep alive with $thumb
+			$tmpFile->bind( $this ); // keep alive with $this
 			wfProfileOut( __METHOD__ );
 			return $tmpFile->getPath();
 		}
@@ -347,14 +357,14 @@ abstract class UploadBase {
 	 *
 	 * @note Only checks that it is not an evil mime. The does it have
 	 *  correct extension given its mime type check is in verifyFile.
-	 * @param $mime string representing the mime
+	 * @param string $mime representing the mime
 	 * @return mixed true if the file is verified, an array otherwise
 	 */
 	protected function verifyMimeType( $mime ) {
 		global $wgVerifyMimeType;
 		wfProfileIn( __METHOD__ );
 		if ( $wgVerifyMimeType ) {
-			wfDebug ( "\n\nmime: <$mime> extension: <{$this->mFinalExtension}>\n\n");
+			wfDebug ( "\n\nmime: <$mime> extension: <{$this->mFinalExtension}>\n\n" );
 			global $wgMimeTypeBlacklist;
 			if ( $this->checkFileExtension( $mime, $wgMimeTypeBlacklist ) ) {
 				wfProfileOut( __METHOD__ );
@@ -447,7 +457,7 @@ abstract class UploadBase {
 		$this->mFileProps = FSFile::getPropsFromPath( $this->mTempPath, $this->mFinalExtension );
 
 		# check mime type, if desired
-		$mime = $this->mFileProps[ 'file-mime' ];
+		$mime = $this->mFileProps['file-mime'];
 		$status = $this->verifyMimeType( $mime );
 		if ( $status !== true ) {
 			wfProfileOut( __METHOD__ );
@@ -575,7 +585,9 @@ abstract class UploadBase {
 	}
 
 	/**
-	 * Check for non fatal problems with the file
+	 * Check for non fatal problems with the file.
+	 *
+	 * This should not assume that mTempPath is set.
 	 *
 	 * @return Array of warnings
 	 */
@@ -610,7 +622,7 @@ abstract class UploadBase {
 
 		global $wgUploadSizeWarning;
 		if ( $wgUploadSizeWarning && ( $this->mFileSize > $wgUploadSizeWarning ) ) {
-			$warnings['large-file'] = $wgUploadSizeWarning;
+			$warnings['large-file'] = array( $wgUploadSizeWarning, $this->mFileSize );
 		}
 
 		if ( $this->mFileSize == 0 ) {
@@ -623,7 +635,7 @@ abstract class UploadBase {
 		}
 
 		// Check dupes against existing files
-		$hash = FSFile::getSha1Base36FromPath( $this->mTempPath );
+		$hash = $this->getTempFileSha1Base36();
 		$dupes = RepoGroup::singleton()->findBySha1( $hash );
 		$title = $this->getTitle();
 		// Remove all matches against self
@@ -723,8 +735,6 @@ abstract class UploadBase {
 		}
 		$this->mFilteredName = $nt->getDBkey();
 
-
-
 		/**
 		 * We'll want to blacklist against *any* 'extension', and use
 		 * only the final one for the whitelist.
@@ -752,7 +762,6 @@ abstract class UploadBase {
 					$ext = array( $this->mFinalExtension );
 				}
 			}
-
 		}
 
 		/* Don't allow users to override the blacklist (check file extension) */
@@ -787,7 +796,7 @@ abstract class UploadBase {
 		}
 
 		if( strlen( $partname ) < 1 ) {
-			$this->mTitleError =  self::MIN_LENGTH_PARTNAME;
+			$this->mTitleError = self::MIN_LENGTH_PARTNAME;
 			return $this->mTitle = null;
 		}
 
@@ -816,13 +825,14 @@ abstract class UploadBase {
 	 * This method returns the file object, which also has a 'fileKey' property which can be passed through a form or
 	 * API request to find this stashed file again.
 	 *
+	 * @param $user User
 	 * @return UploadStashFile stashed file
 	 */
-	public function stashFile() {
+	public function stashFile( User $user = null ) {
 		// was stashSessionFile
 		wfProfileIn( __METHOD__ );
 
-		$stash = RepoGroup::singleton()->getLocalRepo()->getUploadStash();
+		$stash = RepoGroup::singleton()->getLocalRepo()->getUploadStash( $user );
 		$file = $stash->stashFile( $this->mTempPath, $this->getSourceType() );
 		$this->mLocalFile = $file;
 
@@ -905,8 +915,8 @@ abstract class UploadBase {
 	/**
 	 * Checks if the mime type of the uploaded file matches the file extension.
 	 *
-	 * @param $mime String: the mime type of the uploaded file
-	 * @param $extension String: the filename extension that the file is to be served with
+	 * @param string $mime the mime type of the uploaded file
+	 * @param string $extension the filename extension that the file is to be served with
 	 * @return Boolean
 	 */
 	public static function verifyExtension( $mime, $extension ) {
@@ -946,9 +956,9 @@ abstract class UploadBase {
 	 * potentially harmful. The present implementation will produce false
 	 * positives in some situations.
 	 *
-	 * @param $file String: pathname to the temporary upload file
-	 * @param $mime String: the mime type of the file
-	 * @param $extension String: the extension of the file
+	 * @param string $file pathname to the temporary upload file
+	 * @param string $mime the mime type of the file
+	 * @param string $extension the extension of the file
 	 * @return Boolean: true if the file contains something looking like embedded scripts
 	 */
 	public static function detectScript( $file, $mime, $extension ) {
@@ -958,7 +968,7 @@ abstract class UploadBase {
 		# ugly hack: for text files, always look at the entire file.
 		# For binary field, just check the first K.
 
-		if( strpos( $mime,'text/' ) === 0 ) {
+		if( strpos( $mime, 'text/' ) === 0 ) {
 			$chunk = file_get_contents( $file );
 		} else {
 			$fp = fopen( $file, 'rb' );
@@ -988,7 +998,7 @@ abstract class UploadBase {
 
 		$chunk = trim( $chunk );
 
-		# @todo FIXME: Convert from UTF-16 if necessarry!
+		# @todo FIXME: Convert from UTF-16 if necessary!
 		wfDebug( __METHOD__ . ": checking for embedded scripts and HTML stuff\n" );
 
 		# check for HTML doctype
@@ -1173,7 +1183,7 @@ abstract class UploadBase {
 
 		foreach( $attribs as $attrib => $value ) {
 			$stripped = $this->stripXmlNamespace( $attrib );
-			$value = strtolower($value);
+			$value = strtolower( $value );
 
 			if( substr( $stripped, 0, 2 ) == 'on' ) {
 				wfDebug( __METHOD__ . ": Found event-handler attribute '$attrib'='$value' in uploaded file.\n" );
@@ -1186,13 +1196,13 @@ abstract class UploadBase {
 				return true;
 			}
 
-			# href with embeded svg as target
+			# href with embedded svg as target
 			if( $stripped == 'href' && preg_match( '!data:[^,]*image/svg[^,]*,!sim', $value ) ) {
 				wfDebug( __METHOD__ . ": Found href to embedded svg \"<$strippedElement '$attrib'='$value'...\" in uploaded file.\n" );
 				return true;
 			}
 
-			# href with embeded (text/xml) svg as target
+			# href with embedded (text/xml) svg as target
 			if( $stripped == 'href' && preg_match( '!data:[^,]*text/xml[^,]*,!sim', $value ) ) {
 				wfDebug( __METHOD__ . ": Found href to embedded svg \"<$strippedElement '$attrib'='$value'...\" in uploaded file.\n" );
 				return true;
@@ -1206,19 +1216,18 @@ abstract class UploadBase {
 
 			# use set to add href attribute to parent element
 			if( $strippedElement == 'set' && $stripped == 'attributename' && strpos( $value, 'href' ) !== false ) {
-				wfDebug( __METHOD__ . ": Found svg setting href attibute '$value' in uploaded file.\n" );
+				wfDebug( __METHOD__ . ": Found svg setting href attribute '$value' in uploaded file.\n" );
 				return true;
 			}
 
 			# use set to add a remote / data / script target to an element
-			if( $strippedElement == 'set' && $stripped == 'to' &&  preg_match( '!(http|https|data|script):!sim', $value ) ) {
-				wfDebug( __METHOD__ . ": Found svg setting attibute to '$value' in uploaded file.\n" );
+			if( $strippedElement == 'set' && $stripped == 'to' && preg_match( '!(http|https|data|script):!sim', $value ) ) {
+				wfDebug( __METHOD__ . ": Found svg setting attribute to '$value' in uploaded file.\n" );
 				return true;
 			}
 
-
 			# use handler attribute with remote / data / script
-			if( $stripped == 'handler' &&  preg_match( '!(http|https|data|script):!sim', $value ) ) {
+			if( $stripped == 'handler' && preg_match( '!(http|https|data|script):!sim', $value ) ) {
 				wfDebug( __METHOD__ . ": Found svg setting handler with remote/data/script '$attrib'='$value' in uploaded file.\n" );
 				return true;
 			}
@@ -1226,8 +1235,8 @@ abstract class UploadBase {
 			# use CSS styles to bring in remote code
 			# catch url("http:..., url('http:..., url(http:..., but not url("#..., url('#..., url(#....
 			if( $stripped == 'style' && preg_match_all( '!((?:font|clip-path|fill|filter|marker|marker-end|marker-mid|marker-start|mask|stroke)\s*:\s*url\s*\(\s*["\']?\s*[^#]+.*?\))!sim', $value, $matches ) ) {
-				foreach ($matches[1] as $match) {
-					if (!preg_match( '!(?:font|clip-path|fill|filter|marker|marker-end|marker-mid|marker-start|mask|stroke)\s*:\s*url\s*\(\s*(#|\'#|"#)!sim', $match ) ) {
+				foreach ( $matches[1] as $match ) {
+					if ( !preg_match( '!(?:font|clip-path|fill|filter|marker|marker-end|marker-mid|marker-start|mask|stroke)\s*:\s*url\s*\(\s*(#|\'#|"#)!sim', $match ) ) {
 						wfDebug( __METHOD__ . ": Found svg setting a style with remote url '$attrib'='$value' in uploaded file.\n" );
 						return true;
 					}
@@ -1260,7 +1269,7 @@ abstract class UploadBase {
 	 * This relies on the $wgAntivirus and $wgAntivirusSetup variables.
 	 * $wgAntivirusRequired may be used to deny upload if the scan fails.
 	 *
-	 * @param $file String: pathname to the temporary upload file
+	 * @param string $file pathname to the temporary upload file
 	 * @return mixed false if not virus is found, NULL if the scan fails or is disabled,
 	 *         or a string containing feedback from the virus scanner if a virus was found.
 	 *         If textual feedback is missing but a virus was found, this function returns true.
@@ -1317,27 +1326,22 @@ abstract class UploadBase {
 			}
 		}
 
+		/* NB: AV_NO_VIRUS is 0 but AV_SCAN_FAILED is false,
+		 * so we need the strict equalities === and thus can't use a switch here
+		 */
 		if ( $mappedCode === AV_SCAN_FAILED ) {
 			# scan failed (code was mapped to false by $exitCodeMap)
 			wfDebug( __METHOD__ . ": failed to scan $file (code $exitCode).\n" );
 
-			if ( $wgAntivirusRequired ) {
-				wfProfileOut( __METHOD__ );
-				return wfMessage( 'virus-scanfailed', array( $exitCode ) )->text();
-			} else {
-				wfProfileOut( __METHOD__ );
-				return null;
-			}
+			$output = $wgAntivirusRequired ? wfMessage( 'virus-scanfailed', array( $exitCode ) )->text() : null;
 		} elseif ( $mappedCode === AV_SCAN_ABORTED ) {
 			# scan failed because filetype is unknown (probably imune)
 			wfDebug( __METHOD__ . ": unsupported file type $file (code $exitCode).\n" );
-			wfProfileOut( __METHOD__ );
-			return null;
+			$output = null;
 		} elseif ( $mappedCode === AV_NO_VIRUS ) {
 			# no virus found
 			wfDebug( __METHOD__ . ": file passed virus scan.\n" );
-			wfProfileOut( __METHOD__ );
-			return false;
+			$output = false;
 		} else {
 			$output = trim( $output );
 
@@ -1353,9 +1357,10 @@ abstract class UploadBase {
 			}
 
 			wfDebug( __METHOD__ . ": FOUND VIRUS! scanner feedback: $output \n" );
-			wfProfileOut( __METHOD__ );
-			return $output;
 		}
+
+		wfProfileOut( __METHOD__ );
+		return $output;
 	}
 
 	/**
@@ -1392,7 +1397,7 @@ abstract class UploadBase {
 	 * Check if a user is the last uploader
 	 *
 	 * @param $user User object
-	 * @param $img String: image name
+	 * @param string $img image name
 	 * @return Boolean
 	 */
 	public static function userCanReUpload( User $user, $img ) {
@@ -1464,9 +1469,20 @@ abstract class UploadBase {
 			}
 		}
 
+		// Check for files with the same name but a different extension
+		$similarFiles = RepoGroup::singleton()->getLocalRepo()->findFilesByPrefix(
+				"{$partname}.", 1 );
+		if ( count( $similarFiles ) ) {
+			return array(
+				'warning' => 'exists-normalized',
+				'file' => $file,
+				'normalizedFile' => $similarFiles[0],
+			);
+		}
+
 		if ( self::isThumbName( $file->getName() ) ) {
 			# Check for filenames like 50px- or 180px-, these are mostly thumbnails
-			$nt_thb = Title::newFromText( substr( $partname , strpos( $partname , '-' ) +1 ) . '.' . $extension, NS_FILE );
+			$nt_thb = Title::newFromText( substr( $partname, strpos( $partname, '-' ) + 1 ) . '.' . $extension, NS_FILE );
 			$file_thb = wfLocalFile( $nt_thb );
 			if( $file_thb->exists() ) {
 				return array(
@@ -1484,7 +1500,6 @@ abstract class UploadBase {
 			}
 		}
 
-
 		foreach( self::getFilenamePrefixBlacklist() as $prefix ) {
 			if ( substr( $partname, 0, strlen( $prefix ) ) == $prefix ) {
 				return array(
@@ -1507,10 +1522,10 @@ abstract class UploadBase {
 		$n = strrpos( $filename, '.' );
 		$partname = $n ? substr( $filename, 0, $n ) : $filename;
 		return (
-					substr( $partname , 3, 3 ) == 'px-' ||
-					substr( $partname , 2, 3 ) == 'px-'
+					substr( $partname, 3, 3 ) == 'px-' ||
+					substr( $partname, 2, 3 ) == 'px-'
 				) &&
-				preg_match( "/[0-9]{2}/" , substr( $partname , 0, 2 ) );
+				preg_match( "/[0-9]{2}/", substr( $partname, 0, 2 ) );
 	}
 
 	/**
@@ -1590,6 +1605,32 @@ abstract class UploadBase {
 		} else {
 			return intval( $wgMaxUploadSize );
 		}
+	}
+
+	/**
+	 * Get the current status of a chunked upload (used for polling).
+	 * The status will be read from the *current* user session.
+	 * @param $statusKey string
+	 * @return Array|bool
+	 */
+	public static function getSessionStatus( $statusKey ) {
+		return isset( $_SESSION[self::SESSION_STATUS_KEY][$statusKey] )
+			? $_SESSION[self::SESSION_STATUS_KEY][$statusKey]
+			: false;
+	}
 
+	/**
+	 * Set the current status of a chunked upload (used for polling).
+	 * The status will be stored in the *current* user session.
+	 * @param $statusKey string
+	 * @param $value array|false
+	 * @return void
+	 */
+	public static function setSessionStatus( $statusKey, $value ) {
+		if ( $value === false ) {
+			unset( $_SESSION[self::SESSION_STATUS_KEY][$statusKey] );
+		} else {
+			$_SESSION[self::SESSION_STATUS_KEY][$statusKey] = $value;
+		}
 	}
 }
diff --git a/includes/upload/UploadFromChunks.php b/includes/upload/UploadFromChunks.php
index 531f7be4..4b331e98 100644
--- a/includes/upload/UploadFromChunks.php
+++ b/includes/upload/UploadFromChunks.php
@@ -37,7 +37,7 @@ class UploadFromChunks extends UploadFromFile {
 	 * @param $stash UploadStash
 	 * @param $repo FileRepo
 	 */
-	public function __construct( $user = false, $stash = false, $repo = false ) {
+	public function __construct( $user = null, $stash = false, $repo = false ) {
 		// user object. sometimes this won't exist, as when running from cron.
 		$this->user = $user;
 
@@ -60,12 +60,13 @@ class UploadFromChunks extends UploadFromFile {
 
 		return true;
 	}
+
 	/**
 	 * Calls the parent stashFile and updates the uploadsession table to handle "chunks"
 	 *
 	 * @return UploadStashFile stashed file
 	 */
-	public function stashFile() {
+	public function stashFile( User $user = null ) {
 		// Stash file is the called on creating a new chunk session:
 		$this->mChunkIndex = 0;
 		$this->mOffset = 0;
@@ -78,7 +79,7 @@ class UploadFromChunks extends UploadFromFile {
 		$this->mFileKey = $this->mLocalFile->getFileKey();
 
 		// Output a copy of this first to chunk 0 location:
-		$status = $this->outputChunk( $this->mLocalFile->getPath() );
+		$this->outputChunk( $this->mLocalFile->getPath() );
 
 		// Update db table to reflect initial "chunk" state
 		$this->updateChunkStatus();
@@ -113,7 +114,7 @@ class UploadFromChunks extends UploadFromFile {
 		// Concatenate all the chunks to mVirtualTempPath
 		$fileList = Array();
 		// The first chunk is stored at the mVirtualTempPath path so we start on "chunk 1"
-		for( $i = 0; $i <= $this->getChunkIndex(); $i++ ){
+		for( $i = 0; $i <= $this->getChunkIndex(); $i++ ) {
 			$fileList[] = $this->getVirtualChunkLocation( $i );
 		}
 
@@ -122,13 +123,16 @@ class UploadFromChunks extends UploadFromFile {
 		// Get a 0-byte temp file to perform the concatenation at
 		$tmpFile = TempFSFile::factory( 'chunkedupload_', $ext );
 		$tmpPath = $tmpFile
-			? $tmpFile->getPath()
+			? $tmpFile->bind( $this )->getPath() // keep alive with $this
 			: false; // fail in concatenate()
 		// Concatenate the chunks at the temp file
+		$tStart = microtime( true );
 		$status = $this->repo->concatenate( $fileList, $tmpPath, FileRepo::DELETE_SOURCE );
-		if( !$status->isOk() ){
+		$tAmount = microtime( true ) - $tStart;
+		if( !$status->isOk() ) {
 			return $status;
 		}
+		wfDebugLog( 'fileconcatenate', "Combined $i chunks in $tAmount seconds.\n" );
 
 		$this->mTempPath = $tmpPath; // file system path
 		$this->mFileSize = filesize( $this->mTempPath ); //Since this was set for the last chunk previously
@@ -141,7 +145,11 @@ class UploadFromChunks extends UploadFromFile {
 
 		// Update the mTempPath and mLocalFile
 		// ( for FileUpload or normal Stash to take over )
-		$this->mLocalFile = parent::stashFile();
+		$tStart = microtime( true );
+		$this->mLocalFile = parent::stashFile( $this->user );
+		$tAmount = microtime( true ) - $tStart;
+		$this->mLocalFile->setLocalReference( $tmpFile ); // reuse (e.g. for getImageInfo())
+		wfDebugLog( 'fileconcatenate', "Stashed combined file ($i chunks) in $tAmount seconds.\n" );
 
 		return $status;
 	}
@@ -164,7 +172,7 @@ class UploadFromChunks extends UploadFromFile {
 	 * @param $index
 	 * @return string
 	 */
-	function getVirtualChunkLocation( $index ){
+	function getVirtualChunkLocation( $index ) {
 		return $this->repo->getVirtualUrl( 'temp' ) .
 				'/' .
 				$this->repo->getHashPath(
@@ -176,9 +184,9 @@ class UploadFromChunks extends UploadFromFile {
 	/**
 	 * Add a chunk to the temporary directory
 	 *
-	 * @param $chunkPath string path to temporary chunk file
-	 * @param $chunkSize int size of the current chunk
-	 * @param $offset int offset of current chunk ( mutch match database chunk offset )
+	 * @param string $chunkPath path to temporary chunk file
+	 * @param int $chunkSize size of the current chunk
+	 * @param int $offset offset of current chunk ( mutch match database chunk offset )
 	 * @return Status
 	 */
 	public function addChunk( $chunkPath, $chunkSize, $offset ) {
@@ -202,7 +210,7 @@ class UploadFromChunks extends UploadFromFile {
 					return Status::newFatal( $e->getMessage() );
 				}
 				$status = $this->outputChunk( $chunkPath );
-				if( $status->isGood() ){
+				if( $status->isGood() ) {
 					// Update local offset:
 					$this->mOffset = $preAppendOffset + $chunkSize;
 					// Update chunk table status db
@@ -218,11 +226,14 @@ class UploadFromChunks extends UploadFromFile {
 	/**
 	 * Update the chunk db table with the current status:
 	 */
-	private function updateChunkStatus(){
+	private function updateChunkStatus() {
 		wfDebug( __METHOD__ . " update chunk status for {$this->mFileKey} offset:" .
 					$this->getOffset() . ' inx:' . $this->getChunkIndex() . "\n" );
 
 		$dbw = $this->repo->getMasterDb();
+		// Use a quick transaction since we will upload the full temp file into shared
+		// storage, which takes time for large files. We don't want to hold locks then.
+		$dbw->begin( __METHOD__ );
 		$dbw->update(
 			'uploadstash',
 			array(
@@ -233,12 +244,13 @@ class UploadFromChunks extends UploadFromFile {
 			array( 'us_key' => $this->mFileKey ),
 			__METHOD__
 		);
+		$dbw->commit( __METHOD__ );
 	}
 
 	/**
 	 * Get the chunk db state and populate update relevant local values
 	 */
-	private function getChunkStatus(){
+	private function getChunkStatus() {
 		// get Master db to avoid race conditions.
 		// Otherwise, if chunk upload time < replag there will be spurious errors
 		$dbw = $this->repo->getMasterDb();
@@ -264,8 +276,8 @@ class UploadFromChunks extends UploadFromFile {
 	 * Get the current Chunk index
 	 * @return Integer index of the current chunk
 	 */
-	private function getChunkIndex(){
-		if( $this->mChunkIndex !== null ){
+	private function getChunkIndex() {
+		if( $this->mChunkIndex !== null ) {
 			return $this->mChunkIndex;
 		}
 		return 0;
@@ -275,8 +287,8 @@ class UploadFromChunks extends UploadFromFile {
 	 * Gets the current offset in fromt the stashedupload table
 	 * @return Integer current byte offset of the chunk file set
 	 */
-	private function getOffset(){
-		if ( $this->mOffset !== null ){
+	private function getOffset() {
+		if ( $this->mOffset !== null ) {
 			return $this->mOffset;
 		}
 		return 0;
@@ -289,7 +301,7 @@ class UploadFromChunks extends UploadFromFile {
 	 * @throws UploadChunkFileException
 	 * @return FileRepoStatus
 	 */
-	private function outputChunk( $chunkPath ){
+	private function outputChunk( $chunkPath ) {
 		// Key is fileKey + chunk index
 		$fileKey = $this->getChunkFileKey();
 
@@ -314,11 +326,11 @@ class UploadFromChunks extends UploadFromFile {
 		return $storeStatus;
 	}
 
-	private function getChunkFileKey( $index = null ){
-		if( $index === null ){
+	private function getChunkFileKey( $index = null ) {
+		if( $index === null ) {
 			$index = $this->getChunkIndex();
 		}
-		return $this->mFileKey . '.' . $index ;
+		return $this->mFileKey . '.' . $index;
 	}
 
 	/**
diff --git a/includes/upload/UploadFromFile.php b/includes/upload/UploadFromFile.php
index aa0cc77b..ab2a7a39 100644
--- a/includes/upload/UploadFromFile.php
+++ b/includes/upload/UploadFromFile.php
@@ -79,21 +79,21 @@ class UploadFromFile extends UploadBase {
 	 * @return array
 	 */
 	public function verifyUpload() {
-		# Check for a post_max_size or upload_max_size overflow, so that a 
+		# Check for a post_max_size or upload_max_size overflow, so that a
 		# proper error can be shown to the user
 		if ( is_null( $this->mTempPath ) || $this->isEmptyFile() ) {
 			if ( $this->mUpload->isIniSizeOverflow() ) {
-				return array( 
+				return array(
 					'status' => UploadBase::FILE_TOO_LARGE,
-					'max' => min( 
-						self::getMaxUploadSize( $this->getSourceType() ), 
-						wfShorthandToInteger( ini_get( 'upload_max_filesize' ) ), 
+					'max' => min(
+						self::getMaxUploadSize( $this->getSourceType() ),
+						wfShorthandToInteger( ini_get( 'upload_max_filesize' ) ),
 						wfShorthandToInteger( ini_get( 'post_max_size' ) )
 					),
 				);
 			}
 		}
-		
+
 		return parent::verifyUpload();
 	}
 }
diff --git a/includes/upload/UploadFromStash.php b/includes/upload/UploadFromStash.php
index d79641ce..c82103cb 100644
--- a/includes/upload/UploadFromStash.php
+++ b/includes/upload/UploadFromStash.php
@@ -89,7 +89,7 @@ class UploadFromStash extends UploadBase {
 	 * @param $key string
 	 * @param $name string
 	 */
-	public function initialize( $key, $name = 'upload_file' ) {
+	public function initialize( $key, $name = 'upload_file', $initTempFile = true ) {
 		/**
 		 * Confirming a temporarily stashed upload.
 		 * We don't want path names to be forged, so we keep
@@ -98,7 +98,7 @@ class UploadFromStash extends UploadBase {
 		 */
 		$metadata = $this->stash->getMetadata( $key );
 		$this->initializePathInfo( $name,
-			$this->getRealPath ( $metadata['us_path'] ),
+			$initTempFile ? $this->getRealPath( $metadata['us_path'] ) : false,
 			$metadata['us_size'],
 			false
 		);
@@ -129,6 +129,14 @@ class UploadFromStash extends UploadBase {
 		return $this->mSourceType;
 	}
 
+	/**
+	 * Get the base 36 SHA1 of the file
+	 * @return string
+	 */
+	public function getTempFileSha1Base36() {
+		return $this->mFileProps['sha1'];
+	}
+
 	/*
 	 * protected function verifyFile() inherited
 	 */
@@ -136,12 +144,13 @@ class UploadFromStash extends UploadBase {
 	/**
 	 * Stash the file.
 	 *
+	 * @param $user User
 	 * @return UploadStashFile
 	 */
-	public function stashFile() {
+	public function stashFile( User $user = null ) {
 		// replace mLocalFile with an instance of UploadStashFile, which adds some methods
 		// that are useful for stashed files.
-		$this->mLocalFile = parent::stashFile();
+		$this->mLocalFile = parent::stashFile( $user );
 		return $this->mLocalFile;
 	}
 
diff --git a/includes/upload/UploadFromUrl.php b/includes/upload/UploadFromUrl.php
index 927c3cd9..70b69034 100644
--- a/includes/upload/UploadFromUrl.php
+++ b/includes/upload/UploadFromUrl.php
@@ -61,6 +61,8 @@ class UploadFromUrl extends UploadBase {
 
 	/**
 	 * Checks whether the URL is for an allowed host
+	 * The domains in the whitelist can include wildcard characters (*) in place
+	 * of any of the domain levels, e.g. '*.flickr.com' or 'upload.*.gov.uk'.
 	 *
 	 * @param $url string
 	 * @return bool
@@ -76,10 +78,28 @@ class UploadFromUrl extends UploadBase {
 		}
 		$valid = false;
 		foreach( $wgCopyUploadsDomains as $domain ) {
+			// See if the domain for the upload matches this whitelisted domain
+			$whitelistedDomainPieces = explode( '.', $domain );
+			$uploadDomainPieces = explode( '.', $parsedUrl['host'] );
+			if ( count( $whitelistedDomainPieces ) === count( $uploadDomainPieces ) ) {
+				$valid = true;
+				// See if all the pieces match or not (excluding wildcards)
+				foreach ( $whitelistedDomainPieces as $index => $piece ) {
+					if ( $piece !== '*' && $piece !== $uploadDomainPieces[$index] ) {
+						$valid = false;
+					}
+				}
+				if ( $valid ) {
+					// We found a match, so quit comparing against the list
+					break;
+				}
+			}
+			/* Non-wildcard test
 			if ( $parsedUrl['host'] === $domain ) {
 				$valid = true;
 				break;
 			}
+			*/
 		}
 		return $valid;
 	}
@@ -312,7 +332,7 @@ class UploadFromUrl extends UploadBase {
 			'sessionKey' => $sessionKey,
 		) );
 		$job->initializeSessionData();
-		$job->insert();
+		JobQueueGroup::singleton()->push( $job );
 		return $sessionKey;
 	}
 
diff --git a/includes/upload/UploadStash.php b/includes/upload/UploadStash.php
index 53a90582..089bd8b7 100644
--- a/includes/upload/UploadStash.php
+++ b/includes/upload/UploadStash.php
@@ -97,7 +97,7 @@ class UploadStash {
 	 * Get a file and its metadata from the stash.
 	 * The noAuth param is a bit janky but is required for automated scripts which clean out the stash.
 	 *
-	 * @param $key String: key under which file information is stored
+	 * @param string $key key under which file information is stored
 	 * @param $noAuth Boolean (optional) Don't check authentication. Used by maintenance scripts.
 	 * @throws UploadStashFileNotFoundException
 	 * @throws UploadStashNotLoggedInException
@@ -106,15 +106,13 @@ class UploadStash {
 	 * @return UploadStashFile
 	 */
 	public function getFile( $key, $noAuth = false ) {
-
 		if ( ! preg_match( self::KEY_FORMAT_REGEX, $key ) ) {
 			throw new UploadStashBadPathException( "key '$key' is not in a proper format" );
 		}
 
-		if ( !$noAuth ) {
-			if ( !$this->isLoggedIn ) {
-				throw new UploadStashNotLoggedInException( __METHOD__ . ' No user is logged in, files must belong to users' );
-			}
+		if ( !$noAuth && !$this->isLoggedIn ) {
+			throw new UploadStashNotLoggedInException( __METHOD__ .
+				' No user is logged in, files must belong to users' );
 		}
 
 		if ( !isset( $this->fileMetadata[$key] ) ) {
@@ -131,8 +129,13 @@ class UploadStash {
 			$this->initFile( $key );
 
 			// fetch fileprops
-			$path = $this->fileMetadata[$key]['us_path'];
-			$this->fileProps[$key] = $this->repo->getFileProps( $path );
+			if ( strlen( $this->fileMetadata[$key]['us_props'] ) ) {
+				$this->fileProps[$key] = unserialize( $this->fileMetadata[$key]['us_props'] );
+			} else { // b/c for rows with no us_props
+				wfDebug( __METHOD__ . " fetched props for $key from file\n" );
+				$path = $this->fileMetadata[$key]['us_path'];
+				$this->fileProps[$key] = $this->repo->getFileProps( $path );
+			}
 		}
 
 		if ( ! $this->files[$key]->exists() ) {
@@ -152,7 +155,7 @@ class UploadStash {
 	/**
 	 * Getter for file metadata.
 	 *
-	 * @param key String: key under which file information is stored
+	 * @param string $key key under which file information is stored
 	 * @return Array
 	 */
 	public function getMetadata ( $key ) {
@@ -163,7 +166,7 @@ class UploadStash {
 	/**
 	 * Getter for fileProps
 	 *
-	 * @param key String: key under which file information is stored
+	 * @param string $key key under which file information is stored
 	 * @return Array
 	 */
 	public function getFileProps ( $key ) {
@@ -174,15 +177,15 @@ class UploadStash {
 	/**
 	 * Stash a file in a temp directory and record that we did this in the database, along with other metadata.
 	 *
-	 * @param $path String: path to file you want stashed
-	 * @param $sourceType String: the type of upload that generated this file (currently, I believe, 'file' or null)
+	 * @param string $path path to file you want stashed
+	 * @param string $sourceType the type of upload that generated this file (currently, I believe, 'file' or null)
 	 * @throws UploadStashBadPathException
 	 * @throws UploadStashFileException
 	 * @throws UploadStashNotLoggedInException
 	 * @return UploadStashFile: file, or null on failure
 	 */
 	public function stashFile( $path, $sourceType = null ) {
-		if ( ! file_exists( $path ) ) {
+		if ( !is_file( $path ) ) {
 			wfDebug( __METHOD__ . " tried to stash file at '$path', but it doesn't exist\n" );
 			throw new UploadStashBadPathException( "path doesn't exist" );
 		}
@@ -192,12 +195,10 @@ class UploadStash {
 		// we will be initializing from some tmpnam files that don't have extensions.
 		// most of MediaWiki assumes all uploaded files have good extensions. So, we fix this.
 		$extension = self::getExtensionForPath( $path );
-		if ( ! preg_match( "/\\.\\Q$extension\\E$/", $path ) ) {
+		if ( !preg_match( "/\\.\\Q$extension\\E$/", $path ) ) {
 			$pathWithGoodExtension = "$path.$extension";
-			if ( ! rename( $path, $pathWithGoodExtension ) ) {
-				throw new UploadStashFileException( "couldn't rename $path to have a better extension at $pathWithGoodExtension" );
-			}
-			$path = $pathWithGoodExtension;
+		} else {
+			$pathWithGoodExtension = $path;
 		}
 
 		// If no key was supplied, make one.  a mysql insertid would be totally reasonable here, except
@@ -205,8 +206,8 @@ class UploadStash {
 		//
 		// some things that when combined will make a suitably unique key.
 		// see: http://www.jwz.org/doc/mid.html
-		list ($usec, $sec) = explode( ' ', microtime() );
-		$usec = substr($usec, 2);
+		list( $usec, $sec ) = explode( ' ', microtime() );
+		$usec = substr( $usec, 2 );
 		$key = wfBaseConvert( $sec . $usec, 10, 36 ) . '.' .
 			wfBaseConvert( mt_rand(), 10, 36 ) . '.'.
 			$this->userId . '.' .
@@ -221,7 +222,7 @@ class UploadStash {
 		wfDebug( __METHOD__ . " key for '$path': $key\n" );
 
 		// if not already in a temporary area, put it there
-		$storeStatus = $this->repo->storeTemp( basename( $path ), $path );
+		$storeStatus = $this->repo->storeTemp( basename( $pathWithGoodExtension ), $path );
 
 		if ( ! $storeStatus->isOK() ) {
 			// It is a convention in MediaWiki to only return one error per API exception, even if multiple errors
@@ -244,9 +245,6 @@ class UploadStash {
 		}
 		$stashPath = $storeStatus->value;
 
-		// we have renamed the file so we have to cleanup once done
-		unlink($path);
-
 		// fetch the current user ID
 		if ( !$this->isLoggedIn ) {
 			throw new UploadStashNotLoggedInException( __METHOD__ . ' No user is logged in, files must belong to users' );
@@ -262,6 +260,7 @@ class UploadStash {
 			'us_key' => $key,
 			'us_orig_path' => $path,
 			'us_path' => $stashPath, // virtual URL
+			'us_props' => serialize( $fileProps ),
 			'us_size' => $fileProps['size'],
 			'us_sha1' => $fileProps['sha1'],
 			'us_mime' => $fileProps['mime'],
@@ -319,8 +318,8 @@ class UploadStash {
 	/**
 	 * Remove a particular file from the stash.  Also removes it from the repo.
 	 *
-	 * @throws UploadStashNotLoggedInException
-	 * @throws UploadStashWrongOwnerException
+	 * @param $key
+	 * @throws UploadStashNoSuchKeyException|UploadStashNotLoggedInException|UploadStashWrongOwnerException
 	 * @return boolean: success
 	 */
 	public function removeFile( $key ) {
@@ -350,7 +349,6 @@ class UploadStash {
 		return $this->removeFileNoAuth( $key );
 	}
 
-
 	/**
 	 * Remove a file (see removeFile), but doesn't check ownership first.
 	 *
@@ -359,16 +357,16 @@ class UploadStash {
 	public function removeFileNoAuth( $key ) {
 		wfDebug( __METHOD__ . " clearing row $key\n" );
 
+		// Ensure we have the UploadStashFile loaded for this key
+		$this->getFile( $key );
+
 		$dbw = $this->repo->getMasterDb();
 
-		// this gets its own transaction since it's called serially by the cleanupUploadStash maintenance script
-		$dbw->begin( __METHOD__ );
 		$dbw->delete(
 			'uploadstash',
 			array( 'us_key' => $key ),
 			__METHOD__
 		);
-		$dbw->commit( __METHOD__ );
 
 		// TODO: look into UnregisteredLocalFile and find out why the rv here is sometimes wrong (false when file was removed)
 		// for now, ignore.
@@ -419,6 +417,8 @@ class UploadStash {
 	 * with an extension.
 	 * XXX this is somewhat redundant with the checks that ApiUpload.php does with incoming
 	 * uploads versus the desired filename. Maybe we can get that passed to us...
+	 * @param $path
+	 * @throws UploadStashFileException
 	 * @return string
 	 */
 	public static function getExtensionForPath( $path ) {
@@ -456,7 +456,7 @@ class UploadStash {
 	/**
 	 * Helper function: do the actual database query to fetch file metadata.
 	 *
-	 * @param $key String: key
+	 * @param string $key key
 	 * @param $readFromDB: constant (default: DB_SLAVE)
 	 * @return boolean
 	 */
@@ -490,7 +490,7 @@ class UploadStash {
 	/**
 	 * Helper function: Initialize the UploadStashFile for a given file.
 	 *
-	 * @param $key String: key under which to store the object
+	 * @param string $key key under which to store the object
 	 * @throws UploadStashZeroLengthFileException
 	 * @return bool
 	 */
@@ -514,8 +514,8 @@ class UploadStashFile extends UnregisteredLocalFile {
 	 * Arguably UnregisteredLocalFile should be handling its own file repo but that class is a bit retarded currently
 	 *
 	 * @param $repo FileRepo: repository where we should find the path
-	 * @param $path String: path to file
-	 * @param $key String: key to store the path and any stashed data under
+	 * @param string $path path to file
+	 * @param string $key key to store the path and any stashed data under
 	 * @throws UploadStashBadPathException
 	 * @throws UploadStashFileNotFoundException
 	 */
@@ -564,7 +564,7 @@ class UploadStashFile extends UnregisteredLocalFile {
 	 * The actual argument is the result of thumbName although we seem to have
 	 * buggy code elsewhere that expects a boolean 'suffix'
 	 *
-	 * @param $thumbName String: name of thumbnail (e.g. "120px-123456.jpg" ), or false to just get the path
+	 * @param string $thumbName name of thumbnail (e.g. "120px-123456.jpg" ), or false to just get the path
 	 * @return String: path thumbnail should take on filesystem, or containing directory if thumbname is false
 	 */
 	public function getThumbPath( $thumbName = false ) {
@@ -580,7 +580,7 @@ class UploadStashFile extends UnregisteredLocalFile {
 	 * We override this because we want to use the pretty url name instead of the
 	 * ugly file name.
 	 *
-	 * @param $params Array: handler-specific parameters
+	 * @param array $params handler-specific parameters
 	 * @param $flags integer Bitfield that supports THUMB_* constants
 	 * @return String: base name for URL, like '120px-12345.jpg', or null if there is no handler
 	 */
@@ -603,7 +603,7 @@ class UploadStashFile extends UnregisteredLocalFile {
 	 * the thumbnail urls be predictable. However, in our model the URL is not based on the filename
 	 * (that's hidden in the db)
 	 *
-	 * @param $thumbName String: basename of thumbnail file -- however, we don't want to use the file exactly
+	 * @param string $thumbName basename of thumbnail file -- however, we don't want to use the file exactly
 	 * @return String: URL to access thumbnail, or URL with partial path
 	 */
 	public function getThumbUrl( $thumbName = false ) {