summaryrefslogtreecommitdiff
path: root/includes/upload
diff options
context:
space:
mode:
Diffstat (limited to 'includes/upload')
-rw-r--r--includes/upload/UploadBase.php159
-rw-r--r--includes/upload/UploadFromChunks.php56
-rw-r--r--includes/upload/UploadFromFile.php12
-rw-r--r--includes/upload/UploadFromStash.php17
-rw-r--r--includes/upload/UploadFromUrl.php22
-rw-r--r--includes/upload/UploadStash.php74
6 files changed, 211 insertions, 129 deletions
diff --git a/includes/upload/UploadBase.php b/includes/upload/UploadBase.php
index 0848780f..5a823622 100644
--- a/includes/upload/UploadBase.php
+++ b/includes/upload/UploadBase.php
@@ -65,6 +65,8 @@ abstract class UploadBase {
const WINDOWS_NONASCII_FILENAME = 13;
const FILENAME_TOO_LONG = 14;
+ const SESSION_STATUS_KEY = 'wsUploadStatusData';
+
/**
* @param $error int
* @return string
@@ -78,7 +80,7 @@ abstract class UploadBase {
self::ILLEGAL_FILENAME => 'illegal-filename',
self::OVERWRITE_EXISTING_FILE => 'overwrite',
self::VERIFICATION_ERROR => 'verification-error',
- self::HOOK_ABORTED => 'hookaborted',
+ self::HOOK_ABORTED => 'hookaborted',
self::WINDOWS_NONASCII_FILENAME => 'windows-nonascii-filename',
self::FILENAME_TOO_LONG => 'filename-toolong',
);
@@ -108,7 +110,7 @@ abstract class UploadBase {
/**
* Returns true if the user can use this upload module or else a string
* identifying the missing permission.
- * Can be overriden by subclasses.
+ * Can be overridden by subclasses.
*
* @param $user User
* @return bool
@@ -190,10 +192,10 @@ abstract class UploadBase {
/**
* Initialize the path information
- * @param $name string the desired destination name
- * @param $tempPath string the temporary path
- * @param $fileSize int the file size
- * @param $removeTempFile bool (false) remove the temporary file?
+ * @param string $name the desired destination name
+ * @param string $tempPath the temporary path
+ * @param int $fileSize the file size
+ * @param bool $removeTempFile (false) remove the temporary file?
* @throws MWException
*/
public function initializePathInfo( $name, $tempPath, $fileSize, $removeTempFile = false ) {
@@ -209,7 +211,7 @@ abstract class UploadBase {
/**
* Initialize from a WebRequest. Override this in a subclass.
*/
- public abstract function initializeFromRequest( &$request );
+ abstract public function initializeFromRequest( &$request );
/**
* Fetch the file. Usually a no-op
@@ -236,7 +238,15 @@ abstract class UploadBase {
}
/**
- * @param $srcPath String: the source path
+ * Get the base 36 SHA1 of the file
+ * @return string
+ */
+ public function getTempFileSha1Base36() {
+ return FSFile::getSha1Base36FromPath( $this->mTempPath );
+ }
+
+ /**
+ * @param string $srcPath the source path
* @return string the real path if it was a virtual URL
*/
function getRealPath( $srcPath ) {
@@ -244,9 +254,9 @@ abstract class UploadBase {
$repo = RepoGroup::singleton()->getLocalRepo();
if ( $repo->isVirtualUrl( $srcPath ) ) {
// @TODO: just make uploads work with storage paths
- // UploadFromStash loads files via virtuals URLs
+ // UploadFromStash loads files via virtual URLs
$tmpFile = $repo->getLocalCopy( $srcPath );
- $tmpFile->bind( $this ); // keep alive with $thumb
+ $tmpFile->bind( $this ); // keep alive with $this
wfProfileOut( __METHOD__ );
return $tmpFile->getPath();
}
@@ -347,14 +357,14 @@ abstract class UploadBase {
*
* @note Only checks that it is not an evil mime. The does it have
* correct extension given its mime type check is in verifyFile.
- * @param $mime string representing the mime
+ * @param string $mime representing the mime
* @return mixed true if the file is verified, an array otherwise
*/
protected function verifyMimeType( $mime ) {
global $wgVerifyMimeType;
wfProfileIn( __METHOD__ );
if ( $wgVerifyMimeType ) {
- wfDebug ( "\n\nmime: <$mime> extension: <{$this->mFinalExtension}>\n\n");
+ wfDebug ( "\n\nmime: <$mime> extension: <{$this->mFinalExtension}>\n\n" );
global $wgMimeTypeBlacklist;
if ( $this->checkFileExtension( $mime, $wgMimeTypeBlacklist ) ) {
wfProfileOut( __METHOD__ );
@@ -447,7 +457,7 @@ abstract class UploadBase {
$this->mFileProps = FSFile::getPropsFromPath( $this->mTempPath, $this->mFinalExtension );
# check mime type, if desired
- $mime = $this->mFileProps[ 'file-mime' ];
+ $mime = $this->mFileProps['file-mime'];
$status = $this->verifyMimeType( $mime );
if ( $status !== true ) {
wfProfileOut( __METHOD__ );
@@ -575,7 +585,9 @@ abstract class UploadBase {
}
/**
- * Check for non fatal problems with the file
+ * Check for non fatal problems with the file.
+ *
+ * This should not assume that mTempPath is set.
*
* @return Array of warnings
*/
@@ -610,7 +622,7 @@ abstract class UploadBase {
global $wgUploadSizeWarning;
if ( $wgUploadSizeWarning && ( $this->mFileSize > $wgUploadSizeWarning ) ) {
- $warnings['large-file'] = $wgUploadSizeWarning;
+ $warnings['large-file'] = array( $wgUploadSizeWarning, $this->mFileSize );
}
if ( $this->mFileSize == 0 ) {
@@ -623,7 +635,7 @@ abstract class UploadBase {
}
// Check dupes against existing files
- $hash = FSFile::getSha1Base36FromPath( $this->mTempPath );
+ $hash = $this->getTempFileSha1Base36();
$dupes = RepoGroup::singleton()->findBySha1( $hash );
$title = $this->getTitle();
// Remove all matches against self
@@ -723,8 +735,6 @@ abstract class UploadBase {
}
$this->mFilteredName = $nt->getDBkey();
-
-
/**
* We'll want to blacklist against *any* 'extension', and use
* only the final one for the whitelist.
@@ -752,7 +762,6 @@ abstract class UploadBase {
$ext = array( $this->mFinalExtension );
}
}
-
}
/* Don't allow users to override the blacklist (check file extension) */
@@ -787,7 +796,7 @@ abstract class UploadBase {
}
if( strlen( $partname ) < 1 ) {
- $this->mTitleError = self::MIN_LENGTH_PARTNAME;
+ $this->mTitleError = self::MIN_LENGTH_PARTNAME;
return $this->mTitle = null;
}
@@ -816,13 +825,14 @@ abstract class UploadBase {
* This method returns the file object, which also has a 'fileKey' property which can be passed through a form or
* API request to find this stashed file again.
*
+ * @param $user User
* @return UploadStashFile stashed file
*/
- public function stashFile() {
+ public function stashFile( User $user = null ) {
// was stashSessionFile
wfProfileIn( __METHOD__ );
- $stash = RepoGroup::singleton()->getLocalRepo()->getUploadStash();
+ $stash = RepoGroup::singleton()->getLocalRepo()->getUploadStash( $user );
$file = $stash->stashFile( $this->mTempPath, $this->getSourceType() );
$this->mLocalFile = $file;
@@ -905,8 +915,8 @@ abstract class UploadBase {
/**
* Checks if the mime type of the uploaded file matches the file extension.
*
- * @param $mime String: the mime type of the uploaded file
- * @param $extension String: the filename extension that the file is to be served with
+ * @param string $mime the mime type of the uploaded file
+ * @param string $extension the filename extension that the file is to be served with
* @return Boolean
*/
public static function verifyExtension( $mime, $extension ) {
@@ -946,9 +956,9 @@ abstract class UploadBase {
* potentially harmful. The present implementation will produce false
* positives in some situations.
*
- * @param $file String: pathname to the temporary upload file
- * @param $mime String: the mime type of the file
- * @param $extension String: the extension of the file
+ * @param string $file pathname to the temporary upload file
+ * @param string $mime the mime type of the file
+ * @param string $extension the extension of the file
* @return Boolean: true if the file contains something looking like embedded scripts
*/
public static function detectScript( $file, $mime, $extension ) {
@@ -958,7 +968,7 @@ abstract class UploadBase {
# ugly hack: for text files, always look at the entire file.
# For binary field, just check the first K.
- if( strpos( $mime,'text/' ) === 0 ) {
+ if( strpos( $mime, 'text/' ) === 0 ) {
$chunk = file_get_contents( $file );
} else {
$fp = fopen( $file, 'rb' );
@@ -988,7 +998,7 @@ abstract class UploadBase {
$chunk = trim( $chunk );
- # @todo FIXME: Convert from UTF-16 if necessarry!
+ # @todo FIXME: Convert from UTF-16 if necessary!
wfDebug( __METHOD__ . ": checking for embedded scripts and HTML stuff\n" );
# check for HTML doctype
@@ -1173,7 +1183,7 @@ abstract class UploadBase {
foreach( $attribs as $attrib => $value ) {
$stripped = $this->stripXmlNamespace( $attrib );
- $value = strtolower($value);
+ $value = strtolower( $value );
if( substr( $stripped, 0, 2 ) == 'on' ) {
wfDebug( __METHOD__ . ": Found event-handler attribute '$attrib'='$value' in uploaded file.\n" );
@@ -1186,13 +1196,13 @@ abstract class UploadBase {
return true;
}
- # href with embeded svg as target
+ # href with embedded svg as target
if( $stripped == 'href' && preg_match( '!data:[^,]*image/svg[^,]*,!sim', $value ) ) {
wfDebug( __METHOD__ . ": Found href to embedded svg \"<$strippedElement '$attrib'='$value'...\" in uploaded file.\n" );
return true;
}
- # href with embeded (text/xml) svg as target
+ # href with embedded (text/xml) svg as target
if( $stripped == 'href' && preg_match( '!data:[^,]*text/xml[^,]*,!sim', $value ) ) {
wfDebug( __METHOD__ . ": Found href to embedded svg \"<$strippedElement '$attrib'='$value'...\" in uploaded file.\n" );
return true;
@@ -1206,19 +1216,18 @@ abstract class UploadBase {
# use set to add href attribute to parent element
if( $strippedElement == 'set' && $stripped == 'attributename' && strpos( $value, 'href' ) !== false ) {
- wfDebug( __METHOD__ . ": Found svg setting href attibute '$value' in uploaded file.\n" );
+ wfDebug( __METHOD__ . ": Found svg setting href attribute '$value' in uploaded file.\n" );
return true;
}
# use set to add a remote / data / script target to an element
- if( $strippedElement == 'set' && $stripped == 'to' && preg_match( '!(http|https|data|script):!sim', $value ) ) {
- wfDebug( __METHOD__ . ": Found svg setting attibute to '$value' in uploaded file.\n" );
+ if( $strippedElement == 'set' && $stripped == 'to' && preg_match( '!(http|https|data|script):!sim', $value ) ) {
+ wfDebug( __METHOD__ . ": Found svg setting attribute to '$value' in uploaded file.\n" );
return true;
}
-
# use handler attribute with remote / data / script
- if( $stripped == 'handler' && preg_match( '!(http|https|data|script):!sim', $value ) ) {
+ if( $stripped == 'handler' && preg_match( '!(http|https|data|script):!sim', $value ) ) {
wfDebug( __METHOD__ . ": Found svg setting handler with remote/data/script '$attrib'='$value' in uploaded file.\n" );
return true;
}
@@ -1226,8 +1235,8 @@ abstract class UploadBase {
# use CSS styles to bring in remote code
# catch url("http:..., url('http:..., url(http:..., but not url("#..., url('#..., url(#....
if( $stripped == 'style' && preg_match_all( '!((?:font|clip-path|fill|filter|marker|marker-end|marker-mid|marker-start|mask|stroke)\s*:\s*url\s*\(\s*["\']?\s*[^#]+.*?\))!sim', $value, $matches ) ) {
- foreach ($matches[1] as $match) {
- if (!preg_match( '!(?:font|clip-path|fill|filter|marker|marker-end|marker-mid|marker-start|mask|stroke)\s*:\s*url\s*\(\s*(#|\'#|"#)!sim', $match ) ) {
+ foreach ( $matches[1] as $match ) {
+ if ( !preg_match( '!(?:font|clip-path|fill|filter|marker|marker-end|marker-mid|marker-start|mask|stroke)\s*:\s*url\s*\(\s*(#|\'#|"#)!sim', $match ) ) {
wfDebug( __METHOD__ . ": Found svg setting a style with remote url '$attrib'='$value' in uploaded file.\n" );
return true;
}
@@ -1260,7 +1269,7 @@ abstract class UploadBase {
* This relies on the $wgAntivirus and $wgAntivirusSetup variables.
* $wgAntivirusRequired may be used to deny upload if the scan fails.
*
- * @param $file String: pathname to the temporary upload file
+ * @param string $file pathname to the temporary upload file
* @return mixed false if not virus is found, NULL if the scan fails or is disabled,
* or a string containing feedback from the virus scanner if a virus was found.
* If textual feedback is missing but a virus was found, this function returns true.
@@ -1317,27 +1326,22 @@ abstract class UploadBase {
}
}
+ /* NB: AV_NO_VIRUS is 0 but AV_SCAN_FAILED is false,
+ * so we need the strict equalities === and thus can't use a switch here
+ */
if ( $mappedCode === AV_SCAN_FAILED ) {
# scan failed (code was mapped to false by $exitCodeMap)
wfDebug( __METHOD__ . ": failed to scan $file (code $exitCode).\n" );
- if ( $wgAntivirusRequired ) {
- wfProfileOut( __METHOD__ );
- return wfMessage( 'virus-scanfailed', array( $exitCode ) )->text();
- } else {
- wfProfileOut( __METHOD__ );
- return null;
- }
+ $output = $wgAntivirusRequired ? wfMessage( 'virus-scanfailed', array( $exitCode ) )->text() : null;
} elseif ( $mappedCode === AV_SCAN_ABORTED ) {
# scan failed because filetype is unknown (probably imune)
wfDebug( __METHOD__ . ": unsupported file type $file (code $exitCode).\n" );
- wfProfileOut( __METHOD__ );
- return null;
+ $output = null;
} elseif ( $mappedCode === AV_NO_VIRUS ) {
# no virus found
wfDebug( __METHOD__ . ": file passed virus scan.\n" );
- wfProfileOut( __METHOD__ );
- return false;
+ $output = false;
} else {
$output = trim( $output );
@@ -1353,9 +1357,10 @@ abstract class UploadBase {
}
wfDebug( __METHOD__ . ": FOUND VIRUS! scanner feedback: $output \n" );
- wfProfileOut( __METHOD__ );
- return $output;
}
+
+ wfProfileOut( __METHOD__ );
+ return $output;
}
/**
@@ -1392,7 +1397,7 @@ abstract class UploadBase {
* Check if a user is the last uploader
*
* @param $user User object
- * @param $img String: image name
+ * @param string $img image name
* @return Boolean
*/
public static function userCanReUpload( User $user, $img ) {
@@ -1464,9 +1469,20 @@ abstract class UploadBase {
}
}
+ // Check for files with the same name but a different extension
+ $similarFiles = RepoGroup::singleton()->getLocalRepo()->findFilesByPrefix(
+ "{$partname}.", 1 );
+ if ( count( $similarFiles ) ) {
+ return array(
+ 'warning' => 'exists-normalized',
+ 'file' => $file,
+ 'normalizedFile' => $similarFiles[0],
+ );
+ }
+
if ( self::isThumbName( $file->getName() ) ) {
# Check for filenames like 50px- or 180px-, these are mostly thumbnails
- $nt_thb = Title::newFromText( substr( $partname , strpos( $partname , '-' ) +1 ) . '.' . $extension, NS_FILE );
+ $nt_thb = Title::newFromText( substr( $partname, strpos( $partname, '-' ) + 1 ) . '.' . $extension, NS_FILE );
$file_thb = wfLocalFile( $nt_thb );
if( $file_thb->exists() ) {
return array(
@@ -1484,7 +1500,6 @@ abstract class UploadBase {
}
}
-
foreach( self::getFilenamePrefixBlacklist() as $prefix ) {
if ( substr( $partname, 0, strlen( $prefix ) ) == $prefix ) {
return array(
@@ -1507,10 +1522,10 @@ abstract class UploadBase {
$n = strrpos( $filename, '.' );
$partname = $n ? substr( $filename, 0, $n ) : $filename;
return (
- substr( $partname , 3, 3 ) == 'px-' ||
- substr( $partname , 2, 3 ) == 'px-'
+ substr( $partname, 3, 3 ) == 'px-' ||
+ substr( $partname, 2, 3 ) == 'px-'
) &&
- preg_match( "/[0-9]{2}/" , substr( $partname , 0, 2 ) );
+ preg_match( "/[0-9]{2}/", substr( $partname, 0, 2 ) );
}
/**
@@ -1590,6 +1605,32 @@ abstract class UploadBase {
} else {
return intval( $wgMaxUploadSize );
}
+ }
+
+ /**
+ * Get the current status of a chunked upload (used for polling).
+ * The status will be read from the *current* user session.
+ * @param $statusKey string
+ * @return Array|bool
+ */
+ public static function getSessionStatus( $statusKey ) {
+ return isset( $_SESSION[self::SESSION_STATUS_KEY][$statusKey] )
+ ? $_SESSION[self::SESSION_STATUS_KEY][$statusKey]
+ : false;
+ }
+ /**
+ * Set the current status of a chunked upload (used for polling).
+ * The status will be stored in the *current* user session.
+ * @param $statusKey string
+ * @param $value array|false
+ * @return void
+ */
+ public static function setSessionStatus( $statusKey, $value ) {
+ if ( $value === false ) {
+ unset( $_SESSION[self::SESSION_STATUS_KEY][$statusKey] );
+ } else {
+ $_SESSION[self::SESSION_STATUS_KEY][$statusKey] = $value;
+ }
}
}
diff --git a/includes/upload/UploadFromChunks.php b/includes/upload/UploadFromChunks.php
index 531f7be4..4b331e98 100644
--- a/includes/upload/UploadFromChunks.php
+++ b/includes/upload/UploadFromChunks.php
@@ -37,7 +37,7 @@ class UploadFromChunks extends UploadFromFile {
* @param $stash UploadStash
* @param $repo FileRepo
*/
- public function __construct( $user = false, $stash = false, $repo = false ) {
+ public function __construct( $user = null, $stash = false, $repo = false ) {
// user object. sometimes this won't exist, as when running from cron.
$this->user = $user;
@@ -60,12 +60,13 @@ class UploadFromChunks extends UploadFromFile {
return true;
}
+
/**
* Calls the parent stashFile and updates the uploadsession table to handle "chunks"
*
* @return UploadStashFile stashed file
*/
- public function stashFile() {
+ public function stashFile( User $user = null ) {
// Stash file is the called on creating a new chunk session:
$this->mChunkIndex = 0;
$this->mOffset = 0;
@@ -78,7 +79,7 @@ class UploadFromChunks extends UploadFromFile {
$this->mFileKey = $this->mLocalFile->getFileKey();
// Output a copy of this first to chunk 0 location:
- $status = $this->outputChunk( $this->mLocalFile->getPath() );
+ $this->outputChunk( $this->mLocalFile->getPath() );
// Update db table to reflect initial "chunk" state
$this->updateChunkStatus();
@@ -113,7 +114,7 @@ class UploadFromChunks extends UploadFromFile {
// Concatenate all the chunks to mVirtualTempPath
$fileList = Array();
// The first chunk is stored at the mVirtualTempPath path so we start on "chunk 1"
- for( $i = 0; $i <= $this->getChunkIndex(); $i++ ){
+ for( $i = 0; $i <= $this->getChunkIndex(); $i++ ) {
$fileList[] = $this->getVirtualChunkLocation( $i );
}
@@ -122,13 +123,16 @@ class UploadFromChunks extends UploadFromFile {
// Get a 0-byte temp file to perform the concatenation at
$tmpFile = TempFSFile::factory( 'chunkedupload_', $ext );
$tmpPath = $tmpFile
- ? $tmpFile->getPath()
+ ? $tmpFile->bind( $this )->getPath() // keep alive with $this
: false; // fail in concatenate()
// Concatenate the chunks at the temp file
+ $tStart = microtime( true );
$status = $this->repo->concatenate( $fileList, $tmpPath, FileRepo::DELETE_SOURCE );
- if( !$status->isOk() ){
+ $tAmount = microtime( true ) - $tStart;
+ if( !$status->isOk() ) {
return $status;
}
+ wfDebugLog( 'fileconcatenate', "Combined $i chunks in $tAmount seconds.\n" );
$this->mTempPath = $tmpPath; // file system path
$this->mFileSize = filesize( $this->mTempPath ); //Since this was set for the last chunk previously
@@ -141,7 +145,11 @@ class UploadFromChunks extends UploadFromFile {
// Update the mTempPath and mLocalFile
// ( for FileUpload or normal Stash to take over )
- $this->mLocalFile = parent::stashFile();
+ $tStart = microtime( true );
+ $this->mLocalFile = parent::stashFile( $this->user );
+ $tAmount = microtime( true ) - $tStart;
+ $this->mLocalFile->setLocalReference( $tmpFile ); // reuse (e.g. for getImageInfo())
+ wfDebugLog( 'fileconcatenate', "Stashed combined file ($i chunks) in $tAmount seconds.\n" );
return $status;
}
@@ -164,7 +172,7 @@ class UploadFromChunks extends UploadFromFile {
* @param $index
* @return string
*/
- function getVirtualChunkLocation( $index ){
+ function getVirtualChunkLocation( $index ) {
return $this->repo->getVirtualUrl( 'temp' ) .
'/' .
$this->repo->getHashPath(
@@ -176,9 +184,9 @@ class UploadFromChunks extends UploadFromFile {
/**
* Add a chunk to the temporary directory
*
- * @param $chunkPath string path to temporary chunk file
- * @param $chunkSize int size of the current chunk
- * @param $offset int offset of current chunk ( mutch match database chunk offset )
+ * @param string $chunkPath path to temporary chunk file
+ * @param int $chunkSize size of the current chunk
+ * @param int $offset offset of current chunk ( mutch match database chunk offset )
* @return Status
*/
public function addChunk( $chunkPath, $chunkSize, $offset ) {
@@ -202,7 +210,7 @@ class UploadFromChunks extends UploadFromFile {
return Status::newFatal( $e->getMessage() );
}
$status = $this->outputChunk( $chunkPath );
- if( $status->isGood() ){
+ if( $status->isGood() ) {
// Update local offset:
$this->mOffset = $preAppendOffset + $chunkSize;
// Update chunk table status db
@@ -218,11 +226,14 @@ class UploadFromChunks extends UploadFromFile {
/**
* Update the chunk db table with the current status:
*/
- private function updateChunkStatus(){
+ private function updateChunkStatus() {
wfDebug( __METHOD__ . " update chunk status for {$this->mFileKey} offset:" .
$this->getOffset() . ' inx:' . $this->getChunkIndex() . "\n" );
$dbw = $this->repo->getMasterDb();
+ // Use a quick transaction since we will upload the full temp file into shared
+ // storage, which takes time for large files. We don't want to hold locks then.
+ $dbw->begin( __METHOD__ );
$dbw->update(
'uploadstash',
array(
@@ -233,12 +244,13 @@ class UploadFromChunks extends UploadFromFile {
array( 'us_key' => $this->mFileKey ),
__METHOD__
);
+ $dbw->commit( __METHOD__ );
}
/**
* Get the chunk db state and populate update relevant local values
*/
- private function getChunkStatus(){
+ private function getChunkStatus() {
// get Master db to avoid race conditions.
// Otherwise, if chunk upload time < replag there will be spurious errors
$dbw = $this->repo->getMasterDb();
@@ -264,8 +276,8 @@ class UploadFromChunks extends UploadFromFile {
* Get the current Chunk index
* @return Integer index of the current chunk
*/
- private function getChunkIndex(){
- if( $this->mChunkIndex !== null ){
+ private function getChunkIndex() {
+ if( $this->mChunkIndex !== null ) {
return $this->mChunkIndex;
}
return 0;
@@ -275,8 +287,8 @@ class UploadFromChunks extends UploadFromFile {
* Gets the current offset in fromt the stashedupload table
* @return Integer current byte offset of the chunk file set
*/
- private function getOffset(){
- if ( $this->mOffset !== null ){
+ private function getOffset() {
+ if ( $this->mOffset !== null ) {
return $this->mOffset;
}
return 0;
@@ -289,7 +301,7 @@ class UploadFromChunks extends UploadFromFile {
* @throws UploadChunkFileException
* @return FileRepoStatus
*/
- private function outputChunk( $chunkPath ){
+ private function outputChunk( $chunkPath ) {
// Key is fileKey + chunk index
$fileKey = $this->getChunkFileKey();
@@ -314,11 +326,11 @@ class UploadFromChunks extends UploadFromFile {
return $storeStatus;
}
- private function getChunkFileKey( $index = null ){
- if( $index === null ){
+ private function getChunkFileKey( $index = null ) {
+ if( $index === null ) {
$index = $this->getChunkIndex();
}
- return $this->mFileKey . '.' . $index ;
+ return $this->mFileKey . '.' . $index;
}
/**
diff --git a/includes/upload/UploadFromFile.php b/includes/upload/UploadFromFile.php
index aa0cc77b..ab2a7a39 100644
--- a/includes/upload/UploadFromFile.php
+++ b/includes/upload/UploadFromFile.php
@@ -79,21 +79,21 @@ class UploadFromFile extends UploadBase {
* @return array
*/
public function verifyUpload() {
- # Check for a post_max_size or upload_max_size overflow, so that a
+ # Check for a post_max_size or upload_max_size overflow, so that a
# proper error can be shown to the user
if ( is_null( $this->mTempPath ) || $this->isEmptyFile() ) {
if ( $this->mUpload->isIniSizeOverflow() ) {
- return array(
+ return array(
'status' => UploadBase::FILE_TOO_LARGE,
- 'max' => min(
- self::getMaxUploadSize( $this->getSourceType() ),
- wfShorthandToInteger( ini_get( 'upload_max_filesize' ) ),
+ 'max' => min(
+ self::getMaxUploadSize( $this->getSourceType() ),
+ wfShorthandToInteger( ini_get( 'upload_max_filesize' ) ),
wfShorthandToInteger( ini_get( 'post_max_size' ) )
),
);
}
}
-
+
return parent::verifyUpload();
}
}
diff --git a/includes/upload/UploadFromStash.php b/includes/upload/UploadFromStash.php
index d79641ce..c82103cb 100644
--- a/includes/upload/UploadFromStash.php
+++ b/includes/upload/UploadFromStash.php
@@ -89,7 +89,7 @@ class UploadFromStash extends UploadBase {
* @param $key string
* @param $name string
*/
- public function initialize( $key, $name = 'upload_file' ) {
+ public function initialize( $key, $name = 'upload_file', $initTempFile = true ) {
/**
* Confirming a temporarily stashed upload.
* We don't want path names to be forged, so we keep
@@ -98,7 +98,7 @@ class UploadFromStash extends UploadBase {
*/
$metadata = $this->stash->getMetadata( $key );
$this->initializePathInfo( $name,
- $this->getRealPath ( $metadata['us_path'] ),
+ $initTempFile ? $this->getRealPath( $metadata['us_path'] ) : false,
$metadata['us_size'],
false
);
@@ -129,6 +129,14 @@ class UploadFromStash extends UploadBase {
return $this->mSourceType;
}
+ /**
+ * Get the base 36 SHA1 of the file
+ * @return string
+ */
+ public function getTempFileSha1Base36() {
+ return $this->mFileProps['sha1'];
+ }
+
/*
* protected function verifyFile() inherited
*/
@@ -136,12 +144,13 @@ class UploadFromStash extends UploadBase {
/**
* Stash the file.
*
+ * @param $user User
* @return UploadStashFile
*/
- public function stashFile() {
+ public function stashFile( User $user = null ) {
// replace mLocalFile with an instance of UploadStashFile, which adds some methods
// that are useful for stashed files.
- $this->mLocalFile = parent::stashFile();
+ $this->mLocalFile = parent::stashFile( $user );
return $this->mLocalFile;
}
diff --git a/includes/upload/UploadFromUrl.php b/includes/upload/UploadFromUrl.php
index 927c3cd9..70b69034 100644
--- a/includes/upload/UploadFromUrl.php
+++ b/includes/upload/UploadFromUrl.php
@@ -61,6 +61,8 @@ class UploadFromUrl extends UploadBase {
/**
* Checks whether the URL is for an allowed host
+ * The domains in the whitelist can include wildcard characters (*) in place
+ * of any of the domain levels, e.g. '*.flickr.com' or 'upload.*.gov.uk'.
*
* @param $url string
* @return bool
@@ -76,10 +78,28 @@ class UploadFromUrl extends UploadBase {
}
$valid = false;
foreach( $wgCopyUploadsDomains as $domain ) {
+ // See if the domain for the upload matches this whitelisted domain
+ $whitelistedDomainPieces = explode( '.', $domain );
+ $uploadDomainPieces = explode( '.', $parsedUrl['host'] );
+ if ( count( $whitelistedDomainPieces ) === count( $uploadDomainPieces ) ) {
+ $valid = true;
+ // See if all the pieces match or not (excluding wildcards)
+ foreach ( $whitelistedDomainPieces as $index => $piece ) {
+ if ( $piece !== '*' && $piece !== $uploadDomainPieces[$index] ) {
+ $valid = false;
+ }
+ }
+ if ( $valid ) {
+ // We found a match, so quit comparing against the list
+ break;
+ }
+ }
+ /* Non-wildcard test
if ( $parsedUrl['host'] === $domain ) {
$valid = true;
break;
}
+ */
}
return $valid;
}
@@ -312,7 +332,7 @@ class UploadFromUrl extends UploadBase {
'sessionKey' => $sessionKey,
) );
$job->initializeSessionData();
- $job->insert();
+ JobQueueGroup::singleton()->push( $job );
return $sessionKey;
}
diff --git a/includes/upload/UploadStash.php b/includes/upload/UploadStash.php
index 53a90582..089bd8b7 100644
--- a/includes/upload/UploadStash.php
+++ b/includes/upload/UploadStash.php
@@ -97,7 +97,7 @@ class UploadStash {
* Get a file and its metadata from the stash.
* The noAuth param is a bit janky but is required for automated scripts which clean out the stash.
*
- * @param $key String: key under which file information is stored
+ * @param string $key key under which file information is stored
* @param $noAuth Boolean (optional) Don't check authentication. Used by maintenance scripts.
* @throws UploadStashFileNotFoundException
* @throws UploadStashNotLoggedInException
@@ -106,15 +106,13 @@ class UploadStash {
* @return UploadStashFile
*/
public function getFile( $key, $noAuth = false ) {
-
if ( ! preg_match( self::KEY_FORMAT_REGEX, $key ) ) {
throw new UploadStashBadPathException( "key '$key' is not in a proper format" );
}
- if ( !$noAuth ) {
- if ( !$this->isLoggedIn ) {
- throw new UploadStashNotLoggedInException( __METHOD__ . ' No user is logged in, files must belong to users' );
- }
+ if ( !$noAuth && !$this->isLoggedIn ) {
+ throw new UploadStashNotLoggedInException( __METHOD__ .
+ ' No user is logged in, files must belong to users' );
}
if ( !isset( $this->fileMetadata[$key] ) ) {
@@ -131,8 +129,13 @@ class UploadStash {
$this->initFile( $key );
// fetch fileprops
- $path = $this->fileMetadata[$key]['us_path'];
- $this->fileProps[$key] = $this->repo->getFileProps( $path );
+ if ( strlen( $this->fileMetadata[$key]['us_props'] ) ) {
+ $this->fileProps[$key] = unserialize( $this->fileMetadata[$key]['us_props'] );
+ } else { // b/c for rows with no us_props
+ wfDebug( __METHOD__ . " fetched props for $key from file\n" );
+ $path = $this->fileMetadata[$key]['us_path'];
+ $this->fileProps[$key] = $this->repo->getFileProps( $path );
+ }
}
if ( ! $this->files[$key]->exists() ) {
@@ -152,7 +155,7 @@ class UploadStash {
/**
* Getter for file metadata.
*
- * @param key String: key under which file information is stored
+ * @param string $key key under which file information is stored
* @return Array
*/
public function getMetadata ( $key ) {
@@ -163,7 +166,7 @@ class UploadStash {
/**
* Getter for fileProps
*
- * @param key String: key under which file information is stored
+ * @param string $key key under which file information is stored
* @return Array
*/
public function getFileProps ( $key ) {
@@ -174,15 +177,15 @@ class UploadStash {
/**
* Stash a file in a temp directory and record that we did this in the database, along with other metadata.
*
- * @param $path String: path to file you want stashed
- * @param $sourceType String: the type of upload that generated this file (currently, I believe, 'file' or null)
+ * @param string $path path to file you want stashed
+ * @param string $sourceType the type of upload that generated this file (currently, I believe, 'file' or null)
* @throws UploadStashBadPathException
* @throws UploadStashFileException
* @throws UploadStashNotLoggedInException
* @return UploadStashFile: file, or null on failure
*/
public function stashFile( $path, $sourceType = null ) {
- if ( ! file_exists( $path ) ) {
+ if ( !is_file( $path ) ) {
wfDebug( __METHOD__ . " tried to stash file at '$path', but it doesn't exist\n" );
throw new UploadStashBadPathException( "path doesn't exist" );
}
@@ -192,12 +195,10 @@ class UploadStash {
// we will be initializing from some tmpnam files that don't have extensions.
// most of MediaWiki assumes all uploaded files have good extensions. So, we fix this.
$extension = self::getExtensionForPath( $path );
- if ( ! preg_match( "/\\.\\Q$extension\\E$/", $path ) ) {
+ if ( !preg_match( "/\\.\\Q$extension\\E$/", $path ) ) {
$pathWithGoodExtension = "$path.$extension";
- if ( ! rename( $path, $pathWithGoodExtension ) ) {
- throw new UploadStashFileException( "couldn't rename $path to have a better extension at $pathWithGoodExtension" );
- }
- $path = $pathWithGoodExtension;
+ } else {
+ $pathWithGoodExtension = $path;
}
// If no key was supplied, make one. a mysql insertid would be totally reasonable here, except
@@ -205,8 +206,8 @@ class UploadStash {
//
// some things that when combined will make a suitably unique key.
// see: http://www.jwz.org/doc/mid.html
- list ($usec, $sec) = explode( ' ', microtime() );
- $usec = substr($usec, 2);
+ list( $usec, $sec ) = explode( ' ', microtime() );
+ $usec = substr( $usec, 2 );
$key = wfBaseConvert( $sec . $usec, 10, 36 ) . '.' .
wfBaseConvert( mt_rand(), 10, 36 ) . '.'.
$this->userId . '.' .
@@ -221,7 +222,7 @@ class UploadStash {
wfDebug( __METHOD__ . " key for '$path': $key\n" );
// if not already in a temporary area, put it there
- $storeStatus = $this->repo->storeTemp( basename( $path ), $path );
+ $storeStatus = $this->repo->storeTemp( basename( $pathWithGoodExtension ), $path );
if ( ! $storeStatus->isOK() ) {
// It is a convention in MediaWiki to only return one error per API exception, even if multiple errors
@@ -244,9 +245,6 @@ class UploadStash {
}
$stashPath = $storeStatus->value;
- // we have renamed the file so we have to cleanup once done
- unlink($path);
-
// fetch the current user ID
if ( !$this->isLoggedIn ) {
throw new UploadStashNotLoggedInException( __METHOD__ . ' No user is logged in, files must belong to users' );
@@ -262,6 +260,7 @@ class UploadStash {
'us_key' => $key,
'us_orig_path' => $path,
'us_path' => $stashPath, // virtual URL
+ 'us_props' => serialize( $fileProps ),
'us_size' => $fileProps['size'],
'us_sha1' => $fileProps['sha1'],
'us_mime' => $fileProps['mime'],
@@ -319,8 +318,8 @@ class UploadStash {
/**
* Remove a particular file from the stash. Also removes it from the repo.
*
- * @throws UploadStashNotLoggedInException
- * @throws UploadStashWrongOwnerException
+ * @param $key
+ * @throws UploadStashNoSuchKeyException|UploadStashNotLoggedInException|UploadStashWrongOwnerException
* @return boolean: success
*/
public function removeFile( $key ) {
@@ -350,7 +349,6 @@ class UploadStash {
return $this->removeFileNoAuth( $key );
}
-
/**
* Remove a file (see removeFile), but doesn't check ownership first.
*
@@ -359,16 +357,16 @@ class UploadStash {
public function removeFileNoAuth( $key ) {
wfDebug( __METHOD__ . " clearing row $key\n" );
+ // Ensure we have the UploadStashFile loaded for this key
+ $this->getFile( $key );
+
$dbw = $this->repo->getMasterDb();
- // this gets its own transaction since it's called serially by the cleanupUploadStash maintenance script
- $dbw->begin( __METHOD__ );
$dbw->delete(
'uploadstash',
array( 'us_key' => $key ),
__METHOD__
);
- $dbw->commit( __METHOD__ );
// TODO: look into UnregisteredLocalFile and find out why the rv here is sometimes wrong (false when file was removed)
// for now, ignore.
@@ -419,6 +417,8 @@ class UploadStash {
* with an extension.
* XXX this is somewhat redundant with the checks that ApiUpload.php does with incoming
* uploads versus the desired filename. Maybe we can get that passed to us...
+ * @param $path
+ * @throws UploadStashFileException
* @return string
*/
public static function getExtensionForPath( $path ) {
@@ -456,7 +456,7 @@ class UploadStash {
/**
* Helper function: do the actual database query to fetch file metadata.
*
- * @param $key String: key
+ * @param string $key key
* @param $readFromDB: constant (default: DB_SLAVE)
* @return boolean
*/
@@ -490,7 +490,7 @@ class UploadStash {
/**
* Helper function: Initialize the UploadStashFile for a given file.
*
- * @param $key String: key under which to store the object
+ * @param string $key key under which to store the object
* @throws UploadStashZeroLengthFileException
* @return bool
*/
@@ -514,8 +514,8 @@ class UploadStashFile extends UnregisteredLocalFile {
* Arguably UnregisteredLocalFile should be handling its own file repo but that class is a bit retarded currently
*
* @param $repo FileRepo: repository where we should find the path
- * @param $path String: path to file
- * @param $key String: key to store the path and any stashed data under
+ * @param string $path path to file
+ * @param string $key key to store the path and any stashed data under
* @throws UploadStashBadPathException
* @throws UploadStashFileNotFoundException
*/
@@ -564,7 +564,7 @@ class UploadStashFile extends UnregisteredLocalFile {
* The actual argument is the result of thumbName although we seem to have
* buggy code elsewhere that expects a boolean 'suffix'
*
- * @param $thumbName String: name of thumbnail (e.g. "120px-123456.jpg" ), or false to just get the path
+ * @param string $thumbName name of thumbnail (e.g. "120px-123456.jpg" ), or false to just get the path
* @return String: path thumbnail should take on filesystem, or containing directory if thumbname is false
*/
public function getThumbPath( $thumbName = false ) {
@@ -580,7 +580,7 @@ class UploadStashFile extends UnregisteredLocalFile {
* We override this because we want to use the pretty url name instead of the
* ugly file name.
*
- * @param $params Array: handler-specific parameters
+ * @param array $params handler-specific parameters
* @param $flags integer Bitfield that supports THUMB_* constants
* @return String: base name for URL, like '120px-12345.jpg', or null if there is no handler
*/
@@ -603,7 +603,7 @@ class UploadStashFile extends UnregisteredLocalFile {
* the thumbnail urls be predictable. However, in our model the URL is not based on the filename
* (that's hidden in the db)
*
- * @param $thumbName String: basename of thumbnail file -- however, we don't want to use the file exactly
+ * @param string $thumbName basename of thumbnail file -- however, we don't want to use the file exactly
* @return String: URL to access thumbnail, or URL with partial path
*/
public function getThumbUrl( $thumbName = false ) {