From d9022f63880ce039446fba8364f68e656b7bf4cb Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Thu, 3 May 2012 13:01:35 +0200 Subject: Update to MediaWiki 1.19.0 --- includes/upload/UploadBase.php | 100 ++++++------- includes/upload/UploadFromChunks.php | 276 +++++++++++++++++++++++++++++++++++ includes/upload/UploadFromFile.php | 12 +- includes/upload/UploadFromStash.php | 4 +- includes/upload/UploadFromUrl.php | 26 ++-- includes/upload/UploadStash.php | 22 +-- 6 files changed, 352 insertions(+), 88 deletions(-) create mode 100644 includes/upload/UploadFromChunks.php (limited to 'includes/upload') diff --git a/includes/upload/UploadBase.php b/includes/upload/UploadBase.php index a97edbc7..32eeeb38 100644 --- a/includes/upload/UploadBase.php +++ b/includes/upload/UploadBase.php @@ -1,7 +1,10 @@ 'empty-file', @@ -49,6 +52,7 @@ abstract class UploadBase { self::VERIFICATION_ERROR => 'verification-error', self::HOOK_ABORTED => 'hookaborted', self::WINDOWS_NONASCII_FILENAME => 'windows-nonascii-filename', + self::FILENAME_TOO_LONG => 'filename-toolong', ); if( isset( $code_to_status[$error] ) ) { return $code_to_status[$error]; @@ -161,6 +165,9 @@ abstract class UploadBase { */ public function initializePathInfo( $name, $tempPath, $fileSize, $removeTempFile = false ) { $this->mDesiredDestName = $name; + if ( FileBackend::isStoragePath( $tempPath ) ) { + throw new MWException( __METHOD__ . " given storage path `$tempPath`." ); + } $this->mTempPath = $tempPath; $this->mFileSize = $fileSize; $this->mRemoveTempFile = $removeTempFile; @@ -194,32 +201,6 @@ abstract class UploadBase { return $this->mFileSize; } - /** - * Append a file to the Repo file - * - * @param $srcPath String: path to source file - * @param $toAppendPath String: path to the Repo file that will be appended to. - * @return Status Status - */ - protected function appendToUploadFile( $srcPath, $toAppendPath ) { - $repo = RepoGroup::singleton()->getLocalRepo(); - $status = $repo->append( $srcPath, $toAppendPath ); - return $status; - } - - /** - * Finish appending to the Repo file - * - * @param $toAppendPath String: path to the Repo file that will be appended to. - * @return Status Status - */ - protected function appendFinish( $toAppendPath ) { - $repo = RepoGroup::singleton()->getLocalRepo(); - $status = $repo->appendFinish( $toAppendPath ); - return $status; - } - - /** * @param $srcPath String: the source path * @return the real path if it was a virtual URL @@ -227,7 +208,11 @@ abstract class UploadBase { function getRealPath( $srcPath ) { $repo = RepoGroup::singleton()->getLocalRepo(); if ( $repo->isVirtualUrl( $srcPath ) ) { - return $repo->resolveVirtualUrl( $srcPath ); + // @TODO: just make uploads work with storage paths + // UploadFromStash loads files via virtuals URLs + $tmpFile = $repo->getLocalCopy( $srcPath ); + $tmpFile->bind( $this ); // keep alive with $thumb + return $tmpFile->getPath(); } return $srcPath; } @@ -355,12 +340,12 @@ abstract class UploadBase { * @return mixed true of the file is verified, array otherwise. */ protected function verifyFile() { - global $wgAllowJavaUploads; + global $wgAllowJavaUploads, $wgDisableUploadScriptChecks; # get the title, even though we are doing nothing with it, because # we need to populate mFinalExtension $this->getTitle(); - $this->mFileProps = File::getPropsFromPath( $this->mTempPath, $this->mFinalExtension ); + $this->mFileProps = FSFile::getPropsFromPath( $this->mTempPath, $this->mFinalExtension ); # check mime type, if desired $mime = $this->mFileProps[ 'file-mime' ]; @@ -370,13 +355,15 @@ abstract class UploadBase { } # check for htmlish code and javascript - if( self::detectScript( $this->mTempPath, $mime, $this->mFinalExtension ) ) { - return array( 'uploadscripted' ); - } - if( $this->mFinalExtension == 'svg' || $mime == 'image/svg+xml' ) { - if( $this->detectScriptInSvg( $this->mTempPath ) ) { + if ( !$wgDisableUploadScriptChecks ) { + if( self::detectScript( $this->mTempPath, $mime, $this->mFinalExtension ) ) { return array( 'uploadscripted' ); } + if( $this->mFinalExtension == 'svg' || $mime == 'image/svg+xml' ) { + if( $this->detectScriptInSvg( $this->mTempPath ) ) { + return array( 'uploadscripted' ); + } + } } # Check for Java applets, which if uploaded can bypass cross-site @@ -445,7 +432,7 @@ abstract class UploadBase { } /** - * Alias for verifyTitlePermissions. The function was originally 'verifyPermissions' + * Alias for verifyTitlePermissions. The function was originally 'verifyPermissions' * but that suggests it's checking the user, when it's really checking the title + user combination. * @param $user User object to verify the permissions against * @return mixed An array as returned by getUserPermissionsErrors or true @@ -478,7 +465,7 @@ abstract class UploadBase { $permErrors = $nt->getUserPermissionsErrors( 'edit', $user ); $permErrorsUpload = $nt->getUserPermissionsErrors( 'upload', $user ); if ( !$nt->exists() ) { - $permErrorsCreate = $nt->getUserPermissionsErrors( 'createpage', $user ); + $permErrorsCreate = $nt->getUserPermissionsErrors( 'create', $user ); } else { $permErrorsCreate = array(); } @@ -544,7 +531,7 @@ abstract class UploadBase { } // Check dupes against existing files - $hash = File::sha1Base36( $this->mTempPath ); + $hash = FSFile::getSha1Base36FromPath( $this->mTempPath ); $dupes = RepoGroup::singleton()->findBySha1( $hash ); $title = $this->getTitle(); // Remove all matches against self @@ -589,7 +576,7 @@ abstract class UploadBase { if ( $watch ) { $user->addWatch( $this->getLocalFile()->getTitle() ); } - + wfRunHooks( 'UploadComplete', array( &$this ) ); } @@ -606,7 +593,7 @@ abstract class UploadBase { if ( $this->mTitle !== false ) { return $this->mTitle; } - + /* Assume that if a user specified File:Something.jpg, this is an error * and that the namespace prefix needs to be stripped of. */ @@ -617,6 +604,13 @@ abstract class UploadBase { $this->mFilteredName = $this->mDesiredDestName; } + # oi_archive_name is max 255 bytes, which include a timestamp and an + # exclamation mark, so restrict file name to 240 bytes. + if ( strlen( $this->mFilteredName ) > 240 ) { + $this->mTitleError = self::FILENAME_TOO_LONG; + return $this->mTitle = null; + } + /** * Chop off any directories in the given filename. Then * filter out illegal characters, and try to make a legible name @@ -631,6 +625,8 @@ abstract class UploadBase { } $this->mFilteredName = $nt->getDBkey(); + + /** * We'll want to blacklist against *any* 'extension', and use * only the final one for the whitelist. @@ -677,7 +673,7 @@ abstract class UploadBase { $this->mTitleError = self::FILETYPE_BADTYPE; return $this->mTitle = null; } - + // Windows may be broken with special characters, see bug XXX if ( wfIsWindows() && !preg_match( '/^[\x0-\x7f]*$/', $nt->getText() ) ) { $this->mTitleError = self::WINDOWS_NONASCII_FILENAME; @@ -742,14 +738,12 @@ abstract class UploadBase { * This method returns the file object, which also has a 'fileKey' property which can be passed through a form or * API request to find this stashed file again. * - * @param $key String: (optional) the file key used to find the file info again. If not supplied, a key will be autogenerated. * @return UploadStashFile stashed file */ - public function stashFile( $key = null ) { + public function stashFile() { // was stashSessionFile $stash = RepoGroup::singleton()->getLocalRepo()->getUploadStash(); - - $file = $stash->stashFile( $this->mTempPath, $this->getSourceType(), $key ); + $file = $stash->stashFile( $this->mTempPath, $this->getSourceType() ); $this->mLocalFile = $file; return $file; } @@ -757,21 +751,19 @@ abstract class UploadBase { /** * Stash a file in a temporary directory, returning a key which can be used to find the file again. See stashFile(). * - * @param $key String: (optional) the file key used to find the file info again. If not supplied, a key will be autogenerated. * @return String: file key */ - public function stashFileGetKey( $key = null ) { - return $this->stashFile( $key )->getFileKey(); + public function stashFileGetKey() { + return $this->stashFile()->getFileKey(); } - /** + /** * alias for stashFileGetKey, for backwards compatibility * - * @param $key String: (optional) the file key used to find the file info again. If not supplied, a key will be autogenerated. * @return String: file key */ - public function stashSession( $key = null ) { - return $this->stashFileGetKey( $key ); + public function stashSession() { + return $this->stashFileGetKey(); } /** diff --git a/includes/upload/UploadFromChunks.php b/includes/upload/UploadFromChunks.php new file mode 100644 index 00000000..ec83f7d3 --- /dev/null +++ b/includes/upload/UploadFromChunks.php @@ -0,0 +1,276 @@ +user = $user; + + if( $repo ) { + $this->repo = $repo; + } else { + $this->repo = RepoGroup::singleton()->getLocalRepo(); + } + + if( $stash ) { + $this->stash = $stash; + } else { + if( $user ) { + wfDebug( __METHOD__ . " creating new UploadFromChunks instance for " . $user->getId() . "\n" ); + } else { + wfDebug( __METHOD__ . " creating new UploadFromChunks instance with no user\n" ); + } + $this->stash = new UploadStash( $this->repo, $this->user ); + } + + return true; + } + /** + * Calls the parent stashFile and updates the uploadsession table to handle "chunks" + * + * @return UploadStashFile stashed file + */ + public function stashFile() { + // Stash file is the called on creating a new chunk session: + $this->mChunkIndex = 0; + $this->mOffset = 0; + // Create a local stash target + $this->mLocalFile = parent::stashFile(); + // Update the initial file offset ( based on file size ) + $this->mOffset = $this->mLocalFile->getSize(); + $this->mFileKey = $this->mLocalFile->getFileKey(); + + // Output a copy of this first to chunk 0 location: + $status = $this->outputChunk( $this->mLocalFile->getPath() ); + + // Update db table to reflect initial "chunk" state + $this->updateChunkStatus(); + return $this->mLocalFile; + } + + /** + * Continue chunk uploading + */ + public function continueChunks( $name, $key, $webRequestUpload ) { + $this->mFileKey = $key; + $this->mUpload = $webRequestUpload; + // Get the chunk status form the db: + $this->getChunkStatus(); + + $metadata = $this->stash->getMetadata( $key ); + $this->initializePathInfo( $name, + $this->getRealPath( $metadata['us_path'] ), + $metadata['us_size'], + false + ); + } + + /** + * Append the final chunk and ready file for parent::performUpload() + * @return FileRepoStatus + */ + public function concatenateChunks() { + wfDebug( __METHOD__ . " concatenate {$this->mChunkIndex} chunks:" . + $this->getOffset() . ' inx:' . $this->getChunkIndex() . "\n" ); + + // Concatenate all the chunks to mVirtualTempPath + $fileList = Array(); + // The first chunk is stored at the mVirtualTempPath path so we start on "chunk 1" + for( $i = 0; $i <= $this->getChunkIndex(); $i++ ){ + $fileList[] = $this->getVirtualChunkLocation( $i ); + } + + // Get the file extension from the last chunk + $ext = FileBackend::extensionFromPath( $this->mVirtualTempPath ); + // Get a 0-byte temp file to perform the concatenation at + $tmpFile = TempFSFile::factory( 'chunkedupload_', $ext ); + $tmpPath = $tmpFile + ? $tmpFile->getPath() + : false; // fail in concatenate() + // Concatenate the chunks at the temp file + $status = $this->repo->concatenate( $fileList, $tmpPath, FileRepo::DELETE_SOURCE ); + if( !$status->isOk() ){ + return $status; + } + // Update the mTempPath and mLocalFile + // ( for FileUpload or normal Stash to take over ) + $this->mTempPath = $tmpPath; // file system path + $this->mLocalFile = parent::stashFile(); + + return $status; + } + + /** + * Perform the upload, then remove the temp copy afterward + * @param $comment string + * @param $pageText string + * @param $watch bool + * @param $user User + * @return Status + */ + public function performUpload( $comment, $pageText, $watch, $user ) { + $rv = parent::performUpload( $comment, $pageText, $watch, $user ); + return $rv; + } + + /** + * Returns the virtual chunk location: + * @param unknown_type $index + */ + function getVirtualChunkLocation( $index ){ + return $this->repo->getVirtualUrl( 'temp' ) . + '/' . + $this->repo->getHashPath( + $this->getChunkFileKey( $index ) + ) . + $this->getChunkFileKey( $index ); + } + /** + * Add a chunk to the temporary directory + * + * @param $chunkPath path to temporary chunk file + * @param $chunkSize size of the current chunk + * @param $offset offset of current chunk ( mutch match database chunk offset ) + * @return Status + */ + public function addChunk( $chunkPath, $chunkSize, $offset ) { + // Get the offset before we add the chunk to the file system + $preAppendOffset = $this->getOffset(); + + if ( $preAppendOffset + $chunkSize > $this->getMaxUploadSize()) { + $status = Status::newFatal( 'file-too-large' ); + } else { + // Make sure the client is uploading the correct chunk with a matching offset. + if ( $preAppendOffset == $offset ) { + // Update local chunk index for the current chunk + $this->mChunkIndex++; + $status = $this->outputChunk( $chunkPath ); + if( $status->isGood() ){ + // Update local offset: + $this->mOffset = $preAppendOffset + $chunkSize; + // Update chunk table status db + $this->updateChunkStatus(); + } + } else { + $status = Status::newFatal( 'invalid-chunk-offset' ); + } + } + return $status; + } + + /** + * Update the chunk db table with the current status: + */ + private function updateChunkStatus(){ + wfDebug( __METHOD__ . " update chunk status for {$this->mFileKey} offset:" . + $this->getOffset() . ' inx:' . $this->getChunkIndex() . "\n" ); + + $dbw = $this->repo->getMasterDb(); + $dbw->update( + 'uploadstash', + array( + 'us_status' => 'chunks', + 'us_chunk_inx' => $this->getChunkIndex(), + 'us_size' => $this->getOffset() + ), + array( 'us_key' => $this->mFileKey ), + __METHOD__ + ); + } + /** + * Get the chunk db state and populate update relevant local values + */ + private function getChunkStatus(){ + // get Master db to avoid race conditions. + // Otherwise, if chunk upload time < replag there will be spurious errors + $dbw = $this->repo->getMasterDb(); + $row = $dbw->selectRow( + 'uploadstash', + array( + 'us_chunk_inx', + 'us_size', + 'us_path', + ), + array( 'us_key' => $this->mFileKey ), + __METHOD__ + ); + // Handle result: + if ( $row ) { + $this->mChunkIndex = $row->us_chunk_inx; + $this->mOffset = $row->us_size; + $this->mVirtualTempPath = $row->us_path; + } + } + /** + * Get the current Chunk index + * @return Integer index of the current chunk + */ + private function getChunkIndex(){ + if( $this->mChunkIndex !== null ){ + return $this->mChunkIndex; + } + return 0; + } + + /** + * Gets the current offset in fromt the stashedupload table + * @return Integer current byte offset of the chunk file set + */ + private function getOffset(){ + if ( $this->mOffset !== null ){ + return $this->mOffset; + } + return 0; + } + + /** + * Output the chunk to disk + * + * @param $chunkPath string + */ + private function outputChunk( $chunkPath ){ + // Key is fileKey + chunk index + $fileKey = $this->getChunkFileKey(); + + // Store the chunk per its indexed fileKey: + $hashPath = $this->repo->getHashPath( $fileKey ); + $storeStatus = $this->repo->store( $chunkPath, 'temp', "$hashPath$fileKey" ); + + // Check for error in stashing the chunk: + if ( ! $storeStatus->isOK() ) { + $error = $storeStatus->getErrorsArray(); + $error = reset( $error ); + if ( ! count( $error ) ) { + $error = $storeStatus->getWarningsArray(); + $error = reset( $error ); + if ( ! count( $error ) ) { + $error = array( 'unknown', 'no error recorded' ); + } + } + throw new UploadChunkFileException( "error storing file in '$chunkPath': " . implode( '; ', $error ) ); + } + return $storeStatus; + } + private function getChunkFileKey( $index = null ){ + if( $index === null ){ + $index = $this->getChunkIndex(); + } + return $this->mFileKey . '.' . $index ; + } +} + +class UploadChunkZeroLengthFileException extends MWException {}; +class UploadChunkFileException extends MWException {}; diff --git a/includes/upload/UploadFromFile.php b/includes/upload/UploadFromFile.php index c2ab6467..23ec2ef4 100644 --- a/includes/upload/UploadFromFile.php +++ b/includes/upload/UploadFromFile.php @@ -2,11 +2,9 @@ /** * Implements regular file uploads * - * @file - * @ingroup upload + * @ingroup Upload * @author Bryan Tong Minh */ - class UploadFromFile extends UploadBase { /** @@ -75,12 +73,4 @@ class UploadFromFile extends UploadBase { return parent::verifyUpload(); } - - /** - * Get the path to the file underlying the upload - * @return String path to file - */ - public function getFileTempname() { - return $this->mUpload->getTempname(); - } } diff --git a/includes/upload/UploadFromStash.php b/includes/upload/UploadFromStash.php index f34f156d..f7589bd2 100644 --- a/includes/upload/UploadFromStash.php +++ b/includes/upload/UploadFromStash.php @@ -2,11 +2,9 @@ /** * Implements uploading from previously stored file. * - * @file - * @ingroup upload + * @ingroup Upload * @author Bryan Tong Minh */ - class UploadFromStash extends UploadBase { protected $mFileKey, $mVirtualTempPath, $mFileProps, $mSourceType; diff --git a/includes/upload/UploadFromUrl.php b/includes/upload/UploadFromUrl.php index 8178988f..da772fe2 100644 --- a/includes/upload/UploadFromUrl.php +++ b/includes/upload/UploadFromUrl.php @@ -2,27 +2,28 @@ /** * Implements uploading from a HTTP resource. * - * @file - * @ingroup upload + * @ingroup Upload * @author Bryan Tong Minh * @author Michael Dale */ - class UploadFromUrl extends UploadBase { protected $mAsync, $mUrl; protected $mIgnoreWarnings = true; - protected $mTempPath; + protected $mTempPath, $mTmpHandle; /** * Checks if the user is allowed to use the upload-by-URL feature. If the * user is allowed, pass on permissions checking to the parent. * * @param $user User + * + * @return bool */ public static function isAllowed( $user ) { - if ( !$user->isAllowed( 'upload_by_url' ) ) + if ( !$user->isAllowed( 'upload_by_url' ) ) { return 'upload_by_url'; + } return parent::isAllowed( $user ); } @@ -64,8 +65,9 @@ class UploadFromUrl extends UploadBase { */ public function initializeFromRequest( &$request ) { $desiredDestName = $request->getText( 'wpDestFile' ); - if ( !$desiredDestName ) + if ( !$desiredDestName ) { $desiredDestName = $request->getText( 'wpUploadFileURL' ); + } return $this->initialize( $desiredDestName, trim( $request->getVal( 'wpUploadFileURL' ) ), @@ -75,6 +77,7 @@ class UploadFromUrl extends UploadBase { /** * @param $request WebRequest object + * @return bool */ public static function isValidRequest( $request ) { global $wgUser; @@ -85,8 +88,14 @@ class UploadFromUrl extends UploadBase { && $wgUser->isAllowed( 'upload_by_url' ); } + /** + * @return string + */ public function getSourceType() { return 'url'; } + /** + * @return Status + */ public function fetchFile() { if ( !Http::isValidURI( $this->mUrl ) ) { return Status::newFatal( 'http-invalid-url' ); @@ -130,6 +139,7 @@ class UploadFromUrl extends UploadBase { /** * Download the file, save it to the temporary file and update the file * size and set $mRemoveTempFile to true. + * @return Status */ protected function reallyFetchFile() { if ( $this->mTempPath === false ) { @@ -209,9 +219,7 @@ class UploadFromUrl extends UploadBase { if ( $this->mAsync ) { $sessionKey = $this->insertJob( $comment, $pageText, $watch, $user ); - $status = new Status; - $status->error( 'async', $sessionKey ); - return $status; + return Status::newFatal( 'async', $sessionKey ); } return parent::performUpload( $comment, $pageText, $watch, $user ); diff --git a/includes/upload/UploadStash.php b/includes/upload/UploadStash.php index 217b84dc..ad153d2f 100644 --- a/includes/upload/UploadStash.php +++ b/includes/upload/UploadStash.php @@ -16,6 +16,8 @@ * UploadStash represents the entire stash of temporary files. * UploadStashFile is a filestore for the actual physical disk files. * UploadFromStash extends UploadBase, and represents a single stashed file as it is moved from the stash to the regular file repository + * + * @ingroup Upload */ class UploadStash { @@ -48,7 +50,7 @@ class UploadStash { * * @param $repo FileRepo */ - public function __construct( $repo, $user = null ) { + public function __construct( FileRepo $repo, $user = null ) { // this might change based on wiki's configuration. $this->repo = $repo; @@ -106,10 +108,7 @@ class UploadStash { // fetch fileprops $path = $this->fileMetadata[$key]['us_path']; - if ( $this->repo->isVirtualUrl( $path ) ) { - $path = $this->repo->resolveVirtualUrl( $path ); - } - $this->fileProps[$key] = File::getPropsFromPath( $path ); + $this->fileProps[$key] = $this->repo->getFileProps( $path ); } if ( ! $this->files[$key]->exists() ) { @@ -163,7 +162,7 @@ class UploadStash { wfDebug( __METHOD__ . " tried to stash file at '$path', but it doesn't exist\n" ); throw new UploadStashBadPathException( "path doesn't exist" ); } - $fileProps = File::getPropsFromPath( $path ); + $fileProps = FSFile::getPropsFromPath( $path ); wfDebug( __METHOD__ . " stashing file at '$path'\n" ); // we will be initializing from some tmpnam files that don't have extensions. @@ -215,7 +214,8 @@ class UploadStash { $error = array( 'unknown', 'no error recorded' ); } } - throw new UploadStashFileException( "error storing file in '$path': " . implode( '; ', $error ) ); + // at this point, $error should contain the single "most important" error, plus any parameters. + throw new UploadStashFileException( "Error storing file in '$path': " . wfMessage( $error )->text() ); } $stashPath = $storeStatus->value; @@ -233,7 +233,7 @@ class UploadStash { 'us_user' => $this->userId, 'us_key' => $key, 'us_orig_path' => $path, - 'us_path' => $stashPath, + 'us_path' => $stashPath, // virtual URL 'us_size' => $fileProps['size'], 'us_sha1' => $fileProps['sha1'], 'us_mime' => $fileProps['mime'], @@ -334,13 +334,13 @@ class UploadStash { $dbw = $this->repo->getMasterDb(); // this gets its own transaction since it's called serially by the cleanupUploadStash maintenance script - $dbw->begin(); + $dbw->begin( __METHOD__ ); $dbw->delete( 'uploadstash', array( 'us_key' => $key ), __METHOD__ ); - $dbw->commit(); + $dbw->commit( __METHOD__ ); // TODO: look into UnregisteredLocalFile and find out why the rv here is sometimes wrong (false when file was removed) // for now, ignore. @@ -475,7 +475,7 @@ class UploadStashFile extends UnregisteredLocalFile { * A LocalFile wrapper around a file that has been temporarily stashed, so we can do things like create thumbnails for it * Arguably UnregisteredLocalFile should be handling its own file repo but that class is a bit retarded currently * - * @param $repo FSRepo: repository where we should find the path + * @param $repo FileRepo: repository where we should find the path * @param $path String: path to file * @param $key String: key to store the path and any stashed data under * @throws UploadStashBadPathException -- cgit v1.2.2