From 4ac9fa081a7c045f6a9f1cfc529d82423f485b2e Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 8 Dec 2013 09:55:49 +0100 Subject: Update to MediaWiki 1.22.0 --- includes/externalstore/ExternalStore.php | 57 +++++++++- includes/externalstore/ExternalStoreDB.php | 142 ++++++++++++++++++++---- includes/externalstore/ExternalStoreMedium.php | 19 ++++ includes/externalstore/ExternalStoreMwstore.php | 23 ++++ 4 files changed, 217 insertions(+), 24 deletions(-) (limited to 'includes/externalstore') diff --git a/includes/externalstore/ExternalStore.php b/includes/externalstore/ExternalStore.php index 4ca193d4..462b0b90 100644 --- a/includes/externalstore/ExternalStore.php +++ b/includes/externalstore/ExternalStore.php @@ -60,7 +60,7 @@ class ExternalStore { $class = 'ExternalStore' . ucfirst( $proto ); // Any custom modules should be added to $wgAutoLoadClasses for on-demand loading - return MWInit::classExists( $class ) ? new $class( $params ) : false; + return class_exists( $class ) ? new $class( $params ) : false; } /** @@ -90,6 +90,39 @@ class ExternalStore { return $store->fetchFromURL( $url ); } + /** + * Fetch data from multiple URLs with a minimum of round trips + * + * @param array $urls The URLs of the text to get + * @return array Map from url to its data. Data is either string when found + * or false on failure. + */ + public static function batchFetchFromURLs( array $urls ) { + $batches = array(); + foreach ( $urls as $url ) { + $scheme = parse_url( $url, PHP_URL_SCHEME ); + if ( $scheme ) { + $batches[$scheme][] = $url; + } + } + $retval = array(); + foreach ( $batches as $proto => $batchedUrls ) { + $store = self::getStoreObject( $proto ); + if ( $store === false ) { + continue; + } + $retval += $store->batchFetchFromURLs( $batchedUrls ); + } + // invalid, not found, db dead, etc. + $missing = array_diff( $urls, array_keys( $retval ) ); + if ( $missing ) { + foreach ( $missing as $url ) { + $retval[$url] = false; + } + } + return $retval; + } + /** * Store a data item to an external store, identified by a partial URL * The protocol part is used to identify the class, the rest is passed to the @@ -123,9 +156,10 @@ class ExternalStore { /** * Like insert() above, but does more of the work for us. * This function does not need a url param, it builds it by - * itself. It also fails-over to the next possible clusters. + * itself. It also fails-over to the next possible clusters + * provided by $wgDefaultExternalStore. * - * @param $data string + * @param string $data * @param array $params Associative array of ExternalStoreMedium parameters * @return string|bool The URL of the stored data item, or false on error * @throws MWException @@ -133,8 +167,23 @@ class ExternalStore { public static function insertToDefault( $data, array $params = array() ) { global $wgDefaultExternalStore; + return self::insertWithFallback( (array)$wgDefaultExternalStore, $data, $params ); + } + + /** + * Like insert() above, but does more of the work for us. + * This function does not need a url param, it builds it by + * itself. It also fails-over to the next possible clusters + * as provided in the first parameter. + * + * @param array $tryStores refer to $wgDefaultExternalStore + * @param string $data + * @param array $params Associative array of ExternalStoreMedium parameters + * @return string|bool The URL of the stored data item, or false on error + * @throws MWException + */ + public static function insertWithFallback( array $tryStores, $data, array $params = array() ) { $error = false; - $tryStores = (array)$wgDefaultExternalStore; while ( count( $tryStores ) > 0 ) { $index = mt_rand( 0, count( $tryStores ) - 1 ); $storeUrl = $tryStores[$index]; diff --git a/includes/externalstore/ExternalStoreDB.php b/includes/externalstore/ExternalStoreDB.php index 196e7f2c..46a89379 100644 --- a/includes/externalstore/ExternalStoreDB.php +++ b/includes/externalstore/ExternalStoreDB.php @@ -30,21 +30,13 @@ */ class ExternalStoreDB extends ExternalStoreMedium { /** - * The URL returned is of the form of the form DB://cluster/id + * The provided URL is in the form of DB://cluster/id * or DB://cluster/id/itemid for concatened storage. * * @see ExternalStoreMedium::fetchFromURL() */ public function fetchFromURL( $url ) { - $path = explode( '/', $url ); - $cluster = $path[2]; - $id = $path[3]; - if ( isset( $path[4] ) ) { - $itemID = $path[4]; - } else { - $itemID = false; - } - + list( $cluster, $id, $itemID ) = $this->parseURL( $url ); $ret =& $this->fetchBlob( $cluster, $id, $itemID ); if ( $itemID !== false && $ret !== false ) { @@ -53,6 +45,41 @@ class ExternalStoreDB extends ExternalStoreMedium { return $ret; } + /** + * Fetch data from given external store URLs. + * The provided URLs are in the form of DB://cluster/id + * or DB://cluster/id/itemid for concatened storage. + * + * @param array $urls An array of external store URLs + * @return array A map from url to stored content. Failed results + * are not represented. + */ + public function batchFetchFromURLs( array $urls ) { + $batched = $inverseUrlMap = array(); + foreach ( $urls as $url ) { + list( $cluster, $id, $itemID ) = $this->parseURL( $url ); + $batched[$cluster][$id][] = $itemID; + // false $itemID gets cast to int, but should be ok + // since we do === from the $itemID in $batched + $inverseUrlMap[$cluster][$id][$itemID] = $url; + } + $ret = array(); + foreach ( $batched as $cluster => $batchByCluster ) { + $res = $this->batchFetchBlobs( $cluster, $batchByCluster ); + foreach ( $res as $id => $blob ) { + foreach ( $batchByCluster[$id] as $itemID ) { + $url = $inverseUrlMap[$cluster][$id][$itemID]; + if ( $itemID === false ) { + $ret[$url] = $blob; + } else { + $ret[$url] = $blob->getItem( $itemID ); + } + } + } + } + return $ret; + } + /** * @see ExternalStoreMedium::store() */ @@ -64,7 +91,7 @@ class ExternalStoreDB extends ExternalStoreMedium { __METHOD__ ); $id = $dbw->insertId(); if ( !$id ) { - throw new MWException( __METHOD__.': no insert ID' ); + throw new MWException( __METHOD__ . ': no insert ID' ); } if ( $dbw->getFlag( DBO_TRX ) ) { $dbw->commit( __METHOD__ ); @@ -152,25 +179,31 @@ class ExternalStoreDB extends ExternalStoreMedium { static $externalBlobCache = array(); $cacheID = ( $itemID === false ) ? "$cluster/$id" : "$cluster/$id/"; - if( isset( $externalBlobCache[$cacheID] ) ) { - wfDebugLog( 'ExternalStoreDB-cache', "ExternalStoreDB::fetchBlob cache hit on $cacheID\n" ); + if ( isset( $externalBlobCache[$cacheID] ) ) { + wfDebugLog( 'ExternalStoreDB-cache', + "ExternalStoreDB::fetchBlob cache hit on $cacheID\n" ); return $externalBlobCache[$cacheID]; } - wfDebugLog( 'ExternalStoreDB-cache', "ExternalStoreDB::fetchBlob cache miss on $cacheID\n" ); + wfDebugLog( 'ExternalStoreDB-cache', + "ExternalStoreDB::fetchBlob cache miss on $cacheID\n" ); $dbr =& $this->getSlave( $cluster ); - $ret = $dbr->selectField( $this->getTable( $dbr ), 'blob_text', array( 'blob_id' => $id ), __METHOD__ ); + $ret = $dbr->selectField( $this->getTable( $dbr ), + 'blob_text', array( 'blob_id' => $id ), __METHOD__ ); if ( $ret === false ) { - wfDebugLog( 'ExternalStoreDB', "ExternalStoreDB::fetchBlob master fallback on $cacheID\n" ); + wfDebugLog( 'ExternalStoreDB', + "ExternalStoreDB::fetchBlob master fallback on $cacheID\n" ); // Try the master $dbw =& $this->getMaster( $cluster ); - $ret = $dbw->selectField( $this->getTable( $dbw ), 'blob_text', array( 'blob_id' => $id ), __METHOD__ ); - if( $ret === false) { - wfDebugLog( 'ExternalStoreDB', "ExternalStoreDB::fetchBlob master failed to find $cacheID\n" ); + $ret = $dbw->selectField( $this->getTable( $dbw ), + 'blob_text', array( 'blob_id' => $id ), __METHOD__ ); + if ( $ret === false ) { + wfDebugLog( 'ExternalStoreDB', + "ExternalStoreDB::fetchBlob master failed to find $cacheID\n" ); } } - if( $itemID !== false && $ret !== false ) { + if ( $itemID !== false && $ret !== false ) { // Unserialise object; caller extracts item $ret = unserialize( $ret ); } @@ -178,4 +211,73 @@ class ExternalStoreDB extends ExternalStoreMedium { $externalBlobCache = array( $cacheID => &$ret ); return $ret; } + + /** + * Fetch multiple blob items out of the database + * + * @param string $cluster A cluster name valid for use with LBFactory + * @param array $ids A map from the blob_id's to look for to the requested itemIDs in the blobs + * @return array A map from the blob_id's requested to their content. Unlocated ids are not represented + */ + function batchFetchBlobs( $cluster, array $ids ) { + $dbr = $this->getSlave( $cluster ); + $res = $dbr->select( $this->getTable( $dbr ), + array( 'blob_id', 'blob_text' ), array( 'blob_id' => array_keys( $ids ) ), __METHOD__ ); + $ret = array(); + if ( $res !== false ) { + $this->mergeBatchResult( $ret, $ids, $res ); + } + if ( $ids ) { + wfDebugLog( __CLASS__, __METHOD__ . + " master fallback on '$cluster' for: " . + implode( ',', array_keys( $ids ) ) . "\n" ); + // Try the master + $dbw = $this->getMaster( $cluster ); + $res = $dbw->select( $this->getTable( $dbr ), + array( 'blob_id', 'blob_text' ), + array( 'blob_id' => array_keys( $ids ) ), + __METHOD__ ); + if ( $res === false ) { + wfDebugLog( __CLASS__, __METHOD__ . " master failed on '$cluster'\n" ); + } else { + $this->mergeBatchResult( $ret, $ids, $res ); + } + } + if ( $ids ) { + wfDebugLog( __CLASS__, __METHOD__ . + " master on '$cluster' failed locating items: " . + implode( ',', array_keys( $ids ) ) . "\n" ); + } + return $ret; + } + + /** + * Helper function for self::batchFetchBlobs for merging master/slave results + * @param array &$ret Current self::batchFetchBlobs return value + * @param array &$ids Map from blob_id to requested itemIDs + * @param mixed $res DB result from DatabaseBase::select + */ + private function mergeBatchResult( array &$ret, array &$ids, $res ) { + foreach ( $res as $row ) { + $id = $row->blob_id; + $itemIDs = $ids[$id]; + unset( $ids[$id] ); // to track if everything is found + if ( count( $itemIDs ) === 1 && reset( $itemIDs ) === false ) { + // single result stored per blob + $ret[$id] = $row->blob_text; + } else { + // multi result stored per blob + $ret[$id] = unserialize( $row->blob_text ); + } + } + } + + protected function parseURL( $url ) { + $path = explode( '/', $url ); + return array( + $path[2], // cluster + $path[3], // id + isset( $path[4] ) ? $path[4] : false // itemID + ); + } } diff --git a/includes/externalstore/ExternalStoreMedium.php b/includes/externalstore/ExternalStoreMedium.php index 41af7d87..02bdcb51 100644 --- a/includes/externalstore/ExternalStoreMedium.php +++ b/includes/externalstore/ExternalStoreMedium.php @@ -48,6 +48,25 @@ abstract class ExternalStoreMedium { */ abstract public function fetchFromURL( $url ); + /** + * Fetch data from given external store URLs. + * + * @param array $urls A list of external store URLs + * @return array Map from the url to the text stored. Unfound data is not represented + */ + public function batchFetchFromURLs( array $urls ) { + $retval = array(); + foreach ( $urls as $url ) { + $data = $this->fetchFromURL( $url ); + // Dont return when false to allow for simpler implementations. + // errored urls are handled in ExternalStore::batchFetchFromURLs + if ( $data !== false ) { + $retval[$urls] = $data; + } + } + return $retval; + } + /** * Insert a data item into a given location * diff --git a/includes/externalstore/ExternalStoreMwstore.php b/includes/externalstore/ExternalStoreMwstore.php index 0911cca1..aa486796 100644 --- a/includes/externalstore/ExternalStoreMwstore.php +++ b/includes/externalstore/ExternalStoreMwstore.php @@ -46,6 +46,29 @@ class ExternalStoreMwstore extends ExternalStoreMedium { return false; } + /** + * Fetch data from given external store URLs. + * The URL returned is of the form of the form mwstore://backend/container/wiki/id + * + * @param array $urls An array of external store URLs + * @return array A map from url to stored content. Failed results are not represented. + */ + public function batchFetchFromURLs( array $urls ) { + $pathsByBackend = array(); + foreach ( $urls as $url ) { + $be = FileBackendGroup::singleton()->backendFromPath( $url ); + if ( $be instanceof FileBackend ) { + $pathsByBackend[$be->getName()][] = $url; + } + } + $blobs = array(); + foreach ( $pathsByBackend as $backendName => $paths ) { + $be = FileBackendGroup::get( $backendName ); + $blobs = $blobs + $be->getFileContentsMulti( array( 'srcs' => $paths ) ); + } + return $blobs; + } + /** * @see ExternalStoreMedium::store() */ -- cgit v1.2.2