summaryrefslogtreecommitdiff
path: root/includes/cache
diff options
context:
space:
mode:
authorPierre Schmitz <pierre@archlinux.de>2014-12-27 15:41:37 +0100
committerPierre Schmitz <pierre@archlinux.de>2014-12-31 11:43:28 +0100
commitc1f9b1f7b1b77776192048005dcc66dcf3df2bfb (patch)
tree2b38796e738dd74cb42ecd9bfd151803108386bc /includes/cache
parentb88ab0086858470dd1f644e64cb4e4f62bb2be9b (diff)
Update to MediaWiki 1.24.1
Diffstat (limited to 'includes/cache')
-rw-r--r--includes/cache/BacklinkCache.php125
-rw-r--r--includes/cache/CacheDependency.php230
-rw-r--r--includes/cache/CacheHelper.php386
-rw-r--r--includes/cache/FileCacheBase.php15
-rw-r--r--includes/cache/GenderCache.php28
-rw-r--r--includes/cache/HTMLCacheUpdate.php73
-rw-r--r--includes/cache/HTMLFileCache.php47
-rw-r--r--includes/cache/LinkBatch.php47
-rw-r--r--includes/cache/LinkCache.php75
-rw-r--r--includes/cache/LocalisationCache.php571
-rw-r--r--includes/cache/MapCacheLRU.php (renamed from includes/cache/ProcessCacheLRU.php)64
-rw-r--r--includes/cache/MessageCache.php83
-rw-r--r--includes/cache/ObjectFileCache.php4
-rw-r--r--includes/cache/ResourceFileCache.php7
-rw-r--r--includes/cache/SquidUpdate.php300
-rw-r--r--includes/cache/UserCache.php16
-rw-r--r--includes/cache/bloom/BloomCache.php323
-rw-r--r--includes/cache/bloom/BloomCacheRedis.php370
-rw-r--r--includes/cache/bloom/BloomFilters.php79
19 files changed, 1819 insertions, 1024 deletions
diff --git a/includes/cache/BacklinkCache.php b/includes/cache/BacklinkCache.php
index 193f20fe..ed62bba0 100644
--- a/includes/cache/BacklinkCache.php
+++ b/includes/cache/BacklinkCache.php
@@ -48,6 +48,7 @@ class BacklinkCache {
/**
* Multi dimensions array representing batches. Keys are:
* > (string) links table name
+ * > (int) batch size
* > 'numRows' : Number of rows for this link table
* > 'batches' : array( $start, $end )
*
@@ -96,7 +97,7 @@ class BacklinkCache {
* Currently, only one cache instance can exist; callers that
* need multiple backlink cache objects should keep them in scope.
*
- * @param Title $title : Title object to get a backlink cache for
+ * @param Title $title Title object to get a backlink cache for
* @return BacklinkCache
*/
public static function get( Title $title ) {
@@ -107,6 +108,7 @@ class BacklinkCache {
if ( !self::$cache->has( $dbKey, 'obj', 3600 ) ) {
self::$cache->set( $dbKey, 'obj', new self( $title ) );
}
+
return self::$cache->get( $dbKey, 'obj' );
}
@@ -133,7 +135,7 @@ class BacklinkCache {
/**
* Set the Database object to use
*
- * @param $db DatabaseBase
+ * @param DatabaseBase $db
*/
public function setDB( $db ) {
$this->db = $db;
@@ -142,21 +144,22 @@ class BacklinkCache {
/**
* Get the slave connection to the database
* When non existing, will initialize the connection.
- * @return DatabaseBase object
+ * @return DatabaseBase
*/
protected function getDB() {
if ( !isset( $this->db ) ) {
$this->db = wfGetDB( DB_SLAVE );
}
+
return $this->db;
}
/**
* Get the backlinks for a given table. Cached in process memory only.
- * @param $table String
- * @param $startId Integer|false
- * @param $endId Integer|false
- * @param $max Integer|INF
+ * @param string $table
+ * @param int|bool $startId
+ * @param int|bool $endId
+ * @param int|INF $max
* @return TitleArrayFromResult
*/
public function getLinks( $table, $startId = false, $endId = false, $max = INF ) {
@@ -165,20 +168,21 @@ class BacklinkCache {
/**
* Get the backlinks for a given table. Cached in process memory only.
- * @param $table String
- * @param $startId Integer|false
- * @param $endId Integer|false
- * @param $max Integer|INF
+ * @param string $table
+ * @param int|bool $startId
+ * @param int|bool $endId
+ * @param int|INF $max
+ * @param string $select 'all' or 'ids'
* @return ResultWrapper
*/
- protected function queryLinks( $table, $startId, $endId, $max ) {
+ protected function queryLinks( $table, $startId, $endId, $max, $select = 'all' ) {
wfProfileIn( __METHOD__ );
$fromField = $this->getPrefix( $table ) . '_from';
if ( !$startId && !$endId && is_infinite( $max )
- && isset( $this->fullResultCache[$table] ) )
- {
+ && isset( $this->fullResultCache[$table] )
+ ) {
wfDebug( __METHOD__ . ": got results from cache\n" );
$res = $this->fullResultCache[$table];
} else {
@@ -192,20 +196,34 @@ class BacklinkCache {
if ( $endId ) {
$conds[] = "$fromField <= " . intval( $endId );
}
- $options = array( 'STRAIGHT_JOIN', 'ORDER BY' => $fromField );
+ $options = array( 'ORDER BY' => $fromField );
if ( is_finite( $max ) && $max > 0 ) {
$options['LIMIT'] = $max;
}
- $res = $this->getDB()->select(
- array( $table, 'page' ),
- array( 'page_namespace', 'page_title', 'page_id' ),
- $conds,
- __METHOD__,
- $options
- );
+ if ( $select === 'ids' ) {
+ // Just select from the backlink table and ignore the page JOIN
+ $res = $this->getDB()->select(
+ $table,
+ array( $this->getPrefix( $table ) . '_from AS page_id' ),
+ array_filter( $conds, function ( $clause ) { // kind of janky
+ return !preg_match( '/(\b|=)page_id(\b|=)/', $clause );
+ } ),
+ __METHOD__,
+ $options
+ );
+ } else {
+ // Select from the backlink table and JOIN with page title information
+ $res = $this->getDB()->select(
+ array( $table, 'page' ),
+ array( 'page_namespace', 'page_title', 'page_id' ),
+ $conds,
+ __METHOD__,
+ array_merge( array( 'STRAIGHT_JOIN' ), $options )
+ );
+ }
- if ( !$startId && !$endId && $res->numRows() < $max ) {
+ if ( $select === 'all' && !$startId && !$endId && $res->numRows() < $max ) {
// The full results fit within the limit, so cache them
$this->fullResultCache[$table] = $res;
} else {
@@ -214,12 +232,13 @@ class BacklinkCache {
}
wfProfileOut( __METHOD__ );
+
return $res;
}
/**
* Get the field name prefix for a given table
- * @param $table String
+ * @param string $table
* @throws MWException
* @return null|string
*/
@@ -248,15 +267,13 @@ class BacklinkCache {
/**
* Get the SQL condition array for selecting backlinks, with a join
* on the page table.
- * @param $table String
+ * @param string $table
* @throws MWException
* @return array|null
*/
protected function getConditions( $table ) {
$prefix = $this->getPrefix( $table );
- // @todo FIXME: imagelinks and categorylinks do not rely on getNamespace,
- // they could be moved up for nicer case statements
switch ( $table ) {
case 'pagelinks':
case 'templatelinks':
@@ -278,15 +295,10 @@ class BacklinkCache {
);
break;
case 'imagelinks':
- $conds = array(
- 'il_to' => $this->title->getDBkey(),
- 'page_id=il_from'
- );
- break;
case 'categorylinks':
$conds = array(
- 'cl_to' => $this->title->getDBkey(),
- 'page_id=cl_from',
+ "{$prefix}_to" => $this->title->getDBkey(),
+ "page_id={$prefix}_from"
);
break;
default:
@@ -302,7 +314,7 @@ class BacklinkCache {
/**
* Check if there are any backlinks
- * @param $table String
+ * @param string $table
* @return bool
*/
public function hasLinks( $table ) {
@@ -311,16 +323,17 @@ class BacklinkCache {
/**
* Get the approximate number of backlinks
- * @param $table String
- * @param $max integer|INF Only count up to this many backlinks
- * @return integer
+ * @param string $table
+ * @param int|INF $max Only count up to this many backlinks
+ * @return int
*/
public function getNumLinks( $table, $max = INF ) {
- global $wgMemc;
+ global $wgMemc, $wgUpdateRowsPerJob;
// 1) try partition cache ...
if ( isset( $this->partitionCache[$table] ) ) {
$entry = reset( $this->partitionCache[$table] );
+
return min( $max, $entry['numRows'] );
}
@@ -338,9 +351,17 @@ class BacklinkCache {
}
// 4) fetch from the database ...
- $count = $this->getLinks( $table, false, false, $max )->count();
- if ( $count < $max ) { // full count
- $wgMemc->set( $memcKey, $count, self::CACHE_EXPIRY );
+ if ( is_infinite( $max ) ) { // no limit at all
+ // Use partition() since it will batch the query and skip the JOIN.
+ // Use $wgUpdateRowsPerJob just to encourage cache reuse for jobs.
+ $this->partition( $table, $wgUpdateRowsPerJob ); // updates $this->partitionCache
+ return $this->partitionCache[$table][$wgUpdateRowsPerJob]['numRows'];
+ } else { // probably some sane limit
+ // Fetch the full title info, since the caller will likely need it next
+ $count = $this->getLinks( $table, false, false, $max )->count();
+ if ( $count < $max ) { // full count
+ $wgMemc->set( $memcKey, $count, self::CACHE_EXPIRY );
+ }
}
return min( $max, $count );
@@ -351,9 +372,9 @@ class BacklinkCache {
* Returns an array giving the start and end of each range. The first
* batch has a start of false, and the last batch has an end of false.
*
- * @param string $table the links table name
- * @param $batchSize Integer
- * @return Array
+ * @param string $table The links table name
+ * @param int $batchSize
+ * @return array
*/
public function partition( $table, $batchSize ) {
global $wgMemc;
@@ -361,6 +382,7 @@ class BacklinkCache {
// 1) try partition cache ...
if ( isset( $this->partitionCache[$table][$batchSize] ) ) {
wfDebug( __METHOD__ . ": got from partition cache\n" );
+
return $this->partitionCache[$table][$batchSize]['batches'];
}
@@ -371,6 +393,7 @@ class BacklinkCache {
if ( isset( $this->fullResultCache[$table] ) ) {
$cacheEntry = $this->partitionResult( $this->fullResultCache[$table], $batchSize );
wfDebug( __METHOD__ . ": got from full result cache\n" );
+
return $cacheEntry['batches'];
}
@@ -386,6 +409,7 @@ class BacklinkCache {
if ( is_array( $memcValue ) ) {
$cacheEntry = $memcValue;
wfDebug( __METHOD__ . ": got from memcached $memcKey\n" );
+
return $cacheEntry['batches'];
}
@@ -396,13 +420,13 @@ class BacklinkCache {
$selectSize = max( $batchSize, 200000 - ( 200000 % $batchSize ) );
$start = false;
do {
- $res = $this->queryLinks( $table, $start, false, $selectSize );
+ $res = $this->queryLinks( $table, $start, false, $selectSize, 'ids' );
$partitions = $this->partitionResult( $res, $batchSize, false );
// Merge the link count and range partitions for this chunk
$cacheEntry['numRows'] += $partitions['numRows'];
$cacheEntry['batches'] = array_merge( $cacheEntry['batches'], $partitions['batches'] );
if ( count( $partitions['batches'] ) ) {
- list( $lStart, $lEnd ) = end( $partitions['batches'] );
+ list( , $lEnd ) = end( $partitions['batches'] );
$start = $lEnd + 1; // pick up after this inclusive range
}
} while ( $partitions['numRows'] >= $selectSize );
@@ -420,14 +444,15 @@ class BacklinkCache {
$wgMemc->set( $memcKey, $cacheEntry['numRows'], self::CACHE_EXPIRY );
wfDebug( __METHOD__ . ": got from database\n" );
+
return $cacheEntry['batches'];
}
/**
* Partition a DB result with backlinks in it into batches
- * @param $res ResultWrapper database result
- * @param $batchSize integer
- * @param $isComplete bool Whether $res includes all the backlinks
+ * @param ResultWrapper $res Database result
+ * @param int $batchSize
+ * @param bool $isComplete Whether $res includes all the backlinks
* @throws MWException
* @return array
*/
diff --git a/includes/cache/CacheDependency.php b/includes/cache/CacheDependency.php
index 32bcdf7f..9b48ecb7 100644
--- a/includes/cache/CacheDependency.php
+++ b/includes/cache/CacheDependency.php
@@ -28,14 +28,15 @@
* @ingroup Cache
*/
class DependencyWrapper {
- var $value;
- var $deps;
+ private $value;
+ /** @var CacheDependency[] */
+ private $deps;
/**
* Create an instance.
- * @param $value Mixed: the user-supplied value
- * @param $deps Mixed: a dependency or dependency array. All dependencies
- * must be objects implementing CacheDependency.
+ * @param mixed $value The user-supplied value
+ * @param CacheDependency|CacheDependency[] $deps A dependency or dependency
+ * array. All dependencies must be objects implementing CacheDependency.
*/
function __construct( $value = false, $deps = array() ) {
$this->value = $value;
@@ -74,7 +75,7 @@ class DependencyWrapper {
/**
* Get the user-defined value
- * @return bool|Mixed
+ * @return bool|mixed
*/
function getValue() {
return $this->value;
@@ -83,9 +84,9 @@ class DependencyWrapper {
/**
* Store the wrapper to a cache
*
- * @param $cache BagOStuff
- * @param $key
- * @param $expiry
+ * @param BagOStuff $cache
+ * @param string $key
+ * @param int $expiry
*/
function storeToCache( $cache, $key, $expiry = 0 ) {
$this->initialiseDeps();
@@ -97,12 +98,12 @@ class DependencyWrapper {
* it will be generated with the callback function (if present), and the newly
* calculated value will be stored to the cache in a wrapper.
*
- * @param $cache BagOStuff a cache object such as $wgMemc
- * @param string $key the cache key
- * @param $expiry Integer: the expiry timestamp or interval in seconds
- * @param $callback Mixed: the callback for generating the value, or false
- * @param array $callbackParams the function parameters for the callback
- * @param array $deps the dependencies to store on a cache miss. Note: these
+ * @param BagOStuff $cache A cache object such as $wgMemc
+ * @param string $key The cache key
+ * @param int $expiry The expiry timestamp or interval in seconds
+ * @param bool|callable $callback The callback for generating the value, or false
+ * @param array $callbackParams The function parameters for the callback
+ * @param array $deps The dependencies to store on a cache miss. Note: these
* are not the dependencies used on a cache hit! Cache hits use the stored
* dependency array.
*
@@ -110,8 +111,8 @@ class DependencyWrapper {
* callback was defined.
*/
static function getValueFromCache( $cache, $key, $expiry = 0, $callback = false,
- $callbackParams = array(), $deps = array() )
- {
+ $callbackParams = array(), $deps = array()
+ ) {
$obj = $cache->get( $key );
if ( is_object( $obj ) && $obj instanceof DependencyWrapper && !$obj->isExpired() ) {
@@ -141,20 +142,22 @@ abstract class CacheDependency {
/**
* Hook to perform any expensive pre-serialize loading of dependency values.
*/
- function loadDependencyValues() { }
+ function loadDependencyValues() {
+ }
}
/**
* @ingroup Cache
*/
class FileDependency extends CacheDependency {
- var $filename, $timestamp;
+ private $filename;
+ private $timestamp;
/**
* Create a file dependency
*
- * @param string $filename the name of the file, preferably fully qualified
- * @param $timestamp Mixed: the unix last modified timestamp, or false if the
+ * @param string $filename The name of the file, preferably fully qualified
+ * @param null|bool|int $timestamp The unix last modified timestamp, or false if the
* file does not exist. If omitted, the timestamp will be loaded from
* the file.
*
@@ -172,6 +175,7 @@ class FileDependency extends CacheDependency {
*/
function __sleep() {
$this->loadDependencyValues();
+
return array( 'filename', 'timestamp' );
}
@@ -198,6 +202,7 @@ class FileDependency extends CacheDependency {
} else {
# Deleted
wfDebug( "Dependency triggered: {$this->filename} deleted.\n" );
+
return true;
}
} else {
@@ -205,6 +210,7 @@ class FileDependency extends CacheDependency {
if ( $lastmod > $this->timestamp ) {
# Modified or created
wfDebug( "Dependency triggered: {$this->filename} changed.\n" );
+
return true;
} else {
# Not modified
@@ -217,183 +223,9 @@ class FileDependency extends CacheDependency {
/**
* @ingroup Cache
*/
-class TitleDependency extends CacheDependency {
- var $titleObj;
- var $ns, $dbk;
- var $touched;
-
- /**
- * Construct a title dependency
- * @param $title Title
- */
- function __construct( Title $title ) {
- $this->titleObj = $title;
- $this->ns = $title->getNamespace();
- $this->dbk = $title->getDBkey();
- }
-
- function loadDependencyValues() {
- $this->touched = $this->getTitle()->getTouched();
- }
-
- /**
- * Get rid of bulky Title object for sleep
- *
- * @return array
- */
- function __sleep() {
- return array( 'ns', 'dbk', 'touched' );
- }
-
- /**
- * @return Title
- */
- function getTitle() {
- if ( !isset( $this->titleObj ) ) {
- $this->titleObj = Title::makeTitle( $this->ns, $this->dbk );
- }
-
- return $this->titleObj;
- }
-
- /**
- * @return bool
- */
- function isExpired() {
- $touched = $this->getTitle()->getTouched();
-
- if ( $this->touched === false ) {
- if ( $touched === false ) {
- # Still missing
- return false;
- } else {
- # Created
- return true;
- }
- } elseif ( $touched === false ) {
- # Deleted
- return true;
- } elseif ( $touched > $this->touched ) {
- # Updated
- return true;
- } else {
- # Unmodified
- return false;
- }
- }
-}
-
-/**
- * @ingroup Cache
- */
-class TitleListDependency extends CacheDependency {
- var $linkBatch;
- var $timestamps;
-
- /**
- * Construct a dependency on a list of titles
- * @param $linkBatch LinkBatch
- */
- function __construct( LinkBatch $linkBatch ) {
- $this->linkBatch = $linkBatch;
- }
-
- /**
- * @return array
- */
- function calculateTimestamps() {
- # Initialise values to false
- $timestamps = array();
-
- foreach ( $this->getLinkBatch()->data as $ns => $dbks ) {
- if ( count( $dbks ) > 0 ) {
- $timestamps[$ns] = array();
-
- foreach ( $dbks as $dbk => $value ) {
- $timestamps[$ns][$dbk] = false;
- }
- }
- }
-
- # Do the query
- if ( count( $timestamps ) ) {
- $dbr = wfGetDB( DB_SLAVE );
- $where = $this->getLinkBatch()->constructSet( 'page', $dbr );
- $res = $dbr->select(
- 'page',
- array( 'page_namespace', 'page_title', 'page_touched' ),
- $where,
- __METHOD__
- );
-
- foreach ( $res as $row ) {
- $timestamps[$row->page_namespace][$row->page_title] = $row->page_touched;
- }
- }
-
- return $timestamps;
- }
-
- function loadDependencyValues() {
- $this->timestamps = $this->calculateTimestamps();
- }
-
- /**
- * @return array
- */
- function __sleep() {
- return array( 'timestamps' );
- }
-
- /**
- * @return LinkBatch
- */
- function getLinkBatch() {
- if ( !isset( $this->linkBatch ) ) {
- $this->linkBatch = new LinkBatch;
- $this->linkBatch->setArray( $this->timestamps );
- }
- return $this->linkBatch;
- }
-
- /**
- * @return bool
- */
- function isExpired() {
- $newTimestamps = $this->calculateTimestamps();
-
- foreach ( $this->timestamps as $ns => $dbks ) {
- foreach ( $dbks as $dbk => $oldTimestamp ) {
- $newTimestamp = $newTimestamps[$ns][$dbk];
-
- if ( $oldTimestamp === false ) {
- if ( $newTimestamp === false ) {
- # Still missing
- } else {
- # Created
- return true;
- }
- } elseif ( $newTimestamp === false ) {
- # Deleted
- return true;
- } elseif ( $newTimestamp > $oldTimestamp ) {
- # Updated
- return true;
- } else {
- # Unmodified
- }
- }
- }
-
- return false;
- }
-}
-
-/**
- * @ingroup Cache
- */
class GlobalDependency extends CacheDependency {
- var $name, $value;
+ private $name;
+ private $value;
function __construct( $name ) {
$this->name = $name;
@@ -407,6 +239,7 @@ class GlobalDependency extends CacheDependency {
if ( !isset( $GLOBALS[$this->name] ) ) {
return true;
}
+
return $GLOBALS[$this->name] != $this->value;
}
}
@@ -415,7 +248,8 @@ class GlobalDependency extends CacheDependency {
* @ingroup Cache
*/
class ConstantDependency extends CacheDependency {
- var $name, $value;
+ private $name;
+ private $value;
function __construct( $name ) {
$this->name = $name;
diff --git a/includes/cache/CacheHelper.php b/includes/cache/CacheHelper.php
new file mode 100644
index 00000000..401c2b96
--- /dev/null
+++ b/includes/cache/CacheHelper.php
@@ -0,0 +1,386 @@
+<?php
+/**
+ * Cache of various elements in a single cache entry.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ * http://www.gnu.org/copyleft/gpl.html
+ *
+ * @file
+ * @license GNU GPL v2 or later
+ * @author Jeroen De Dauw < jeroendedauw@gmail.com >
+ */
+
+/**
+ * Interface for all classes implementing CacheHelper functionality.
+ *
+ * @since 1.20
+ */
+interface ICacheHelper {
+ /**
+ * Sets if the cache should be enabled or not.
+ *
+ * @since 1.20
+ * @param bool $cacheEnabled
+ */
+ function setCacheEnabled( $cacheEnabled );
+
+ /**
+ * Initializes the caching.
+ * Should be called before the first time anything is added via addCachedHTML.
+ *
+ * @since 1.20
+ *
+ * @param int|null $cacheExpiry Sets the cache expiry, either ttl in seconds or unix timestamp.
+ * @param bool|null $cacheEnabled Sets if the cache should be enabled or not.
+ */
+ function startCache( $cacheExpiry = null, $cacheEnabled = null );
+
+ /**
+ * Get a cached value if available or compute it if not and then cache it if possible.
+ * The provided $computeFunction is only called when the computation needs to happen
+ * and should return a result value. $args are arguments that will be passed to the
+ * compute function when called.
+ *
+ * @since 1.20
+ *
+ * @param callable $computeFunction
+ * @param array|mixed $args
+ * @param string|null $key
+ *
+ * @return mixed
+ */
+ function getCachedValue( $computeFunction, $args = array(), $key = null );
+
+ /**
+ * Saves the HTML to the cache in case it got recomputed.
+ * Should be called after the last time anything is added via addCachedHTML.
+ *
+ * @since 1.20
+ */
+ function saveCache();
+
+ /**
+ * Sets the time to live for the cache, in seconds or a unix timestamp
+ * indicating the point of expiry...
+ *
+ * @since 1.20
+ *
+ * @param int $cacheExpiry
+ */
+ function setExpiry( $cacheExpiry );
+}
+
+/**
+ * Helper class for caching various elements in a single cache entry.
+ *
+ * To get a cached value or compute it, use getCachedValue like this:
+ * $this->getCachedValue( $callback );
+ *
+ * To add HTML that should be cached, use addCachedHTML like this:
+ * $this->addCachedHTML( $callback );
+ *
+ * The callback function is only called when needed, so do all your expensive
+ * computations here. This function should returns the HTML to be cached.
+ * It should not add anything to the PageOutput object!
+ *
+ * Before the first addCachedHTML call, you should call $this->startCache();
+ * After adding the last HTML that should be cached, call $this->saveCache();
+ *
+ * @since 1.20
+ */
+class CacheHelper implements ICacheHelper {
+ /**
+ * The time to live for the cache, in seconds or a unix timestamp indicating the point of expiry.
+ *
+ * @since 1.20
+ * @var int
+ */
+ protected $cacheExpiry = 3600;
+
+ /**
+ * List of HTML chunks to be cached (if !hasCached) or that where cached (of hasCached).
+ * If not cached already, then the newly computed chunks are added here,
+ * if it as cached already, chunks are removed from this list as they are needed.
+ *
+ * @since 1.20
+ * @var array
+ */
+ protected $cachedChunks;
+
+ /**
+ * Indicates if the to be cached content was already cached.
+ * Null if this information is not available yet.
+ *
+ * @since 1.20
+ * @var bool|null
+ */
+ protected $hasCached = null;
+
+ /**
+ * If the cache is enabled or not.
+ *
+ * @since 1.20
+ * @var bool
+ */
+ protected $cacheEnabled = true;
+
+ /**
+ * Function that gets called when initialization is done.
+ *
+ * @since 1.20
+ * @var callable
+ */
+ protected $onInitHandler = false;
+
+ /**
+ * Elements to build a cache key with.
+ *
+ * @since 1.20
+ * @var array
+ */
+ protected $cacheKey = array();
+
+ /**
+ * Sets if the cache should be enabled or not.
+ *
+ * @since 1.20
+ * @param bool $cacheEnabled
+ */
+ public function setCacheEnabled( $cacheEnabled ) {
+ $this->cacheEnabled = $cacheEnabled;
+ }
+
+ /**
+ * Initializes the caching.
+ * Should be called before the first time anything is added via addCachedHTML.
+ *
+ * @since 1.20
+ *
+ * @param int|null $cacheExpiry Sets the cache expiry, either ttl in seconds or unix timestamp.
+ * @param bool|null $cacheEnabled Sets if the cache should be enabled or not.
+ */
+ public function startCache( $cacheExpiry = null, $cacheEnabled = null ) {
+ if ( is_null( $this->hasCached ) ) {
+ if ( !is_null( $cacheExpiry ) ) {
+ $this->cacheExpiry = $cacheExpiry;
+ }
+
+ if ( !is_null( $cacheEnabled ) ) {
+ $this->setCacheEnabled( $cacheEnabled );
+ }
+
+ $this->initCaching();
+ }
+ }
+
+ /**
+ * Returns a message that notifies the user he/she is looking at
+ * a cached version of the page, including a refresh link.
+ *
+ * @since 1.20
+ *
+ * @param IContextSource $context
+ * @param bool $includePurgeLink
+ *
+ * @return string
+ */
+ public function getCachedNotice( IContextSource $context, $includePurgeLink = true ) {
+ if ( $this->cacheExpiry < 86400 * 3650 ) {
+ $message = $context->msg(
+ 'cachedspecial-viewing-cached-ttl',
+ $context->getLanguage()->formatDuration( $this->cacheExpiry )
+ )->escaped();
+ } else {
+ $message = $context->msg(
+ 'cachedspecial-viewing-cached-ts'
+ )->escaped();
+ }
+
+ if ( $includePurgeLink ) {
+ $refreshArgs = $context->getRequest()->getQueryValues();
+ unset( $refreshArgs['title'] );
+ $refreshArgs['action'] = 'purge';
+
+ $subPage = $context->getTitle()->getFullText();
+ $subPage = explode( '/', $subPage, 2 );
+ $subPage = count( $subPage ) > 1 ? $subPage[1] : false;
+
+ $message .= ' ' . Linker::link(
+ $context->getTitle( $subPage ),
+ $context->msg( 'cachedspecial-refresh-now' )->escaped(),
+ array(),
+ $refreshArgs
+ );
+ }
+
+ return $message;
+ }
+
+ /**
+ * Initializes the caching if not already done so.
+ * Should be called before any of the caching functionality is used.
+ *
+ * @since 1.20
+ */
+ protected function initCaching() {
+ if ( $this->cacheEnabled && is_null( $this->hasCached ) ) {
+ $cachedChunks = wfGetCache( CACHE_ANYTHING )->get( $this->getCacheKeyString() );
+
+ $this->hasCached = is_array( $cachedChunks );
+ $this->cachedChunks = $this->hasCached ? $cachedChunks : array();
+
+ if ( $this->onInitHandler !== false ) {
+ call_user_func( $this->onInitHandler, $this->hasCached );
+ }
+ }
+ }
+
+ /**
+ * Get a cached value if available or compute it if not and then cache it if possible.
+ * The provided $computeFunction is only called when the computation needs to happen
+ * and should return a result value. $args are arguments that will be passed to the
+ * compute function when called.
+ *
+ * @since 1.20
+ *
+ * @param callable $computeFunction
+ * @param array|mixed $args
+ * @param string|null $key
+ *
+ * @return mixed
+ */
+ public function getCachedValue( $computeFunction, $args = array(), $key = null ) {
+ $this->initCaching();
+
+ if ( $this->cacheEnabled && $this->hasCached ) {
+ $value = null;
+
+ if ( is_null( $key ) ) {
+ $itemKey = array_keys( array_slice( $this->cachedChunks, 0, 1 ) );
+ $itemKey = array_shift( $itemKey );
+
+ if ( !is_integer( $itemKey ) ) {
+ wfWarn( "Attempted to get item with non-numeric key while " .
+ "the next item in the queue has a key ($itemKey) in " . __METHOD__ );
+ } elseif ( is_null( $itemKey ) ) {
+ wfWarn( "Attempted to get an item while the queue is empty in " . __METHOD__ );
+ } else {
+ $value = array_shift( $this->cachedChunks );
+ }
+ } else {
+ if ( array_key_exists( $key, $this->cachedChunks ) ) {
+ $value = $this->cachedChunks[$key];
+ unset( $this->cachedChunks[$key] );
+ } else {
+ wfWarn( "There is no item with key '$key' in this->cachedChunks in " . __METHOD__ );
+ }
+ }
+ } else {
+ if ( !is_array( $args ) ) {
+ $args = array( $args );
+ }
+
+ $value = call_user_func_array( $computeFunction, $args );
+
+ if ( $this->cacheEnabled ) {
+ if ( is_null( $key ) ) {
+ $this->cachedChunks[] = $value;
+ } else {
+ $this->cachedChunks[$key] = $value;
+ }
+ }
+ }
+
+ return $value;
+ }
+
+ /**
+ * Saves the HTML to the cache in case it got recomputed.
+ * Should be called after the last time anything is added via addCachedHTML.
+ *
+ * @since 1.20
+ */
+ public function saveCache() {
+ if ( $this->cacheEnabled && $this->hasCached === false && !empty( $this->cachedChunks ) ) {
+ wfGetCache( CACHE_ANYTHING )->set(
+ $this->getCacheKeyString(),
+ $this->cachedChunks,
+ $this->cacheExpiry
+ );
+ }
+ }
+
+ /**
+ * Sets the time to live for the cache, in seconds or a unix timestamp
+ * indicating the point of expiry...
+ *
+ * @since 1.20
+ *
+ * @param int $cacheExpiry
+ */
+ public function setExpiry( $cacheExpiry ) {
+ $this->cacheExpiry = $cacheExpiry;
+ }
+
+ /**
+ * Returns the cache key to use to cache this page's HTML output.
+ * Is constructed from the special page name and language code.
+ *
+ * @since 1.20
+ *
+ * @return string
+ * @throws MWException
+ */
+ protected function getCacheKeyString() {
+ if ( $this->cacheKey === array() ) {
+ throw new MWException( 'No cache key set, so cannot obtain or save the CacheHelper values.' );
+ }
+
+ return call_user_func_array( 'wfMemcKey', $this->cacheKey );
+ }
+
+ /**
+ * Sets the cache key that should be used.
+ *
+ * @since 1.20
+ *
+ * @param array $cacheKey
+ */
+ public function setCacheKey( array $cacheKey ) {
+ $this->cacheKey = $cacheKey;
+ }
+
+ /**
+ * Rebuild the content, even if it's already cached.
+ * This effectively has the same effect as purging the cache,
+ * since it will be overridden with the new value on the next request.
+ *
+ * @since 1.20
+ */
+ public function rebuildOnDemand() {
+ $this->hasCached = false;
+ }
+
+ /**
+ * Sets a function that gets called when initialization of the cache is done.
+ *
+ * @since 1.20
+ *
+ * @param callable $handlerFunction
+ */
+ public function setOnInitializedHandler( $handlerFunction ) {
+ $this->onInitHandler = $handlerFunction;
+ }
+}
diff --git a/includes/cache/FileCacheBase.php b/includes/cache/FileCacheBase.php
index d4bf5ee6..4bf36114 100644
--- a/includes/cache/FileCacheBase.php
+++ b/includes/cache/FileCacheBase.php
@@ -51,6 +51,7 @@ abstract class FileCacheBase {
*/
final protected function baseCacheDirectory() {
global $wgFileCacheDirectory;
+
return $wgFileCacheDirectory;
}
@@ -91,6 +92,7 @@ abstract class FileCacheBase {
if ( $this->mCached === null ) {
$this->mCached = file_exists( $this->cachePath() );
}
+
return $this->mCached;
}
@@ -100,6 +102,7 @@ abstract class FileCacheBase {
*/
public function cacheTimestamp() {
$timestamp = filemtime( $this->cachePath() );
+
return ( $timestamp !== false )
? wfTimestamp( TS_MW, $timestamp )
: false;
@@ -120,7 +123,8 @@ abstract class FileCacheBase {
$cachetime = $this->cacheTimestamp();
$good = ( $timestamp <= $cachetime && $wgCacheEpoch <= $cachetime );
- wfDebug( __METHOD__ . ": cachetime $cachetime, touched '{$timestamp}' epoch {$wgCacheEpoch}, good $good\n" );
+ wfDebug( __METHOD__ .
+ ": cachetime $cachetime, touched '{$timestamp}' epoch {$wgCacheEpoch}, good $good\n" );
return $good;
}
@@ -140,6 +144,7 @@ abstract class FileCacheBase {
public function fetchText() {
if ( $this->useGzip() ) {
$fh = gzopen( $this->cachePath(), 'rb' );
+
return stream_get_contents( $fh );
} else {
return file_get_contents( $this->cachePath() );
@@ -148,7 +153,8 @@ abstract class FileCacheBase {
/**
* Save and compress text to the cache
- * @return string compressed text
+ * @param string $text
+ * @return string Compressed text
*/
public function saveText( $text ) {
global $wgUseFileCache;
@@ -165,10 +171,12 @@ abstract class FileCacheBase {
if ( !file_put_contents( $this->cachePath(), $text, LOCK_EX ) ) {
wfDebug( __METHOD__ . "() failed saving " . $this->cachePath() . "\n" );
$this->mCached = null;
+
return false;
}
$this->mCached = true;
+
return $text;
}
@@ -223,7 +231,7 @@ abstract class FileCacheBase {
/**
* Roughly increments the cache misses in the last hour by unique visitors
- * @param $request WebRequest
+ * @param WebRequest $request
* @return void
*/
public function incrMissesRecent( WebRequest $request ) {
@@ -262,6 +270,7 @@ abstract class FileCacheBase {
*/
public function getMissesRecent() {
global $wgMemc;
+
return self::MISS_FACTOR * $wgMemc->get( $this->cacheMissKey() );
}
diff --git a/includes/cache/GenderCache.php b/includes/cache/GenderCache.php
index a933527a..63e7bfd7 100644
--- a/includes/cache/GenderCache.php
+++ b/includes/cache/GenderCache.php
@@ -41,27 +41,30 @@ class GenderCache {
if ( $that === null ) {
$that = new self();
}
+
return $that;
}
- protected function __construct() {}
+ protected function __construct() {
+ }
/**
* Returns the default gender option in this wiki.
- * @return String
+ * @return string
*/
protected function getDefault() {
if ( $this->default === null ) {
$this->default = User::getDefaultOption( 'gender' );
}
+
return $this->default;
}
/**
* Returns the gender for given username.
- * @param string $username or User: username
- * @param string $caller the calling method
- * @return String
+ * @param string|User $username Username
+ * @param string $caller The calling method
+ * @return string
*/
public function getGenderOf( $username, $caller = '' ) {
global $wgUser;
@@ -77,8 +80,8 @@ class GenderCache {
$this->misses++;
wfDebug( __METHOD__ . ": too many misses, returning default onwards\n" );
}
- return $this->getDefault();
+ return $this->getDefault();
} else {
$this->misses++;
$this->doQuery( $username, $caller );
@@ -94,8 +97,8 @@ class GenderCache {
/**
* Wrapper for doQuery that processes raw LinkBatch data.
*
- * @param $data
- * @param $caller
+ * @param array $data
+ * @param string $caller
*/
public function doLinkBatch( $data, $caller = '' ) {
$users = array();
@@ -115,8 +118,8 @@ class GenderCache {
* Wrapper for doQuery that processes a title or string array.
*
* @since 1.20
- * @param $titles List: array of Title objects or strings
- * @param string $caller the calling method
+ * @param array $titles Array of Title objects or strings
+ * @param string $caller The calling method
*/
public function doTitlesArray( $titles, $caller = '' ) {
$users = array();
@@ -136,8 +139,8 @@ class GenderCache {
/**
* Preloads genders for given list of users.
- * @param $users List|String: usernames
- * @param string $caller the calling method
+ * @param array|string $users Usernames
+ * @param string $caller The calling method
*/
public function doQuery( $users, $caller = '' ) {
$default = $this->getDefault();
@@ -184,6 +187,7 @@ class GenderCache {
if ( $indexSlash !== false ) {
$username = substr( $username, 0, $indexSlash );
}
+
// normalize underscore/spaces
return strtr( $username, '_', ' ' );
}
diff --git a/includes/cache/HTMLCacheUpdate.php b/includes/cache/HTMLCacheUpdate.php
deleted file mode 100644
index 992809ef..00000000
--- a/includes/cache/HTMLCacheUpdate.php
+++ /dev/null
@@ -1,73 +0,0 @@
-<?php
-/**
- * HTML cache invalidation of all pages linking to a given title.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
- * http://www.gnu.org/copyleft/gpl.html
- *
- * @file
- * @ingroup Cache
- */
-
-/**
- * Class to invalidate the HTML cache of all the pages linking to a given title.
- *
- * @ingroup Cache
- */
-class HTMLCacheUpdate implements DeferrableUpdate {
- /**
- * @var Title
- */
- public $mTitle;
-
- public $mTable;
-
- /**
- * @param $titleTo
- * @param $table
- * @param $start bool
- * @param $end bool
- */
- function __construct( Title $titleTo, $table ) {
- $this->mTitle = $titleTo;
- $this->mTable = $table;
- }
-
- public function doUpdate() {
- wfProfileIn( __METHOD__ );
-
- $job = new HTMLCacheUpdateJob(
- $this->mTitle,
- array(
- 'table' => $this->mTable,
- ) + Job::newRootJobParams( // "overall" refresh links job info
- "htmlCacheUpdate:{$this->mTable}:{$this->mTitle->getPrefixedText()}"
- )
- );
-
- $count = $this->mTitle->getBacklinkCache()->getNumLinks( $this->mTable, 200 );
- if ( $count >= 200 ) { // many backlinks
- JobQueueGroup::singleton()->push( $job );
- JobQueueGroup::singleton()->deduplicateRootJob( $job );
- } else { // few backlinks ($count might be off even if 0)
- $dbw = wfGetDB( DB_MASTER );
- $dbw->onTransactionIdle( function() use ( $job ) {
- $job->run(); // just do the purge query now
- } );
- }
-
- wfProfileOut( __METHOD__ );
- }
-}
diff --git a/includes/cache/HTMLFileCache.php b/includes/cache/HTMLFileCache.php
index ab379116..58ca2dcd 100644
--- a/includes/cache/HTMLFileCache.php
+++ b/includes/cache/HTMLFileCache.php
@@ -31,25 +31,32 @@
class HTMLFileCache extends FileCacheBase {
/**
* Construct an ObjectFileCache from a Title and an action
- * @param $title Title|string Title object or prefixed DB key string
- * @param $action string
+ * @param Title|string $title Title object or prefixed DB key string
+ * @param string $action
* @throws MWException
* @return HTMLFileCache
+ *
+ * @deprecated Since 1.24, instantiate this class directly
*/
public static function newFromTitle( $title, $action ) {
- $cache = new self();
+ return new self( $title, $action );
+ }
+ /**
+ * @param Title|string $title Title object or prefixed DB key string
+ * @param string $action
+ * @throws MWException
+ */
+ public function __construct( $title, $action ) {
$allowedTypes = self::cacheablePageActions();
if ( !in_array( $action, $allowedTypes ) ) {
- throw new MWException( "Invalid filecache type given." );
+ throw new MWException( 'Invalid file cache type given.' );
}
- $cache->mKey = ( $title instanceof Title )
+ $this->mKey = ( $title instanceof Title )
? $title->getPrefixedDBkey()
: (string)$title;
- $cache->mType = (string)$action;
- $cache->mExt = 'html';
-
- return $cache;
+ $this->mType = (string)$action;
+ $this->mExt = 'html';
}
/**
@@ -84,7 +91,7 @@ class HTMLFileCache extends FileCacheBase {
/**
* Check if pages can be cached for this request/user
- * @param $context IContextSource
+ * @param IContextSource $context
* @return bool
*/
public static function useFileCache( IContextSource $context ) {
@@ -94,6 +101,7 @@ class HTMLFileCache extends FileCacheBase {
}
if ( $wgShowIPinHeader || $wgDebugToolbar ) {
wfDebug( "HTML file cache skipped. Either \$wgShowIPinHeader and/or \$wgDebugToolbar on\n" );
+
return false;
}
@@ -109,6 +117,7 @@ class HTMLFileCache extends FileCacheBase {
} elseif ( $query === 'maxage' || $query === 'smaxage' ) {
continue;
}
+
return false;
}
$user = $context->getUser();
@@ -116,13 +125,18 @@ class HTMLFileCache extends FileCacheBase {
// and extensions for auto-detecting user language.
$ulang = $context->getLanguage()->getCode();
$clang = $wgContLang->getCode();
+
// Check that there are no other sources of variation
- return !$user->getId() && !$user->getNewtalk() && $ulang == $clang;
+ if ( $user->getId() || $user->getNewtalk() || $ulang != $clang ) {
+ return false;
+ }
+ // Allow extensions to disable caching
+ return wfRunHooks( 'HTMLFileCache::useFileCache', array( $context ) );
}
/**
* Read from cache to context output
- * @param $context IContextSource
+ * @param IContextSource $context
* @return void
*/
public function loadFromFileCache( IContextSource $context ) {
@@ -152,7 +166,7 @@ class HTMLFileCache extends FileCacheBase {
/**
* Save this cache object with the given text.
* Use this as an ob_start() handler.
- * @param $text string
+ * @param string $text
* @return bool Whether $wgUseFileCache is enabled
*/
public function saveToFileCache( $text ) {
@@ -163,7 +177,7 @@ class HTMLFileCache extends FileCacheBase {
return $text;
}
- wfDebug( __METHOD__ . "()\n", false );
+ wfDebug( __METHOD__ . "()\n", 'log' );
$now = wfTimestampNow();
if ( $this->useGzip() ) {
@@ -185,6 +199,7 @@ class HTMLFileCache extends FileCacheBase {
// @todo Ugly wfClientAcceptsGzip() function - use context!
if ( wfClientAcceptsGzip() ) {
header( 'Content-Encoding: gzip' );
+
return $compressed;
} else {
return $text;
@@ -196,7 +211,7 @@ class HTMLFileCache extends FileCacheBase {
/**
* Clear the file caches for a page for all actions
- * @param $title Title
+ * @param Title $title
* @return bool Whether $wgUseFileCache is enabled
*/
public static function clearFileCache( Title $title ) {
@@ -207,7 +222,7 @@ class HTMLFileCache extends FileCacheBase {
}
foreach ( self::cacheablePageActions() as $type ) {
- $fc = self::newFromTitle( $title, $type );
+ $fc = new self( $title, $type );
$fc->clearCache();
}
diff --git a/includes/cache/LinkBatch.php b/includes/cache/LinkBatch.php
index 48b60aa9..48c063f4 100644
--- a/includes/cache/LinkBatch.php
+++ b/includes/cache/LinkBatch.php
@@ -31,7 +31,7 @@ class LinkBatch {
/**
* 2-d array, first index namespace, second index dbkey, value arbitrary
*/
- var $data = array();
+ public $data = array();
/**
* For debugging which method is using this class.
@@ -49,14 +49,14 @@ class LinkBatch {
* class. Only used in debugging output.
* @since 1.17
*
- * @param $caller
+ * @param string $caller
*/
public function setCaller( $caller ) {
$this->caller = $caller;
}
/**
- * @param $title Title
+ * @param Title $title
*/
public function addObj( $title ) {
if ( is_object( $title ) ) {
@@ -67,9 +67,8 @@ class LinkBatch {
}
/**
- * @param $ns int
- * @param $dbkey string
- * @return mixed
+ * @param int $ns
+ * @param string $dbkey
*/
public function add( $ns, $dbkey ) {
if ( $ns < 0 ) {
@@ -86,7 +85,7 @@ class LinkBatch {
* Set the link list to a given 2-d array
* First key is the namespace, second is the DB key, value arbitrary
*
- * @param $array array
+ * @param array $array
*/
public function setArray( $array ) {
$this->data = $array;
@@ -113,10 +112,11 @@ class LinkBatch {
/**
* Do the query and add the results to the LinkCache object
*
- * @return Array mapping PDBK to ID
+ * @return array Mapping PDBK to ID
*/
public function execute() {
$linkCache = LinkCache::singleton();
+
return $this->executeInto( $linkCache );
}
@@ -124,8 +124,8 @@ class LinkBatch {
* Do the query and add the results to a given LinkCache object
* Return an array mapping PDBK to ID
*
- * @param $cache LinkCache
- * @return Array remaining IDs
+ * @param LinkCache $cache
+ * @return array Remaining IDs
*/
protected function executeInto( &$cache ) {
wfProfileIn( __METHOD__ );
@@ -133,6 +133,7 @@ class LinkBatch {
$this->doGenderQuery();
$ids = $this->addResultToCache( $cache, $res );
wfProfileOut( __METHOD__ );
+
return $ids;
}
@@ -142,9 +143,9 @@ class LinkBatch {
* This function *also* stores extra fields of the title used for link
* parsing to avoid extra DB queries.
*
- * @param $cache LinkCache
- * @param $res
- * @return Array of remaining titles
+ * @param LinkCache $cache
+ * @param ResultWrapper $res
+ * @return array Array of remaining titles
*/
public function addResultToCache( $cache, $res ) {
if ( !$res ) {
@@ -170,14 +171,17 @@ class LinkBatch {
$ids[$title->getPrefixedDBkey()] = 0;
}
}
+
return $ids;
}
/**
* Perform the existence test query, return a ResultWrapper with page_id fields
- * @return Bool|ResultWrapper
+ * @return bool|ResultWrapper
*/
public function doQuery() {
+ global $wgContentHandlerUseDB;
+
if ( $this->isEmpty() ) {
return false;
}
@@ -188,6 +192,11 @@ class LinkBatch {
$table = 'page';
$fields = array( 'page_id', 'page_namespace', 'page_title', 'page_len',
'page_is_redirect', 'page_latest' );
+
+ if ( $wgContentHandlerUseDB ) {
+ $fields[] = 'page_content_model';
+ }
+
$conds = $this->constructSet( 'page', $dbr );
// Do query
@@ -197,13 +206,14 @@ class LinkBatch {
}
$res = $dbr->select( $table, $fields, $conds, $caller );
wfProfileOut( __METHOD__ );
+
return $res;
}
/**
* Do (and cache) {{GENDER:...}} information for userpages in this LinkBatch
*
- * @return bool whether the query was successful
+ * @return bool Whether the query was successful
*/
public function doGenderQuery() {
if ( $this->isEmpty() ) {
@@ -217,15 +227,16 @@ class LinkBatch {
$genderCache = GenderCache::singleton();
$genderCache->doLinkBatch( $this->data, $this->caller );
+
return true;
}
/**
* Construct a WHERE clause which will match all the given titles.
*
- * @param string $prefix the appropriate table's field name prefix ('page', 'pl', etc)
- * @param $db DatabaseBase object to use
- * @return mixed string with SQL where clause fragment, or false if no items.
+ * @param string $prefix The appropriate table's field name prefix ('page', 'pl', etc)
+ * @param DatabaseBase $db DatabaseBase object to use
+ * @return string|bool String with SQL where clause fragment, or false if no items.
*/
public function constructSet( $prefix, $db ) {
return $db->makeWhereFrom2d( $this->data, "{$prefix}_namespace", "{$prefix}_title" );
diff --git a/includes/cache/LinkCache.php b/includes/cache/LinkCache.php
index 54de1989..6925df90 100644
--- a/includes/cache/LinkCache.php
+++ b/includes/cache/LinkCache.php
@@ -35,7 +35,6 @@ class LinkCache {
private $mGoodLinkFields = array();
private $mBadLinks = array();
private $mForUpdate = false;
- private $useDatabase = true;
/**
* @var LinkCache
@@ -52,6 +51,7 @@ class LinkCache {
return self::$instance;
}
self::$instance = new LinkCache;
+
return self::$instance;
}
@@ -78,6 +78,7 @@ class LinkCache {
/**
* General accessor to get/set whether SELECT FOR UPDATE should be used
*
+ * @param bool $update
* @return bool
*/
public function forUpdate( $update = null ) {
@@ -85,8 +86,8 @@ class LinkCache {
}
/**
- * @param $title
- * @return array|int
+ * @param string $title
+ * @return int
*/
public function getGoodLinkID( $title ) {
if ( array_key_exists( $title, $this->mGoodLinks ) ) {
@@ -99,9 +100,9 @@ class LinkCache {
/**
* Get a field of a title object from cache.
* If this link is not good, it will return NULL.
- * @param $title Title
+ * @param Title $title
* @param string $field ('length','redirect','revision','model')
- * @return mixed
+ * @return string|null
*/
public function getGoodLinkFieldObj( $title, $field ) {
$dbkey = $title->getPrefixedDBkey();
@@ -113,7 +114,7 @@ class LinkCache {
}
/**
- * @param $title
+ * @param string $title
* @return bool
*/
public function isBadLink( $title ) {
@@ -123,28 +124,31 @@ class LinkCache {
/**
* Add a link for the title to the link cache
*
- * @param $id Integer: page's ID
- * @param $title Title object
- * @param $len Integer: text's length
- * @param $redir Integer: whether the page is a redirect
- * @param $revision Integer: latest revision's ID
- * @param $model Integer: latest revision's content model ID
+ * @param int $id Page's ID
+ * @param Title $title
+ * @param int $len Text's length
+ * @param int $redir Whether the page is a redirect
+ * @param int $revision Latest revision's ID
+ * @param string|null $model Latest revision's content model ID
*/
- public function addGoodLinkObj( $id, $title, $len = -1, $redir = null, $revision = false, $model = false ) {
+ public function addGoodLinkObj( $id, $title, $len = -1, $redir = null,
+ $revision = 0, $model = null
+ ) {
$dbkey = $title->getPrefixedDBkey();
- $this->mGoodLinks[$dbkey] = intval( $id );
+ $this->mGoodLinks[$dbkey] = (int)$id;
$this->mGoodLinkFields[$dbkey] = array(
- 'length' => intval( $len ),
- 'redirect' => intval( $redir ),
- 'revision' => intval( $revision ),
- 'model' => intval( $model ) );
+ 'length' => (int)$len,
+ 'redirect' => (int)$redir,
+ 'revision' => (int)$revision,
+ 'model' => $model ? (string)$model : null,
+ );
}
/**
* Same as above with better interface.
* @since 1.19
- * @param $title Title
- * @param $row object which has the fields page_id, page_is_redirect,
+ * @param Title $title
+ * @param stdClass $row Object which has the fields page_id, page_is_redirect,
* page_latest and page_content_model
*/
public function addGoodLinkObjFromRow( $title, $row ) {
@@ -159,7 +163,7 @@ class LinkCache {
}
/**
- * @param $title Title
+ * @param Title $title
*/
public function addBadLinkObj( $title ) {
$dbkey = $title->getPrefixedDBkey();
@@ -173,7 +177,7 @@ class LinkCache {
}
/**
- * @param $title Title
+ * @param Title $title
*/
public function clearLink( $title ) {
$dbkey = $title->getPrefixedDBkey();
@@ -193,8 +197,8 @@ class LinkCache {
/**
* Add a title to the link cache, return the page_id or zero if non-existent
*
- * @param string $title title to add
- * @return Integer
+ * @param string $title Title to add
+ * @return int
*/
public function addLink( $title ) {
$nt = Title::newFromDBkey( $title );
@@ -206,23 +210,10 @@ class LinkCache {
}
/**
- * Enable or disable database use.
- * @since 1.22
- * @param $value Boolean
- * @return Boolean
- */
- public function useDatabase( $value = null ) {
- if ( $value !== null ) {
- $this->useDatabase = (bool)$value;
- }
- return $this->useDatabase;
- }
-
- /**
* Add a title to the link cache, return the page_id or zero if non-existent
*
- * @param $nt Title object to add
- * @return Integer
+ * @param Title $nt Title object to add
+ * @return int
*/
public function addLinkObj( $nt ) {
global $wgAntiLockFlags, $wgContentHandlerUseDB;
@@ -232,20 +223,19 @@ class LinkCache {
$key = $nt->getPrefixedDBkey();
if ( $this->isBadLink( $key ) || $nt->isExternal() ) {
wfProfileOut( __METHOD__ );
+
return 0;
}
$id = $this->getGoodLinkID( $key );
if ( $id != 0 ) {
wfProfileOut( __METHOD__ );
+
return $id;
}
if ( $key === '' ) {
wfProfileOut( __METHOD__ );
- return 0;
- }
- if( !$this->useDatabase ) {
return 0;
}
@@ -280,6 +270,7 @@ class LinkCache {
}
wfProfileOut( __METHOD__ );
+
return $id;
}
diff --git a/includes/cache/LocalisationCache.php b/includes/cache/LocalisationCache.php
index 25a1e196..ae27fba3 100644
--- a/includes/cache/LocalisationCache.php
+++ b/includes/cache/LocalisationCache.php
@@ -20,8 +20,6 @@
* @file
*/
-define( 'MW_LC_VERSION', 2 );
-
/**
* Class for caching the contents of localisation files, Messages*.php
* and *.i18n.php.
@@ -35,20 +33,22 @@ define( 'MW_LC_VERSION', 2 );
* as grammatical transformation, is done by the caller.
*/
class LocalisationCache {
+ const VERSION = 2;
+
/** Configuration associative array */
- var $conf;
+ private $conf;
/**
* True if recaching should only be done on an explicit call to recache().
* Setting this reduces the overhead of cache freshness checking, which
* requires doing a stat() for every extension i18n file.
*/
- var $manualRecache = false;
+ private $manualRecache = false;
/**
* True to treat all files as expired until they are regenerated by this object.
*/
- var $forceRecache = false;
+ private $forceRecache = false;
/**
* The cache data. 3-d array, where the first key is the language code,
@@ -56,14 +56,14 @@ class LocalisationCache {
* an item specific subkey index. Some items are not arrays and so for those
* items, there are no subkeys.
*/
- var $data = array();
+ protected $data = array();
/**
* The persistent store object. An instance of LCStore.
*
* @var LCStore
*/
- var $store;
+ private $store;
/**
* A 2-d associative array, code/key, where presence indicates that the item
@@ -72,32 +72,32 @@ class LocalisationCache {
* For split items, if set, this indicates that all of the subitems have been
* loaded.
*/
- var $loadedItems = array();
+ private $loadedItems = array();
/**
* A 3-d associative array, code/key/subkey, where presence indicates that
* the subitem is loaded. Only used for the split items, i.e. messages.
*/
- var $loadedSubitems = array();
+ private $loadedSubitems = array();
/**
* An array where presence of a key indicates that that language has been
* initialised. Initialisation includes checking for cache expiry and doing
* any necessary updates.
*/
- var $initialisedLangs = array();
+ private $initialisedLangs = array();
/**
* An array mapping non-existent pseudo-languages to fallback languages. This
* is filled by initShallowFallback() when data is requested from a language
* that lacks a Messages*.php file.
*/
- var $shallowFallbacks = array();
+ private $shallowFallbacks = array();
/**
* An array where the keys are codes that have been recached by this instance.
*/
- var $recachedLangs = array();
+ private $recachedLangs = array();
/**
* All item keys
@@ -106,7 +106,7 @@ class LocalisationCache {
'fallback', 'namespaceNames', 'bookstoreList',
'magicWords', 'messages', 'rtl', 'capitalizeAllNouns', 'digitTransformTable',
'separatorTransformTable', 'fallback8bitEncoding', 'linkPrefixExtension',
- 'linkTrail', 'namespaceAliases',
+ 'linkTrail', 'linkPrefixCharset', 'namespaceAliases',
'dateFormats', 'datePreferences', 'datePreferenceMigrationMap',
'defaultDateFormat', 'extraUserToggles', 'specialPageAliases',
'imageFiles', 'preloadedMessages', 'namespaceGenderAliases',
@@ -158,7 +158,7 @@ class LocalisationCache {
* Associative array of cached plural rules. The key is the language code,
* the value is an array of plural rules for that language.
*/
- var $pluralRules = null;
+ private $pluralRules = null;
/**
* Associative array of cached plural rule types. The key is the language
@@ -172,16 +172,16 @@ class LocalisationCache {
* example, {{plural:count|wordform1|wordform2|wordform3}}, rather than
* {{plural:count|one=wordform1|two=wordform2|many=wordform3}}.
*/
- var $pluralRuleTypes = null;
+ private $pluralRuleTypes = null;
- var $mergeableKeys = null;
+ private $mergeableKeys = null;
/**
* Constructor.
* For constructor parameters, see the documentation in DefaultSettings.php
* for $wgLocalisationCacheConf.
*
- * @param $conf Array
+ * @param array $conf
* @throws MWException
*/
function __construct( $conf ) {
@@ -195,16 +195,13 @@ class LocalisationCache {
switch ( $conf['store'] ) {
case 'files':
case 'file':
- $storeClass = 'LCStore_CDB';
+ $storeClass = 'LCStoreCDB';
break;
case 'db':
- $storeClass = 'LCStore_DB';
- break;
- case 'accel':
- $storeClass = 'LCStore_Accel';
+ $storeClass = 'LCStoreDB';
break;
case 'detect':
- $storeClass = $wgCacheDirectory ? 'LCStore_CDB' : 'LCStore_DB';
+ $storeClass = $wgCacheDirectory ? 'LCStoreCDB' : 'LCStoreDB';
break;
default:
throw new MWException(
@@ -212,7 +209,7 @@ class LocalisationCache {
}
}
- wfDebug( get_class( $this ) . ": using store $storeClass\n" );
+ wfDebugLog( 'caches', get_class( $this ) . ": using store $storeClass" );
if ( !empty( $conf['storeDirectory'] ) ) {
$storeConf['directory'] = $conf['storeDirectory'];
}
@@ -228,7 +225,7 @@ class LocalisationCache {
/**
* Returns true if the given key is mergeable, that is, if it is an associative
* array which can be merged through a fallback sequence.
- * @param $key
+ * @param string $key
* @return bool
*/
public function isMergeableKey( $key ) {
@@ -241,6 +238,7 @@ class LocalisationCache {
self::$magicWordKeys
) );
}
+
return isset( $this->mergeableKeys[$key] );
}
@@ -249,8 +247,8 @@ class LocalisationCache {
*
* Warning: this may be slow for split items (messages), since it will
* need to fetch all of the subitems from the cache individually.
- * @param $code
- * @param $key
+ * @param string $code
+ * @param string $key
* @return mixed
*/
public function getItem( $code, $key ) {
@@ -269,14 +267,15 @@ class LocalisationCache {
/**
* Get a subitem, for instance a single message for a given language.
- * @param $code
- * @param $key
- * @param $subkey
- * @return null
+ * @param string $code
+ * @param string $key
+ * @param string $subkey
+ * @return mixed|null
*/
public function getSubitem( $code, $key, $subkey ) {
if ( !isset( $this->loadedSubitems[$code][$key][$subkey] ) &&
- !isset( $this->loadedItems[$code][$key] ) ) {
+ !isset( $this->loadedItems[$code][$key] )
+ ) {
wfProfileIn( __METHOD__ . '-load' );
$this->loadSubitem( $code, $key, $subkey );
wfProfileOut( __METHOD__ . '-load' );
@@ -297,8 +296,8 @@ class LocalisationCache {
*
* Will return null if the item is not found, or false if the item is not an
* array.
- * @param $code
- * @param $key
+ * @param string $code
+ * @param string $key
* @return bool|null|string
*/
public function getSubitemList( $code, $key ) {
@@ -316,8 +315,8 @@ class LocalisationCache {
/**
* Load an item into the cache.
- * @param $code
- * @param $key
+ * @param string $code
+ * @param string $key
*/
protected function loadItem( $code, $key ) {
if ( !isset( $this->initialisedLangs[$code] ) ) {
@@ -331,6 +330,7 @@ class LocalisationCache {
if ( isset( $this->shallowFallbacks[$code] ) ) {
$this->loadItem( $this->shallowFallbacks[$code], $key );
+
return;
}
@@ -351,13 +351,14 @@ class LocalisationCache {
/**
* Load a subitem into the cache
- * @param $code
- * @param $key
- * @param $subkey
+ * @param string $code
+ * @param string $key
+ * @param string $subkey
*/
protected function loadSubitem( $code, $key, $subkey ) {
if ( !in_array( $key, self::$splitKeys ) ) {
$this->loadItem( $code, $key );
+
return;
}
@@ -367,12 +368,14 @@ class LocalisationCache {
// Check to see if initLanguage() loaded it for us
if ( isset( $this->loadedItems[$code][$key] ) ||
- isset( $this->loadedSubitems[$code][$key][$subkey] ) ) {
+ isset( $this->loadedSubitems[$code][$key][$subkey] )
+ ) {
return;
}
if ( isset( $this->shallowFallbacks[$code] ) ) {
$this->loadSubitem( $this->shallowFallbacks[$code], $key, $subkey );
+
return;
}
@@ -391,15 +394,17 @@ class LocalisationCache {
public function isExpired( $code ) {
if ( $this->forceRecache && !isset( $this->recachedLangs[$code] ) ) {
wfDebug( __METHOD__ . "($code): forced reload\n" );
+
return true;
}
$deps = $this->store->get( $code, 'deps' );
$keys = $this->store->get( $code, 'list' );
$preload = $this->store->get( $code, 'preload' );
- // Different keys may expire separately, at least in LCStore_Accel
+ // Different keys may expire separately for some stores
if ( $deps === null || $keys === null || $preload === null ) {
wfDebug( __METHOD__ . "($code): cache missing, need to make one\n" );
+
return true;
}
@@ -411,6 +416,7 @@ class LocalisationCache {
if ( !$dep instanceof CacheDependency || $dep->isExpired() ) {
wfDebug( __METHOD__ . "($code): cache for $code expired due to " .
get_class( $dep ) . "\n" );
+
return true;
}
}
@@ -420,7 +426,7 @@ class LocalisationCache {
/**
* Initialise a language in this object. Rebuild the cache if necessary.
- * @param $code
+ * @param string $code
* @throws MWException
*/
protected function initLanguage( $code ) {
@@ -433,18 +439,20 @@ class LocalisationCache {
# If the code is of the wrong form for a Messages*.php file, do a shallow fallback
if ( !Language::isValidBuiltInCode( $code ) ) {
$this->initShallowFallback( $code, 'en' );
+
return;
}
# Recache the data if necessary
if ( !$this->manualRecache && $this->isExpired( $code ) ) {
- if ( file_exists( Language::getMessagesFileName( $code ) ) ) {
+ if ( Language::isSupportedLanguage( $code ) ) {
$this->recache( $code );
} elseif ( $code === 'en' ) {
throw new MWException( 'MessagesEn.php is missing.' );
} else {
$this->initShallowFallback( $code, 'en' );
}
+
return;
}
@@ -458,6 +466,7 @@ class LocalisationCache {
'Please run maintenance/rebuildLocalisationCache.php.' );
}
$this->initShallowFallback( $code, 'en' );
+
return;
} else {
throw new MWException( 'Invalid or missing localisation cache.' );
@@ -478,8 +487,8 @@ class LocalisationCache {
/**
* Create a fallback from one language to another, without creating a
* complete persistent cache.
- * @param $primaryCode
- * @param $fallbackCode
+ * @param string $primaryCode
+ * @param string $fallbackCode
*/
public function initShallowFallback( $primaryCode, $fallbackCode ) {
$this->data[$primaryCode] =& $this->data[$fallbackCode];
@@ -490,17 +499,23 @@ class LocalisationCache {
/**
* Read a PHP file containing localisation data.
- * @param $_fileName
- * @param $_fileType
+ * @param string $_fileName
+ * @param string $_fileType
* @throws MWException
* @return array
*/
protected function readPHPFile( $_fileName, $_fileType ) {
wfProfileIn( __METHOD__ );
// Disable APC caching
+ wfSuppressWarnings();
$_apcEnabled = ini_set( 'apc.cache_by_default', '0' );
+ wfRestoreWarnings();
+
include $_fileName;
+
+ wfSuppressWarnings();
ini_set( 'apc.cache_by_default', $_apcEnabled );
+ wfRestoreWarnings();
if ( $_fileType == 'core' || $_fileType == 'extension' ) {
$data = compact( self::$allKeys );
@@ -511,12 +526,57 @@ class LocalisationCache {
throw new MWException( __METHOD__ . ": Invalid file type: $_fileType" );
}
wfProfileOut( __METHOD__ );
+
return $data;
}
/**
+ * Read a JSON file containing localisation messages.
+ * @param string $fileName Name of file to read
+ * @throws MWException If there is a syntax error in the JSON file
+ * @return array Array with a 'messages' key, or empty array if the file doesn't exist
+ */
+ public function readJSONFile( $fileName ) {
+ wfProfileIn( __METHOD__ );
+
+ if ( !is_readable( $fileName ) ) {
+ wfProfileOut( __METHOD__ );
+
+ return array();
+ }
+
+ $json = file_get_contents( $fileName );
+ if ( $json === false ) {
+ wfProfileOut( __METHOD__ );
+
+ return array();
+ }
+
+ $data = FormatJson::decode( $json, true );
+ if ( $data === null ) {
+ wfProfileOut( __METHOD__ );
+
+ throw new MWException( __METHOD__ . ": Invalid JSON file: $fileName" );
+ }
+
+ // Remove keys starting with '@', they're reserved for metadata and non-message data
+ foreach ( $data as $key => $unused ) {
+ if ( $key === '' || $key[0] === '@' ) {
+ unset( $data[$key] );
+ }
+ }
+
+ wfProfileOut( __METHOD__ );
+
+ // The JSON format only supports messages, none of the other variables, so wrap the data
+ return array( 'messages' => $data );
+ }
+
+ /**
* Get the compiled plural rules for a given language from the XML files.
* @since 1.20
+ * @param string $code
+ * @return array|null
*/
public function getCompiledPluralRules( $code ) {
$rules = $this->getPluralRules( $code );
@@ -526,9 +586,11 @@ class LocalisationCache {
try {
$compiledRules = CLDRPluralRuleEvaluator::compile( $rules );
} catch ( CLDRPluralRuleError $e ) {
- wfDebugLog( 'l10n', $e->getMessage() . "\n" );
+ wfDebugLog( 'l10n', $e->getMessage() );
+
return array();
}
+
return $compiledRules;
}
@@ -536,6 +598,8 @@ class LocalisationCache {
* Get the plural rules for a given language from the XML files.
* Cached.
* @since 1.20
+ * @param string $code
+ * @return array|null
*/
public function getPluralRules( $code ) {
if ( $this->pluralRules === null ) {
@@ -552,6 +616,8 @@ class LocalisationCache {
* Get the plural rule types for a given language from the XML files.
* Cached.
* @since 1.22
+ * @param string $code
+ * @return array|null
*/
public function getPluralRuleTypes( $code ) {
if ( $this->pluralRuleTypes === null ) {
@@ -582,6 +648,8 @@ class LocalisationCache {
/**
* Load a plural XML file with the given filename, compile the relevant
* rules, and save the compiled rules in a process-local cache.
+ *
+ * @param string $fileName
*/
protected function loadPluralFile( $fileName ) {
$doc = new DOMDocument;
@@ -612,20 +680,24 @@ class LocalisationCache {
* Read the data from the source files for a given language, and register
* the relevant dependencies in the $deps array. If the localisation
* exists, the data array is returned, otherwise false is returned.
+ *
+ * @param string $code
+ * @param array $deps
+ * @return array
*/
protected function readSourceFilesAndRegisterDeps( $code, &$deps ) {
global $IP;
wfProfileIn( __METHOD__ );
+ // This reads in the PHP i18n file with non-messages l10n data
$fileName = Language::getMessagesFileName( $code );
if ( !file_exists( $fileName ) ) {
- wfProfileOut( __METHOD__ );
- return false;
+ $data = array();
+ } else {
+ $deps[] = new FileDependency( $fileName );
+ $data = $this->readPHPFile( $fileName, 'core' );
}
- $deps[] = new FileDependency( $fileName );
- $data = $this->readPHPFile( $fileName, 'core' );
-
# Load CLDR plural rules for JavaScript
$data['pluralRules'] = $this->getPluralRules( $code );
# And for PHP
@@ -637,15 +709,16 @@ class LocalisationCache {
$deps['plurals-mw'] = new FileDependency( "$IP/languages/data/plurals-mediawiki.xml" );
wfProfileOut( __METHOD__ );
+
return $data;
}
/**
* Merge two localisation values, a primary and a fallback, overwriting the
* primary value in place.
- * @param $key
- * @param $value
- * @param $fallbackValue
+ * @param string $key
+ * @param mixed $value
+ * @param mixed $fallbackValue
*/
protected function mergeItem( $key, &$value, $fallbackValue ) {
if ( !is_null( $value ) ) {
@@ -674,8 +747,8 @@ class LocalisationCache {
}
/**
- * @param $value
- * @param $fallbackValue
+ * @param mixed $value
+ * @param mixed $fallbackValue
*/
protected function mergeMagicWords( &$value, $fallbackValue ) {
foreach ( $fallbackValue as $magicName => $fallbackInfo ) {
@@ -698,10 +771,10 @@ class LocalisationCache {
*
* Returns true if any data from the extension array was used, false
* otherwise.
- * @param $codeSequence
- * @param $key
- * @param $value
- * @param $fallbackValue
+ * @param array $codeSequence
+ * @param string $key
+ * @param mixed $value
+ * @param mixed $fallbackValue
* @return bool
*/
protected function mergeExtensionItem( $codeSequence, $key, &$value, $fallbackValue ) {
@@ -719,11 +792,11 @@ class LocalisationCache {
/**
* Load localisation data for a given language for both core and extensions
* and save it to the persistent cache store and the process cache
- * @param $code
+ * @param string $code
* @throws MWException
*/
public function recache( $code ) {
- global $wgExtensionMessagesFiles;
+ global $wgExtensionMessagesFiles, $wgMessagesDirs;
wfProfileIn( __METHOD__ );
if ( !$code ) {
@@ -751,7 +824,6 @@ class LocalisationCache {
foreach ( $data as $key => $value ) {
$this->mergeItem( $key, $coreData[$key], $value );
}
-
}
# Fill in the fallback if it's not there already
@@ -768,43 +840,31 @@ class LocalisationCache {
if ( $coreData['fallbackSequence'][$len - 1] !== 'en' ) {
$coreData['fallbackSequence'][] = 'en';
}
+ }
- # Load the fallback localisation item by item and merge it
- foreach ( $coreData['fallbackSequence'] as $fbCode ) {
- # Load the secondary localisation from the source file to
- # avoid infinite cycles on cyclic fallbacks
- $fbData = $this->readSourceFilesAndRegisterDeps( $fbCode, $deps );
- if ( $fbData === false ) {
- continue;
- }
+ $codeSequence = array_merge( array( $code ), $coreData['fallbackSequence'] );
- foreach ( self::$allKeys as $key ) {
- if ( !isset( $fbData[$key] ) ) {
- continue;
- }
+ wfProfileIn( __METHOD__ . '-fallbacks' );
- if ( is_null( $coreData[$key] ) || $this->isMergeableKey( $key ) ) {
- $this->mergeItem( $key, $coreData[$key], $fbData[$key] );
- }
- }
+ # Load non-JSON localisation data for extensions
+ $extensionData = array_combine(
+ $codeSequence,
+ array_fill( 0, count( $codeSequence ), $initialData ) );
+ foreach ( $wgExtensionMessagesFiles as $extension => $fileName ) {
+ if ( isset( $wgMessagesDirs[$extension] ) ) {
+ # This extension has JSON message data; skip the PHP shim
+ continue;
}
- }
- $codeSequence = array_merge( array( $code ), $coreData['fallbackSequence'] );
-
- # Load the extension localisations
- # This is done after the core because we know the fallback sequence now.
- # But it has a higher precedence for merging so that we can support things
- # like site-specific message overrides.
- wfProfileIn( __METHOD__ . '-extensions' );
- $allData = $initialData;
- foreach ( $wgExtensionMessagesFiles as $fileName ) {
$data = $this->readPHPFile( $fileName, 'extension' );
$used = false;
foreach ( $data as $key => $item ) {
- if ( $this->mergeExtensionItem( $codeSequence, $key, $allData[$key], $item ) ) {
- $used = true;
+ foreach ( $codeSequence as $csCode ) {
+ if ( isset( $item[$csCode] ) ) {
+ $this->mergeItem( $key, $extensionData[$csCode][$key], $item[$csCode] );
+ $used = true;
+ }
}
}
@@ -813,15 +873,81 @@ class LocalisationCache {
}
}
- # Merge core data into extension data
- foreach ( $coreData as $key => $item ) {
- $this->mergeItem( $key, $allData[$key], $item );
+ # Load the localisation data for each fallback, then merge it into the full array
+ $allData = $initialData;
+ foreach ( $codeSequence as $csCode ) {
+ $csData = $initialData;
+
+ # Load core messages and the extension localisations.
+ foreach ( $wgMessagesDirs as $dirs ) {
+ foreach ( (array)$dirs as $dir ) {
+ $fileName = "$dir/$csCode.json";
+ $data = $this->readJSONFile( $fileName );
+
+ foreach ( $data as $key => $item ) {
+ $this->mergeItem( $key, $csData[$key], $item );
+ }
+
+ $deps[] = new FileDependency( $fileName );
+ }
+ }
+
+ # Merge non-JSON extension data
+ if ( isset( $extensionData[$csCode] ) ) {
+ foreach ( $extensionData[$csCode] as $key => $item ) {
+ $this->mergeItem( $key, $csData[$key], $item );
+ }
+ }
+
+ if ( $csCode === $code ) {
+ # Merge core data into extension data
+ foreach ( $coreData as $key => $item ) {
+ $this->mergeItem( $key, $csData[$key], $item );
+ }
+ } else {
+ # Load the secondary localisation from the source file to
+ # avoid infinite cycles on cyclic fallbacks
+ $fbData = $this->readSourceFilesAndRegisterDeps( $csCode, $deps );
+ if ( $fbData !== false ) {
+ # Only merge the keys that make sense to merge
+ foreach ( self::$allKeys as $key ) {
+ if ( !isset( $fbData[$key] ) ) {
+ continue;
+ }
+
+ if ( is_null( $coreData[$key] ) || $this->isMergeableKey( $key ) ) {
+ $this->mergeItem( $key, $csData[$key], $fbData[$key] );
+ }
+ }
+ }
+ }
+
+ # Allow extensions an opportunity to adjust the data for this
+ # fallback
+ wfRunHooks( 'LocalisationCacheRecacheFallback', array( $this, $csCode, &$csData ) );
+
+ # Merge the data for this fallback into the final array
+ if ( $csCode === $code ) {
+ $allData = $csData;
+ } else {
+ foreach ( self::$allKeys as $key ) {
+ if ( !isset( $csData[$key] ) ) {
+ continue;
+ }
+
+ if ( is_null( $allData[$key] ) || $this->isMergeableKey( $key ) ) {
+ $this->mergeItem( $key, $allData[$key], $csData[$key] );
+ }
+ }
+ }
}
- wfProfileOut( __METHOD__ . '-extensions' );
+
+ wfProfileOut( __METHOD__ . '-fallbacks' );
# Add cache dependencies for any referenced globals
$deps['wgExtensionMessagesFiles'] = new GlobalDependency( 'wgExtensionMessagesFiles' );
- $deps['version'] = new ConstantDependency( 'MW_LC_VERSION' );
+ $deps['wgMessagesDirs'] = new GlobalDependency( 'wgMessagesDirs' );
+ $deps['version'] = new ConstantDependency( 'LocalisationCache::VERSION' );
# Add dependencies to the cache entry
$allData['deps'] = $deps;
@@ -854,7 +980,8 @@ class LocalisationCache {
$allData['list'][$key] = array_keys( $allData[$key] );
}
# Run hooks
- wfRunHooks( 'LocalisationCacheRecache', array( $this, $code, &$allData ) );
+ $purgeBlobs = true;
+ wfRunHooks( 'LocalisationCacheRecache', array( $this, $code, &$allData, &$purgeBlobs ) );
if ( is_null( $allData['namespaceNames'] ) ) {
wfProfileOut( __METHOD__ );
@@ -889,8 +1016,8 @@ class LocalisationCache {
# Clear out the MessageBlobStore
# HACK: If using a null (i.e. disabled) storage backend, we
# can't write to the MessageBlobStore either
- if ( !$this->store instanceof LCStore_Null ) {
- MessageBlobStore::clear();
+ if ( $purgeBlobs && !$this->store instanceof LCStoreNull ) {
+ MessageBlobStore::getInstance()->clear();
}
wfProfileOut( __METHOD__ );
@@ -901,7 +1028,7 @@ class LocalisationCache {
*
* The preload item will be loaded automatically, improving performance
* for the commonly-requested items it contains.
- * @param $data
+ * @param array $data
* @return array
*/
protected function buildPreload( $data ) {
@@ -925,7 +1052,7 @@ class LocalisationCache {
/**
* Unload the data for a given language from the object cache.
* Reduces memory usage.
- * @param $code
+ * @param string $code
*/
public function unload( $code ) {
unset( $this->data[$code] );
@@ -954,28 +1081,9 @@ class LocalisationCache {
* Disable the storage backend
*/
public function disableBackend() {
- $this->store = new LCStore_Null;
+ $this->store = new LCStoreNull;
$this->manualRecache = false;
}
-
- /**
- * Return an array with initialised languages.
- *
- * @return array
- */
- public function getInitialisedLanguages() {
- return $this->initialisedLangs;
- }
-
- /**
- * Set initialised languages.
- *
- * @param array $languages Optional array of initialised languages.
- */
- public function setInitialisedLanguages( $languages = array() ) {
- $this->initialisedLangs = $languages;
- }
-
}
/**
@@ -1024,68 +1132,19 @@ interface LCStore {
}
/**
- * LCStore implementation which uses PHP accelerator to store data.
- * This will work if one of XCache, WinCache or APC cacher is configured.
- * (See ObjectCache.php)
- */
-class LCStore_Accel implements LCStore {
- var $currentLang;
- var $keys;
-
- public function __construct() {
- $this->cache = wfGetCache( CACHE_ACCEL );
- }
-
- public function get( $code, $key ) {
- $k = wfMemcKey( 'l10n', $code, 'k', $key );
- $r = $this->cache->get( $k );
- return $r === false ? null : $r;
- }
-
- public function startWrite( $code ) {
- $k = wfMemcKey( 'l10n', $code, 'l' );
- $keys = $this->cache->get( $k );
- if ( $keys ) {
- foreach ( $keys as $k ) {
- $this->cache->delete( $k );
- }
- }
- $this->currentLang = $code;
- $this->keys = array();
- }
-
- public function finishWrite() {
- if ( $this->currentLang ) {
- $k = wfMemcKey( 'l10n', $this->currentLang, 'l' );
- $this->cache->set( $k, array_keys( $this->keys ) );
- }
- $this->currentLang = null;
- $this->keys = array();
- }
-
- public function set( $key, $value ) {
- if ( $this->currentLang ) {
- $k = wfMemcKey( 'l10n', $this->currentLang, 'k', $key );
- $this->keys[$k] = true;
- $this->cache->set( $k, $value );
- }
- }
-}
-
-/**
* LCStore implementation which uses the standard DB functions to store data.
* This will work on any MediaWiki installation.
*/
-class LCStore_DB implements LCStore {
- var $currentLang;
- var $writesDone = false;
+class LCStoreDB implements LCStore {
+ private $currentLang;
+ private $writesDone = false;
- /**
- * @var DatabaseBase
- */
- var $dbw;
- var $batch;
- var $readOnly = false;
+ /** @var DatabaseBase */
+ private $dbw;
+ /** @var array */
+ private $batch = array();
+
+ private $readOnly = false;
public function get( $code, $key ) {
if ( $this->writesDone ) {
@@ -1096,7 +1155,7 @@ class LCStore_DB implements LCStore {
$row = $db->selectRow( 'l10n_cache', array( 'lc_value' ),
array( 'lc_lang' => $code, 'lc_key' => $key ), __METHOD__ );
if ( $row ) {
- return unserialize( $row->lc_value );
+ return unserialize( $db->decodeBlob( $row->lc_value ) );
} else {
return null;
}
@@ -1105,25 +1164,11 @@ class LCStore_DB implements LCStore {
public function startWrite( $code ) {
if ( $this->readOnly ) {
return;
- }
-
- if ( !$code ) {
+ } elseif ( !$code ) {
throw new MWException( __METHOD__ . ": Invalid language \"$code\"" );
}
$this->dbw = wfGetDB( DB_MASTER );
- try {
- $this->dbw->begin( __METHOD__ );
- $this->dbw->delete( 'l10n_cache', array( 'lc_lang' => $code ), __METHOD__ );
- } catch ( DBQueryError $e ) {
- if ( $this->dbw->wasReadOnlyError() ) {
- $this->readOnly = true;
- $this->dbw->rollback( __METHOD__ );
- return;
- } else {
- throw $e;
- }
- }
$this->currentLang = $code;
$this->batch = array();
@@ -1132,37 +1177,42 @@ class LCStore_DB implements LCStore {
public function finishWrite() {
if ( $this->readOnly ) {
return;
+ } elseif ( is_null( $this->currentLang ) ) {
+ throw new MWException( __CLASS__ . ': must call startWrite() before finishWrite()' );
}
- if ( $this->batch ) {
- $this->dbw->insert( 'l10n_cache', $this->batch, __METHOD__ );
+ $this->dbw->begin( __METHOD__ );
+ try {
+ $this->dbw->delete( 'l10n_cache',
+ array( 'lc_lang' => $this->currentLang ), __METHOD__ );
+ foreach ( array_chunk( $this->batch, 500 ) as $rows ) {
+ $this->dbw->insert( 'l10n_cache', $rows, __METHOD__ );
+ }
+ $this->writesDone = true;
+ } catch ( DBQueryError $e ) {
+ if ( $this->dbw->wasReadOnlyError() ) {
+ $this->readOnly = true; // just avoid site down time
+ } else {
+ throw $e;
+ }
}
-
$this->dbw->commit( __METHOD__ );
+
$this->currentLang = null;
- $this->dbw = null;
$this->batch = array();
- $this->writesDone = true;
}
public function set( $key, $value ) {
if ( $this->readOnly ) {
return;
- }
-
- if ( is_null( $this->currentLang ) ) {
- throw new MWException( __CLASS__ . ': must call startWrite() before calling set()' );
+ } elseif ( is_null( $this->currentLang ) ) {
+ throw new MWException( __CLASS__ . ': must call startWrite() before set()' );
}
$this->batch[] = array(
'lc_lang' => $this->currentLang,
'lc_key' => $key,
- 'lc_value' => serialize( $value ) );
-
- if ( count( $this->batch ) >= 100 ) {
- $this->dbw->insert( 'l10n_cache', $this->batch, __METHOD__ );
- $this->batch = array();
- }
+ 'lc_value' => $this->dbw->encodeBlob( serialize( $value ) ) );
}
}
@@ -1178,8 +1228,18 @@ class LCStore_DB implements LCStore {
*
* See Cdb.php and http://cr.yp.to/cdb.html
*/
-class LCStore_CDB implements LCStore {
- var $readers, $writer, $currentLang, $directory;
+class LCStoreCDB implements LCStore {
+ /** @var CdbReader[] */
+ private $readers;
+
+ /** @var CdbWriter */
+ private $writer;
+
+ /** @var string Current language code */
+ private $currentLang;
+
+ /** @var bool|string Cache directory. False if not set */
+ private $directory;
function __construct( $conf = array() ) {
global $wgCacheDirectory;
@@ -1195,21 +1255,30 @@ class LCStore_CDB implements LCStore {
if ( !isset( $this->readers[$code] ) ) {
$fileName = $this->getFileName( $code );
- if ( !file_exists( $fileName ) ) {
- $this->readers[$code] = false;
- } else {
- $this->readers[$code] = CdbReader::open( $fileName );
+ $this->readers[$code] = false;
+ if ( file_exists( $fileName ) ) {
+ try {
+ $this->readers[$code] = CdbReader::open( $fileName );
+ } catch ( CdbException $e ) {
+ wfDebug( __METHOD__ . ": unable to open cdb file for reading\n" );
+ }
}
}
if ( !$this->readers[$code] ) {
return null;
} else {
- $value = $this->readers[$code]->get( $key );
-
+ $value = false;
+ try {
+ $value = $this->readers[$code]->get( $key );
+ } catch ( CdbException $e ) {
+ wfDebug( __METHOD__ . ": CdbException caught, error message was "
+ . $e->getMessage() . "\n" );
+ }
if ( $value === false ) {
return null;
}
+
return unserialize( $value );
}
}
@@ -1227,13 +1296,21 @@ class LCStore_CDB implements LCStore {
$this->readers[$code]->close();
}
- $this->writer = CdbWriter::open( $this->getFileName( $code ) );
+ try {
+ $this->writer = CdbWriter::open( $this->getFileName( $code ) );
+ } catch ( CdbException $e ) {
+ throw new MWException( $e->getMessage() );
+ }
$this->currentLang = $code;
}
public function finishWrite() {
// Close the writer
- $this->writer->close();
+ try {
+ $this->writer->close();
+ } catch ( CdbException $e ) {
+ throw new MWException( $e->getMessage() );
+ }
$this->writer = null;
unset( $this->readers[$this->currentLang] );
$this->currentLang = null;
@@ -1243,13 +1320,18 @@ class LCStore_CDB implements LCStore {
if ( is_null( $this->writer ) ) {
throw new MWException( __CLASS__ . ': must call startWrite() before calling set()' );
}
- $this->writer->set( $key, serialize( $value ) );
+ try {
+ $this->writer->set( $key, serialize( $value ) );
+ } catch ( CdbException $e ) {
+ throw new MWException( $e->getMessage() );
+ }
}
protected function getFileName( $code ) {
if ( strval( $code ) === '' || strpos( $code, '/' ) !== false ) {
throw new MWException( __METHOD__ . ": Invalid language \"$code\"" );
}
+
return "{$this->directory}/l10n_cache-$code.cdb";
}
}
@@ -1257,42 +1339,47 @@ class LCStore_CDB implements LCStore {
/**
* Null store backend, used to avoid DB errors during install
*/
-class LCStore_Null implements LCStore {
+class LCStoreNull implements LCStore {
public function get( $code, $key ) {
return null;
}
- public function startWrite( $code ) {}
- public function finishWrite() {}
- public function set( $key, $value ) {}
+ public function startWrite( $code ) {
+ }
+
+ public function finishWrite() {
+ }
+
+ public function set( $key, $value ) {
+ }
}
/**
* A localisation cache optimised for loading large amounts of data for many
* languages. Used by rebuildLocalisationCache.php.
*/
-class LocalisationCache_BulkLoad extends LocalisationCache {
+class LocalisationCacheBulkLoad extends LocalisationCache {
/**
* A cache of the contents of data files.
* Core files are serialized to avoid using ~1GB of RAM during a recache.
*/
- var $fileCache = array();
+ private $fileCache = array();
/**
* Most recently used languages. Uses the linked-list aspect of PHP hashtables
* to keep the most recently used language codes at the end of the array, and
* the language codes that are ready to be deleted at the beginning.
*/
- var $mruLangs = array();
+ private $mruLangs = array();
/**
* Maximum number of languages that may be loaded into $this->data
*/
- var $maxLoadedLangs = 10;
+ private $maxLoadedLangs = 10;
/**
- * @param $fileName
- * @param $fileType
+ * @param string $fileName
+ * @param string $fileType
* @return array|mixed
*/
protected function readPHPFile( $fileName, $fileType ) {
@@ -1317,30 +1404,32 @@ class LocalisationCache_BulkLoad extends LocalisationCache {
}
/**
- * @param $code
- * @param $key
+ * @param string $code
+ * @param string $key
* @return mixed
*/
public function getItem( $code, $key ) {
unset( $this->mruLangs[$code] );
$this->mruLangs[$code] = true;
+
return parent::getItem( $code, $key );
}
/**
- * @param $code
- * @param $key
- * @param $subkey
- * @return
+ * @param string $code
+ * @param string $key
+ * @param string $subkey
+ * @return mixed
*/
public function getSubitem( $code, $key, $subkey ) {
unset( $this->mruLangs[$code] );
$this->mruLangs[$code] = true;
+
return parent::getSubitem( $code, $key, $subkey );
}
/**
- * @param $code
+ * @param string $code
*/
public function recache( $code ) {
parent::recache( $code );
@@ -1350,7 +1439,7 @@ class LocalisationCache_BulkLoad extends LocalisationCache {
}
/**
- * @param $code
+ * @param string $code
*/
public function unload( $code ) {
unset( $this->mruLangs[$code] );
diff --git a/includes/cache/ProcessCacheLRU.php b/includes/cache/MapCacheLRU.php
index 76c76f37..95e3af76 100644
--- a/includes/cache/ProcessCacheLRU.php
+++ b/includes/cache/MapCacheLRU.php
@@ -22,19 +22,22 @@
*/
/**
- * Handles per process caching of items
+ * Handles a simple LRU key/value map with a maximum number of entries
+ *
+ * Use ProcessCacheLRU if hierarchical purging is needed or objects can become stale
+ *
+ * @see ProcessCacheLRU
* @ingroup Cache
+ * @since 1.23
*/
-class ProcessCacheLRU {
- /** @var Array */
- protected $cache = array(); // (key => prop => value)
- /** @var Array */
- protected $cacheTimes = array(); // (key => prop => UNIX timestamp)
+class MapCacheLRU {
+ /** @var array */
+ protected $cache = array(); // (key => value)
protected $maxCacheKeys; // integer; max entries
/**
- * @param $maxKeys integer Maximum number of entries allowed (min 1).
+ * @param int $maxKeys Maximum number of entries allowed (min 1).
* @throws MWException When $maxCacheKeys is not an int or =< 0.
*/
public function __construct( $maxKeys ) {
@@ -45,56 +48,47 @@ class ProcessCacheLRU {
}
/**
- * Set a property field for a cache entry.
+ * Set a key/value pair.
* This will prune the cache if it gets too large based on LRU.
* If the item is already set, it will be pushed to the top of the cache.
*
- * @param $key string
- * @param $prop string
- * @param $value mixed
+ * @param string $key
+ * @param mixed $value
* @return void
*/
- public function set( $key, $prop, $value ) {
- if ( isset( $this->cache[$key] ) ) {
+ public function set( $key, $value ) {
+ if ( array_key_exists( $key, $this->cache ) ) {
$this->ping( $key ); // push to top
} elseif ( count( $this->cache ) >= $this->maxCacheKeys ) {
reset( $this->cache );
$evictKey = key( $this->cache );
unset( $this->cache[$evictKey] );
- unset( $this->cacheTimes[$evictKey] );
}
- $this->cache[$key][$prop] = $value;
- $this->cacheTimes[$key][$prop] = time();
+ $this->cache[$key] = $value;
}
/**
- * Check if a property field exists for a cache entry.
+ * Check if a key exists
*
- * @param $key string
- * @param $prop string
- * @param $maxAge integer Ignore items older than this many seconds (since 1.21)
+ * @param string $key
* @return bool
*/
- public function has( $key, $prop, $maxAge = 0 ) {
- if ( isset( $this->cache[$key][$prop] ) ) {
- return ( $maxAge <= 0 || ( time() - $this->cacheTimes[$key][$prop] ) <= $maxAge );
- }
- return false;
+ public function has( $key ) {
+ return array_key_exists( $key, $this->cache );
}
/**
- * Get a property field for a cache entry.
- * This returns null if the property is not set.
+ * Get the value for a key.
+ * This returns null if the key is not set.
* If the item is already set, it will be pushed to the top of the cache.
*
- * @param $key string
- * @param $prop string
+ * @param string $key
* @return mixed
*/
- public function get( $key, $prop ) {
- if ( isset( $this->cache[$key][$prop] ) ) {
+ public function get( $key ) {
+ if ( array_key_exists( $key, $this->cache ) ) {
$this->ping( $key ); // push to top
- return $this->cache[$key][$prop];
+ return $this->cache[$key];
} else {
return null;
}
@@ -103,17 +97,15 @@ class ProcessCacheLRU {
/**
* Clear one or several cache entries, or all cache entries
*
- * @param $keys string|Array
+ * @param string|array $keys
* @return void
*/
public function clear( $keys = null ) {
if ( $keys === null ) {
$this->cache = array();
- $this->cacheTimes = array();
} else {
foreach ( (array)$keys as $key ) {
unset( $this->cache[$key] );
- unset( $this->cacheTimes[$key] );
}
}
}
@@ -121,7 +113,7 @@ class ProcessCacheLRU {
/**
* Push an entry to the top of the cache
*
- * @param $key string
+ * @param string $key
*/
protected function ping( $key ) {
$item = $this->cache[$key];
diff --git a/includes/cache/MessageCache.php b/includes/cache/MessageCache.php
index a92c87f4..1ef7cc58 100644
--- a/includes/cache/MessageCache.php
+++ b/includes/cache/MessageCache.php
@@ -110,6 +110,7 @@ class MessageCache {
$wgMsgCacheExpiry
);
}
+
return self::$instance;
}
@@ -123,7 +124,7 @@ class MessageCache {
}
/**
- * @param ObjectCache $memCached A cache instance. If none, fall back to CACHE_NONE.
+ * @param BagOStuff $memCached A cache instance. If none, fall back to CACHE_NONE.
* @param bool $useDB
* @param int $expiry Lifetime for cache. @see $mExpiry.
*/
@@ -147,14 +148,15 @@ class MessageCache {
$this->mParserOptions = new ParserOptions;
$this->mParserOptions->setEditSection( false );
}
+
return $this->mParserOptions;
}
/**
* Try to load the cache from a local file.
*
- * @param string $hash the hash of contents, to check validity.
- * @param Mixed $code Optional language code, see documenation of load().
+ * @param string $hash The hash of contents, to check validity.
+ * @param string $code Optional language code, see documenation of load().
* @return array The cache array
*/
function getLocalCache( $hash, $code ) {
@@ -179,15 +181,20 @@ class MessageCache {
$serialized .= fread( $file, 100000 );
}
fclose( $file );
+
return unserialize( $serialized );
} else {
fclose( $file );
+
return false; // Wrong hash
}
}
/**
* Save the cache to a local file.
+ * @param string $serialized
+ * @param string $hash
+ * @param string $code
*/
function saveToLocal( $serialized, $hash, $code ) {
global $wgCacheDirectory;
@@ -201,6 +208,7 @@ class MessageCache {
if ( !$file ) {
wfDebug( "Unable to open local cache file for writing\n" );
+
return;
}
@@ -227,7 +235,7 @@ class MessageCache {
* or false if populating empty cache fails. Also returns true if MessageCache
* is disabled.
*
- * @param bool|String $code Language to which load messages
+ * @param bool|string $code Language to which load messages
* @throws MWException
* @return bool
*/
@@ -253,6 +261,7 @@ class MessageCache {
wfDebug( __METHOD__ . ": disabled\n" );
$shownDisabled = true;
}
+
return true;
}
@@ -415,6 +424,7 @@ class MessageCache {
$info = implode( ', ', $where );
wfDebug( __METHOD__ . ": Loading $code... $info\n" );
wfProfileOut( __METHOD__ );
+
return $success;
}
@@ -502,6 +512,7 @@ class MessageCache {
$cache['VERSION'] = MSG_CACHE_VERSION;
$cache['EXPIRY'] = wfTimestamp( TS_MW, time() + $this->mExpiry );
wfProfileOut( __METHOD__ );
+
return $cache;
}
@@ -517,6 +528,7 @@ class MessageCache {
if ( $this->mDisable ) {
wfProfileOut( __METHOD__ );
+
return;
}
@@ -561,7 +573,7 @@ class MessageCache {
// Update the message in the message blob store
global $wgContLang;
- MessageBlobStore::updateMessage( $wgContLang->lcfirst( $msg ) );
+ MessageBlobStore::getInstance()->updateMessage( $wgContLang->lcfirst( $msg ) );
wfRunHooks( 'MessageCacheReplace', array( $title, $text ) );
@@ -571,7 +583,7 @@ class MessageCache {
/**
* Is the given cache array expired due to time passing or a version change?
*
- * @param $cache
+ * @param array $cache
* @return bool
*/
protected function isCacheExpired( $cache ) {
@@ -584,6 +596,7 @@ class MessageCache {
if ( wfTimestampNow() >= $cache['EXPIRY'] ) {
return true;
}
+
return false;
}
@@ -617,6 +630,7 @@ class MessageCache {
}
wfProfileOut( __METHOD__ );
+
return $success;
}
@@ -627,7 +641,7 @@ class MessageCache {
* a timeout of MessageCache::MSG_LOCK_TIMEOUT.
*
* @param string $key
- * @return Boolean: success
+ * @return bool Success
*/
function lock( $key ) {
$lockKey = $key . ':lock';
@@ -676,20 +690,20 @@ class MessageCache {
* * Fallbacks will be just that: fallbacks. A fallback language will never be reached if
* the message is available *anywhere* in the language for which it is a fallback.
*
- * @param string $key the message key
+ * @param string $key The message key
* @param bool $useDB If true, look for the message in the DB, false
- * to use only the compiled l10n cache.
+ * to use only the compiled l10n cache.
* @param bool|string|object $langcode Code of the language to get the message for.
- * - If string and a valid code, will create a standard language object
- * - If string but not a valid code, will create a basic language object
- * - If boolean and false, create object from the current users language
- * - If boolean and true, create object from the wikis content language
- * - If language object, use it as given
- * @param bool $isFullKey specifies whether $key is a two part key
- * "msg/lang".
+ * - If string and a valid code, will create a standard language object
+ * - If string but not a valid code, will create a basic language object
+ * - If boolean and false, create object from the current users language
+ * - If boolean and true, create object from the wikis content language
+ * - If language object, use it as given
+ * @param bool $isFullKey Specifies whether $key is a two part key "msg/lang".
*
- * @throws MWException when given an invalid key
- * @return string|bool False if the message doesn't exist, otherwise the message (which can be empty)
+ * @throws MWException When given an invalid key
+ * @return string|bool False if the message doesn't exist, otherwise the
+ * message (which can be empty)
*/
function get( $key, $useDB = true, $langcode = true, $isFullKey = false ) {
global $wgContLang;
@@ -716,17 +730,28 @@ class MessageCache {
// Normalise title-case input (with some inlining)
$lckey = strtr( $key, ' ', '_' );
- if ( ord( $key ) < 128 ) {
+ if ( ord( $lckey ) < 128 ) {
$lckey[0] = strtolower( $lckey[0] );
- $uckey = ucfirst( $lckey );
} else {
$lckey = $wgContLang->lcfirst( $lckey );
+ }
+
+ wfRunHooks( 'MessageCache::get', array( &$lckey ) );
+
+ if ( ord( $lckey ) < 128 ) {
+ $uckey = ucfirst( $lckey );
+ } else {
$uckey = $wgContLang->ucfirst( $lckey );
}
// Loop through each language in the fallback list until we find something useful
$lang = wfGetLangObj( $langcode );
- $message = $this->getMessageFromFallbackChain( $lang, $lckey, $uckey, !$this->mDisable && $useDB );
+ $message = $this->getMessageFromFallbackChain(
+ $lang,
+ $lckey,
+ $uckey,
+ !$this->mDisable && $useDB
+ );
// If we still have no message, maybe the key was in fact a full key so try that
if ( $message === false ) {
@@ -804,7 +829,8 @@ class MessageCache {
return $message;
}
- list( $fallbackChain, $siteFallbackChain ) = Language::getFallbacksIncludingSiteLanguage( $langcode );
+ list( $fallbackChain, $siteFallbackChain ) =
+ Language::getFallbacksIncludingSiteLanguage( $langcode );
// Next try checking the database for all of the fallback languages of the requested language.
if ( $useDB ) {
@@ -897,11 +923,13 @@ class MessageCache {
if ( $entry ) {
if ( substr( $entry, 0, 1 ) === ' ' ) {
$this->mCache[$code][$title] = $entry;
+
// The message exists, so make sure a string
// is returned.
return (string)substr( $entry, 1 );
} elseif ( $entry === '!NONEXISTENT' ) {
$this->mCache[$code][$title] = '!NONEXISTENT';
+
return false;
} else {
# Corrupt/obsolete entry, delete it
@@ -983,6 +1011,7 @@ class MessageCache {
$this->mInParser = false;
$popts->setUserLang( $userlang );
}
+
return $message;
}
@@ -996,13 +1025,14 @@ class MessageCache {
$wgParser->firstCallInit();
# Clone it and store it
$class = $wgParserConf['class'];
- if ( $class == 'Parser_DiffTest' ) {
+ if ( $class == 'ParserDiffTest' ) {
# Uncloneable
$this->mParser = new $class( $wgParserConf );
} else {
$this->mParser = clone $wgParser;
}
}
+
return $this->mParser;
}
@@ -1043,6 +1073,7 @@ class MessageCache {
$this->mInParser = false;
wfProfileOut( __METHOD__ );
+
return $res;
}
@@ -1069,7 +1100,7 @@ class MessageCache {
}
/**
- * @param $key
+ * @param string $key
* @return array
*/
public function figureMessage( $key ) {
@@ -1085,6 +1116,7 @@ class MessageCache {
}
$message = implode( '/', $pieces );
+
return array( $message, $lang );
}
@@ -1094,7 +1126,7 @@ class MessageCache {
* for which MediaWiki:msgkey exists. If $code is another language code, this
* will ONLY return message keys for which MediaWiki:msgkey/$code exists.
* @param string $code Language code
- * @return array of message keys (strings)
+ * @return array Array of message keys (strings)
*/
public function getAllMessageKeys( $code ) {
global $wgContLang;
@@ -1109,6 +1141,7 @@ class MessageCache {
unset( $cache['EXPIRY'] );
// Remove any !NONEXISTENT keys
$cache = array_diff( $cache, array( '!NONEXISTENT' ) );
+
// Keys may appear with a capital first letter. lcfirst them.
return array_map( array( $wgContLang, 'lcfirst' ), array_keys( $cache ) );
}
diff --git a/includes/cache/ObjectFileCache.php b/includes/cache/ObjectFileCache.php
index ed1e49a6..c7ef0443 100644
--- a/includes/cache/ObjectFileCache.php
+++ b/includes/cache/ObjectFileCache.php
@@ -29,8 +29,8 @@
class ObjectFileCache extends FileCacheBase {
/**
* Construct an ObjectFileCache from a key and a type
- * @param $key string
- * @param $type string
+ * @param string $key
+ * @param string $type
* @return ObjectFileCache
*/
public static function newFromKey( $key, $type ) {
diff --git a/includes/cache/ResourceFileCache.php b/includes/cache/ResourceFileCache.php
index 2ad7b853..55da52c5 100644
--- a/includes/cache/ResourceFileCache.php
+++ b/includes/cache/ResourceFileCache.php
@@ -34,7 +34,7 @@ class ResourceFileCache extends FileCacheBase {
/**
* Construct an ResourceFileCache from a context
- * @param $context ResourceLoaderContext
+ * @param ResourceLoaderContext $context
* @return ResourceFileCache
*/
public static function newFromContext( ResourceLoaderContext $context ) {
@@ -58,7 +58,7 @@ class ResourceFileCache extends FileCacheBase {
/**
* Check if an RL request can be cached.
* Caller is responsible for checking if any modules are private.
- * @param $context ResourceLoaderContext
+ * @param ResourceLoaderContext $context
* @return bool
*/
public static function useFileCache( ResourceLoaderContext $context ) {
@@ -80,8 +80,10 @@ class ResourceFileCache extends FileCacheBase {
} elseif ( $query === 'debug' && $val === 'false' ) {
continue;
}
+
return false;
}
+
return true; // cacheable
}
@@ -104,6 +106,7 @@ class ResourceFileCache extends FileCacheBase {
$this->getMissesRecent() >= self::MISS_THRESHOLD // many misses
);
}
+
return $this->mCacheWorthy;
}
}
diff --git a/includes/cache/SquidUpdate.php b/includes/cache/SquidUpdate.php
deleted file mode 100644
index 71afeba9..00000000
--- a/includes/cache/SquidUpdate.php
+++ /dev/null
@@ -1,300 +0,0 @@
-<?php
-/**
- * Squid cache purging.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
- * http://www.gnu.org/copyleft/gpl.html
- *
- * @file
- * @ingroup Cache
- */
-
-/**
- * Handles purging appropriate Squid URLs given a title (or titles)
- * @ingroup Cache
- */
-class SquidUpdate {
-
- /**
- * Collection of URLs to purge.
- * @var array
- */
- protected $urlArr;
-
- /**
- * @param array $urlArr Collection of URLs to purge
- * @param bool|int $maxTitles Maximum number of unique URLs to purge
- */
- public function __construct( $urlArr = array(), $maxTitles = false ) {
- global $wgMaxSquidPurgeTitles;
- if ( $maxTitles === false ) {
- $maxTitles = $wgMaxSquidPurgeTitles;
- }
-
- // Remove duplicate URLs from list
- $urlArr = array_unique( $urlArr );
- if ( count( $urlArr ) > $maxTitles ) {
- // Truncate to desired maximum URL count
- $urlArr = array_slice( $urlArr, 0, $maxTitles );
- }
- $this->urlArr = $urlArr;
- }
-
- /**
- * Create a SquidUpdate from the given Title object.
- *
- * The resulting SquidUpdate will purge the given Title's URLs as well as
- * the pages that link to it. Capped at $wgMaxSquidPurgeTitles total URLs.
- *
- * @param Title $title
- * @return SquidUpdate
- */
- public static function newFromLinksTo( Title $title ) {
- global $wgMaxSquidPurgeTitles;
- wfProfileIn( __METHOD__ );
-
- # Get a list of URLs linking to this page
- $dbr = wfGetDB( DB_SLAVE );
- $res = $dbr->select( array( 'links', 'page' ),
- array( 'page_namespace', 'page_title' ),
- array(
- 'pl_namespace' => $title->getNamespace(),
- 'pl_title' => $title->getDBkey(),
- 'pl_from=page_id' ),
- __METHOD__ );
- $blurlArr = $title->getSquidURLs();
- if ( $res->numRows() <= $wgMaxSquidPurgeTitles ) {
- foreach ( $res as $BL ) {
- $tobj = Title::makeTitle( $BL->page_namespace, $BL->page_title );
- $blurlArr[] = $tobj->getInternalURL();
- }
- }
-
- wfProfileOut( __METHOD__ );
- return new SquidUpdate( $blurlArr );
- }
-
- /**
- * Create a SquidUpdate from an array of Title objects, or a TitleArray object
- *
- * @param array $titles
- * @param array $urlArr
- * @return SquidUpdate
- */
- public static function newFromTitles( $titles, $urlArr = array() ) {
- global $wgMaxSquidPurgeTitles;
- $i = 0;
- foreach ( $titles as $title ) {
- $urlArr[] = $title->getInternalURL();
- if ( $i++ > $wgMaxSquidPurgeTitles ) {
- break;
- }
- }
- return new SquidUpdate( $urlArr );
- }
-
- /**
- * @param Title $title
- * @return SquidUpdate
- */
- public static function newSimplePurge( Title $title ) {
- $urlArr = $title->getSquidURLs();
- return new SquidUpdate( $urlArr );
- }
-
- /**
- * Purges the list of URLs passed to the constructor.
- */
- public function doUpdate() {
- self::purge( $this->urlArr );
- }
-
- /**
- * Purges a list of Squids defined in $wgSquidServers.
- * $urlArr should contain the full URLs to purge as values
- * (example: $urlArr[] = 'http://my.host/something')
- * XXX report broken Squids per mail or log
- *
- * @param array $urlArr List of full URLs to purge
- */
- public static function purge( $urlArr ) {
- global $wgSquidServers, $wgHTCPRouting;
-
- if ( !$urlArr ) {
- return;
- }
-
- wfDebugLog( 'squid', __METHOD__ . ': ' . implode( ' ', $urlArr ) . "\n" );
-
- if ( $wgHTCPRouting ) {
- self::HTCPPurge( $urlArr );
- }
-
- wfProfileIn( __METHOD__ );
-
- // Remove duplicate URLs
- $urlArr = array_unique( $urlArr );
- // Maximum number of parallel connections per squid
- $maxSocketsPerSquid = 8;
- // Number of requests to send per socket
- // 400 seems to be a good tradeoff, opening a socket takes a while
- $urlsPerSocket = 400;
- $socketsPerSquid = ceil( count( $urlArr ) / $urlsPerSocket );
- if ( $socketsPerSquid > $maxSocketsPerSquid ) {
- $socketsPerSquid = $maxSocketsPerSquid;
- }
-
- $pool = new SquidPurgeClientPool;
- $chunks = array_chunk( $urlArr, ceil( count( $urlArr ) / $socketsPerSquid ) );
- foreach ( $wgSquidServers as $server ) {
- foreach ( $chunks as $chunk ) {
- $client = new SquidPurgeClient( $server );
- foreach ( $chunk as $url ) {
- $client->queuePurge( $url );
- }
- $pool->addClient( $client );
- }
- }
- $pool->run();
-
- wfProfileOut( __METHOD__ );
- }
-
- /**
- * Send Hyper Text Caching Protocol (HTCP) CLR requests.
- *
- * @throws MWException
- * @param array $urlArr Collection of URLs to purge
- */
- public static function HTCPPurge( $urlArr ) {
- global $wgHTCPRouting, $wgHTCPMulticastTTL;
- wfProfileIn( __METHOD__ );
-
- // HTCP CLR operation
- $htcpOpCLR = 4;
-
- // @todo FIXME: PHP doesn't support these socket constants (include/linux/in.h)
- if ( !defined( "IPPROTO_IP" ) ) {
- define( "IPPROTO_IP", 0 );
- define( "IP_MULTICAST_LOOP", 34 );
- define( "IP_MULTICAST_TTL", 33 );
- }
-
- // pfsockopen doesn't work because we need set_sock_opt
- $conn = socket_create( AF_INET, SOCK_DGRAM, SOL_UDP );
- if ( ! $conn ) {
- $errstr = socket_strerror( socket_last_error() );
- wfDebugLog( 'squid', __METHOD__ .
- ": Error opening UDP socket: $errstr\n" );
- wfProfileOut( __METHOD__ );
- return;
- }
-
- // Set socket options
- socket_set_option( $conn, IPPROTO_IP, IP_MULTICAST_LOOP, 0 );
- if ( $wgHTCPMulticastTTL != 1 ) {
- // Set multicast time to live (hop count) option on socket
- socket_set_option( $conn, IPPROTO_IP, IP_MULTICAST_TTL,
- $wgHTCPMulticastTTL );
- }
-
- // Remove duplicate URLs from collection
- $urlArr = array_unique( $urlArr );
- foreach ( $urlArr as $url ) {
- if ( !is_string( $url ) ) {
- wfProfileOut( __METHOD__ );
- throw new MWException( 'Bad purge URL' );
- }
- $url = self::expand( $url );
- $conf = self::getRuleForURL( $url, $wgHTCPRouting );
- if ( !$conf ) {
- wfDebugLog( 'squid', __METHOD__ .
- "No HTCP rule configured for URL {$url} , skipping\n" );
- continue;
- }
-
- if ( isset( $conf['host'] ) && isset( $conf['port'] ) ) {
- // Normalize single entries
- $conf = array( $conf );
- }
- foreach ( $conf as $subconf ) {
- if ( !isset( $subconf['host'] ) || !isset( $subconf['port'] ) ) {
- wfProfileOut( __METHOD__ );
- throw new MWException( "Invalid HTCP rule for URL $url\n" );
- }
- }
-
- // Construct a minimal HTCP request diagram
- // as per RFC 2756
- // Opcode 'CLR', no response desired, no auth
- $htcpTransID = rand();
-
- $htcpSpecifier = pack( 'na4na*na8n',
- 4, 'HEAD', strlen( $url ), $url,
- 8, 'HTTP/1.0', 0 );
-
- $htcpDataLen = 8 + 2 + strlen( $htcpSpecifier );
- $htcpLen = 4 + $htcpDataLen + 2;
-
- // Note! Squid gets the bit order of the first
- // word wrong, wrt the RFC. Apparently no other
- // implementation exists, so adapt to Squid
- $htcpPacket = pack( 'nxxnCxNxxa*n',
- $htcpLen, $htcpDataLen, $htcpOpCLR,
- $htcpTransID, $htcpSpecifier, 2 );
-
- wfDebugLog( 'squid', __METHOD__ .
- "Purging URL $url via HTCP\n" );
- foreach ( $conf as $subconf ) {
- socket_sendto( $conn, $htcpPacket, $htcpLen, 0,
- $subconf['host'], $subconf['port'] );
- }
- }
- wfProfileOut( __METHOD__ );
- }
-
- /**
- * Expand local URLs to fully-qualified URLs using the internal protocol
- * and host defined in $wgInternalServer. Input that's already fully-
- * qualified will be passed through unchanged.
- *
- * This is used to generate purge URLs that may be either local to the
- * main wiki or include a non-native host, such as images hosted on a
- * second internal server.
- *
- * Client functions should not need to call this.
- *
- * @param string $url
- * @return string
- */
- public static function expand( $url ) {
- return wfExpandUrl( $url, PROTO_INTERNAL );
- }
-
- /**
- * Find the HTCP routing rule to use for a given URL.
- * @param string $url URL to match
- * @param array $rules Array of rules, see $wgHTCPRouting for format and behavior
- * @return mixed Element of $rules that matched, or false if nothing matched
- */
- private static function getRuleForURL( $url, $rules ) {
- foreach ( $rules as $regex => $routing ) {
- if ( $regex === '' || preg_match( $regex, $url ) ) {
- return $routing;
- }
- }
- return false;
- }
-}
diff --git a/includes/cache/UserCache.php b/includes/cache/UserCache.php
index 6085f586..7f36f5a6 100644
--- a/includes/cache/UserCache.php
+++ b/includes/cache/UserCache.php
@@ -36,23 +36,26 @@ class UserCache {
if ( $instance === null ) {
$instance = new self();
}
+
return $instance;
}
- protected function __construct() {}
+ protected function __construct() {
+ }
/**
* Get a property of a user based on their user ID
*
- * @param $userId integer User ID
+ * @param int $userId User ID
* @param string $prop User property
- * @return mixed The property or false if the user does not exist
+ * @return mixed|bool The property or false if the user does not exist
*/
public function getProp( $userId, $prop ) {
if ( !isset( $this->cache[$userId][$prop] ) ) {
wfDebug( __METHOD__ . ": querying DB for prop '$prop' for user ID '$userId'.\n" );
$this->doQuery( array( $userId ) ); // cache miss
}
+
return isset( $this->cache[$userId][$prop] )
? $this->cache[$userId][$prop]
: false; // user does not exist?
@@ -61,8 +64,9 @@ class UserCache {
/**
* Get the name of a user or return $ip if the user ID is 0
*
- * @param integer $userId
+ * @param int $userId
* @param string $ip
+ * @return string
* @since 1.22
*/
public function getUserName( $userId, $ip ) {
@@ -73,7 +77,7 @@ class UserCache {
* Preloads user names for given list of users.
* @param array $userIds List of user IDs
* @param array $options Option flags; include 'userpage' and 'usertalk'
- * @param string $caller the calling method
+ * @param string $caller The calling method
*/
public function doQuery( array $userIds, $options = array(), $caller = '' ) {
wfProfileIn( __METHOD__ );
@@ -136,7 +140,7 @@ class UserCache {
/**
* Check if a cache type is in $options and was not loaded for this user
*
- * @param $uid integer user ID
+ * @param int $uid User ID
* @param string $type Cache type
* @param array $options Requested cache types
* @return bool
diff --git a/includes/cache/bloom/BloomCache.php b/includes/cache/bloom/BloomCache.php
new file mode 100644
index 00000000..236db954
--- /dev/null
+++ b/includes/cache/bloom/BloomCache.php
@@ -0,0 +1,323 @@
+<?php
+/**
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ * http://www.gnu.org/copyleft/gpl.html
+ *
+ * @file
+ * @author Aaron Schulz
+ */
+
+/**
+ * Persistent bloom filter used to avoid expensive lookups
+ *
+ * @since 1.24
+ */
+abstract class BloomCache {
+ /** @var string Unique ID for key namespacing */
+ protected $cacheID;
+
+ /** @var array Map of (id => BloomCache) */
+ protected static $instances = array();
+
+ /**
+ * @param string $id
+ * @return BloomCache
+ */
+ final public static function get( $id ) {
+ global $wgBloomFilterStores;
+
+ if ( !isset( self::$instances[$id] ) ) {
+ if ( isset( $wgBloomFilterStores[$id] ) ) {
+ $class = $wgBloomFilterStores[$id]['class'];
+ self::$instances[$id] = new $class( $wgBloomFilterStores[$id] );
+ } else {
+ wfDebug( "No bloom filter store '$id'; using EmptyBloomCache." );
+ return new EmptyBloomCache( array() );
+ }
+ }
+
+ return self::$instances[$id];
+ }
+
+ /**
+ * Create a new bloom cache instance from configuration.
+ * This should only be called from within BloomCache.
+ *
+ * @param array $config Parameters include:
+ * - cacheID : Prefix to all bloom filter names that is unique to this cache.
+ * It should only consist of alphanumberic, '-', and '_' characters.
+ * This ID is what avoids collisions if multiple logical caches
+ * use the same storage system, so this should be set carefully.
+ */
+ public function __construct( array $config ) {
+ $this->cacheID = $config['cacheId'];
+ if ( !preg_match( '!^[a-zA-Z0-9-_]{1,32}$!', $this->cacheID ) ) {
+ throw new MWException( "Cache ID '{$this->cacheID}' is invalid." );
+ }
+ }
+
+ /**
+ * Check if a member is set in the bloom filter
+ *
+ * A member being set means that it *might* have been added.
+ * A member not being set means it *could not* have been added.
+ *
+ * This abstracts over isHit() to deal with filter updates and readiness.
+ * A class must exist with the name BloomFilter<type> and a static public
+ * mergeAndCheck() method. The later takes the following arguments:
+ * (BloomCache $bcache, $domain, $virtualKey, array $status)
+ * The method should return a bool indicating whether to use the filter.
+ *
+ * The 'shared' bloom key must be used for any updates and will be used
+ * for the membership check if the method returns true. Since the key is shared,
+ * the method should never use delete(). The filter cannot be used in cases where
+ * membership in the filter needs to be taken away. In such cases, code *cannot*
+ * use this method - instead, it can directly use the other BloomCache methods
+ * to manage custom filters with their own keys (e.g. not 'shared').
+ *
+ * @param string $domain
+ * @param string $type
+ * @param string $member
+ * @return bool True if set, false if not (also returns true on error)
+ */
+ final public function check( $domain, $type, $member ) {
+ $section = new ProfileSection( get_class( $this ) . '::' . __FUNCTION__ );
+
+ if ( method_exists( "BloomFilter{$type}", 'mergeAndCheck' ) ) {
+ try {
+ $virtualKey = "$domain:$type";
+
+ $status = $this->getStatus( $virtualKey );
+ if ( $status == false ) {
+ wfDebug( "Could not query virtual bloom filter '$virtualKey'." );
+ return null;
+ }
+
+ $useFilter = call_user_func_array(
+ array( "BloomFilter{$type}", 'mergeAndCheck' ),
+ array( $this, $domain, $virtualKey, $status )
+ );
+
+ if ( $useFilter ) {
+ return ( $this->isHit( 'shared', "$virtualKey:$member" ) !== false );
+ }
+ } catch ( MWException $e ) {
+ MWExceptionHandler::logException( $e );
+ return true;
+ }
+ }
+
+ return true;
+ }
+
+ /**
+ * Inform the bloom filter of a new member in order to keep it up to date
+ *
+ * @param string $domain
+ * @param string $type
+ * @param string|array $members
+ * @return bool Success
+ */
+ final public function insert( $domain, $type, $members ) {
+ $section = new ProfileSection( get_class( $this ) . '::' . __FUNCTION__ );
+
+ if ( method_exists( "BloomFilter{$type}", 'mergeAndCheck' ) ) {
+ try {
+ $virtualKey = "$domain:$type";
+ $prefixedMembers = array();
+ foreach ( (array)$members as $member ) {
+ $prefixedMembers[] = "$virtualKey:$member";
+ }
+
+ return $this->add( 'shared', $prefixedMembers );
+ } catch ( MWException $e ) {
+ MWExceptionHandler::logException( $e );
+ return false;
+ }
+ }
+
+ return true;
+ }
+
+ /**
+ * Create a new bloom filter at $key (if one does not exist yet)
+ *
+ * @param string $key
+ * @param integer $size Bit length [default: 1000000]
+ * @param float $precision [default: .001]
+ * @return bool Success
+ */
+ final public function init( $key, $size = 1000000, $precision = .001 ) {
+ $section = new ProfileSection( get_class( $this ) . '::' . __FUNCTION__ );
+
+ return $this->doInit( "{$this->cacheID}:$key", $size, min( .1, $precision ) );
+ }
+
+ /**
+ * Add a member to the bloom filter at $key
+ *
+ * @param string $key
+ * @param string|array $members
+ * @return bool Success
+ */
+ final public function add( $key, $members ) {
+ $section = new ProfileSection( get_class( $this ) . '::' . __FUNCTION__ );
+
+ return $this->doAdd( "{$this->cacheID}:$key", (array)$members );
+ }
+
+ /**
+ * Check if a member is set in the bloom filter.
+ *
+ * A member being set means that it *might* have been added.
+ * A member not being set means it *could not* have been added.
+ *
+ * If this returns true, then the caller usually should do the
+ * expensive check (whatever that may be). It can be avoided otherwise.
+ *
+ * @param string $key
+ * @param string $member
+ * @return bool|null True if set, false if not, null on error
+ */
+ final public function isHit( $key, $member ) {
+ $section = new ProfileSection( get_class( $this ) . '::' . __FUNCTION__ );
+
+ return $this->doIsHit( "{$this->cacheID}:$key", $member );
+ }
+
+ /**
+ * Destroy a bloom filter at $key
+ *
+ * @param string $key
+ * @return bool Success
+ */
+ final public function delete( $key ) {
+ $section = new ProfileSection( get_class( $this ) . '::' . __FUNCTION__ );
+
+ return $this->doDelete( "{$this->cacheID}:$key" );
+ }
+
+ /**
+ * Set the status map of the virtual bloom filter at $key
+ *
+ * @param string $virtualKey
+ * @param array $values Map including some of (lastID, asOfTime, epoch)
+ * @return bool Success
+ */
+ final public function setStatus( $virtualKey, array $values ) {
+ $section = new ProfileSection( get_class( $this ) . '::' . __FUNCTION__ );
+
+ return $this->doSetStatus( "{$this->cacheID}:$virtualKey", $values );
+ }
+
+ /**
+ * Get the status map of the virtual bloom filter at $key
+ *
+ * The map includes:
+ * - lastID : the highest ID of the items merged in
+ * - asOfTime : UNIX timestamp that the filter is up-to-date as of
+ * - epoch : UNIX timestamp that filter started being populated
+ * Unset fields will have a null value.
+ *
+ * @param string $virtualKey
+ * @return array|bool False on failure
+ */
+ final public function getStatus( $virtualKey ) {
+ $section = new ProfileSection( get_class( $this ) . '::' . __FUNCTION__ );
+
+ return $this->doGetStatus( "{$this->cacheID}:$virtualKey" );
+ }
+
+ /**
+ * Get an exclusive lock on a filter for updates
+ *
+ * @param string $virtualKey
+ * @return ScopedCallback|ScopedLock|null Returns null if acquisition failed
+ */
+ public function getScopedLock( $virtualKey ) {
+ return null;
+ }
+
+ /**
+ * @param string $key
+ * @param integer $size Bit length
+ * @param float $precision
+ * @return bool Success
+ */
+ abstract protected function doInit( $key, $size, $precision );
+
+ /**
+ * @param string $key
+ * @param array $members
+ * @return bool Success
+ */
+ abstract protected function doAdd( $key, array $members );
+
+ /**
+ * @param string $key
+ * @param string $member
+ * @return bool|null
+ */
+ abstract protected function doIsHit( $key, $member );
+
+ /**
+ * @param string $key
+ * @return bool Success
+ */
+ abstract protected function doDelete( $key );
+
+ /**
+ * @param string $virtualKey
+ * @param array $values
+ * @return bool Success
+ */
+ abstract protected function doSetStatus( $virtualKey, array $values );
+
+ /**
+ * @param string $key
+ * @return array|bool
+ */
+ abstract protected function doGetStatus( $key );
+}
+
+class EmptyBloomCache extends BloomCache {
+ public function __construct( array $config ) {
+ parent::__construct( array( 'cacheId' => 'none' ) );
+ }
+
+ protected function doInit( $key, $size, $precision ) {
+ return true;
+ }
+
+ protected function doAdd( $key, array $members ) {
+ return true;
+ }
+
+ protected function doIsHit( $key, $member ) {
+ return true;
+ }
+
+ protected function doDelete( $key ) {
+ return true;
+ }
+
+ protected function doSetStatus( $virtualKey, array $values ) {
+ return true;
+ }
+
+ protected function doGetStatus( $virtualKey ) {
+ return array( 'lastID' => null, 'asOfTime' => null, 'epoch' => null ) ;
+ }
+}
diff --git a/includes/cache/bloom/BloomCacheRedis.php b/includes/cache/bloom/BloomCacheRedis.php
new file mode 100644
index 00000000..212e5e8b
--- /dev/null
+++ b/includes/cache/bloom/BloomCacheRedis.php
@@ -0,0 +1,370 @@
+<?php
+/**
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ * http://www.gnu.org/copyleft/gpl.html
+ *
+ * @file
+ * @author Aaron Schulz
+ */
+
+/**
+ * Bloom filter implemented using Redis
+ *
+ * The Redis server must be >= 2.6 and should have volatile-lru or volatile-ttl
+ * if there is any eviction policy. It should not be allkeys-* in any case. Also,
+ * this can be used in a simple master/slave setup or with Redis Sentinel preferably.
+ *
+ * Some bits are based on https://github.com/ErikDubbelboer/redis-lua-scaling-bloom-filter
+ * but are simplified to use a single filter instead of up to 32 filters.
+ *
+ * @since 1.24
+ */
+class BloomCacheRedis extends BloomCache {
+ /** @var RedisConnectionPool */
+ protected $redisPool;
+ /** @var RedisLockManager */
+ protected $lockMgr;
+ /** @var array */
+ protected $servers;
+ /** @var integer Federate each filter into this many redis bitfield objects */
+ protected $segments = 128;
+
+ /**
+ * @params include:
+ * - redisServers : list of servers (address:<port>) (the first is the master)
+ * - redisConf : additional redis configuration
+ *
+ * @param array $config
+ */
+ public function __construct( array $config ) {
+ parent::__construct( $config );
+
+ $redisConf = $config['redisConfig'];
+ $redisConf['serializer'] = 'none'; // manage that in this class
+ $this->redisPool = RedisConnectionPool::singleton( $redisConf );
+ $this->servers = $config['redisServers'];
+ $this->lockMgr = new RedisLockManager( array(
+ 'lockServers' => array( 'srv1' => $this->servers[0] ),
+ 'srvsByBucket' => array( 0 => array( 'srv1' ) ),
+ 'redisConfig' => $config['redisConfig']
+ ) );
+ }
+
+ protected function doInit( $key, $size, $precision ) {
+ $conn = $this->getConnection( 'master' );
+ if ( !$conn ) {
+ return false;
+ }
+
+ // 80000000 items at p = .001 take up 500MB and fit into one value.
+ // Do not hit the 512MB redis value limit by reducing the demands.
+ $size = min( $size, 80000000 * $this->segments );
+ $precision = max( round( $precision, 3 ), .001 );
+ $epoch = microtime( true );
+
+ static $script =
+<<<LUA
+ local kMetadata, kData = unpack(KEYS)
+ local aEntries, aPrec, aEpoch = unpack(ARGV)
+ if redis.call('EXISTS',kMetadata) == 0 or redis.call('EXISTS',kData) == 0 then
+ redis.call('DEL',kMetadata)
+ redis.call('HSET',kMetadata,'entries',aEntries)
+ redis.call('HSET',kMetadata,'precision',aPrec)
+ redis.call('HSET',kMetadata,'epoch',aEpoch)
+ redis.call('SET',kData,'')
+ return 1
+ end
+ return 0
+LUA;
+
+ $res = false;
+ try {
+ $conn->script( 'load', $script );
+ $conn->multi( Redis::MULTI );
+ for ( $i = 0; $i < $this->segments; ++$i ) {
+ $res = $conn->luaEval( $script,
+ array(
+ "$key:$i:bloom-metadata", # KEYS[1]
+ "$key:$i:bloom-data", # KEYS[2]
+ ceil( $size / $this->segments ), # ARGV[1]
+ $precision, # ARGV[2]
+ $epoch # ARGV[3]
+ ),
+ 2 # number of first argument(s) that are keys
+ );
+ }
+ $results = $conn->exec();
+ $res = $results && !in_array( false, $results, true );
+ } catch ( RedisException $e ) {
+ $this->handleException( $conn, $e );
+ }
+
+ return ( $res !== false );
+ }
+
+ protected function doAdd( $key, array $members ) {
+ $conn = $this->getConnection( 'master' );
+ if ( !$conn ) {
+ return false;
+ }
+
+ static $script =
+<<<LUA
+ local kMetadata, kData = unpack(KEYS)
+ local aMember = unpack(ARGV)
+
+ -- Check if the filter was initialized
+ if redis.call('EXISTS',kMetadata) == 0 or redis.call('EXISTS',kData) == 0 then
+ return false
+ end
+
+ -- Initial expected entries and desired precision
+ local entries = 1*redis.call('HGET',kMetadata,'entries')
+ local precision = 1*redis.call('HGET',kMetadata,'precision')
+ local hash = redis.sha1hex(aMember)
+
+ -- Based on the math from: http://en.wikipedia.org/wiki/Bloom_filter#Probability_of_false_positives
+ -- 0.480453013 = ln(2)^2
+ local bits = math.ceil((entries * math.log(precision)) / -0.480453013)
+
+ -- 0.693147180 = ln(2)
+ local k = math.floor(0.693147180 * bits / entries)
+
+ -- This uses a variation on:
+ -- 'Less Hashing, Same Performance: Building a Better Bloom Filter'
+ -- http://www.eecs.harvard.edu/~kirsch/pubs/bbbf/esa06.pdf
+ local h = { }
+ h[0] = tonumber(string.sub(hash, 1, 8 ), 16)
+ h[1] = tonumber(string.sub(hash, 9, 16), 16)
+ h[2] = tonumber(string.sub(hash, 17, 24), 16)
+ h[3] = tonumber(string.sub(hash, 25, 32), 16)
+
+ for i=1, k do
+ local pos = (h[i % 2] + i * h[2 + (((i + (i % 2)) % 4) / 2)]) % bits
+ redis.call('SETBIT', kData, pos, 1)
+ end
+
+ return 1
+LUA;
+
+ $res = false;
+ try {
+ $conn->script( 'load', $script );
+ $conn->multi( Redis::PIPELINE );
+ foreach ( $members as $member ) {
+ $i = $this->getSegment( $member );
+ $conn->luaEval( $script,
+ array(
+ "$key:$i:bloom-metadata", # KEYS[1],
+ "$key:$i:bloom-data", # KEYS[2]
+ $member # ARGV[1]
+ ),
+ 2 # number of first argument(s) that are keys
+ );
+ }
+ $results = $conn->exec();
+ $res = $results && !in_array( false, $results, true );
+ } catch ( RedisException $e ) {
+ $this->handleException( $conn, $e );
+ }
+
+ if ( $res === false ) {
+ wfDebug( "Could not add to the '$key' bloom filter; it may be missing." );
+ }
+
+ return ( $res !== false );
+ }
+
+ protected function doSetStatus( $virtualKey, array $values ) {
+ $conn = $this->getConnection( 'master' );
+ if ( !$conn ) {
+ return null;
+ }
+
+ $res = false;
+ try {
+ $res = $conn->hMSet( "$virtualKey:filter-metadata", $values );
+ } catch ( RedisException $e ) {
+ $this->handleException( $conn, $e );
+ }
+
+ return ( $res !== false );
+ }
+
+ protected function doGetStatus( $virtualKey ) {
+ $conn = $this->getConnection( 'slave' );
+ if ( !$conn ) {
+ return false;
+ }
+
+ $res = false;
+ try {
+ $res = $conn->hGetAll( "$virtualKey:filter-metadata" );
+ } catch ( RedisException $e ) {
+ $this->handleException( $conn, $e );
+ }
+
+ if ( is_array( $res ) ) {
+ $res['lastID'] = isset( $res['lastID'] ) ? $res['lastID'] : null;
+ $res['asOfTime'] = isset( $res['asOfTime'] ) ? $res['asOfTime'] : null;
+ $res['epoch'] = isset( $res['epoch'] ) ? $res['epoch'] : null;
+ }
+
+ return $res;
+ }
+
+ protected function doIsHit( $key, $member ) {
+ $conn = $this->getConnection( 'slave' );
+ if ( !$conn ) {
+ return null;
+ }
+
+ static $script =
+<<<LUA
+ local kMetadata, kData = unpack(KEYS)
+ local aMember = unpack(ARGV)
+
+ -- Check if the filter was initialized
+ if redis.call('EXISTS',kMetadata) == 0 or redis.call('EXISTS',kData) == 0 then
+ return false
+ end
+
+ -- Initial expected entries and desired precision.
+ -- This determines the size of the first and subsequent filters.
+ local entries = redis.call('HGET',kMetadata,'entries')
+ local precision = redis.call('HGET',kMetadata,'precision')
+ local hash = redis.sha1hex(aMember)
+
+ -- This uses a variation on:
+ -- 'Less Hashing, Same Performance: Building a Better Bloom Filter'
+ -- http://www.eecs.harvard.edu/~kirsch/pubs/bbbf/esa06.pdf
+ local h = { }
+ h[0] = tonumber(string.sub(hash, 1, 8 ), 16)
+ h[1] = tonumber(string.sub(hash, 9, 16), 16)
+ h[2] = tonumber(string.sub(hash, 17, 24), 16)
+ h[3] = tonumber(string.sub(hash, 25, 32), 16)
+
+ -- 0.480453013 = ln(2)^2
+ local bits = math.ceil((entries * math.log(precision)) / -0.480453013)
+
+ -- 0.693147180 = ln(2)
+ local k = math.floor(0.693147180 * bits / entries)
+
+ local found = 1
+ for i=1, k do
+ local pos = (h[i % 2] + i * h[2 + (((i + (i % 2)) % 4) / 2)]) % bits
+ if redis.call('GETBIT', kData, pos) == 0 then
+ found = 0
+ break
+ end
+ end
+
+ return found
+LUA;
+
+ $res = null;
+ try {
+ $i = $this->getSegment( $member );
+ $res = $conn->luaEval( $script,
+ array(
+ "$key:$i:bloom-metadata", # KEYS[1],
+ "$key:$i:bloom-data", # KEYS[2]
+ $member # ARGV[1]
+ ),
+ 2 # number of first argument(s) that are keys
+ );
+ } catch ( RedisException $e ) {
+ $this->handleException( $conn, $e );
+ }
+
+ return is_int( $res ) ? (bool)$res : null;
+ }
+
+ protected function doDelete( $key ) {
+ $conn = $this->getConnection( 'master' );
+ if ( !$conn ) {
+ return false;
+ }
+
+ $res = false;
+ try {
+ $keys = array();
+ for ( $i = 0; $i < $this->segments; ++$i ) {
+ $keys[] = "$key:$i:bloom-metadata";
+ $keys[] = "$key:$i:bloom-data";
+ }
+ $res = $conn->delete( $keys );
+ } catch ( RedisException $e ) {
+ $this->handleException( $conn, $e );
+ }
+
+ return ( $res !== false );
+ }
+
+ public function getScopedLock( $virtualKey ) {
+ $status = Status::newGood();
+ return ScopedLock::factory( $this->lockMgr,
+ array( $virtualKey ), LockManager::LOCK_EX, $status );
+ }
+
+ /**
+ * @param string $member
+ * @return integer
+ */
+ protected function getSegment( $member ) {
+ return hexdec( substr( md5( $member ), 0, 2 ) ) % $this->segments;
+ }
+
+ /**
+ * $param string $to (master/slave)
+ * @return RedisConnRef|bool Returns false on failure
+ */
+ protected function getConnection( $to ) {
+ if ( $to === 'master' ) {
+ $conn = $this->redisPool->getConnection( $this->servers[0] );
+ } else {
+ static $lastServer = null;
+
+ $conn = false;
+ if ( $lastServer ) {
+ $conn = $this->redisPool->getConnection( $lastServer );
+ if ( $conn ) {
+ return $conn; // reuse connection
+ }
+ }
+ $servers = $this->servers;
+ $attempts = min( 3, count( $servers ) );
+ for ( $i = 1; $i <= $attempts; ++$i ) {
+ $index = mt_rand( 0, count( $servers ) - 1 );
+ $conn = $this->redisPool->getConnection( $servers[$index] );
+ if ( $conn ) {
+ $lastServer = $servers[$index];
+ return $conn;
+ }
+ unset( $servers[$index] ); // skip next time
+ }
+ }
+
+ return $conn;
+ }
+
+ /**
+ * @param RedisConnRef $conn
+ * @param Exception $e
+ */
+ protected function handleException( RedisConnRef $conn, $e ) {
+ $this->redisPool->handleError( $conn, $e );
+ }
+}
diff --git a/includes/cache/bloom/BloomFilters.php b/includes/cache/bloom/BloomFilters.php
new file mode 100644
index 00000000..9b710d79
--- /dev/null
+++ b/includes/cache/bloom/BloomFilters.php
@@ -0,0 +1,79 @@
+<?php
+/**
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ * http://www.gnu.org/copyleft/gpl.html
+ *
+ * @file
+ * @author Aaron Schulz
+ */
+
+/**
+ * @since 1.24
+ */
+class BloomFilterTitleHasLogs {
+ public static function mergeAndCheck(
+ BloomCache $bcache, $domain, $virtualKey, array $status
+ ) {
+ $age = microtime( true ) - $status['asOfTime']; // seconds
+ $scopedLock = ( mt_rand( 1, (int)pow( 3, max( 0, 5 - $age ) ) ) == 1 )
+ ? $bcache->getScopedLock( $virtualKey )
+ : false;
+
+ if ( $scopedLock ) {
+ $updates = self::merge( $bcache, $domain, $virtualKey, $status );
+ if ( isset( $updates['asOfTime'] ) ) {
+ $age = ( microtime( true ) - $updates['asOfTime'] );
+ }
+ }
+
+ return ( $age < 30 );
+ }
+
+ public static function merge(
+ BloomCache $bcache, $domain, $virtualKey, array $status
+ ) {
+ $limit = 1000;
+ $dbr = wfGetDB( DB_SLAVE, array(), $domain );
+ $res = $dbr->select( 'logging',
+ array( 'log_namespace', 'log_title', 'log_id', 'log_timestamp' ),
+ array( 'log_id > ' . $dbr->addQuotes( (int)$status['lastID'] ) ),
+ __METHOD__,
+ array( 'ORDER BY' => 'log_id', 'LIMIT' => $limit )
+ );
+
+ $updates = array();
+ if ( $res->numRows() > 0 ) {
+ $members = array();
+ foreach ( $res as $row ) {
+ $members[] = "$virtualKey:{$row->log_namespace}:{$row->log_title}";
+ }
+ $lastID = $row->log_id;
+ $lastTime = $row->log_timestamp;
+ if ( !$bcache->add( 'shared', $members ) ) {
+ return false;
+ }
+ $updates['lastID'] = $lastID;
+ $updates['asOfTime'] = wfTimestamp( TS_UNIX, $lastTime );
+ } else {
+ $updates['asOfTime'] = microtime( true );
+ }
+
+ $updates['epoch'] = $status['epoch'] ?: microtime( true );
+
+ $bcache->setStatus( $virtualKey, $updates );
+
+ return $updates;
+ }
+}