From ca32f08966f1b51fcb19460f0996bb0c4048e6fe Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sat, 3 Dec 2011 13:29:22 +0100 Subject: Update to MediaWiki 1.18.0 * also update ArchLinux skin to chagnes in MonoBook * Use only css to hide our menu bar when printing --- includes/objectcache/APCBagOStuff.php | 43 + includes/objectcache/BagOStuff.php | 164 ++++ includes/objectcache/DBABagOStuff.php | 194 +++++ includes/objectcache/EhcacheBagOStuff.php | 230 +++++ includes/objectcache/EmptyBagOStuff.php | 27 + includes/objectcache/HashBagOStuff.php | 58 ++ includes/objectcache/MemcachedClient.php | 1115 ++++++++++++++++++++++++ includes/objectcache/MemcachedPhpBagOStuff.php | 178 ++++ includes/objectcache/MultiWriteBagOStuff.php | 113 +++ includes/objectcache/ObjectCache.php | 119 +++ includes/objectcache/SqlBagOStuff.php | 432 +++++++++ includes/objectcache/WinCacheBagOStuff.php | 71 ++ includes/objectcache/XCacheBagOStuff.php | 51 ++ includes/objectcache/eAccelBagOStuff.php | 46 + 14 files changed, 2841 insertions(+) create mode 100644 includes/objectcache/APCBagOStuff.php create mode 100644 includes/objectcache/BagOStuff.php create mode 100644 includes/objectcache/DBABagOStuff.php create mode 100644 includes/objectcache/EhcacheBagOStuff.php create mode 100644 includes/objectcache/EmptyBagOStuff.php create mode 100644 includes/objectcache/HashBagOStuff.php create mode 100644 includes/objectcache/MemcachedClient.php create mode 100644 includes/objectcache/MemcachedPhpBagOStuff.php create mode 100644 includes/objectcache/MultiWriteBagOStuff.php create mode 100644 includes/objectcache/ObjectCache.php create mode 100644 includes/objectcache/SqlBagOStuff.php create mode 100644 includes/objectcache/WinCacheBagOStuff.php create mode 100644 includes/objectcache/XCacheBagOStuff.php create mode 100644 includes/objectcache/eAccelBagOStuff.php (limited to 'includes/objectcache') diff --git a/includes/objectcache/APCBagOStuff.php b/includes/objectcache/APCBagOStuff.php new file mode 100644 index 00000000..dd4a76e1 --- /dev/null +++ b/includes/objectcache/APCBagOStuff.php @@ -0,0 +1,43 @@ + + * http://www.mediawiki.org/ + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @ingroup Cache + */ + +/** + * @defgroup Cache Cache + */ + +/** + * interface is intended to be more or less compatible with + * the PHP memcached client. + * + * backends for local hash array and SQL table included: + * + * $bag = new HashBagOStuff(); + * $bag = new SqlBagOStuff(); # connect to db first + * + * + * @ingroup Cache + */ +abstract class BagOStuff { + private $debugMode = false; + + /** + * @param $bool bool + */ + public function setDebug( $bool ) { + $this->debugMode = $bool; + } + + /* *** THE GUTS OF THE OPERATION *** */ + /* Override these with functional things in subclasses */ + + /** + * Get an item with the given key. Returns false if it does not exist. + * @param $key string + * + * @return bool|Object + */ + abstract public function get( $key ); + + /** + * Set an item. + * @param $key string + * @param $value mixed + * @param $exptime int Either an interval in seconds or a unix timestamp for expiry + */ + abstract public function set( $key, $value, $exptime = 0 ); + + /** + * Delete an item. + * @param $key string + * @param $time int Amount of time to delay the operation (mostly memcached-specific) + */ + abstract public function delete( $key, $time = 0 ); + + public function lock( $key, $timeout = 0 ) { + /* stub */ + return true; + } + + public function unlock( $key ) { + /* stub */ + return true; + } + + public function keys() { + /* stub */ + return array(); + } + + /** + * Delete all objects expiring before a certain date. + * + * @return true on success, false if unimplemented + */ + public function deleteObjectsExpiringBefore( $date ) { + // stub + return false; + } + + /* *** Emulated functions *** */ + + public function add( $key, $value, $exptime = 0 ) { + if ( !$this->get( $key ) ) { + $this->set( $key, $value, $exptime ); + + return true; + } + } + + public function replace( $key, $value, $exptime = 0 ) { + if ( $this->get( $key ) !== false ) { + $this->set( $key, $value, $exptime ); + } + } + + /** + * @param $key String: Key to increase + * @param $value Integer: Value to add to $key (Default 1) + * @return null if lock is not possible else $key value increased by $value + */ + public function incr( $key, $value = 1 ) { + if ( !$this->lock( $key ) ) { + return null; + } + + $value = intval( $value ); + + if ( ( $n = $this->get( $key ) ) !== false ) { + $n += $value; + $this->set( $key, $n ); // exptime? + } + $this->unlock( $key ); + + return $n; + } + + public function decr( $key, $value = 1 ) { + return $this->incr( $key, - $value ); + } + + public function debug( $text ) { + if ( $this->debugMode ) { + $class = get_class( $this ); + wfDebug( "$class debug: $text\n" ); + } + } + + /** + * Convert an optionally relative time to an absolute time + */ + protected function convertExpiry( $exptime ) { + if ( ( $exptime != 0 ) && ( $exptime < 86400 * 3650 /* 10 years */ ) ) { + return time() + $exptime; + } else { + return $exptime; + } + } +} + + diff --git a/includes/objectcache/DBABagOStuff.php b/includes/objectcache/DBABagOStuff.php new file mode 100644 index 00000000..783cd22b --- /dev/null +++ b/includes/objectcache/DBABagOStuff.php @@ -0,0 +1,194 @@ +mFile = "$dir/mw-cache-" . wfWikiID(); + $this->mFile .= '.db'; + wfDebug( __CLASS__ . ": using cache file {$this->mFile}\n" ); + $this->mHandler = $wgDBAhandler; + } + + /** + * Encode value and expiry for storage + * @param $value + * @param $expiry + * + * @return string + */ + function encode( $value, $expiry ) { + # Convert to absolute time + $expiry = $this->convertExpiry( $expiry ); + + return sprintf( '%010u', intval( $expiry ) ) . ' ' . serialize( $value ); + } + + /** + * @return array list containing value first and expiry second + */ + function decode( $blob ) { + if ( !is_string( $blob ) ) { + return array( null, 0 ); + } else { + return array( + unserialize( substr( $blob, 11 ) ), + intval( substr( $blob, 0, 10 ) ) + ); + } + } + + function getReader() { + if ( file_exists( $this->mFile ) ) { + $handle = dba_open( $this->mFile, 'rl', $this->mHandler ); + } else { + $handle = $this->getWriter(); + } + + if ( !$handle ) { + wfDebug( "Unable to open DBA cache file {$this->mFile}\n" ); + } + + return $handle; + } + + function getWriter() { + $handle = dba_open( $this->mFile, 'cl', $this->mHandler ); + + if ( !$handle ) { + wfDebug( "Unable to open DBA cache file {$this->mFile}\n" ); + } + + return $handle; + } + + function get( $key ) { + wfProfileIn( __METHOD__ ); + wfDebug( __METHOD__ . "($key)\n" ); + + $handle = $this->getReader(); + if ( !$handle ) { + wfProfileOut( __METHOD__ ); + return null; + } + + $val = dba_fetch( $key, $handle ); + list( $val, $expiry ) = $this->decode( $val ); + + # Must close ASAP because locks are held + dba_close( $handle ); + + if ( !is_null( $val ) && $expiry && $expiry < time() ) { + # Key is expired, delete it + $handle = $this->getWriter(); + dba_delete( $key, $handle ); + dba_close( $handle ); + wfDebug( __METHOD__ . ": $key expired\n" ); + $val = null; + } + + wfProfileOut( __METHOD__ ); + return $val; + } + + function set( $key, $value, $exptime = 0 ) { + wfProfileIn( __METHOD__ ); + wfDebug( __METHOD__ . "($key)\n" ); + + $blob = $this->encode( $value, $exptime ); + + $handle = $this->getWriter(); + if ( !$handle ) { + wfProfileOut( __METHOD__ ); + return false; + } + + $ret = dba_replace( $key, $blob, $handle ); + dba_close( $handle ); + + wfProfileOut( __METHOD__ ); + return $ret; + } + + function delete( $key, $time = 0 ) { + wfProfileIn( __METHOD__ ); + wfDebug( __METHOD__ . "($key)\n" ); + + $handle = $this->getWriter(); + if ( !$handle ) { + wfProfileOut( __METHOD__ ); + return false; + } + + $ret = dba_delete( $key, $handle ); + dba_close( $handle ); + + wfProfileOut( __METHOD__ ); + return $ret; + } + + function add( $key, $value, $exptime = 0 ) { + wfProfileIn( __METHOD__ ); + + $blob = $this->encode( $value, $exptime ); + + $handle = $this->getWriter(); + + if ( !$handle ) { + wfProfileOut( __METHOD__ ); + return false; + } + + $ret = dba_insert( $key, $blob, $handle ); + + # Insert failed, check to see if it failed due to an expired key + if ( !$ret ) { + list( $value, $expiry ) = $this->decode( dba_fetch( $key, $handle ) ); + + if ( $expiry < time() ) { + # Yes expired, delete and try again + dba_delete( $key, $handle ); + $ret = dba_insert( $key, $blob, $handle ); + # This time if it failed then it will be handled by the caller like any other race + } + } + + dba_close( $handle ); + + wfProfileOut( __METHOD__ ); + return $ret; + } + + function keys() { + $reader = $this->getReader(); + $k1 = dba_firstkey( $reader ); + + if ( !$k1 ) { + return array(); + } + + $result[] = $k1; + + while ( $key = dba_nextkey( $reader ) ) { + $result[] = $key; + } + + return $result; + } +} + diff --git a/includes/objectcache/EhcacheBagOStuff.php b/includes/objectcache/EhcacheBagOStuff.php new file mode 100644 index 00000000..75aad27a --- /dev/null +++ b/includes/objectcache/EhcacheBagOStuff.php @@ -0,0 +1,230 @@ +servers = $params['servers']; + $this->cacheName = isset( $params['cache'] ) ? $params['cache'] : 'mw'; + $this->connectTimeout = isset( $params['connectTimeout'] ) + ? $params['connectTimeout'] : 1; + $this->timeout = isset( $params['timeout'] ) ? $params['timeout'] : 1; + $this->curlOptions = array( + CURLOPT_CONNECTTIMEOUT_MS => intval( $this->connectTimeout * 1000 ), + CURLOPT_TIMEOUT_MS => intval( $this->timeout * 1000 ), + CURLOPT_RETURNTRANSFER => 1, + CURLOPT_CUSTOMREQUEST => 'GET', + CURLOPT_POST => 0, + CURLOPT_POSTFIELDS => '', + CURLOPT_HTTPHEADER => array(), + ); + } + + public function get( $key ) { + wfProfileIn( __METHOD__ ); + $response = $this->doItemRequest( $key ); + if ( !$response || $response['http_code'] == 404 ) { + wfProfileOut( __METHOD__ ); + return false; + } + if ( $response['http_code'] >= 300 ) { + wfDebug( __METHOD__.": GET failure, got HTTP {$response['http_code']}\n" ); + wfProfileOut( __METHOD__ ); + return false; + } + $body = $response['body']; + $type = $response['content_type']; + if ( $type == 'application/vnd.php.serialized+deflate' ) { + $body = gzinflate( $body ); + if ( !$body ) { + wfDebug( __METHOD__.": error inflating $key\n" ); + wfProfileOut( __METHOD__ ); + return false; + } + $data = unserialize( $body ); + } elseif ( $type == 'application/vnd.php.serialized' ) { + $data = unserialize( $body ); + } else { + wfDebug( __METHOD__.": unknown content type \"$type\"\n" ); + wfProfileOut( __METHOD__ ); + return false; + } + + wfProfileOut( __METHOD__ ); + return $data; + } + + public function set( $key, $value, $expiry = 0 ) { + wfProfileIn( __METHOD__ ); + $expiry = $this->convertExpiry( $expiry ); + $ttl = $expiry ? $expiry - time() : 2147483647; + $blob = serialize( $value ); + if ( strlen( $blob ) > 100 ) { + $blob = gzdeflate( $blob ); + $contentType = 'application/vnd.php.serialized+deflate'; + } else { + $contentType = 'application/vnd.php.serialized'; + } + + $code = $this->attemptPut( $key, $blob, $contentType, $ttl ); + + if ( $code == 404 ) { + // Maybe the cache does not exist yet, let's try creating it + if ( !$this->createCache( $key ) ) { + wfDebug( __METHOD__.": cache creation failed\n" ); + wfProfileOut( __METHOD__ ); + return false; + } + $code = $this->attemptPut( $key, $blob, $contentType, $ttl ); + } + + $result = false; + if ( !$code ) { + wfDebug( __METHOD__.": PUT failure for key $key\n" ); + } elseif ( $code >= 300 ) { + wfDebug( __METHOD__.": PUT failure for key $key: HTTP $code\n" ); + } else { + $result = true; + } + + wfProfileOut( __METHOD__ ); + return $result; + } + + public function delete( $key, $time = 0 ) { + wfProfileIn( __METHOD__ ); + $response = $this->doItemRequest( $key, + array( CURLOPT_CUSTOMREQUEST => 'DELETE' ) ); + $code = isset( $response['http_code'] ) ? $response['http_code'] : 0; + if ( !$response || ( $code != 404 && $code >= 300 ) ) { + wfDebug( __METHOD__.": DELETE failure for key $key\n" ); + $result = false; + } else { + $result = true; + } + wfProfileOut( __METHOD__ ); + return $result; + } + + protected function getCacheUrl( $key ) { + if ( count( $this->servers ) == 1 ) { + $server = reset( $this->servers ); + } else { + // Use consistent hashing + $hashes = array(); + foreach ( $this->servers as $server ) { + $hashes[$server] = md5( $server . '/' . $key ); + } + asort( $hashes ); + reset( $hashes ); + $server = key( $hashes ); + } + return "http://$server/ehcache/rest/{$this->cacheName}"; + } + + /** + * Get a cURL handle for the given cache URL. + * We cache the handles to allow keepalive. + */ + protected function getCurl( $cacheUrl ) { + if ( !isset( $this->curls[$cacheUrl] ) ) { + $this->curls[$cacheUrl] = curl_init(); + } + return $this->curls[$cacheUrl]; + } + + protected function attemptPut( $key, $data, $type, $ttl ) { + // In initial benchmarking, it was 30 times faster to use CURLOPT_POST + // than CURLOPT_UPLOAD with CURLOPT_READFUNCTION. This was because + // CURLOPT_UPLOAD was pushing the request headers first, then waiting + // for an ACK packet, then sending the data, whereas CURLOPT_POST just + // sends the headers and the data in a single send(). + $response = $this->doItemRequest( $key, + array( + CURLOPT_POST => 1, + CURLOPT_CUSTOMREQUEST => 'PUT', + CURLOPT_POSTFIELDS => $data, + CURLOPT_HTTPHEADER => array( + 'Content-Type: ' . $type, + 'ehcacheTimeToLiveSeconds: ' . $ttl + ) + ) + ); + if ( !$response ) { + return 0; + } else { + return $response['http_code']; + } + } + + protected function createCache( $key ) { + wfDebug( __METHOD__.": creating cache for $key\n" ); + $response = $this->doCacheRequest( $key, + array( + CURLOPT_POST => 1, + CURLOPT_CUSTOMREQUEST => 'PUT', + CURLOPT_POSTFIELDS => '', + ) ); + if ( !$response ) { + wfDebug( __CLASS__.": failed to create cache for $key\n" ); + return false; + } + if ( $response['http_code'] == 201 /* created */ + || $response['http_code'] == 409 /* already there */ ) + { + return true; + } else { + return false; + } + } + + protected function doCacheRequest( $key, $curlOptions = array() ) { + $cacheUrl = $this->getCacheUrl( $key ); + $curl = $this->getCurl( $cacheUrl ); + return $this->doRequest( $curl, $cacheUrl, $curlOptions ); + } + + protected function doItemRequest( $key, $curlOptions = array() ) { + $cacheUrl = $this->getCacheUrl( $key ); + $curl = $this->getCurl( $cacheUrl ); + $url = $cacheUrl . '/' . rawurlencode( $key ); + return $this->doRequest( $curl, $url, $curlOptions ); + } + + protected function doRequest( $curl, $url, $curlOptions = array() ) { + if ( array_diff_key( $curlOptions, $this->curlOptions ) ) { + // var_dump( array_diff_key( $curlOptions, $this->curlOptions ) ); + throw new MWException( __METHOD__.": to prevent options set in one doRequest() " . + "call from affecting subsequent doRequest() calls, only options listed " . + "in \$this->curlOptions may be specified in the \$curlOptions parameter." ); + } + $curlOptions += $this->curlOptions; + $curlOptions[CURLOPT_URL] = $url; + + curl_setopt_array( $curl, $curlOptions ); + $result = curl_exec( $curl ); + if ( $result === false ) { + wfDebug( __CLASS__.": curl error: " . curl_error( $curl ) . "\n" ); + return false; + } + $info = curl_getinfo( $curl ); + $info['body'] = $result; + return $info; + } +} diff --git a/includes/objectcache/EmptyBagOStuff.php b/includes/objectcache/EmptyBagOStuff.php new file mode 100644 index 00000000..e956e2ee --- /dev/null +++ b/includes/objectcache/EmptyBagOStuff.php @@ -0,0 +1,27 @@ +bag = array(); + } + + protected function expire( $key ) { + $et = $this->bag[$key][1]; + + if ( ( $et == 0 ) || ( $et > time() ) ) { + return false; + } + + $this->delete( $key ); + + return true; + } + + function get( $key ) { + if ( !isset( $this->bag[$key] ) ) { + return false; + } + + if ( $this->expire( $key ) ) { + return false; + } + + return $this->bag[$key][0]; + } + + function set( $key, $value, $exptime = 0 ) { + $this->bag[$key] = array( $value, $this->convertExpiry( $exptime ) ); + } + + function delete( $key, $time = 0 ) { + if ( !isset( $this->bag[$key] ) ) { + return false; + } + + unset( $this->bag[$key] ); + + return true; + } + + function keys() { + return array_keys( $this->bag ); + } +} + diff --git a/includes/objectcache/MemcachedClient.php b/includes/objectcache/MemcachedClient.php new file mode 100644 index 00000000..dd4401a8 --- /dev/null +++ b/includes/objectcache/MemcachedClient.php @@ -0,0 +1,1115 @@ + | + * | All rights reserved. | + * | | + * | Redistribution and use in source and binary forms, with or without | + * | modification, are permitted provided that the following conditions | + * | are met: | + * | | + * | 1. Redistributions of source code must retain the above copyright | + * | notice, this list of conditions and the following disclaimer. | + * | 2. Redistributions in binary form must reproduce the above copyright | + * | notice, this list of conditions and the following disclaimer in the | + * | documentation and/or other materials provided with the distribution. | + * | | + * | THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR | + * | IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES | + * | OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. | + * | IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, | + * | INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT | + * | NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | + * | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | + * | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | + * | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF | + * | THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | + * +---------------------------------------------------------------------------+ + * | Author: Ryan T. Dean | + * | Heavily influenced by the Perl memcached client by Brad Fitzpatrick. | + * | Permission granted by Brad Fitzpatrick for relicense of ported Perl | + * | client logic under 2-clause BSD license. | + * +---------------------------------------------------------------------------+ + * + * @file + * $TCAnet$ + */ + +/** + * This is the PHP client for memcached - a distributed memory cache daemon. + * More information is available at http://www.danga.com/memcached/ + * + * Usage example: + * + * require_once 'memcached.php'; + * + * $mc = new MWMemcached(array( + * 'servers' => array('127.0.0.1:10000', + * array('192.0.0.1:10010', 2), + * '127.0.0.1:10020'), + * 'debug' => false, + * 'compress_threshold' => 10240, + * 'persistent' => true)); + * + * $mc->add('key', array('some', 'array')); + * $mc->replace('key', 'some random string'); + * $val = $mc->get('key'); + * + * @author Ryan T. Dean + * @version 0.1.2 + */ + +// {{{ requirements +// }}} + +// {{{ class MWMemcached +/** + * memcached client class implemented using (p)fsockopen() + * + * @author Ryan T. Dean + * @ingroup Cache + */ +class MWMemcached { + // {{{ properties + // {{{ public + + // {{{ constants + // {{{ flags + + /** + * Flag: indicates data is serialized + */ + const SERIALIZED = 1; + + /** + * Flag: indicates data is compressed + */ + const COMPRESSED = 2; + + // }}} + + /** + * Minimum savings to store data compressed + */ + const COMPRESSION_SAVINGS = 0.20; + + // }}} + + + /** + * Command statistics + * + * @var array + * @access public + */ + var $stats; + + // }}} + // {{{ private + + /** + * Cached Sockets that are connected + * + * @var array + * @access private + */ + var $_cache_sock; + + /** + * Current debug status; 0 - none to 9 - profiling + * + * @var boolean + * @access private + */ + var $_debug; + + /** + * Dead hosts, assoc array, 'host'=>'unixtime when ok to check again' + * + * @var array + * @access private + */ + var $_host_dead; + + /** + * Is compression available? + * + * @var boolean + * @access private + */ + var $_have_zlib; + + /** + * Do we want to use compression? + * + * @var boolean + * @access private + */ + var $_compress_enable; + + /** + * At how many bytes should we compress? + * + * @var integer + * @access private + */ + var $_compress_threshold; + + /** + * Are we using persistent links? + * + * @var boolean + * @access private + */ + var $_persistent; + + /** + * If only using one server; contains ip:port to connect to + * + * @var string + * @access private + */ + var $_single_sock; + + /** + * Array containing ip:port or array(ip:port, weight) + * + * @var array + * @access private + */ + var $_servers; + + /** + * Our bit buckets + * + * @var array + * @access private + */ + var $_buckets; + + /** + * Total # of bit buckets we have + * + * @var integer + * @access private + */ + var $_bucketcount; + + /** + * # of total servers we have + * + * @var integer + * @access private + */ + var $_active; + + /** + * Stream timeout in seconds. Applies for example to fread() + * + * @var integer + * @access private + */ + var $_timeout_seconds; + + /** + * Stream timeout in microseconds + * + * @var integer + * @access private + */ + var $_timeout_microseconds; + + /** + * Connect timeout in seconds + */ + var $_connect_timeout; + + /** + * Number of connection attempts for each server + */ + var $_connect_attempts; + + // }}} + // }}} + // {{{ methods + // {{{ public functions + // {{{ memcached() + + /** + * Memcache initializer + * + * @param $args Array Associative array of settings + * + * @return mixed + */ + public function __construct( $args ) { + $this->set_servers( isset( $args['servers'] ) ? $args['servers'] : array() ); + $this->_debug = isset( $args['debug'] ) ? $args['debug'] : false; + $this->stats = array(); + $this->_compress_threshold = isset( $args['compress_threshold'] ) ? $args['compress_threshold'] : 0; + $this->_persistent = isset( $args['persistent'] ) ? $args['persistent'] : false; + $this->_compress_enable = true; + $this->_have_zlib = function_exists( 'gzcompress' ); + + $this->_cache_sock = array(); + $this->_host_dead = array(); + + $this->_timeout_seconds = 0; + $this->_timeout_microseconds = isset( $args['timeout'] ) ? $args['timeout'] : 100000; + + $this->_connect_timeout = isset( $args['connect_timeout'] ) ? $args['connect_timeout'] : 0.1; + $this->_connect_attempts = 2; + } + + // }}} + // {{{ add() + + /** + * Adds a key/value to the memcache server if one isn't already set with + * that key + * + * @param $key String: key to set with data + * @param $val Mixed: value to store + * @param $exp Integer: (optional) Expiration time. This can be a number of seconds + * to cache for (up to 30 days inclusive). Any timespans of 30 days + 1 second or + * longer must be the timestamp of the time at which the mapping should expire. It + * is safe to use timestamps in all cases, regardless of exipration + * eg: strtotime("+3 hour") + * + * @return Boolean + */ + public function add( $key, $val, $exp = 0 ) { + return $this->_set( 'add', $key, $val, $exp ); + } + + // }}} + // {{{ decr() + + /** + * Decrease a value stored on the memcache server + * + * @param $key String: key to decrease + * @param $amt Integer: (optional) amount to decrease + * + * @return Mixed: FALSE on failure, value on success + */ + public function decr( $key, $amt = 1 ) { + return $this->_incrdecr( 'decr', $key, $amt ); + } + + // }}} + // {{{ delete() + + /** + * Deletes a key from the server, optionally after $time + * + * @param $key String: key to delete + * @param $time Integer: (optional) how long to wait before deleting + * + * @return Boolean: TRUE on success, FALSE on failure + */ + public function delete( $key, $time = 0 ) { + if ( !$this->_active ) { + return false; + } + + $sock = $this->get_sock( $key ); + if ( !is_resource( $sock ) ) { + return false; + } + + $key = is_array( $key ) ? $key[1] : $key; + + if ( isset( $this->stats['delete'] ) ) { + $this->stats['delete']++; + } else { + $this->stats['delete'] = 1; + } + $cmd = "delete $key $time\r\n"; + if( !$this->_safe_fwrite( $sock, $cmd, strlen( $cmd ) ) ) { + $this->_dead_sock( $sock ); + return false; + } + $res = trim( fgets( $sock ) ); + + if ( $this->_debug ) { + $this->_debugprint( sprintf( "MemCache: delete %s (%s)\n", $key, $res ) ); + } + + if ( $res == "DELETED" ) { + return true; + } + return false; + } + + public function lock( $key, $timeout = 0 ) { + /* stub */ + return true; + } + + public function unlock( $key ) { + /* stub */ + return true; + } + + // }}} + // {{{ disconnect_all() + + /** + * Disconnects all connected sockets + */ + public function disconnect_all() { + foreach ( $this->_cache_sock as $sock ) { + fclose( $sock ); + } + + $this->_cache_sock = array(); + } + + // }}} + // {{{ enable_compress() + + /** + * Enable / Disable compression + * + * @param $enable Boolean: TRUE to enable, FALSE to disable + */ + public function enable_compress( $enable ) { + $this->_compress_enable = $enable; + } + + // }}} + // {{{ forget_dead_hosts() + + /** + * Forget about all of the dead hosts + */ + public function forget_dead_hosts() { + $this->_host_dead = array(); + } + + // }}} + // {{{ get() + + /** + * Retrieves the value associated with the key from the memcache server + * + * @param $key Mixed: key to retrieve + * + * @return Mixed + */ + public function get( $key ) { + wfProfileIn( __METHOD__ ); + + if ( $this->_debug ) { + $this->_debugprint( "get($key)\n" ); + } + + if ( !$this->_active ) { + wfProfileOut( __METHOD__ ); + return false; + } + + $sock = $this->get_sock( $key ); + + if ( !is_resource( $sock ) ) { + wfProfileOut( __METHOD__ ); + return false; + } + + if ( isset( $this->stats['get'] ) ) { + $this->stats['get']++; + } else { + $this->stats['get'] = 1; + } + + $cmd = "get $key\r\n"; + if ( !$this->_safe_fwrite( $sock, $cmd, strlen( $cmd ) ) ) { + $this->_dead_sock( $sock ); + wfProfileOut( __METHOD__ ); + return false; + } + + $val = array(); + $this->_load_items( $sock, $val ); + + if ( $this->_debug ) { + foreach ( $val as $k => $v ) { + $this->_debugprint( sprintf( "MemCache: sock %s got %s\n", serialize( $sock ), $k ) ); + } + } + + $value = false; + if ( isset( $val[$key] ) ) { + $value = $val[$key]; + } + wfProfileOut( __METHOD__ ); + return $value; + } + + // }}} + // {{{ get_multi() + + /** + * Get multiple keys from the server(s) + * + * @param $keys Array: keys to retrieve + * + * @return Array + */ + public function get_multi( $keys ) { + if ( !$this->_active ) { + return false; + } + + if ( isset( $this->stats['get_multi'] ) ) { + $this->stats['get_multi']++; + } else { + $this->stats['get_multi'] = 1; + } + $sock_keys = array(); + + foreach ( $keys as $key ) { + $sock = $this->get_sock( $key ); + if ( !is_resource( $sock ) ) { + continue; + } + $key = is_array( $key ) ? $key[1] : $key; + if ( !isset( $sock_keys[$sock] ) ) { + $sock_keys[$sock] = array(); + $socks[] = $sock; + } + $sock_keys[$sock][] = $key; + } + + // Send out the requests + foreach ( $socks as $sock ) { + $cmd = 'get'; + foreach ( $sock_keys[$sock] as $key ) { + $cmd .= ' ' . $key; + } + $cmd .= "\r\n"; + + if ( $this->_safe_fwrite( $sock, $cmd, strlen( $cmd ) ) ) { + $gather[] = $sock; + } else { + $this->_dead_sock( $sock ); + } + } + + // Parse responses + $val = array(); + foreach ( $gather as $sock ) { + $this->_load_items( $sock, $val ); + } + + if ( $this->_debug ) { + foreach ( $val as $k => $v ) { + $this->_debugprint( sprintf( "MemCache: got %s\n", $k ) ); + } + } + + return $val; + } + + // }}} + // {{{ incr() + + /** + * Increments $key (optionally) by $amt + * + * @param $key String: key to increment + * @param $amt Integer: (optional) amount to increment + * + * @return Integer: null if the key does not exist yet (this does NOT + * create new mappings if the key does not exist). If the key does + * exist, this returns the new value for that key. + */ + public function incr( $key, $amt = 1 ) { + return $this->_incrdecr( 'incr', $key, $amt ); + } + + // }}} + // {{{ replace() + + /** + * Overwrites an existing value for key; only works if key is already set + * + * @param $key String: key to set value as + * @param $value Mixed: value to store + * @param $exp Integer: (optional) Expiration time. This can be a number of seconds + * to cache for (up to 30 days inclusive). Any timespans of 30 days + 1 second or + * longer must be the timestamp of the time at which the mapping should expire. It + * is safe to use timestamps in all cases, regardless of exipration + * eg: strtotime("+3 hour") + * + * @return Boolean + */ + public function replace( $key, $value, $exp = 0 ) { + return $this->_set( 'replace', $key, $value, $exp ); + } + + // }}} + // {{{ run_command() + + /** + * Passes through $cmd to the memcache server connected by $sock; returns + * output as an array (null array if no output) + * + * NOTE: due to a possible bug in how PHP reads while using fgets(), each + * line may not be terminated by a \r\n. More specifically, my testing + * has shown that, on FreeBSD at least, each line is terminated only + * with a \n. This is with the PHP flag auto_detect_line_endings set + * to falase (the default). + * + * @param $sock Resource: socket to send command on + * @param $cmd String: command to run + * + * @return Array: output array + */ + public function run_command( $sock, $cmd ) { + if ( !is_resource( $sock ) ) { + return array(); + } + + if ( !$this->_safe_fwrite( $sock, $cmd, strlen( $cmd ) ) ) { + return array(); + } + + while ( true ) { + $res = fgets( $sock ); + $ret[] = $res; + if ( preg_match( '/^END/', $res ) ) { + break; + } + if ( strlen( $res ) == 0 ) { + break; + } + } + return $ret; + } + + // }}} + // {{{ set() + + /** + * Unconditionally sets a key to a given value in the memcache. Returns true + * if set successfully. + * + * @param $key String: key to set value as + * @param $value Mixed: value to set + * @param $exp Integer: (optional) Expiration time. This can be a number of seconds + * to cache for (up to 30 days inclusive). Any timespans of 30 days + 1 second or + * longer must be the timestamp of the time at which the mapping should expire. It + * is safe to use timestamps in all cases, regardless of exipration + * eg: strtotime("+3 hour") + * + * @return Boolean: TRUE on success + */ + public function set( $key, $value, $exp = 0 ) { + return $this->_set( 'set', $key, $value, $exp ); + } + + // }}} + // {{{ set_compress_threshold() + + /** + * Sets the compression threshold + * + * @param $thresh Integer: threshold to compress if larger than + */ + public function set_compress_threshold( $thresh ) { + $this->_compress_threshold = $thresh; + } + + // }}} + // {{{ set_debug() + + /** + * Sets the debug flag + * + * @param $dbg Boolean: TRUE for debugging, FALSE otherwise + * + * @see MWMemcached::__construct + */ + public function set_debug( $dbg ) { + $this->_debug = $dbg; + } + + // }}} + // {{{ set_servers() + + /** + * Sets the server list to distribute key gets and puts between + * + * @param $list Array of servers to connect to + * + * @see MWMemcached::__construct() + */ + public function set_servers( $list ) { + $this->_servers = $list; + $this->_active = count( $list ); + $this->_buckets = null; + $this->_bucketcount = 0; + + $this->_single_sock = null; + if ( $this->_active == 1 ) { + $this->_single_sock = $this->_servers[0]; + } + } + + /** + * Sets the timeout for new connections + * + * @param $seconds Integer: number of seconds + * @param $microseconds Integer: number of microseconds + */ + public function set_timeout( $seconds, $microseconds ) { + $this->_timeout_seconds = $seconds; + $this->_timeout_microseconds = $microseconds; + } + + // }}} + // }}} + // {{{ private methods + // {{{ _close_sock() + + /** + * Close the specified socket + * + * @param $sock String: socket to close + * + * @access private + */ + function _close_sock( $sock ) { + $host = array_search( $sock, $this->_cache_sock ); + fclose( $this->_cache_sock[$host] ); + unset( $this->_cache_sock[$host] ); + } + + // }}} + // {{{ _connect_sock() + + /** + * Connects $sock to $host, timing out after $timeout + * + * @param $sock Integer: socket to connect + * @param $host String: Host:IP to connect to + * + * @return boolean + * @access private + */ + function _connect_sock( &$sock, $host ) { + list( $ip, $port ) = explode( ':', $host ); + $sock = false; + $timeout = $this->_connect_timeout; + $errno = $errstr = null; + for( $i = 0; !$sock && $i < $this->_connect_attempts; $i++ ) { + wfSuppressWarnings(); + if ( $this->_persistent == 1 ) { + $sock = pfsockopen( $ip, $port, $errno, $errstr, $timeout ); + } else { + $sock = fsockopen( $ip, $port, $errno, $errstr, $timeout ); + } + wfRestoreWarnings(); + } + if ( !$sock ) { + if ( $this->_debug ) { + $this->_debugprint( "Error connecting to $host: $errstr\n" ); + } + return false; + } + + // Initialise timeout + stream_set_timeout( $sock, $this->_timeout_seconds, $this->_timeout_microseconds ); + + return true; + } + + // }}} + // {{{ _dead_sock() + + /** + * Marks a host as dead until 30-40 seconds in the future + * + * @param $sock String: socket to mark as dead + * + * @access private + */ + function _dead_sock( $sock ) { + $host = array_search( $sock, $this->_cache_sock ); + $this->_dead_host( $host ); + } + + function _dead_host( $host ) { + $parts = explode( ':', $host ); + $ip = $parts[0]; + $this->_host_dead[$ip] = time() + 30 + intval( rand( 0, 10 ) ); + $this->_host_dead[$host] = $this->_host_dead[$ip]; + unset( $this->_cache_sock[$host] ); + } + + // }}} + // {{{ get_sock() + + /** + * get_sock + * + * @param $key String: key to retrieve value for; + * + * @return Mixed: resource on success, false on failure + * @access private + */ + function get_sock( $key ) { + if ( !$this->_active ) { + return false; + } + + if ( $this->_single_sock !== null ) { + $this->_flush_read_buffer( $this->_single_sock ); + return $this->sock_to_host( $this->_single_sock ); + } + + $hv = is_array( $key ) ? intval( $key[0] ) : $this->_hashfunc( $key ); + + if ( $this->_buckets === null ) { + foreach ( $this->_servers as $v ) { + if ( is_array( $v ) ) { + for( $i = 0; $i < $v[1]; $i++ ) { + $bu[] = $v[0]; + } + } else { + $bu[] = $v; + } + } + $this->_buckets = $bu; + $this->_bucketcount = count( $bu ); + } + + $realkey = is_array( $key ) ? $key[1] : $key; + for( $tries = 0; $tries < 20; $tries++ ) { + $host = $this->_buckets[$hv % $this->_bucketcount]; + $sock = $this->sock_to_host( $host ); + if ( is_resource( $sock ) ) { + $this->_flush_read_buffer( $sock ); + return $sock; + } + $hv = $this->_hashfunc( $hv . $realkey ); + } + + return false; + } + + // }}} + // {{{ _hashfunc() + + /** + * Creates a hash integer based on the $key + * + * @param $key String: key to hash + * + * @return Integer: hash value + * @access private + */ + function _hashfunc( $key ) { + # Hash function must on [0,0x7ffffff] + # We take the first 31 bits of the MD5 hash, which unlike the hash + # function used in a previous version of this client, works + return hexdec( substr( md5( $key ), 0, 8 ) ) & 0x7fffffff; + } + + // }}} + // {{{ _incrdecr() + + /** + * Perform increment/decriment on $key + * + * @param $cmd String: command to perform + * @param $key String: key to perform it on + * @param $amt Integer: amount to adjust + * + * @return Integer: new value of $key + * @access private + */ + function _incrdecr( $cmd, $key, $amt = 1 ) { + if ( !$this->_active ) { + return null; + } + + $sock = $this->get_sock( $key ); + if ( !is_resource( $sock ) ) { + return null; + } + + $key = is_array( $key ) ? $key[1] : $key; + if ( isset( $this->stats[$cmd] ) ) { + $this->stats[$cmd]++; + } else { + $this->stats[$cmd] = 1; + } + if ( !$this->_safe_fwrite( $sock, "$cmd $key $amt\r\n" ) ) { + return $this->_dead_sock( $sock ); + } + + $line = fgets( $sock ); + $match = array(); + if ( !preg_match( '/^(\d+)/', $line, $match ) ) { + return null; + } + return $match[1]; + } + + // }}} + // {{{ _load_items() + + /** + * Load items into $ret from $sock + * + * @param $sock Resource: socket to read from + * @param $ret Array: returned values + * + * @access private + */ + function _load_items( $sock, &$ret ) { + while ( 1 ) { + $decl = fgets( $sock ); + if ( $decl == "END\r\n" ) { + return true; + } elseif ( preg_match( '/^VALUE (\S+) (\d+) (\d+)\r\n$/', $decl, $match ) ) { + list( $rkey, $flags, $len ) = array( $match[1], $match[2], $match[3] ); + $bneed = $len + 2; + $offset = 0; + + while ( $bneed > 0 ) { + $data = fread( $sock, $bneed ); + $n = strlen( $data ); + if ( $n == 0 ) { + break; + } + $offset += $n; + $bneed -= $n; + if ( isset( $ret[$rkey] ) ) { + $ret[$rkey] .= $data; + } else { + $ret[$rkey] = $data; + } + } + + if ( $offset != $len + 2 ) { + // Something is borked! + if ( $this->_debug ) { + $this->_debugprint( sprintf( "Something is borked! key %s expecting %d got %d length\n", $rkey, $len + 2, $offset ) ); + } + + unset( $ret[$rkey] ); + $this->_close_sock( $sock ); + return false; + } + + if ( $this->_have_zlib && $flags & self::COMPRESSED ) { + $ret[$rkey] = gzuncompress( $ret[$rkey] ); + } + + $ret[$rkey] = rtrim( $ret[$rkey] ); + + if ( $flags & self::SERIALIZED ) { + $ret[$rkey] = unserialize( $ret[$rkey] ); + } + + } else { + $this->_debugprint( "Error parsing memcached response\n" ); + return 0; + } + } + } + + // }}} + // {{{ _set() + + /** + * Performs the requested storage operation to the memcache server + * + * @param $cmd String: command to perform + * @param $key String: key to act on + * @param $val Mixed: what we need to store + * @param $exp Integer: (optional) Expiration time. This can be a number of seconds + * to cache for (up to 30 days inclusive). Any timespans of 30 days + 1 second or + * longer must be the timestamp of the time at which the mapping should expire. It + * is safe to use timestamps in all cases, regardless of exipration + * eg: strtotime("+3 hour") + * + * @return Boolean + * @access private + */ + function _set( $cmd, $key, $val, $exp ) { + if ( !$this->_active ) { + return false; + } + + $sock = $this->get_sock( $key ); + if ( !is_resource( $sock ) ) { + return false; + } + + if ( isset( $this->stats[$cmd] ) ) { + $this->stats[$cmd]++; + } else { + $this->stats[$cmd] = 1; + } + + // Memcached doesn't seem to handle very high TTL values very well, + // so clamp them at 30 days + if ( $exp > 2592000 ) { + $exp = 2592000; + } + + $flags = 0; + + if ( !is_scalar( $val ) ) { + $val = serialize( $val ); + $flags |= self::SERIALIZED; + if ( $this->_debug ) { + $this->_debugprint( sprintf( "client: serializing data as it is not scalar\n" ) ); + } + } + + $len = strlen( $val ); + + if ( $this->_have_zlib && $this->_compress_enable && + $this->_compress_threshold && $len >= $this->_compress_threshold ) + { + $c_val = gzcompress( $val, 9 ); + $c_len = strlen( $c_val ); + + if ( $c_len < $len * ( 1 - self::COMPRESSION_SAVINGS ) ) { + if ( $this->_debug ) { + $this->_debugprint( sprintf( "client: compressing data; was %d bytes is now %d bytes\n", $len, $c_len ) ); + } + $val = $c_val; + $len = $c_len; + $flags |= self::COMPRESSED; + } + } + if ( !$this->_safe_fwrite( $sock, "$cmd $key $flags $exp $len\r\n$val\r\n" ) ) { + return $this->_dead_sock( $sock ); + } + + $line = trim( fgets( $sock ) ); + + if ( $this->_debug ) { + $this->_debugprint( sprintf( "%s %s (%s)\n", $cmd, $key, $line ) ); + } + if ( $line == "STORED" ) { + return true; + } + return false; + } + + // }}} + // {{{ sock_to_host() + + /** + * Returns the socket for the host + * + * @param $host String: Host:IP to get socket for + * + * @return Mixed: IO Stream or false + * @access private + */ + function sock_to_host( $host ) { + if ( isset( $this->_cache_sock[$host] ) ) { + return $this->_cache_sock[$host]; + } + + $sock = null; + $now = time(); + list( $ip, /* $port */) = explode( ':', $host ); + if ( isset( $this->_host_dead[$host] ) && $this->_host_dead[$host] > $now || + isset( $this->_host_dead[$ip] ) && $this->_host_dead[$ip] > $now + ) { + return null; + } + + if ( !$this->_connect_sock( $sock, $host ) ) { + return $this->_dead_host( $host ); + } + + // Do not buffer writes + stream_set_write_buffer( $sock, 0 ); + + $this->_cache_sock[$host] = $sock; + + return $this->_cache_sock[$host]; + } + + function _debugprint( $str ) { + print( $str ); + } + + /** + * Write to a stream, timing out after the correct amount of time + * + * @return Boolean: false on failure, true on success + */ + /* + function _safe_fwrite( $f, $buf, $len = false ) { + stream_set_blocking( $f, 0 ); + + if ( $len === false ) { + wfDebug( "Writing " . strlen( $buf ) . " bytes\n" ); + $bytesWritten = fwrite( $f, $buf ); + } else { + wfDebug( "Writing $len bytes\n" ); + $bytesWritten = fwrite( $f, $buf, $len ); + } + $n = stream_select( $r = null, $w = array( $f ), $e = null, 10, 0 ); + # $this->_timeout_seconds, $this->_timeout_microseconds ); + + wfDebug( "stream_select returned $n\n" ); + stream_set_blocking( $f, 1 ); + return $n == 1; + return $bytesWritten; + }*/ + + /** + * Original behaviour + */ + function _safe_fwrite( $f, $buf, $len = false ) { + if ( $len === false ) { + $bytesWritten = fwrite( $f, $buf ); + } else { + $bytesWritten = fwrite( $f, $buf, $len ); + } + return $bytesWritten; + } + + /** + * Flush the read buffer of a stream + */ + function _flush_read_buffer( $f ) { + if ( !is_resource( $f ) ) { + return; + } + $n = stream_select( $r = array( $f ), $w = null, $e = null, 0, 0 ); + while ( $n == 1 && !feof( $f ) ) { + fread( $f, 1024 ); + $n = stream_select( $r = array( $f ), $w = null, $e = null, 0, 0 ); + } + } + + // }}} + // }}} + // }}} +} + +// vim: sts=3 sw=3 et + +// }}} + +class MemCachedClientforWiki extends MWMemcached { + function _debugprint( $text ) { + wfDebug( "memcached: $text" ); + } +} diff --git a/includes/objectcache/MemcachedPhpBagOStuff.php b/includes/objectcache/MemcachedPhpBagOStuff.php new file mode 100644 index 00000000..14016683 --- /dev/null +++ b/includes/objectcache/MemcachedPhpBagOStuff.php @@ -0,0 +1,178 @@ +client = new MemCachedClientforWiki( $params ); + $this->client->set_servers( $params['servers'] ); + $this->client->set_debug( $params['debug'] ); + } + + /** + * @param $debug bool + */ + public function setDebug( $debug ) { + $this->client->set_debug( $debug ); + } + + /** + * @param $key string + * @return Mixed + */ + public function get( $key ) { + return $this->client->get( $this->encodeKey( $key ) ); + } + + /** + * @param $key string + * @param $value + * @param $exptime int + * @return bool + */ + public function set( $key, $value, $exptime = 0 ) { + return $this->client->set( $this->encodeKey( $key ), $value, $exptime ); + } + + /** + * @param $key string + * @param $time int + * @return bool + */ + public function delete( $key, $time = 0 ) { + return $this->client->delete( $this->encodeKey( $key ), $time ); + } + + /** + * @param $key + * @param $timeout int + * @return + */ + public function lock( $key, $timeout = 0 ) { + return $this->client->lock( $this->encodeKey( $key ), $timeout ); + } + + /** + * @param $key string + * @return Mixed + */ + public function unlock( $key ) { + return $this->client->unlock( $this->encodeKey( $key ) ); + } + + /** + * @param $key string + * @param $value int + * @return Mixed + */ + public function add( $key, $value, $exptime = 0 ) { + return $this->client->add( $this->encodeKey( $key ), $value, $exptime ); + } + + /** + * @param $key string + * @param $value int + * @param $exptime + * @return Mixed + */ + public function replace( $key, $value, $exptime = 0 ) { + return $this->client->replace( $this->encodeKey( $key ), $value, $exptime ); + } + + /** + * @param $key string + * @param $value int + * @return Mixed + */ + public function incr( $key, $value = 1 ) { + return $this->client->incr( $this->encodeKey( $key ), $value ); + } + + /** + * @param $key string + * @param $value int + * @return Mixed + */ + public function decr( $key, $value = 1 ) { + return $this->client->decr( $this->encodeKey( $key ), $value ); + } + + /** + * Get the underlying client object. This is provided for debugging + * purposes. + * + * @return MemCachedClientforWiki + */ + public function getClient() { + return $this->client; + } + + /** + * Encode a key for use on the wire inside the memcached protocol. + * + * We encode spaces and line breaks to avoid protocol errors. We encode + * the other control characters for compatibility with libmemcached + * verify_key. We leave other punctuation alone, to maximise backwards + * compatibility. + */ + public function encodeKey( $key ) { + return preg_replace_callback( '/[\x00-\x20\x25\x7f]+/', + array( $this, 'encodeKeyCallback' ), $key ); + } + + protected function encodeKeyCallback( $m ) { + return rawurlencode( $m[0] ); + } + + /** + * Decode a key encoded with encodeKey(). This is provided as a convenience + * function for debugging. + * + * @param $key string + * + * @return string + */ + public function decodeKey( $key ) { + return urldecode( $key ); + } +} + diff --git a/includes/objectcache/MultiWriteBagOStuff.php b/includes/objectcache/MultiWriteBagOStuff.php new file mode 100644 index 00000000..2b88b427 --- /dev/null +++ b/includes/objectcache/MultiWriteBagOStuff.php @@ -0,0 +1,113 @@ +caches = array(); + foreach ( $params['caches'] as $cacheInfo ) { + $this->caches[] = ObjectCache::newFromParams( $cacheInfo ); + } + } + + public function setDebug( $debug ) { + $this->doWrite( 'setDebug', $debug ); + } + + public function get( $key ) { + foreach ( $this->caches as $cache ) { + $value = $cache->get( $key ); + if ( $value !== false ) { + return $value; + } + } + return false; + } + + public function set( $key, $value, $exptime = 0 ) { + return $this->doWrite( 'set', $key, $value, $exptime ); + } + + public function delete( $key, $time = 0 ) { + return $this->doWrite( 'delete', $key, $time ); + } + + public function add( $key, $value, $exptime = 0 ) { + return $this->doWrite( 'add', $key, $value, $exptime ); + } + + public function replace( $key, $value, $exptime = 0 ) { + return $this->doWrite( 'replace', $key, $value, $exptime ); + } + + public function incr( $key, $value = 1 ) { + return $this->doWrite( 'incr', $key, $value ); + } + + public function decr( $key, $value = 1 ) { + return $this->doWrite( 'decr', $key, $value ); + } + + public function lock( $key, $timeout = 0 ) { + // Lock only the first cache, to avoid deadlocks + if ( isset( $this->caches[0] ) ) { + return $this->caches[0]->lock( $key, $timeout ); + } else { + return true; + } + } + + public function unlock( $key ) { + if ( isset( $this->caches[0] ) ) { + return $this->caches[0]->unlock( $key ); + } else { + return true; + } + } + + protected function doWrite( $method /*, ... */ ) { + $ret = true; + $args = func_get_args(); + array_shift( $args ); + + foreach ( $this->caches as $cache ) { + if ( !call_user_func_array( array( $cache, $method ), $args ) ) { + $ret = false; + } + } + return $ret; + } + + /** + * Delete objects expiring before a certain date. + * + * Succeed if any of the child caches succeed. + */ + public function deleteObjectsExpiringBefore( $date ) { + $ret = false; + foreach ( $this->caches as $cache ) { + if ( $cache->deleteObjectsExpiringBefore( $date ) ) { + $ret = true; + } + } + return $ret; + } +} diff --git a/includes/objectcache/ObjectCache.php b/includes/objectcache/ObjectCache.php new file mode 100644 index 00000000..99e38953 --- /dev/null +++ b/includes/objectcache/ObjectCache.php @@ -0,0 +1,119 @@ +serverInfo = $params['server']; + $this->serverInfo['load'] = 1; + } + if ( isset( $params['purgePeriod'] ) ) { + $this->purgePeriod = intval( $params['purgePeriod'] ); + } + if ( isset( $params['tableName'] ) ) { + $this->tableName = $params['tableName']; + } + if ( isset( $params['shards'] ) ) { + $this->shards = intval( $params['shards'] ); + } + } + + /** + * @return DatabaseBase + */ + protected function getDB() { + if ( !isset( $this->db ) ) { + # If server connection info was given, use that + if ( $this->serverInfo ) { + $this->lb = new LoadBalancer( array( + 'servers' => array( $this->serverInfo ) ) ); + $this->db = $this->lb->getConnection( DB_MASTER ); + $this->db->clearFlag( DBO_TRX ); + } else { + # We must keep a separate connection to MySQL in order to avoid deadlocks + # However, SQLite has an opposite behaviour. + # @todo Investigate behaviour for other databases + if ( wfGetDB( DB_MASTER )->getType() == 'sqlite' ) { + $this->db = wfGetDB( DB_MASTER ); + } else { + $this->lb = wfGetLBFactory()->newMainLB(); + $this->db = $this->lb->getConnection( DB_MASTER ); + $this->db->clearFlag( DBO_TRX ); + } + } + } + + return $this->db; + } + + /** + * Get the table name for a given key + */ + protected function getTableByKey( $key ) { + if ( $this->shards > 1 ) { + $hash = hexdec( substr( md5( $key ), 0, 8 ) ) & 0x7fffffff; + return $this->getTableByShard( $hash % $this->shards ); + } else { + return $this->tableName; + } + } + + /** + * Get the table name for a given shard index + */ + protected function getTableByShard( $index ) { + if ( $this->shards > 1 ) { + $decimals = strlen( $this->shards - 1 ); + return $this->tableName . + sprintf( "%0{$decimals}d", $index ); + } else { + return $this->tableName; + } + } + + public function get( $key ) { + # expire old entries if any + $this->garbageCollect(); + $db = $this->getDB(); + $tableName = $this->getTableByKey( $key ); + $row = $db->selectRow( $tableName, array( 'value', 'exptime' ), + array( 'keyname' => $key ), __METHOD__ ); + + if ( !$row ) { + $this->debug( 'get: no matching rows' ); + return false; + } + + $this->debug( "get: retrieved data; expiry time is " . $row->exptime ); + + if ( $this->isExpired( $row->exptime ) ) { + $this->debug( "get: key has expired, deleting" ); + try { + $db->begin(); + # Put the expiry time in the WHERE condition to avoid deleting a + # newly-inserted value + $db->delete( $tableName, + array( + 'keyname' => $key, + 'exptime' => $row->exptime + ), __METHOD__ ); + $db->commit(); + } catch ( DBQueryError $e ) { + $this->handleWriteError( $e ); + } + + return false; + } + + return $this->unserialize( $db->decodeBlob( $row->value ) ); + } + + public function set( $key, $value, $exptime = 0 ) { + $db = $this->getDB(); + $exptime = intval( $exptime ); + + if ( $exptime < 0 ) { + $exptime = 0; + } + + if ( $exptime == 0 ) { + $encExpiry = $this->getMaxDateTime(); + } else { + if ( $exptime < 3.16e8 ) { # ~10 years + $exptime += time(); + } + + $encExpiry = $db->timestamp( $exptime ); + } + try { + $db->begin(); + // (bug 24425) use a replace if the db supports it instead of + // delete/insert to avoid clashes with conflicting keynames + $db->replace( + $this->getTableByKey( $key ), + array( 'keyname' ), + array( + 'keyname' => $key, + 'value' => $db->encodeBlob( $this->serialize( $value ) ), + 'exptime' => $encExpiry + ), __METHOD__ ); + $db->commit(); + } catch ( DBQueryError $e ) { + $this->handleWriteError( $e ); + + return false; + } + + return true; + } + + public function delete( $key, $time = 0 ) { + $db = $this->getDB(); + + try { + $db->begin(); + $db->delete( + $this->getTableByKey( $key ), + array( 'keyname' => $key ), + __METHOD__ ); + $db->commit(); + } catch ( DBQueryError $e ) { + $this->handleWriteError( $e ); + + return false; + } + + return true; + } + + public function incr( $key, $step = 1 ) { + $db = $this->getDB(); + $tableName = $this->getTableByKey( $key ); + $step = intval( $step ); + + try { + $db->begin(); + $row = $db->selectRow( + $tableName, + array( 'value', 'exptime' ), + array( 'keyname' => $key ), + __METHOD__, + array( 'FOR UPDATE' ) ); + if ( $row === false ) { + // Missing + $db->commit(); + + return null; + } + $db->delete( $tableName, array( 'keyname' => $key ), __METHOD__ ); + if ( $this->isExpired( $row->exptime ) ) { + // Expired, do not reinsert + $db->commit(); + + return null; + } + + $oldValue = intval( $this->unserialize( $db->decodeBlob( $row->value ) ) ); + $newValue = $oldValue + $step; + $db->insert( $tableName, + array( + 'keyname' => $key, + 'value' => $db->encodeBlob( $this->serialize( $newValue ) ), + 'exptime' => $row->exptime + ), __METHOD__, 'IGNORE' ); + + if ( $db->affectedRows() == 0 ) { + // Race condition. See bug 28611 + $newValue = null; + } + $db->commit(); + } catch ( DBQueryError $e ) { + $this->handleWriteError( $e ); + + return null; + } + + return $newValue; + } + + public function keys() { + $db = $this->getDB(); + $result = array(); + + for ( $i = 0; $i < $this->shards; $i++ ) { + $res = $db->select( $this->getTableByShard( $i ), + array( 'keyname' ), false, __METHOD__ ); + foreach ( $res as $row ) { + $result[] = $row->keyname; + } + } + + return $result; + } + + protected function isExpired( $exptime ) { + return $exptime != $this->getMaxDateTime() && wfTimestamp( TS_UNIX, $exptime ) < time(); + } + + protected function getMaxDateTime() { + if ( time() > 0x7fffffff ) { + return $this->getDB()->timestamp( 1 << 62 ); + } else { + return $this->getDB()->timestamp( 0x7fffffff ); + } + } + + protected function garbageCollect() { + if ( !$this->purgePeriod ) { + // Disabled + return; + } + // Only purge on one in every $this->purgePeriod requests. + if ( $this->purgePeriod !== 1 && mt_rand( 0, $this->purgePeriod - 1 ) ) { + return; + } + $now = time(); + // Avoid repeating the delete within a few seconds + if ( $now > ( $this->lastExpireAll + 1 ) ) { + $this->lastExpireAll = $now; + $this->expireAll(); + } + } + + public function expireAll() { + $this->deleteObjectsExpiringBefore( wfTimestampNow() ); + } + + /** + * Delete objects from the database which expire before a certain date. + */ + public function deleteObjectsExpiringBefore( $timestamp ) { + $db = $this->getDB(); + $dbTimestamp = $db->timestamp( $timestamp ); + + try { + for ( $i = 0; $i < $this->shards; $i++ ) { + $db->begin(); + $db->delete( + $this->getTableByShard( $i ), + array( 'exptime < ' . $db->addQuotes( $dbTimestamp ) ), + __METHOD__ ); + $db->commit(); + } + } catch ( DBQueryError $e ) { + $this->handleWriteError( $e ); + } + return true; + } + + public function deleteAll() { + $db = $this->getDB(); + + try { + for ( $i = 0; $i < $this->shards; $i++ ) { + $db->begin(); + $db->delete( $this->getTableByShard( $i ), '*', __METHOD__ ); + $db->commit(); + } + } catch ( DBQueryError $e ) { + $this->handleWriteError( $e ); + } + } + + /** + * Serialize an object and, if possible, compress the representation. + * On typical message and page data, this can provide a 3X decrease + * in storage requirements. + * + * @param $data mixed + * @return string + */ + protected function serialize( &$data ) { + $serial = serialize( $data ); + + if ( function_exists( 'gzdeflate' ) ) { + return gzdeflate( $serial ); + } else { + return $serial; + } + } + + /** + * Unserialize and, if necessary, decompress an object. + * @param $serial string + * @return mixed + */ + protected function unserialize( $serial ) { + if ( function_exists( 'gzinflate' ) ) { + wfSuppressWarnings(); + $decomp = gzinflate( $serial ); + wfRestoreWarnings(); + + if ( false !== $decomp ) { + $serial = $decomp; + } + } + + $ret = unserialize( $serial ); + + return $ret; + } + + /** + * Handle a DBQueryError which occurred during a write operation. + * Ignore errors which are due to a read-only database, rethrow others. + */ + protected function handleWriteError( $exception ) { + $db = $this->getDB(); + + if ( !$db->wasReadOnlyError() ) { + throw $exception; + } + + try { + $db->rollback(); + } catch ( DBQueryError $e ) { + } + + wfDebug( __METHOD__ . ": ignoring query error\n" ); + $db->ignoreErrors( false ); + } + + /** + * Create shard tables. For use from eval.php. + */ + public function createTables() { + $db = $this->getDB(); + if ( $db->getType() !== 'mysql' + || version_compare( $db->getServerVersion(), '4.1.0', '<' ) ) + { + throw new MWException( __METHOD__ . ' is not supported on this DB server' ); + } + + for ( $i = 0; $i < $this->shards; $i++ ) { + $db->begin(); + $db->query( + 'CREATE TABLE ' . $db->tableName( $this->getTableByShard( $i ) ) . + ' LIKE ' . $db->tableName( 'objectcache' ), + __METHOD__ ); + $db->commit(); + } + } +} + +/** + * Backwards compatibility alias + */ +class MediaWikiBagOStuff extends SqlBagOStuff { } + diff --git a/includes/objectcache/WinCacheBagOStuff.php b/includes/objectcache/WinCacheBagOStuff.php new file mode 100644 index 00000000..7f464946 --- /dev/null +++ b/includes/objectcache/WinCacheBagOStuff.php @@ -0,0 +1,71 @@ +