diff options
Diffstat (limited to 'includes/poolcounter')
| -rw-r--r-- | includes/poolcounter/PoolCounter.php | 173 | ||||
| -rw-r--r-- | includes/poolcounter/PoolCounterRedis.php | 417 | ||||
| -rw-r--r-- | includes/poolcounter/PoolCounterWork.php | 160 | ||||
| -rw-r--r-- | includes/poolcounter/PoolCounterWorkViaCallback.php | 92 | ||||
| -rw-r--r-- | includes/poolcounter/PoolWorkArticleView.php | 208 |
5 files changed, 1050 insertions, 0 deletions
diff --git a/includes/poolcounter/PoolCounter.php b/includes/poolcounter/PoolCounter.php new file mode 100644 index 00000000..e77ffd7c --- /dev/null +++ b/includes/poolcounter/PoolCounter.php @@ -0,0 +1,173 @@ +<?php +/** + * Provides of semaphore semantics for restricting the number + * of workers that may be concurrently performing the same task. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + */ + +/** + * When you have many workers (threads/servers) giving service, and a + * cached item expensive to produce expires, you may get several workers + * doing the job at the same time. + * + * Given enough requests and the item expiring fast (non-cacheable, + * lots of edits...) that single work can end up unfairly using most (all) + * of the cpu of the pool. This is also known as 'Michael Jackson effect' + * since this effect triggered on the english wikipedia on the day Michael + * Jackson died, the biographical article got hit with several edits per + * minutes and hundreds of read hits. + * + * The PoolCounter provides semaphore semantics for restricting the number + * of workers that may be concurrently performing such single task. + * + * By default PoolCounter_Stub is used, which provides no locking. You + * can get a useful one in the PoolCounter extension. + */ +abstract class PoolCounter { + /* Return codes */ + const LOCKED = 1; /* Lock acquired */ + const RELEASED = 2; /* Lock released */ + const DONE = 3; /* Another worker did the work for you */ + + const ERROR = -1; /* Indeterminate error */ + const NOT_LOCKED = -2; /* Called release() with no lock held */ + const QUEUE_FULL = -3; /* There are already maxqueue workers on this lock */ + const TIMEOUT = -4; /* Timeout exceeded */ + const LOCK_HELD = -5; /* Cannot acquire another lock while you have one lock held */ + + /** @var string All workers with the same key share the lock */ + protected $key; + /** @var int Maximum number of workers working on tasks with the same key simultaneously */ + protected $workers; + /** + * Maximum number of workers working on this task type, regardless of key. + * 0 means unlimited. Max allowed value is 65536. + * The way the slot limit is enforced is overzealous - this option should be used with caution. + * @var int + */ + protected $slots = 0; + /** @var int If this number of workers are already working/waiting, fail instead of wait */ + protected $maxqueue; + /** @var float Maximum time in seconds to wait for the lock */ + protected $timeout; + + /** + * @param array $conf + * @param string $type + * @param string $key + */ + protected function __construct( $conf, $type, $key ) { + $this->workers = $conf['workers']; + $this->maxqueue = $conf['maxqueue']; + $this->timeout = $conf['timeout']; + if ( isset( $conf['slots'] ) ) { + $this->slots = $conf['slots']; + } + + if ( $this->slots ) { + $key = $this->hashKeyIntoSlots( $key, $this->slots ); + } + $this->key = $key; + } + + /** + * Create a Pool counter. This should only be called from the PoolWorks. + * + * @param string $type + * @param string $key + * + * @return PoolCounter + */ + public static function factory( $type, $key ) { + global $wgPoolCounterConf; + if ( !isset( $wgPoolCounterConf[$type] ) ) { + return new PoolCounter_Stub; + } + $conf = $wgPoolCounterConf[$type]; + $class = $conf['class']; + + return new $class( $conf, $type, $key ); + } + + /** + * @return string + */ + public function getKey() { + return $this->key; + } + + /** + * I want to do this task and I need to do it myself. + * + * @return Status Value is one of Locked/Error + */ + abstract public function acquireForMe(); + + /** + * I want to do this task, but if anyone else does it + * instead, it's also fine for me. I will read its cached data. + * + * @return Status Value is one of Locked/Done/Error + */ + abstract public function acquireForAnyone(); + + /** + * I have successfully finished my task. + * Lets another one grab the lock, and returns the workers + * waiting on acquireForAnyone() + * + * @return Status Value is one of Released/NotLocked/Error + */ + abstract public function release(); + + /** + * Given a key (any string) and the number of lots, returns a slot number (an integer from the [0..($slots-1)] range). + * This is used for a global limit on the number of instances of a given type that can acquire a lock. + * The hashing is deterministic so that PoolCounter::$workers is always an upper limit of how many instances with + * the same key can acquire a lock. + * + * @param string $key PoolCounter instance key (any string) + * @param int $slots The number of slots (max allowed value is 65536) + * @return int + */ + protected function hashKeyIntoSlots( $key, $slots ) { + return hexdec( substr( sha1( $key ), 0, 4 ) ) % $slots; + } +} + +// @codingStandardsIgnoreStart Squiz.Classes.ValidClassName.NotCamelCaps +class PoolCounter_Stub extends PoolCounter { + // @codingStandardsIgnoreEnd + + public function __construct() { + /* No parameters needed */ + } + + public function acquireForMe() { + return Status::newGood( PoolCounter::LOCKED ); + } + + public function acquireForAnyone() { + return Status::newGood( PoolCounter::LOCKED ); + } + + public function release() { + return Status::newGood( PoolCounter::RELEASED ); + } +} diff --git a/includes/poolcounter/PoolCounterRedis.php b/includes/poolcounter/PoolCounterRedis.php new file mode 100644 index 00000000..d609f614 --- /dev/null +++ b/includes/poolcounter/PoolCounterRedis.php @@ -0,0 +1,417 @@ +<?php +/** + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + * @author Aaron Schulz + */ + +/** + * Version of PoolCounter that uses Redis + * + * There are four main redis keys used to track each pool counter key: + * - poolcounter:l-slots-* : A list of available slot IDs for a pool. + * - poolcounter:z-renewtime-* : A sorted set of (slot ID, UNIX timestamp as score) + * used for tracking the next time a slot should be + * released. This is -1 when a slot is created, and is + * set when released (expired), locked, and unlocked. + * - poolcounter:z-wait-* : A sorted set of (slot ID, UNIX timestamp as score) + * used for tracking waiting processes (and wait time). + * - poolcounter:l-wakeup-* : A list pushed to for the sake of waking up processes + * when a any process in the pool finishes (lasts for 1ms). + * For a given pool key, all the redis keys start off non-existing and are deleted if not + * used for a while to prevent garbage from building up on the server. They are atomically + * re-initialized as needed. The "z-renewtime" key is used for detecting sessions which got + * slots but then disappeared. Stale entries from there have their timestamp updated and the + * corresponding slots freed up. The "z-wait" key is used for detecting processes registered + * as waiting but that disappeared. Stale entries from there are deleted and the corresponding + * slots are freed up. The worker count is included in all the redis key names as it does not + * vary within each $wgPoolCounterConf type and doing so handles configuration changes. + * + * This class requires Redis 2.6 as it makes use Lua scripts for fast atomic operations. + * Also this should be on a server plenty of RAM for the working set to avoid evictions. + * Evictions could temporarily allow wait queues to double in size or temporarily cause + * pools to appear as full when they are not. Using volatile-ttl and bumping memory-samples + * in redis.conf can be helpful otherwise. + * + * @ingroup Redis + * @since 1.23 + */ +class PoolCounterRedis extends PoolCounter { + /** @var HashRing */ + protected $ring; + /** @var RedisConnectionPool */ + protected $pool; + /** @var array (server label => host) map */ + protected $serversByLabel; + /** @var string SHA-1 of the key */ + protected $keySha1; + /** @var int TTL for locks to expire (work should finish in this time) */ + protected $lockTTL; + + /** @var RedisConnRef */ + protected $conn; + /** @var string Pool slot value */ + protected $slot; + /** @var int AWAKE_* constant */ + protected $onRelease; + /** @var string Unique string to identify this process */ + protected $session; + /** @var int UNIX timestamp */ + protected $slotTime; + + const AWAKE_ONE = 1; // wake-up if when a slot can be taken from an existing process + const AWAKE_ALL = 2; // wake-up if an existing process finishes and wake up such others + + /** @var array List of active PoolCounterRedis objects in this script */ + protected static $active = null; + + function __construct( $conf, $type, $key ) { + parent::__construct( $conf, $type, $key ); + + $this->serversByLabel = $conf['servers']; + $this->ring = new HashRing( array_fill_keys( array_keys( $conf['servers'] ), 100 ) ); + + $conf['redisConfig']['serializer'] = 'none'; // for use with Lua + $this->pool = RedisConnectionPool::singleton( $conf['redisConfig'] ); + + $this->keySha1 = sha1( $this->key ); + $met = ini_get( 'max_execution_time' ); // usually 0 in CLI mode + $this->lockTTL = $met ? 2 * $met : 3600; + + if ( self::$active === null ) { + self::$active = array(); + register_shutdown_function( array( __CLASS__, 'releaseAll' ) ); + } + } + + /** + * @return Status Uses RediConnRef as value on success + */ + protected function getConnection() { + if ( !isset( $this->conn ) ) { + $conn = false; + $servers = $this->ring->getLocations( $this->key, 3 ); + ArrayUtils::consistentHashSort( $servers, $this->key ); + foreach ( $servers as $server ) { + $conn = $this->pool->getConnection( $this->serversByLabel[$server] ); + if ( $conn ) { + break; + } + } + if ( !$conn ) { + return Status::newFatal( 'pool-servererror', implode( ', ', $servers ) ); + } + $this->conn = $conn; + } + return Status::newGood( $this->conn ); + } + + function acquireForMe() { + $section = new ProfileSection( __METHOD__ ); + + return $this->waitForSlotOrNotif( self::AWAKE_ONE ); + } + + function acquireForAnyone() { + $section = new ProfileSection( __METHOD__ ); + + return $this->waitForSlotOrNotif( self::AWAKE_ALL ); + } + + function release() { + $section = new ProfileSection( __METHOD__ ); + + if ( $this->slot === null ) { + return Status::newGood( PoolCounter::NOT_LOCKED ); // not locked + } + + $status = $this->getConnection(); + if ( !$status->isOK() ) { + return $status; + } + $conn = $status->value; + + static $script = +<<<LUA + local kSlots,kSlotsNextRelease,kWakeup,kWaiting = unpack(KEYS) + local rMaxWorkers,rExpiry,rSlot,rSlotTime,rAwakeAll,rTime = unpack(ARGV) + -- Add the slots back to the list (if rSlot is "w" then it is not a slot). + -- Treat the list as expired if the "next release" time sorted-set is missing. + if rSlot ~= 'w' and redis.call('exists',kSlotsNextRelease) == 1 then + if 1*redis.call('zScore',kSlotsNextRelease,rSlot) ~= (rSlotTime + rExpiry) then + -- Slot lock expired and was released already + elseif redis.call('lLen',kSlots) >= 1*rMaxWorkers then + -- Slots somehow got out of sync; reset the list for sanity + redis.call('del',kSlots,kSlotsNextRelease) + elseif redis.call('lLen',kSlots) == (1*rMaxWorkers - 1) and redis.call('zCard',kWaiting) == 0 then + -- Slot list will be made full; clear it to save space (it re-inits as needed) + -- since nothing is waiting on being unblocked by a push to the list + redis.call('del',kSlots,kSlotsNextRelease) + else + -- Add slot back to pool and update the "next release" time + redis.call('rPush',kSlots,rSlot) + redis.call('zAdd',kSlotsNextRelease,rTime + 30,rSlot) + -- Always keep renewing the expiry on use + redis.call('expireAt',kSlots,math.ceil(rTime + rExpiry)) + redis.call('expireAt',kSlotsNextRelease,math.ceil(rTime + rExpiry)) + end + end + -- Update an ephemeral list to wake up other clients that can + -- reuse any cached work from this process. Only do this if no + -- slots are currently free (e.g. clients could be waiting). + if 1*rAwakeAll == 1 then + local count = redis.call('zCard',kWaiting) + for i = 1,count do + redis.call('rPush',kWakeup,'w') + end + redis.call('pexpire',kWakeup,1) + end + return 1 +LUA; + try { + $res = $conn->luaEval( $script, + array( + $this->getSlotListKey(), + $this->getSlotRTimeSetKey(), + $this->getWakeupListKey(), + $this->getWaitSetKey(), + $this->workers, + $this->lockTTL, + $this->slot, + $this->slotTime, // used for CAS-style sanity check + ( $this->onRelease === self::AWAKE_ALL ) ? 1 : 0, + microtime( true ) + ), + 4 # number of first argument(s) that are keys + ); + } catch ( RedisException $e ) { + return Status::newFatal( 'pool-error-unknown', $e->getMessage() ); + } + + $this->slot = null; + $this->slotTime = null; + $this->onRelease = null; + unset( self::$active[$this->session] ); + + return Status::newGood( PoolCounter::RELEASED ); + } + + /** + * @param int $doWakeup AWAKE_* constant + * @return Status + */ + protected function waitForSlotOrNotif( $doWakeup ) { + if ( $this->slot !== null ) { + return Status::newGood( PoolCounter::LOCK_HELD ); // already acquired + } + + $status = $this->getConnection(); + if ( !$status->isOK() ) { + return $status; + } + $conn = $status->value; + + $now = microtime( true ); + try { + $slot = $this->initAndPopPoolSlotList( $conn, $now ); + if ( ctype_digit( $slot ) ) { + // Pool slot acquired by this process + $slotTime = $now; + } elseif ( $slot === 'QUEUE_FULL' ) { + // Too many processes are waiting for pooled processes to finish + return Status::newGood( PoolCounter::QUEUE_FULL ); + } elseif ( $slot === 'QUEUE_WAIT' ) { + // This process is now registered as waiting + $keys = ( $doWakeup == self::AWAKE_ALL ) + // Wait for an open slot or wake-up signal (preferring the later) + ? array( $this->getWakeupListKey(), $this->getSlotListKey() ) + // Just wait for an actual pool slot + : array( $this->getSlotListKey() ); + + $res = $conn->blPop( $keys, $this->timeout ); + if ( $res === array() ) { + $conn->zRem( $this->getWaitSetKey(), $this->session ); // no longer waiting + return Status::newGood( PoolCounter::TIMEOUT ); + } + + $slot = $res[1]; // pool slot or "w" for wake-up notifications + $slotTime = microtime( true ); // last microtime() was a few RTTs ago + // Unregister this process as waiting and bump slot "next release" time + $this->registerAcquisitionTime( $conn, $slot, $slotTime ); + } else { + return Status::newFatal( 'pool-error-unknown', "Server gave slot '$slot'." ); + } + } catch ( RedisException $e ) { + return Status::newFatal( 'pool-error-unknown', $e->getMessage() ); + } + + if ( $slot !== 'w' ) { + $this->slot = $slot; + $this->slotTime = $slotTime; + $this->onRelease = $doWakeup; + self::$active[$this->session] = $this; + } + + return Status::newGood( $slot === 'w' ? PoolCounter::DONE : PoolCounter::LOCKED ); + } + + /** + * @param RedisConnRef $conn + * @param float $now UNIX timestamp + * @return string|bool False on failure + */ + protected function initAndPopPoolSlotList( RedisConnRef $conn, $now ) { + static $script = +<<<LUA + local kSlots,kSlotsNextRelease,kSlotWaits = unpack(KEYS) + local rMaxWorkers,rMaxQueue,rTimeout,rExpiry,rSess,rTime = unpack(ARGV) + -- Initialize if the "next release" time sorted-set is empty. The slot key + -- itself is empty if all slots are busy or when nothing is initialized. + -- If the list is empty but the set is not, then it is the later case. + -- For sanity, if the list exists but not the set, then reset everything. + if redis.call('exists',kSlotsNextRelease) == 0 then + redis.call('del',kSlots) + for i = 1,1*rMaxWorkers do + redis.call('rPush',kSlots,i) + redis.call('zAdd',kSlotsNextRelease,-1,i) + end + -- Otherwise do maintenance to clean up after network partitions + else + -- Find stale slot locks and add free them (avoid duplicates for sanity) + local staleLocks = redis.call('zRangeByScore',kSlotsNextRelease,0,rTime) + for k,slot in ipairs(staleLocks) do + redis.call('lRem',kSlots,0,slot) + redis.call('rPush',kSlots,slot) + redis.call('zAdd',kSlotsNextRelease,rTime + 30,slot) + end + -- Find stale wait slot entries and remove them + redis.call('zRemRangeByScore',kSlotWaits,0,rTime - 2*rTimeout) + end + local slot + -- Try to acquire a slot if possible now + if redis.call('lLen',kSlots) > 0 then + slot = redis.call('lPop',kSlots) + -- Update the slot "next release" time + redis.call('zAdd',kSlotsNextRelease,rTime + rExpiry,slot) + elseif redis.call('zCard',kSlotWaits) >= 1*rMaxQueue then + slot = 'QUEUE_FULL' + else + slot = 'QUEUE_WAIT' + -- Register this process as waiting + redis.call('zAdd',kSlotWaits,rTime,rSess) + redis.call('expireAt',kSlotWaits,math.ceil(rTime + 2*rTimeout)) + end + -- Always keep renewing the expiry on use + redis.call('expireAt',kSlots,math.ceil(rTime + rExpiry)) + redis.call('expireAt',kSlotsNextRelease,math.ceil(rTime + rExpiry)) + return slot +LUA; + return $conn->luaEval( $script, + array( + $this->getSlotListKey(), + $this->getSlotRTimeSetKey(), + $this->getWaitSetKey(), + $this->workers, + $this->maxqueue, + $this->timeout, + $this->lockTTL, + $this->session, + $now + ), + 3 # number of first argument(s) that are keys + ); + } + + /** + * @param RedisConnRef $conn + * @param string $slot + * @param float $now + * @return int|bool False on failure + */ + protected function registerAcquisitionTime( RedisConnRef $conn, $slot, $now ) { + static $script = +<<<LUA + local kSlots,kSlotsNextRelease,kSlotWaits = unpack(KEYS) + local rSlot,rExpiry,rSess,rTime = unpack(ARGV) + -- If rSlot is 'w' then the client was told to wake up but got no slot + if rSlot ~= 'w' then + -- Update the slot "next release" time + redis.call('zAdd',kSlotsNextRelease,rTime + rExpiry,rSlot) + -- Always keep renewing the expiry on use + redis.call('expireAt',kSlots,math.ceil(rTime + rExpiry)) + redis.call('expireAt',kSlotsNextRelease,math.ceil(rTime + rExpiry)) + end + -- Unregister this process as waiting + redis.call('zRem',kSlotWaits,rSess) + return 1 +LUA; + return $conn->luaEval( $script, + array( + $this->getSlotListKey(), + $this->getSlotRTimeSetKey(), + $this->getWaitSetKey(), + $slot, + $this->lockTTL, + $this->session, + $now + ), + 3 # number of first argument(s) that are keys + ); + } + + /** + * @return string + */ + protected function getSlotListKey() { + return "poolcounter:l-slots-{$this->keySha1}-{$this->workers}"; + } + + /** + * @return string + */ + protected function getSlotRTimeSetKey() { + return "poolcounter:z-renewtime-{$this->keySha1}-{$this->workers}"; + } + + /** + * @return string + */ + protected function getWaitSetKey() { + return "poolcounter:z-wait-{$this->keySha1}-{$this->workers}"; + } + + /** + * @return string + */ + protected function getWakeupListKey() { + return "poolcounter:l-wakeup-{$this->keySha1}-{$this->workers}"; + } + + /** + * Try to make sure that locks get released (even with exceptions and fatals) + */ + public static function releaseAll() { + foreach ( self::$active as $poolCounter ) { + try { + if ( $poolCounter->slot !== null ) { + $poolCounter->release(); + } + } catch ( Exception $e ) { + } + } + } +} diff --git a/includes/poolcounter/PoolCounterWork.php b/includes/poolcounter/PoolCounterWork.php new file mode 100644 index 00000000..c0be7a1b --- /dev/null +++ b/includes/poolcounter/PoolCounterWork.php @@ -0,0 +1,160 @@ +<?php +/** + * Provides of semaphore semantics for restricting the number + * of workers that may be concurrently performing the same task. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + */ + +/** + * Class for dealing with PoolCounters using class members + */ +abstract class PoolCounterWork { + /** @var string */ + protected $type = 'generic'; + /** @var bool */ + protected $cacheable = false; // does this override getCachedWork() ? + + /** + * @param string $type The type of PoolCounter to use + * @param string $key Key that identifies the queue this work is placed on + */ + public function __construct( $type, $key ) { + $this->type = $type; + $this->poolCounter = PoolCounter::factory( $type, $key ); + } + + /** + * Actually perform the work, caching it if needed + * @return mixed Work result or false + */ + abstract public function doWork(); + + /** + * Retrieve the work from cache + * @return mixed Work result or false + */ + public function getCachedWork() { + return false; + } + + /** + * A work not so good (eg. expired one) but better than an error + * message. + * @return mixed Work result or false + */ + public function fallback() { + return false; + } + + /** + * Do something with the error, like showing it to the user. + * + * @param Status $status + * + * @return bool + */ + public function error( $status ) { + return false; + } + + /** + * Log an error + * + * @param Status $status + * @return void + */ + public function logError( $status ) { + $key = $this->poolCounter->getKey(); + + wfDebugLog( 'poolcounter', "Pool key '$key' ({$this->type}): " + . $status->getMessage()->inLanguage( 'en' )->useDatabase( false )->text() ); + } + + /** + * Get the result of the work (whatever it is), or the result of the error() function. + * This returns the result of the first applicable method that returns a non-false value, + * where the methods are checked in the following order: + * - a) doWork() : Applies if the work is exclusive or no another process + * is doing it, and on the condition that either this process + * successfully entered the pool or the pool counter is down. + * - b) doCachedWork() : Applies if the work is cacheable and this blocked on another + * process which finished the work. + * - c) fallback() : Applies for all remaining cases. + * If these all fall through (by returning false), then the result of error() is returned. + * + * @param bool $skipcache + * @return mixed + */ + public function execute( $skipcache = false ) { + if ( $this->cacheable && !$skipcache ) { + $status = $this->poolCounter->acquireForAnyone(); + } else { + $status = $this->poolCounter->acquireForMe(); + } + + if ( !$status->isOK() ) { + // Respond gracefully to complete server breakage: just log it and do the work + $this->logError( $status ); + return $this->doWork(); + } + + switch ( $status->value ) { + case PoolCounter::LOCK_HELD: + // Better to ignore nesting pool counter limits than to fail. + // Assume that the outer pool limiting is reasonable enough. + /* no break */ + case PoolCounter::LOCKED: + $result = $this->doWork(); + $this->poolCounter->release(); + return $result; + + case PoolCounter::DONE: + $result = $this->getCachedWork(); + if ( $result === false ) { + /* That someone else work didn't serve us. + * Acquire the lock for me + */ + return $this->execute( true ); + } + return $result; + + case PoolCounter::QUEUE_FULL: + case PoolCounter::TIMEOUT: + $result = $this->fallback(); + + if ( $result !== false ) { + return $result; + } + /* no break */ + + /* These two cases should never be hit... */ + case PoolCounter::ERROR: + default: + $errors = array( + PoolCounter::QUEUE_FULL => 'pool-queuefull', + PoolCounter::TIMEOUT => 'pool-timeout' ); + + $status = Status::newFatal( isset( $errors[$status->value] ) + ? $errors[$status->value] + : 'pool-errorunknown' ); + $this->logError( $status ); + return $this->error( $status ); + } + } +} diff --git a/includes/poolcounter/PoolCounterWorkViaCallback.php b/includes/poolcounter/PoolCounterWorkViaCallback.php new file mode 100644 index 00000000..af83d2e0 --- /dev/null +++ b/includes/poolcounter/PoolCounterWorkViaCallback.php @@ -0,0 +1,92 @@ +<?php +/** + * Provides of semaphore semantics for restricting the number + * of workers that may be concurrently performing the same task. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + */ + +/** + * Convenience class for dealing with PoolCounters using callbacks + * @since 1.22 + */ +class PoolCounterWorkViaCallback extends PoolCounterWork { + /** @var callable */ + protected $doWork; + /** @var callable|null */ + protected $doCachedWork; + /** @var callable|null */ + protected $fallback; + /** @var callable|null */ + protected $error; + + /** + * Build a PoolCounterWork class from a type, key, and callback map. + * + * The callback map must at least have a callback for the 'doWork' method. + * Additionally, callbacks can be provided for the 'doCachedWork', 'fallback', + * and 'error' methods. Methods without callbacks will be no-ops that return false. + * If a 'doCachedWork' callback is provided, then execute() may wait for any prior + * process in the pool to finish and reuse its cached result. + * + * @param string $type + * @param string $key + * @param array $callbacks Map of callbacks + * @throws MWException + */ + public function __construct( $type, $key, array $callbacks ) { + parent::__construct( $type, $key ); + foreach ( array( 'doWork', 'doCachedWork', 'fallback', 'error' ) as $name ) { + if ( isset( $callbacks[$name] ) ) { + if ( !is_callable( $callbacks[$name] ) ) { + throw new MWException( "Invalid callback provided for '$name' function." ); + } + $this->$name = $callbacks[$name]; + } + } + if ( !isset( $this->doWork ) ) { + throw new MWException( "No callback provided for 'doWork' function." ); + } + $this->cacheable = isset( $this->doCachedWork ); + } + + public function doWork() { + return call_user_func_array( $this->doWork, array() ); + } + + public function getCachedWork() { + if ( $this->doCachedWork ) { + return call_user_func_array( $this->doCachedWork, array() ); + } + return false; + } + + public function fallback() { + if ( $this->fallback ) { + return call_user_func_array( $this->fallback, array() ); + } + return false; + } + + public function error( $status ) { + if ( $this->error ) { + return call_user_func_array( $this->error, array( $status ) ); + } + return false; + } +} diff --git a/includes/poolcounter/PoolWorkArticleView.php b/includes/poolcounter/PoolWorkArticleView.php new file mode 100644 index 00000000..5e7e3912 --- /dev/null +++ b/includes/poolcounter/PoolWorkArticleView.php @@ -0,0 +1,208 @@ +<?php +/** + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + * http://www.gnu.org/copyleft/gpl.html + * + * @file + */ + +class PoolWorkArticleView extends PoolCounterWork { + /** @var Page */ + private $page; + + /** @var string */ + private $cacheKey; + + /** @var int */ + private $revid; + + /** @var ParserOptions */ + private $parserOptions; + + /** @var Content|null */ + private $content = null; + + /** @var ParserOutput|bool */ + private $parserOutput = false; + + /** @var bool */ + private $isDirty = false; + + /** @var Status|bool */ + private $error = false; + + /** + * @param Page $page + * @param ParserOptions $parserOptions ParserOptions to use for the parse + * @param int $revid ID of the revision being parsed. + * @param bool $useParserCache Whether to use the parser cache. + * operation. + * @param Content|string $content Content to parse or null to load it; may + * also be given as a wikitext string, for BC. + */ + public function __construct( Page $page, ParserOptions $parserOptions, + $revid, $useParserCache, $content = null + ) { + if ( is_string( $content ) ) { // BC: old style call + $modelId = $page->getRevision()->getContentModel(); + $format = $page->getRevision()->getContentFormat(); + $content = ContentHandler::makeContent( $content, $page->getTitle(), $modelId, $format ); + } + + $this->page = $page; + $this->revid = $revid; + $this->cacheable = $useParserCache; + $this->parserOptions = $parserOptions; + $this->content = $content; + $this->cacheKey = ParserCache::singleton()->getKey( $page, $parserOptions ); + parent::__construct( 'ArticleView', $this->cacheKey . ':revid:' . $revid ); + } + + /** + * Get the ParserOutput from this object, or false in case of failure + * + * @return ParserOutput + */ + public function getParserOutput() { + return $this->parserOutput; + } + + /** + * Get whether the ParserOutput is a dirty one (i.e. expired) + * + * @return bool + */ + public function getIsDirty() { + return $this->isDirty; + } + + /** + * Get a Status object in case of error or false otherwise + * + * @return Status|bool + */ + public function getError() { + return $this->error; + } + + /** + * @return bool + */ + public function doWork() { + global $wgUseFileCache; + + // @todo several of the methods called on $this->page are not declared in Page, but present + // in WikiPage and delegated by Article. + + $isCurrent = $this->revid === $this->page->getLatest(); + + if ( $this->content !== null ) { + $content = $this->content; + } elseif ( $isCurrent ) { + // XXX: why use RAW audience here, and PUBLIC (default) below? + $content = $this->page->getContent( Revision::RAW ); + } else { + $rev = Revision::newFromTitle( $this->page->getTitle(), $this->revid ); + + if ( $rev === null ) { + $content = null; + } else { + // XXX: why use PUBLIC audience here (default), and RAW above? + $content = $rev->getContent(); + } + } + + if ( $content === null ) { + return false; + } + + // Reduce effects of race conditions for slow parses (bug 46014) + $cacheTime = wfTimestampNow(); + + $time = - microtime( true ); + $this->parserOutput = $content->getParserOutput( + $this->page->getTitle(), + $this->revid, + $this->parserOptions + ); + $time += microtime( true ); + + // Timing hack + if ( $time > 3 ) { + wfDebugLog( 'slow-parse', sprintf( "%-5.2f %s", $time, + $this->page->getTitle()->getPrefixedDBkey() ) ); + } + + if ( $this->cacheable && $this->parserOutput->isCacheable() && $isCurrent ) { + ParserCache::singleton()->save( + $this->parserOutput, $this->page, $this->parserOptions, $cacheTime, $this->revid ); + } + + // Make sure file cache is not used on uncacheable content. + // Output that has magic words in it can still use the parser cache + // (if enabled), though it will generally expire sooner. + if ( !$this->parserOutput->isCacheable() || $this->parserOutput->containsOldMagic() ) { + $wgUseFileCache = false; + } + + if ( $isCurrent ) { + $this->page->doCascadeProtectionUpdates( $this->parserOutput ); + } + + return true; + } + + /** + * @return bool + */ + public function getCachedWork() { + $this->parserOutput = ParserCache::singleton()->get( $this->page, $this->parserOptions ); + + if ( $this->parserOutput === false ) { + wfDebug( __METHOD__ . ": parser cache miss\n" ); + return false; + } else { + wfDebug( __METHOD__ . ": parser cache hit\n" ); + return true; + } + } + + /** + * @return bool + */ + public function fallback() { + $this->parserOutput = ParserCache::singleton()->getDirty( $this->page, $this->parserOptions ); + + if ( $this->parserOutput === false ) { + wfDebugLog( 'dirty', 'dirty missing' ); + wfDebug( __METHOD__ . ": no dirty cache\n" ); + return false; + } else { + wfDebug( __METHOD__ . ": sending dirty output\n" ); + wfDebugLog( 'dirty', "dirty output {$this->cacheKey}" ); + $this->isDirty = true; + return true; + } + } + + /** + * @param Status $status + * @return bool + */ + public function error( $status ) { + $this->error = $status; + return false; + } +} |