Update to MediaWiki 1.21.1

1 files changed, 311 insertions, 217 deletions

diff --git a/includes/objectcache/SqlBagOStuff.php b/includes/objectcache/SqlBagOStuff.php
index 54051dc1..87f787d8 100644
--- a/includes/objectcache/SqlBagOStuff.php
+++ b/includes/objectcache/SqlBagOStuff.php
@@ -32,23 +32,26 @@ class SqlBagOStuff extends BagOStuff {
 	 */
 	var $lb;
 
-	/**
-	 * @var DatabaseBase
-	 */
-	var $db;
-	var $serverInfo;
+	var $serverInfos;
+	var $serverNames;
+	var $numServers;
+	var $conns;
 	var $lastExpireAll = 0;
 	var $purgePeriod = 100;
 	var $shards = 1;
 	var $tableName = 'objectcache';
 
-	protected $connFailureTime = 0; // UNIX timestamp
-	protected $connFailureError; // exception
+	protected $connFailureTimes = array(); // UNIX timestamps
+	protected $connFailureErrors = array(); // exceptions
 
 	/**
 	 * Constructor. Parameters are:
-	 *   - server:   A server info structure in the format required by each
-	 *               element in $wgDBServers.
+	 *   - server:      A server info structure in the format required by each
+	 *                  element in $wgDBServers.
+	 *
+	 *   - servers:     An array of server info structures describing a set of
+	 *                  database servers to distribute keys to. If this is
+	 *                  specified, the "server" option will be ignored.
 	 *
 	 *   - purgePeriod: The average number of object cache requests in between
 	 *                  garbage collection operations, where expired entries
@@ -59,8 +62,8 @@ class SqlBagOStuff extends BagOStuff {
 	 *
 	 *   - tableName:   The table name to use, default is "objectcache".
 	 *
-	 *   - shards:      The number of tables to use for data storage. If this is
-	 *                  more than 1, table names will be formed in the style
+	 *   - shards:      The number of tables to use for data storage on each server.
+	 *                  If this is more than 1, table names will be formed in the style
 	 *                  objectcacheNNN where NNN is the shard index, between 0 and
 	 *                  shards-1. The number of digits will be the minimum number
 	 *                  required to hold the largest shard index. Data will be
@@ -70,9 +73,19 @@ class SqlBagOStuff extends BagOStuff {
 	 * @param $params array
 	 */
 	public function __construct( $params ) {
-		if ( isset( $params['server'] ) ) {
-			$this->serverInfo = $params['server'];
-			$this->serverInfo['load'] = 1;
+		if ( isset( $params['servers'] ) ) {
+			$this->serverInfos = $params['servers'];
+			$this->numServers = count( $this->serverInfos );
+			$this->serverNames = array();
+			foreach ( $this->serverInfos as $i => $info ) {
+				$this->serverNames[$i] = isset( $info['host'] ) ? $info['host'] : "#$i";
+			}
+		} elseif ( isset( $params['server'] ) ) {
+			$this->serverInfos = array( $params['server'] );
+			$this->numServers = count( $this->serverInfos );
+		} else {
+			$this->serverInfos = false;
+			$this->numServers = 1;
 		}
 		if ( isset( $params['purgePeriod'] ) ) {
 			$this->purgePeriod = intval( $params['purgePeriod'] );
@@ -86,60 +99,81 @@ class SqlBagOStuff extends BagOStuff {
 	}
 
 	/**
+	 * Get a connection to the specified database
+	 *
+	 * @param $serverIndex integer
 	 * @return DatabaseBase
 	 */
-	protected function getDB() {
+	protected function getDB( $serverIndex ) {
 		global $wgDebugDBTransactions;
 
-		# Don't keep timing out trying to connect for each call if the DB is down
-		if ( $this->connFailureError && ( time() - $this->connFailureTime ) < 60 ) {
-			throw $this->connFailureError;
-		}
+		if ( !isset( $this->conns[$serverIndex] ) ) {
+			if ( $serverIndex >= $this->numServers ) {
+				throw new MWException( __METHOD__ . ": Invalid server index \"$serverIndex\"" );
+			}
+
+			# Don't keep timing out trying to connect for each call if the DB is down
+			if ( isset( $this->connFailureErrors[$serverIndex] )
+				&& ( time() - $this->connFailureTimes[$serverIndex] ) < 60 )
+			{
+				throw $this->connFailureErrors[$serverIndex];
+			}
 
-		if ( !isset( $this->db ) ) {
 			# If server connection info was given, use that
-			if ( $this->serverInfo ) {
+			if ( $this->serverInfos ) {
 				if ( $wgDebugDBTransactions ) {
-					wfDebug( sprintf( "Using provided serverInfo for SqlBagOStuff\n" ) );
+					wfDebug( "Using provided serverInfo for SqlBagOStuff\n" );
 				}
-				$this->lb = new LoadBalancer( array(
-					'servers' => array( $this->serverInfo ) ) );
-				$this->db = $this->lb->getConnection( DB_MASTER );
-				$this->db->clearFlag( DBO_TRX );
+				$info = $this->serverInfos[$serverIndex];
+				$type = isset( $info['type'] ) ? $info['type'] : 'mysql';
+				$host = isset( $info['host'] ) ? $info['host'] : '[unknown]';
+				wfDebug( __CLASS__ . ": connecting to $host\n" );
+				$db = DatabaseBase::factory( $type, $info );
+				$db->clearFlag( DBO_TRX );
 			} else {
 				/*
 				 * We must keep a separate connection to MySQL in order to avoid deadlocks
-				 * However, SQLite has an opposite behaviour. And PostgreSQL needs to know
+				 * However, SQLite has an opposite behavior. And PostgreSQL needs to know
 				 * if we are in transaction or no
 				 */
 				if ( wfGetDB( DB_MASTER )->getType() == 'mysql' ) {
 					$this->lb = wfGetLBFactory()->newMainLB();
-					$this->db = $this->lb->getConnection( DB_MASTER );
-					$this->db->clearFlag( DBO_TRX ); // auto-commit mode
+					$db = $this->lb->getConnection( DB_MASTER );
+					$db->clearFlag( DBO_TRX ); // auto-commit mode
 				} else {
-					$this->db = wfGetDB( DB_MASTER );
+					$db = wfGetDB( DB_MASTER );
 				}
 			}
 			if ( $wgDebugDBTransactions ) {
-				wfDebug( sprintf( "Connection %s will be used for SqlBagOStuff\n", $this->db ) );
+				wfDebug( sprintf( "Connection %s will be used for SqlBagOStuff\n", $db ) );
 			}
+			$this->conns[$serverIndex] = $db;
 		}
 
-		return $this->db;
+		return $this->conns[$serverIndex];
 	}
 
 	/**
-	 * Get the table name for a given key
+	 * Get the server index and table name for a given key
 	 * @param $key string
-	 * @return string
+	 * @return Array: server index and table name
 	 */
 	protected function getTableByKey( $key ) {
 		if ( $this->shards > 1 ) {
 			$hash = hexdec( substr( md5( $key ), 0, 8 ) ) & 0x7fffffff;
-			return $this->getTableByShard( $hash % $this->shards );
+			$tableIndex = $hash % $this->shards;
 		} else {
-			return $this->tableName;
+			$tableIndex = 0;
+		}
+		if ( $this->numServers > 1 ) {
+			$sortedServers = $this->serverNames;
+			ArrayUtils::consistentHashSort( $sortedServers, $key );
+			reset( $sortedServers );
+			$serverIndex = key( $sortedServers );
+		} else {
+			$serverIndex = 0;
 		}
+		return array( $serverIndex, $this->getTableNameByShard( $tableIndex ) );
 	}
 
 	/**
@@ -147,7 +181,7 @@ class SqlBagOStuff extends BagOStuff {
 	 * @param $index int
 	 * @return string
 	 */
-	protected function getTableByShard( $index ) {
+	protected function getTableNameByShard( $index ) {
 		if ( $this->shards > 1 ) {
 			$decimals = strlen( $this->shards - 1 );
 			return $this->tableName .
@@ -159,11 +193,16 @@ class SqlBagOStuff extends BagOStuff {
 
 	/**
 	 * @param $key string
+	 * @param $casToken[optional] mixed
 	 * @return mixed
 	 */
-	public function get( $key ) {
+	public function get( $key, &$casToken = null ) {
 		$values = $this->getMulti( array( $key ) );
-		return array_key_exists( $key, $values ) ? $values[$key] : false;
+		if ( array_key_exists( $key, $values ) ) {
+			$casToken = $values[$key];
+			return $values[$key];
+		}
+		return false;
 	}
 
 	/**
@@ -173,59 +212,61 @@ class SqlBagOStuff extends BagOStuff {
 	public function getMulti( array $keys ) {
 		$values = array(); // array of (key => value)
 
-		try {
-			$db = $this->getDB();
-			$keysByTableName = array();
-			foreach ( $keys as $key ) {
-				$tableName = $this->getTableByKey( $key );
-				if ( !isset( $keysByTableName[$tableName] ) ) {
-					$keysByTableName[$tableName] = array();
-				}
-				$keysByTableName[$tableName][] = $key;
-			}
+		$keysByTable = array();
+		foreach ( $keys as $key ) {
+			list( $serverIndex, $tableName ) = $this->getTableByKey( $key );
+			$keysByTable[$serverIndex][$tableName][] = $key;
+		}
 
-			$this->garbageCollect(); // expire old entries if any
+		$this->garbageCollect(); // expire old entries if any
 
-			$dataRows = array();
-			foreach ( $keysByTableName as $tableName => $tableKeys ) {
-				$res = $db->select( $tableName,
-					array( 'keyname', 'value', 'exptime' ),
-					array( 'keyname' => $tableKeys ),
-					__METHOD__ );
-				foreach ( $res as $row ) {
-					$dataRows[$row->keyname] = $row;
+		$dataRows = array();
+		foreach ( $keysByTable as $serverIndex => $serverKeys ) {
+			$db = $this->getDB( $serverIndex );
+			try {
+				foreach ( $serverKeys as $tableName => $tableKeys ) {
+					$res = $db->select( $tableName,
+						array( 'keyname', 'value', 'exptime' ),
+						array( 'keyname' => $tableKeys ),
+						__METHOD__ );
+					foreach ( $res as $row ) {
+						$row->serverIndex = $serverIndex;
+						$row->tableName = $tableName;
+						$dataRows[$row->keyname] = $row;
+					}
 				}
+			} catch ( DBError $e ) {
+				$this->handleReadError( $e, $serverIndex );
 			}
+		}
 
-			foreach ( $keys as $key ) {
-				if ( isset( $dataRows[$key] ) ) { // HIT?
-					$row = $dataRows[$key];
-					$this->debug( "get: retrieved data; expiry time is " . $row->exptime );
-					if ( $this->isExpired( $row->exptime ) ) { // MISS
-						$this->debug( "get: key has expired, deleting" );
-						try {
-							$db->begin( __METHOD__ );
-							# Put the expiry time in the WHERE condition to avoid deleting a
-							# newly-inserted value
-							$db->delete( $this->getTableByKey( $key ),
-								array( 'keyname' => $key, 'exptime' => $row->exptime ),
-								__METHOD__ );
-							$db->commit( __METHOD__ );
-						} catch ( DBQueryError $e ) {
-							$this->handleWriteError( $e );
-						}
-						$values[$key] = false;
-					} else { // HIT
-						$values[$key] = $this->unserialize( $db->decodeBlob( $row->value ) );
+		foreach ( $keys as $key ) {
+			if ( isset( $dataRows[$key] ) ) { // HIT?
+				$row = $dataRows[$key];
+				$this->debug( "get: retrieved data; expiry time is " . $row->exptime );
+				$db = $this->getDB( $row->serverIndex );
+				if ( $this->isExpired( $db, $row->exptime ) ) { // MISS
+					$this->debug( "get: key has expired, deleting" );
+					try {
+						$db->begin( __METHOD__ );
+						# Put the expiry time in the WHERE condition to avoid deleting a
+						# newly-inserted value
+						$db->delete( $row->tableName,
+							array( 'keyname' => $key, 'exptime' => $row->exptime ),
+							__METHOD__ );
+						$db->commit( __METHOD__ );
+					} catch ( DBQueryError $e ) {
+						$this->handleWriteError( $e, $row->serverIndex );
 					}
-				} else { // MISS
 					$values[$key] = false;
-					$this->debug( 'get: no matching rows' );
+				} else { // HIT
+					$values[$key] = $this->unserialize( $db->decodeBlob( $row->value ) );
 				}
+			} else { // MISS
+				$values[$key] = false;
+				$this->debug( 'get: no matching rows' );
 			}
-		} catch ( DBError $e ) {
-			$this->handleReadError( $e );
-		};
+		}
 
 		return $values;
 	}
@@ -237,8 +278,9 @@ class SqlBagOStuff extends BagOStuff {
 	 * @return bool
 	 */
 	public function set( $key, $value, $exptime = 0 ) {
+		list( $serverIndex, $tableName ) = $this->getTableByKey( $key );
 		try {
-			$db = $this->getDB();
+			$db = $this->getDB( $serverIndex );
 			$exptime = intval( $exptime );
 
 			if ( $exptime < 0 ) {
@@ -246,7 +288,7 @@ class SqlBagOStuff extends BagOStuff {
 			}
 
 			if ( $exptime == 0 ) {
-				$encExpiry = $this->getMaxDateTime();
+				$encExpiry = $this->getMaxDateTime( $db );
 			} else {
 				if ( $exptime < 3.16e8 ) { # ~10 years
 					$exptime += time();
@@ -258,7 +300,7 @@ class SqlBagOStuff extends BagOStuff {
 			// (bug 24425) use a replace if the db supports it instead of
 			// delete/insert to avoid clashes with conflicting keynames
 			$db->replace(
-				$this->getTableByKey( $key ),
+				$tableName,
 				array( 'keyname' ),
 				array(
 					'keyname' => $key,
@@ -267,7 +309,7 @@ class SqlBagOStuff extends BagOStuff {
 				), __METHOD__ );
 			$db->commit( __METHOD__ );
 		} catch ( DBError $e ) {
-			$this->handleWriteError( $e );
+			$this->handleWriteError( $e, $serverIndex );
 			return false;
 		}
 
@@ -275,21 +317,73 @@ class SqlBagOStuff extends BagOStuff {
 	}
 
 	/**
+	 * @param $casToken mixed
+	 * @param $key string
+	 * @param $value mixed
+	 * @param $exptime int
+	 * @return bool
+	 */
+	public function cas( $casToken, $key, $value, $exptime = 0 ) {
+		list( $serverIndex, $tableName ) = $this->getTableByKey( $key );
+		try {
+			$db = $this->getDB( $serverIndex );
+			$exptime = intval( $exptime );
+
+			if ( $exptime < 0 ) {
+				$exptime = 0;
+			}
+
+			if ( $exptime == 0 ) {
+				$encExpiry = $this->getMaxDateTime( $db );
+			} else {
+				if ( $exptime < 3.16e8 ) { # ~10 years
+					$exptime += time();
+				}
+				$encExpiry = $db->timestamp( $exptime );
+			}
+			$db->begin( __METHOD__ );
+			// (bug 24425) use a replace if the db supports it instead of
+			// delete/insert to avoid clashes with conflicting keynames
+			$db->update(
+				$tableName,
+				array(
+					'keyname' => $key,
+					'value' => $db->encodeBlob( $this->serialize( $value ) ),
+					'exptime' => $encExpiry
+				),
+				array(
+					'keyname' => $key,
+					'value' => $db->encodeBlob( $this->serialize( $casToken ) )
+				),
+				__METHOD__
+			);
+			$db->commit( __METHOD__ );
+		} catch ( DBQueryError $e ) {
+			$this->handleWriteError( $e, $serverIndex );
+
+			return false;
+		}
+
+		return (bool) $db->affectedRows();
+	}
+
+	/**
 	 * @param $key string
 	 * @param $time int
 	 * @return bool
 	 */
 	public function delete( $key, $time = 0 ) {
+		list( $serverIndex, $tableName ) = $this->getTableByKey( $key );
 		try {
-			$db = $this->getDB();
+			$db = $this->getDB( $serverIndex );
 			$db->begin( __METHOD__ );
 			$db->delete(
-				$this->getTableByKey( $key ),
+				$tableName,
 				array( 'keyname' => $key ),
 				__METHOD__ );
 			$db->commit( __METHOD__ );
 		} catch ( DBError $e ) {
-			$this->handleWriteError( $e );
+			$this->handleWriteError( $e, $serverIndex );
 			return false;
 		}
 
@@ -302,9 +396,9 @@ class SqlBagOStuff extends BagOStuff {
 	 * @return int|null
 	 */
 	public function incr( $key, $step = 1 ) {
+		list( $serverIndex, $tableName ) = $this->getTableByKey( $key );
 		try {
-			$db = $this->getDB();
-			$tableName = $this->getTableByKey( $key );
+			$db = $this->getDB( $serverIndex );
 			$step = intval( $step );
 			$db->begin( __METHOD__ );
 			$row = $db->selectRow(
@@ -320,7 +414,7 @@ class SqlBagOStuff extends BagOStuff {
 				return null;
 			}
 			$db->delete( $tableName, array( 'keyname' => $key ), __METHOD__ );
-			if ( $this->isExpired( $row->exptime ) ) {
+			if ( $this->isExpired( $db, $row->exptime ) ) {
 				// Expired, do not reinsert
 				$db->commit( __METHOD__ );
 
@@ -342,7 +436,7 @@ class SqlBagOStuff extends BagOStuff {
 			}
 			$db->commit( __METHOD__ );
 		} catch ( DBError $e ) {
-			$this->handleWriteError( $e );
+			$this->handleWriteError( $e, $serverIndex );
 			return null;
 		}
 
@@ -350,43 +444,21 @@ class SqlBagOStuff extends BagOStuff {
 	}
 
 	/**
-	 * @return Array
-	 */
-	public function keys() {
-		$result = array();
-
-		try {
-			$db = $this->getDB();
-			for ( $i = 0; $i < $this->shards; $i++ ) {
-				$res = $db->select( $this->getTableByShard( $i ),
-					array( 'keyname' ), false, __METHOD__ );
-				foreach ( $res as $row ) {
-					$result[] = $row->keyname;
-				}
-			}
-		} catch ( DBError $e ) {
-			$this->handleReadError( $e );
-		}
-
-		return $result;
-	}
-
-	/**
 	 * @param $exptime string
 	 * @return bool
 	 */
-	protected function isExpired( $exptime ) {
-		return $exptime != $this->getMaxDateTime() && wfTimestamp( TS_UNIX, $exptime ) < time();
+	protected function isExpired( $db, $exptime ) {
+		return $exptime != $this->getMaxDateTime( $db ) && wfTimestamp( TS_UNIX, $exptime ) < time();
 	}
 
 	/**
 	 * @return string
 	 */
-	protected function getMaxDateTime() {
+	protected function getMaxDateTime( $db ) {
 		if ( time() > 0x7fffffff ) {
-			return $this->getDB()->timestamp( 1 << 62 );
+			return $db->timestamp( 1 << 62 );
 		} else {
-			return $this->getDB()->timestamp( 0x7fffffff );
+			return $db->timestamp( 0x7fffffff );
 		}
 	}
 
@@ -418,87 +490,91 @@ class SqlBagOStuff extends BagOStuff {
 	 * @return bool
 	 */
 	public function deleteObjectsExpiringBefore( $timestamp, $progressCallback = false ) {
-		try {
-			$db = $this->getDB();
-			$dbTimestamp = $db->timestamp( $timestamp );
-			$totalSeconds = false;
-			$baseConds = array( 'exptime < ' . $db->addQuotes( $dbTimestamp ) );
-			for ( $i = 0; $i < $this->shards; $i++ ) {
-				$maxExpTime = false;
-				while ( true ) {
-					$conds = $baseConds;
-					if ( $maxExpTime !== false ) {
-						$conds[] = 'exptime > ' . $db->addQuotes( $maxExpTime );
-					}
-					$rows = $db->select(
-						$this->getTableByShard( $i ),
-						array( 'keyname', 'exptime' ),
-						$conds,
-						__METHOD__,
-						array( 'LIMIT' => 100, 'ORDER BY' => 'exptime' ) );
-					if ( !$rows->numRows() ) {
-						break;
-					}
-					$keys = array();
-					$row = $rows->current();
-					$minExpTime = $row->exptime;
-					if ( $totalSeconds === false ) {
-						$totalSeconds = wfTimestamp( TS_UNIX, $timestamp )
-							- wfTimestamp( TS_UNIX, $minExpTime );
-					}
-					foreach ( $rows as $row ) {
-						$keys[] = $row->keyname;
-						$maxExpTime = $row->exptime;
-					}
-
-					$db->begin( __METHOD__ );
-					$db->delete(
-						$this->getTableByShard( $i ),
-						array(
-							'exptime >= ' . $db->addQuotes( $minExpTime ),
-							'exptime < ' . $db->addQuotes( $dbTimestamp ),
-							'keyname' => $keys
-						),
-						__METHOD__ );
-					$db->commit( __METHOD__ );
+		for ( $serverIndex = 0; $serverIndex < $this->numServers; $serverIndex++ ) {
+			try {
+				$db = $this->getDB( $serverIndex );
+				$dbTimestamp = $db->timestamp( $timestamp );
+				$totalSeconds = false;
+				$baseConds = array( 'exptime < ' . $db->addQuotes( $dbTimestamp ) );
+				for ( $i = 0; $i < $this->shards; $i++ ) {
+					$maxExpTime = false;
+					while ( true ) {
+						$conds = $baseConds;
+						if ( $maxExpTime !== false ) {
+							$conds[] = 'exptime > ' . $db->addQuotes( $maxExpTime );
+						}
+						$rows = $db->select(
+							$this->getTableNameByShard( $i ),
+							array( 'keyname', 'exptime' ),
+							$conds,
+							__METHOD__,
+							array( 'LIMIT' => 100, 'ORDER BY' => 'exptime' ) );
+						if ( !$rows->numRows() ) {
+							break;
+						}
+						$keys = array();
+						$row = $rows->current();
+						$minExpTime = $row->exptime;
+						if ( $totalSeconds === false ) {
+							$totalSeconds = wfTimestamp( TS_UNIX, $timestamp )
+								- wfTimestamp( TS_UNIX, $minExpTime );
+						}
+						foreach ( $rows as $row ) {
+							$keys[] = $row->keyname;
+							$maxExpTime = $row->exptime;
+						}
 
-					if ( $progressCallback ) {
-						if ( intval( $totalSeconds ) === 0 ) {
-							$percent = 0;
-						} else {
-							$remainingSeconds = wfTimestamp( TS_UNIX, $timestamp )
-								- wfTimestamp( TS_UNIX, $maxExpTime );
-							if ( $remainingSeconds > $totalSeconds ) {
-								$totalSeconds = $remainingSeconds;
+						$db->begin( __METHOD__ );
+						$db->delete(
+							$this->getTableNameByShard( $i ),
+							array(
+								'exptime >= ' . $db->addQuotes( $minExpTime ),
+								'exptime < ' . $db->addQuotes( $dbTimestamp ),
+								'keyname' => $keys
+							),
+							__METHOD__ );
+						$db->commit( __METHOD__ );
+
+						if ( $progressCallback ) {
+							if ( intval( $totalSeconds ) === 0 ) {
+								$percent = 0;
+							} else {
+								$remainingSeconds = wfTimestamp( TS_UNIX, $timestamp )
+									- wfTimestamp( TS_UNIX, $maxExpTime );
+								if ( $remainingSeconds > $totalSeconds ) {
+									$totalSeconds = $remainingSeconds;
+								}
+								$percent = ( $i + $remainingSeconds / $totalSeconds )
+									/ $this->shards * 100;
 							}
-							$percent = ( $i + $remainingSeconds / $totalSeconds )
-								/ $this->shards * 100;
+							$percent = ( $percent / $this->numServers )
+								+ ( $serverIndex / $this->numServers * 100 );
+							call_user_func( $progressCallback, $percent );
 						}
-						call_user_func( $progressCallback, $percent );
 					}
 				}
+			} catch ( DBError $e ) {
+				$this->handleWriteError( $e, $serverIndex );
+				return false;
 			}
-		} catch ( DBError $e ) {
-			$this->handleWriteError( $e );
-			return false;
 		}
-
 		return true;
 	}
 
 	public function deleteAll() {
-		try {
-			$db = $this->getDB();
-			for ( $i = 0; $i < $this->shards; $i++ ) {
-				$db->begin( __METHOD__ );
-				$db->delete( $this->getTableByShard( $i ), '*', __METHOD__ );
-				$db->commit( __METHOD__ );
+		for ( $serverIndex = 0; $serverIndex < $this->numServers; $serverIndex++ ) {
+			try {
+				$db = $this->getDB( $serverIndex );
+				for ( $i = 0; $i < $this->shards; $i++ ) {
+					$db->begin( __METHOD__ );
+					$db->delete( $this->getTableNameByShard( $i ), '*', __METHOD__ );
+					$db->commit( __METHOD__ );
+				}
+			} catch ( DBError $e ) {
+				$this->handleWriteError( $e, $serverIndex );
+				return false;
 			}
-		} catch ( DBError $e ) {
-			$this->handleWriteError( $e );
-			return false;
 		}
-
 		return true;
 	}
 
@@ -544,58 +620,77 @@ class SqlBagOStuff extends BagOStuff {
 	/**
 	 * Handle a DBError which occurred during a read operation.
 	 */
-	protected function handleReadError( DBError $exception ) {
+	protected function handleReadError( DBError $exception, $serverIndex ) {
 		if ( $exception instanceof DBConnectionError ) {
-			$this->connFailureTime  = time();
-			$this->connFailureError = $exception;
+			$this->markServerDown( $exception, $serverIndex );
 		}
 		wfDebugLog( 'SQLBagOStuff', "DBError: {$exception->getMessage()}" );
-		if ( $this->db ) {
-			wfDebug( __METHOD__ . ": ignoring query error\n" );
-		} else {
+		if ( $exception instanceof DBConnectionError ) {
 			wfDebug( __METHOD__ . ": ignoring connection error\n" );
+		} else {
+			wfDebug( __METHOD__ . ": ignoring query error\n" );
 		}
 	}
 
 	/**
 	 * Handle a DBQueryError which occurred during a write operation.
 	 */
-	protected function handleWriteError( DBError $exception ) {
+	protected function handleWriteError( DBError $exception, $serverIndex ) {
 		if ( $exception instanceof DBConnectionError ) {
-			$this->connFailureTime  = time();
-			$this->connFailureError = $exception;
+			$this->markServerDown( $exception, $serverIndex );
 		}
-		if ( $this->db && $this->db->wasReadOnlyError() ) {
+		if ( $exception->db && $exception->db->wasReadOnlyError() ) {
 			try {
-				$this->db->rollback( __METHOD__ );
+				$exception->db->rollback( __METHOD__ );
 			} catch ( DBError $e ) {}
 		}
 		wfDebugLog( 'SQLBagOStuff', "DBError: {$exception->getMessage()}" );
-		if ( $this->db ) {
-			wfDebug( __METHOD__ . ": ignoring query error\n" );
-		} else {
+		if ( $exception instanceof DBConnectionError ) {
 			wfDebug( __METHOD__ . ": ignoring connection error\n" );
+		} else {
+			wfDebug( __METHOD__ . ": ignoring query error\n" );
+		}
+	}
+
+	/**
+	 * Mark a server down due to a DBConnectionError exception
+	 */
+	protected function markServerDown( $exception, $serverIndex ) {
+		if ( isset( $this->connFailureTimes[$serverIndex] ) ) {
+			if ( time() - $this->connFailureTimes[$serverIndex] >= 60 ) {
+				unset( $this->connFailureTimes[$serverIndex] );
+				unset( $this->connFailureErrors[$serverIndex] );
+			} else {
+				wfDebug( __METHOD__ . ": Server #$serverIndex already down\n" );
+				return;
+			}
 		}
+		$now = time();
+		wfDebug( __METHOD__ . ": Server #$serverIndex down until " . ( $now + 60 ) . "\n" );
+		$this->connFailureTimes[$serverIndex] = $now;
+		$this->connFailureErrors[$serverIndex] = $exception;
 	}
 
 	/**
 	 * Create shard tables. For use from eval.php.
 	 */
 	public function createTables() {
-		$db = $this->getDB();
-		if ( $db->getType() !== 'mysql'
-			|| version_compare( $db->getServerVersion(), '4.1.0', '<' ) )
-		{
-			throw new MWException( __METHOD__ . ' is not supported on this DB server' );
-		}
+		for ( $serverIndex = 0; $serverIndex < $this->numServers; $serverIndex++ ) {
+			$db = $this->getDB( $serverIndex );
+			if ( $db->getType() !== 'mysql'
+				|| version_compare( $db->getServerVersion(), '4.1.0', '<' ) )
+			{
+				throw new MWException( __METHOD__ . ' is not supported on this DB server' );
+			}
 
-		for ( $i = 0; $i < $this->shards; $i++ ) {
-			$db->begin( __METHOD__ );
-			$db->query(
-				'CREATE TABLE ' . $db->tableName( $this->getTableByShard( $i ) ) .
-				' LIKE ' . $db->tableName( 'objectcache' ),
-				__METHOD__ );
-			$db->commit( __METHOD__ );
+			for ( $i = 0; $i < $this->shards; $i++ ) {
+				$db->begin( __METHOD__ );
+				$db->query(
+					'CREATE TABLE ' . $db->tableName( $this->getTableNameByShard( $i ) ) .
+					' LIKE ' . $db->tableName( 'objectcache' ),
+					__METHOD__ );
+				$db->commit( __METHOD__ );
+			}
 		}
 	}
 }
@@ -604,4 +699,3 @@ class SqlBagOStuff extends BagOStuff {
  * Backwards compatibility alias
  */
 class MediaWikiBagOStuff extends SqlBagOStuff { }
-