1 files changed, 265 insertions, 30 deletions

diff --git a/includes/Collation.php b/includes/Collation.php
index ad2b94b1..6bba019b 100644
--- a/includes/Collation.php
+++ b/includes/Collation.php
@@ -48,8 +48,12 @@ abstract class Collation {
 			case 'uca-default':
 				return new IcuCollation( 'root' );
 			default:
-				# Provide a mechanism for extensions to hook in.
+				$match = array();
+				if ( preg_match( '/^uca-([a-z-]+)$/', $collationName, $match ) ) {
+					return new IcuCollation( $match[1] );
+				}
 
+				# Provide a mechanism for extensions to hook in.
 				$collationObject = null;
 				wfRunHooks( 'Collation::factory', array( $collationName, &$collationObject ) );
 
@@ -144,18 +148,19 @@ class IdentityCollation extends Collation {
 	}
 }
 
-
 class IcuCollation extends Collation {
+	const FIRST_LETTER_VERSION = 1;
+
 	var $primaryCollator, $mainCollator, $locale;
 	var $firstLetterData;
 
 	/**
 	 * Unified CJK blocks.
 	 *
-	 * The same definition of a CJK block must be used for both Collation and 
-	 * generateCollationData.php. These blocks are omitted from the first 
-	 * letter data, as an optimisation measure and because the default UCA table 
-	 * is pretty useless for sorting Chinese text anyway. Japanese and Korean 
+	 * The same definition of a CJK block must be used for both Collation and
+	 * generateCollationData.php. These blocks are omitted from the first
+	 * letter data, as an optimisation measure and because the default UCA table
+	 * is pretty useless for sorting Chinese text anyway. Japanese and Korean
 	 * blocks are not included here, because they are smaller and more useful.
 	 */
 	static $cjkBlocks = array(
@@ -176,11 +181,105 @@ class IcuCollation extends Collation {
 		array( 0x2F800, 0x2FA1F ), // CJK Compatibility Ideographs Supplement
 	);
 
+	/**
+	 * Additional characters (or character groups) to be considered separate
+	 * letters for given languages, or to be removed from the list of such
+	 * letters (denoted by keys starting with '-').
+	 *
+	 * These are additions to (or subtractions from) the data stored in the
+	 * first-letters-root.ser file (which among others includes full basic latin,
+	 * cyrillic and greek alphabets).
+	 *
+	 * "Separate letter" is a letter that would have a separate heading/section
+	 * for it in a dictionary or a phone book in this language. This data isn't
+	 * used for sorting (the ICU library handles that), only for deciding which
+	 * characters (or character groups) to use as headings.
+	 *
+	 * Initially generated based on the primary level of Unicode collation
+	 * tailorings available at http://developer.mimer.com/charts/tailorings.htm ,
+	 * later modified.
+	 *
+	 * Empty arrays are intended; this signifies that the data for the language is
+	 * available and that there are, in fact, no additional letters to consider.
+	 */
+	static $tailoringFirstLetters = array(
+		// Verified by native speakers
+		'be' => array( "Ё" ),
+		'be-tarask' => array( "Ё" ),
+		'en' => array(),
+		'fi' => array( "Å", "Ä", "Ö" ),
+		'hu' => array( "Cs", "Dz", "Dzs", "Gy", "Ly", "Ny", "Ö", "Sz", "Ty", "Ü", "Zs" ),
+		'it' => array(),
+		'pl' => array( "Ą", "Ć", "Ę", "Ł", "Ń", "Ó", "Ś", "Ź", "Ż" ),
+		'pt' => array(),
+		'ru' => array(),
+		'sv' => array( "Å", "Ä", "Ö" ),
+		'uk' => array( "Ґ", "Ь" ),
+		'vi' => array( "Ă", "Â", "Đ", "Ê", "Ô", "Ơ", "Ư" ),
+		// Not verified, but likely correct
+		'af' => array(),
+		'ast' => array( "Ch", "Ll", "Ñ" ),
+		'az' => array( "Ç", "Ə", "Ğ", "İ", "Ö", "Ş", "Ü" ),
+		'bg' => array(),
+		'br' => array( "Ch", "C'h" ),
+		'bs' => array( "Č", "Ć", "Dž", "Đ", "Lj", "Nj", "Š", "Ž" ),
+		'ca' => array(),
+		'co' => array(),
+		'cs' => array( "Č", "Ch", "Ř", "Š", "Ž" ),
+		'cy' => array( "Ch", "Dd", "Ff", "Ng", "Ll", "Ph", "Rh", "Th" ),
+		'da' => array( "Æ", "Ø", "Å" ),
+		'de' => array(),
+		'dsb' => array( "Č", "Ć", "Dź", "Ě", "Ch", "Ł", "Ń", "Ŕ", "Š", "Ś", "Ž", "Ź" ),
+		'el' => array(),
+		'eo' => array( "Ĉ", "Ĝ", "Ĥ", "Ĵ", "Ŝ", "Ŭ" ),
+		'es' => array( "Ñ" ),
+		'et' => array( "Š", "Ž", "Õ", "Ä", "Ö", "Ü" ),
+		'eu' => array( "Ñ" ),
+		'fo' => array( "Á", "Ð", "Í", "Ó", "Ú", "Ý", "Æ", "Ø", "Å" ),
+		'fr' => array(),
+		'fur' => array( "À", "Á", "Â", "È", "Ì", "Ò", "Ù" ),
+		'fy' => array(),
+		'ga' => array(),
+		'gd' => array(),
+		'gl' => array( "Ch", "Ll", "Ñ" ),
+		'hr' => array( "Č", "Ć", "Dž", "Đ", "Lj", "Nj", "Š", "Ž" ),
+		'hsb' => array( "Č", "Dź", "Ě", "Ch", "Ł", "Ń", "Ř", "Š", "Ć", "Ž" ),
+		'is' => array( "Á", "Ð", "É", "Í", "Ó", "Ú", "Ý", "Þ", "Æ", "Ö", "Å" ),
+		'kk' => array( "Ү", "І" ),
+		'kl' => array( "Æ", "Ø", "Å" ),
+		'ku' => array( "Ç", "Ê", "Î", "Ş", "Û" ),
+		'ky' => array( "Ё" ),
+		'la' => array(),
+		'lb' => array(),
+		'lt' => array( "Č", "Š", "Ž" ),
+		'lv' => array( "Č", "Ģ", "Ķ", "Ļ", "Ņ", "Š", "Ž" ),
+		'mk' => array(),
+		'mo' => array( "Ă", "Â", "Î", "Ş", "Ţ" ),
+		'mt' => array( "Ċ", "Ġ", "Għ", "Ħ", "Ż" ),
+		'nl' => array(),
+		'no' => array( "Æ", "Ø", "Å" ),
+		'oc' => array(),
+		'rm' => array(),
+		'ro' => array( "Ă", "Â", "Î", "Ş", "Ţ" ),
+		'rup' => array( "Ă", "Â", "Î", "Ľ", "Ń", "Ş", "Ţ" ),
+		'sco' => array(),
+		'sk' => array( "Ä", "Č", "Ch", "Ô", "Š", "Ž" ),
+		'sl' => array( "Č", "Š", "Ž" ),
+		'smn' => array( "Á", "Č", "Đ", "Ŋ", "Š", "Ŧ", "Ž", "Æ", "Ø", "Å", "Ä", "Ö" ),
+		'sq' => array( "Ç", "Dh", "Ë", "Gj", "Ll", "Nj", "Rr", "Sh", "Th", "Xh", "Zh" ),
+		'sr' => array(),
+		'tk' => array( "Ç", "Ä", "Ž", "Ň", "Ö", "Ş", "Ü", "Ý" ),
+		'tl' => array( "Ñ", "Ng" ),
+		'tr' => array( "Ç", "Ğ", "İ", "Ö", "Ş", "Ü" ),
+		'tt' => array( "Ә", "Ө", "Ү", "Җ", "Ң", "Һ" ),
+		'uz' => array( "Ch", "G'", "Ng", "O'", "Sh" ),
+	);
+
 	const RECORD_LENGTH = 14;
 
 	function __construct( $locale ) {
 		if ( !extension_loaded( 'intl' ) ) {
-			throw new MWException( 'An ICU collation was requested, ' . 
+			throw new MWException( 'An ICU collation was requested, ' .
 				'but the intl extension is not available.' );
 		}
 		$this->locale = $locale;
@@ -218,8 +317,8 @@ class IcuCollation extends Collation {
 
 		// Check for CJK
 		$firstChar = mb_substr( $string, 0, 1, 'UTF-8' );
-		if ( ord( $firstChar ) > 0x7f 
-			&& self::isCjk( utf8ToCodepoint( $firstChar ) ) ) 
+		if ( ord( $firstChar ) > 0x7f
+			&& self::isCjk( utf8ToCodepoint( $firstChar ) ) )
 		{
 			return $firstChar;
 		}
@@ -249,25 +348,37 @@ class IcuCollation extends Collation {
 		$cacheKey = wfMemcKey( 'first-letters', $this->locale );
 		$cacheEntry = $cache->get( $cacheKey );
 
-		if ( $cacheEntry ) {
+		if ( $cacheEntry && isset( $cacheEntry['version'] )
+			&& $cacheEntry['version'] == self::FIRST_LETTER_VERSION ) 
+		{
 			$this->firstLetterData = $cacheEntry;
 			return $this->firstLetterData;
 		}
 
 		// Generate data from serialized data file
 
-		$letters = wfGetPrecompiledData( "first-letters-{$this->locale}.ser" );
-		if ( $letters === false ) {
-			throw new MWException( "MediaWiki does not support ICU locale " .
-				"\"{$this->locale}\"" );
+		if ( isset ( self::$tailoringFirstLetters[$this->locale] ) ) {
+			$letters = wfGetPrecompiledData( "first-letters-root.ser" );
+			// Append additional characters
+			$letters = array_merge( $letters, self::$tailoringFirstLetters[$this->locale] );
+			// Remove unnecessary ones, if any
+			if ( isset( self::$tailoringFirstLetters[ '-' . $this->locale ] ) ) {
+				$letters = array_diff( $letters, self::$tailoringFirstLetters[ '-' . $this->locale ] );
+			}
+		} else {
+			$letters = wfGetPrecompiledData( "first-letters-{$this->locale}.ser" );
+			if ( $letters === false ) {
+				throw new MWException( "MediaWiki does not support ICU locale " .
+					"\"{$this->locale}\"" );
+			}
 		}
 
 		// Sort the letters.
 		//
 		// It's impossible to have the precompiled data file properly sorted,
-		// because the sort order changes depending on ICU version. If the 
-		// array is not properly sorted, the binary search will return random 
-		// results. 
+		// because the sort order changes depending on ICU version. If the
+		// array is not properly sorted, the binary search will return random
+		// results.
 		//
 		// We also take this opportunity to remove primary collisions.
 		$letterMap = array();
@@ -284,9 +395,76 @@ class IcuCollation extends Collation {
 			}
 		}
 		ksort( $letterMap, SORT_STRING );
+		// Remove duplicate prefixes. Basically if something has a sortkey
+		// which is a prefix of some other sortkey, then it is an
+		// expansion and probably should not be considered a section
+		// header.
+		//
+		// For example 'þ' is sometimes sorted as if it is the letters
+		// 'th'. Other times it is its own primary element. Another
+		// example is '₨'. Sometimes its a currency symbol. Sometimes it
+		// is an 'R' followed by an 's'.
+		//
+		// Additionally an expanded element should always sort directly
+		// after its first element due to they way sortkeys work.
+		//
+		// UCA sortkey elements are of variable length but no collation
+		// element should be a prefix of some other element, so I think
+		// this is safe. See:
+		// * https://ssl.icu-project.org/repos/icu/icuhtml/trunk/design/collation/ICU_collation_design.htm
+		// * http://site.icu-project.org/design/collation/uca-weight-allocation
+		//
+		// Additionally, there is something called primary compression to
+		// worry about. Basically, if you have two primary elements that
+		// are more than one byte and both start with the same byte then
+		// the first byte is dropped on the second primary. Additionally
+		// either \x03 or \xFF may be added to mean that the next primary
+		// does not start with the first byte of the first primary.
+		//
+		// This shouldn't matter much, as the first primary is not
+		// changed, and that is what we are comparing against.
+		//
+		// tl;dr: This makes some assumptions about how icu implements
+		// collations. It seems incredibly unlikely these assumptions
+		// will change, but nonetheless they are assumptions.
+
+		$prev = false;
+		$duplicatePrefixes = array();
+		foreach( $letterMap as $key => $value ) {
+			// Remove terminator byte. Otherwise the prefix
+			// comparison will get hung up on that.
+			$trimmedKey = rtrim( $key, "\0" );
+			if ( $prev === false || $prev === '' ) {
+				$prev = $trimmedKey;
+				// We don't yet have a collation element
+				// to compare against, so continue.
+				continue;
+			}
+
+			// Due to the fact the array is sorted, we only have
+			// to compare with the element directly previous
+			// to the current element (skipping expansions).
+			// An element "X" will always sort directly
+			// before "XZ" (Unless we have "XY", but we
+			// do not update $prev in that case).
+			if ( substr( $trimmedKey, 0, strlen( $prev ) ) === $prev ) {
+				$duplicatePrefixes[] = $key;
+				// If this is an expansion, we don't want to
+				// compare the next element to this element,
+				// but to what is currently $prev
+				continue;
+			}
+			$prev = $trimmedKey;
+		}
+		foreach( $duplicatePrefixes as $badKey ) {
+			wfDebug( "Removing '{$letterMap[$badKey]}' from first letters." );
+			unset( $letterMap[$badKey] );
+			// This code assumes that unsetting does not change sort order.
+		}
 		$data = array(
 			'chars' => array_values( $letterMap ),
-			'keys' => array_keys( $letterMap )
+			'keys' => array_keys( $letterMap ),
+			'version' => self::FIRST_LETTER_VERSION,
 		);
 
 		// Reduce memory usage before caching
@@ -320,23 +498,27 @@ class IcuCollation extends Collation {
 	}
 
 	/**
-	 * Do a binary search, and return the index of the largest item that sorts 
+	 * Do a binary search, and return the index of the largest item that sorts
 	 * less than or equal to the target value.
 	 *
-	 * @param $valueCallback array A function to call to get the value with
+	 * @param array $valueCallback A function to call to get the value with
 	 *     a given array index.
-	 * @param $valueCount int The number of items accessible via $valueCallback,
+	 * @param int $valueCount The number of items accessible via $valueCallback,
 	 *     indexed from 0 to $valueCount - 1
-	 * @param $comparisonCallback array A callback to compare two values, returning
+	 * @param array $comparisonCallback A callback to compare two values, returning
 	 *     -1, 0 or 1 in the style of strcmp().
-	 * @param $target string The target value to find.
+	 * @param string $target The target value to find.
 	 *
 	 * @return int|bool The item index of the lower bound, or false if the target value
 	 *     sorts before all items.
 	 */
 	function findLowerBound( $valueCallback, $valueCount, $comparisonCallback, $target ) {
+		if ( $valueCount === 0 ) {
+			return false;
+		}
+
 		$min = 0;
-		$max = $valueCount - 1;
+		$max = $valueCount;
 		do {
 			$mid = $min + ( ( $max - $min ) >> 1 );
 			$item = call_user_func( $valueCallback, $mid );
@@ -351,12 +533,15 @@ class IcuCollation extends Collation {
 			}
 		} while ( $min < $max - 1 );
 
-		if ( $min == 0 && $max == 0 && $comparison > 0 ) {
-			// Before the first item
-			return false;
-		} else {
-			return $min;
+		if ( $min == 0 ) {
+			$item = call_user_func( $valueCallback, $min );
+			$comparison = call_user_func( $comparisonCallback, $target, $item );
+			if ( $comparison < 0 ) {
+				// Before the first item
+				return false;
+			}
 		}
+		return $min;
 	}
 
 	static function isCjk( $codepoint ) {
@@ -367,5 +552,55 @@ class IcuCollation extends Collation {
 		}
 		return false;
 	}
-}
 
+	/**
+	 * Return the version of ICU library used by PHP's intl extension,
+	 * or false when the extension is not installed of the version
+	 * can't be determined.
+	 *
+	 * The constant INTL_ICU_VERSION this function refers to isn't really
+	 * documented. It is available since PHP 5.3.7 (see PHP bug 54561).
+	 * This function will return false on older PHPs.
+	 *
+	 * @since 1.21
+	 * @return string|false
+	 */
+	static function getICUVersion() {
+		return defined( 'INTL_ICU_VERSION' ) ? INTL_ICU_VERSION : false;
+	}
+
+	/**
+	 * Return the version of Unicode appropriate for the version of ICU library
+	 * currently in use, or false when it can't be determined.
+	 *
+	 * @since 1.21
+	 * @return string|false
+	 */
+	static function getUnicodeVersionForICU() {
+		$icuVersion = IcuCollation::getICUVersion();
+		if ( !$icuVersion ) {
+			return false;
+		}
+
+		$versionPrefix = substr( $icuVersion, 0, 3 );
+		// Source: http://site.icu-project.org/download
+		$map = array(
+			'50.' => '6.2',
+			'49.' => '6.1',
+			'4.8' => '6.0',
+			'4.6' => '6.0',
+			'4.4' => '5.2',
+			'4.2' => '5.1',
+			'4.0' => '5.1',
+			'3.8' => '5.0',
+			'3.6' => '5.0',
+			'3.4' => '4.1',
+		);
+
+		if ( isset( $map[$versionPrefix] ) ) {
+			return $map[$versionPrefix];
+		} else {
+			return false;
+		}
+	}
+}