From 4ac9fa081a7c045f6a9f1cfc529d82423f485b2e Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 8 Dec 2013 09:55:49 +0100 Subject: Update to MediaWiki 1.22.0 --- includes/StringUtils.php | 135 +++++++++++++++++++++++++++++++---------------- 1 file changed, 90 insertions(+), 45 deletions(-) (limited to 'includes/StringUtils.php') diff --git a/includes/StringUtils.php b/includes/StringUtils.php index f4c98f1d..c1545e6e 100644 --- a/includes/StringUtils.php +++ b/includes/StringUtils.php @@ -38,6 +38,9 @@ class StringUtils { * unit testing our internal implementation. * * @since 1.21 + * @note In MediaWiki 1.21, this function did not provide proper UTF-8 validation. + * In particular, the pure PHP code path did not in fact check for overlong forms. + * Beware of this when backporting code to that version of MediaWiki. * * @param string $value String to check * @param boolean $disableMbstring Whether to use the pure PHP @@ -47,26 +50,64 @@ class StringUtils { * @return boolean Whether the given $value is a valid UTF-8 encoded string */ static function isUtf8( $value, $disableMbstring = false ) { + $value = (string)$value; - if ( preg_match( '/[\x80-\xff]/', $value ) === 0 ) { - # no high bit set, this is pure ASCII which is de facto - # valid UTF-8 + // If the mbstring extension is loaded, use it. However, before PHP 5.4, values above + // U+10FFFF are incorrectly allowed, so we have to check for them separately. + if ( !$disableMbstring && function_exists( 'mb_check_encoding' ) ) { + static $newPHP; + if ( $newPHP === null ) { + $newPHP = !mb_check_encoding( "\xf4\x90\x80\x80", 'UTF-8' ); + } + + return mb_check_encoding( $value, 'UTF-8' ) && + ( $newPHP || preg_match( "/\xf4[\x90-\xbf]|[\xf5-\xff]/S", $value ) === 0 ); + } + + if ( preg_match( "/[\x80-\xff]/S", $value ) === 0 ) { + // String contains only ASCII characters, has to be valid return true; } - if ( !$disableMbstring && function_exists( 'mb_check_encoding' ) ) { - return mb_check_encoding( $value, 'UTF-8' ); - } else { - $hasUtf8 = preg_match( '/^(?> - [\x00-\x7f] - | [\xc0-\xdf][\x80-\xbf] - | [\xe0-\xef][\x80-\xbf]{2} - | [\xf0-\xf7][\x80-\xbf]{3} - | [\xf8-\xfb][\x80-\xbf]{4} - | \xfc[\x84-\xbf][\x80-\xbf]{4} - )+$/x', $value ); - return ($hasUtf8 > 0 ); + // PCRE implements repetition using recursion; to avoid a stack overflow (and segfault) + // for large input, we check for invalid sequences (<= 5 bytes) rather than valid + // sequences, which can be as long as the input string is. Multiple short regexes are + // used rather than a single long regex for performance. + static $regexes; + if ( $regexes === null ) { + $cont = "[\x80-\xbf]"; + $after = "(?!$cont)"; // "(?:[^\x80-\xbf]|$)" would work here + $regexes = array( + // Continuation byte at the start + "/^$cont/", + + // ASCII byte followed by a continuation byte + "/[\\x00-\x7f]$cont/S", + + // Illegal byte + "/[\xc0\xc1\xf5-\xff]/S", + + // Invalid 2-byte sequence, or valid one then an extra continuation byte + "/[\xc2-\xdf](?!$cont$after)/S", + + // Invalid 3-byte sequence, or valid one then an extra continuation byte + "/\xe0(?![\xa0-\xbf]$cont$after)/", + "/[\xe1-\xec\xee\xef](?!$cont{2}$after)/S", + "/\xed(?![\x80-\x9f]$cont$after)/", + + // Invalid 4-byte sequence, or valid one then an extra continuation byte + "/\xf0(?![\x90-\xbf]$cont{2}$after)/", + "/[\xf1-\xf3](?!$cont{3}$after)/S", + "/\xf4(?![\x80-\x8f]$cont{2}$after)/", + ); } + + foreach ( $regexes as $regex ) { + if ( preg_match( $regex, $value ) !== 0 ) { + return false; + } + } + return true; } /** @@ -217,8 +258,8 @@ class StringUtils { /** * More or less "markup-safe" explode() * Ignores any instances of the separator inside <...> - * @param $separator String - * @param $text String + * @param string $separator + * @param string $text * @return array */ static function explodeMarkup( $separator, $text ) { @@ -233,7 +274,7 @@ class StringUtils { // Explode, then put the replaced separators back in $items = explode( $separator, $cleaned ); - foreach( $items as $i => $str ) { + foreach ( $items as $i => $str ) { $items[$i] = str_replace( $placeholder, $separator, $str ); } @@ -244,8 +285,8 @@ class StringUtils { * Escape a string to make it suitable for inclusion in a preg_replace() * replacement parameter. * - * @param $string String - * @return String + * @param string $string + * @return string */ static function escapeRegexReplacement( $string ) { $string = str_replace( '\\', '\\\\', $string ); @@ -256,8 +297,8 @@ class StringUtils { /** * Workalike for explode() with limited memory usage. * Returns an Iterator - * @param $separator - * @param $subject + * @param string $separator + * @param string $subject * @return ArrayIterator|ExplodeIterator */ static function explode( $separator, $subject ) { @@ -290,14 +331,14 @@ class RegexlikeReplacer extends Replacer { var $r; /** - * @param $r string + * @param string $r */ function __construct( $r ) { $this->r = $r; } /** - * @param $matches array + * @param array $matches * @return string */ function replace( $matches ) { @@ -318,7 +359,7 @@ class DoubleReplacer extends Replacer { /** * @param $from * @param $to - * @param $index int + * @param int $index */ function __construct( $from, $to, $index = 0 ) { $this->from = $from; @@ -327,7 +368,7 @@ class DoubleReplacer extends Replacer { } /** - * @param $matches array + * @param array $matches * @return mixed */ function replace( $matches ) { @@ -343,7 +384,7 @@ class HashtableReplacer extends Replacer { /** * @param $table - * @param $index int + * @param int $index */ function __construct( $table, $index = 0 ) { $this->table = $table; @@ -351,7 +392,7 @@ class HashtableReplacer extends Replacer { } /** - * @param $matches array + * @param array $matches * @return mixed */ function replace( $matches ) { @@ -389,6 +430,7 @@ class ReplacementArray { /** * Set the whole replacement array at once + * @param array $data */ function setArray( $data ) { $this->data = $data; @@ -404,8 +446,8 @@ class ReplacementArray { /** * Set an element of the replacement array - * @param $from string - * @param $to string + * @param string $from + * @param string $to */ function setPair( $from, $to ) { $this->data[$from] = $to; @@ -413,7 +455,7 @@ class ReplacementArray { } /** - * @param $data array + * @param array $data */ function mergeArray( $data ) { $this->data = array_merge( $this->data, $data ); @@ -421,7 +463,7 @@ class ReplacementArray { } /** - * @param $other + * @param ReplacementArray $other */ function merge( $other ) { $this->data = array_merge( $this->data, $other->data ); @@ -429,7 +471,7 @@ class ReplacementArray { } /** - * @param $from string + * @param string $from */ function removePair( $from ) { unset( $this->data[$from] ); @@ -437,31 +479,31 @@ class ReplacementArray { } /** - * @param $data array + * @param array $data */ function removeArray( $data ) { - foreach( $data as $from => $to ) { + foreach ( $data as $from => $to ) { $this->removePair( $from ); } $this->fss = false; } /** - * @param $subject string + * @param string $subject * @return string */ function replace( $subject ) { if ( function_exists( 'fss_prep_replace' ) ) { - wfProfileIn( __METHOD__.'-fss' ); + wfProfileIn( __METHOD__ . '-fss' ); if ( $this->fss === false ) { $this->fss = fss_prep_replace( $this->data ); } $result = fss_exec_replace( $this->fss, $subject ); - wfProfileOut( __METHOD__.'-fss' ); + wfProfileOut( __METHOD__ . '-fss' ); } else { - wfProfileIn( __METHOD__.'-strtr' ); + wfProfileIn( __METHOD__ . '-strtr' ); $result = strtr( $subject, $this->data ); - wfProfileOut( __METHOD__.'-strtr' ); + wfProfileOut( __METHOD__ . '-strtr' ); } return $result; } @@ -494,15 +536,15 @@ class ExplodeIterator implements Iterator { /** * Construct a DelimIterator - * @param $delim string - * @param $s string + * @param string $delim + * @param string $subject */ - function __construct( $delim, $s ) { - $this->subject = $s; + function __construct( $delim, $subject ) { + $this->subject = $subject; $this->delim = $delim; // Micro-optimisation (theoretical) - $this->subjectLength = strlen( $s ); + $this->subjectLength = strlen( $subject ); $this->delimLength = strlen( $delim ); $this->rewind(); @@ -530,6 +572,9 @@ class ExplodeIterator implements Iterator { return $this->current; } + /** + * @return int|bool Current position or boolean false if invalid + */ function key() { return $this->curPos; } -- cgit v1.2.2