diff options
Diffstat (limited to 'extensions/LocalisationUpdate/LocalisationUpdate.class.php')
| -rw-r--r-- | extensions/LocalisationUpdate/LocalisationUpdate.class.php | 605 |
1 files changed, 45 insertions, 560 deletions
diff --git a/extensions/LocalisationUpdate/LocalisationUpdate.class.php b/extensions/LocalisationUpdate/LocalisationUpdate.class.php index 66b63232..349c35f0 100644 --- a/extensions/LocalisationUpdate/LocalisationUpdate.class.php +++ b/extensions/LocalisationUpdate/LocalisationUpdate.class.php @@ -1,596 +1,81 @@ <?php /** - * Class for localization updates. - * - * TODO: refactor code to remove duplication + * Class for localization update hooks and static methods. */ class LocalisationUpdate { - - private static $newHashes = null; - private static $filecache = array(); - - /** - * LocalisationCacheRecache hook handler. - * - * @param $lc LocalisationCache - * @param $langcode String - * @param $cache Array - * - * @return true - */ - public static function onRecache( LocalisationCache $lc, $langcode, array &$cache ) { - // Handle fallback sequence and load all fallback messages from the cache - $codeSequence = array_merge( array( $langcode ), $cache['fallbackSequence'] ); - // Iterate over the fallback sequence in reverse, otherwise the fallback - // language will override the requested language - foreach ( array_reverse( $codeSequence ) as $code ) { - if ( $code == 'en' ) { - // Skip English, otherwise we end up trying to read - // the nonexistent cache file for en a couple hundred times - continue; - } - - $cache['messages'] = array_merge( - $cache['messages'], - self::readFile( $code ) - ); - - $cache['deps'][] = new FileDependency( - self::filename( $code ) - ); - } - - return true; - } + /** @todo Remove this once pre-1.24 versions of MW are no longer supported. */ + private static $onRecacheFallbackCalled = false; /** - * Called from the cronjob to fetch new messages from SVN. - * - * @param $options Array - * - * @return true + * Hook: LocalisationCacheRecacheFallback */ - public static function updateMessages( array $options ) { - global $wgLocalisationUpdateDirectory, $wgLocalisationUpdateCoreURL, - $wgLocalisationUpdateExtensionURL, $wgLocalisationUpdateSVNURL; - - $verbose = !isset( $options['quiet'] ); - $all = isset( $options['all'] ); - $skipCore = isset( $options['skip-core'] ); - $skipExtensions = isset( $options['skip-extensions'] ); - - if( isset( $options['outdir'] ) ) { - $wgLocalisationUpdateDirectory = $options['outdir']; - } - - $coreUrl = $wgLocalisationUpdateCoreURL; - $extUrl = $wgLocalisationUpdateExtensionURL; - - // Some ugly BC - if ( $wgLocalisationUpdateSVNURL ) { - $coreUrl = $wgLocalisationUpdateSVNURL . '/phase3/$2'; - $extUrl = $wgLocalisationUpdateSVNURL . '/extensions/$1/$2'; - } - - // Some more ugly BC - if ( isset( $options['svnurl'] ) ) { - $coreUrl = $options['svnurl'] . '/phase3/$2'; - $extUrl = $options['svnurl'] . '/extensions/$1/$2'; - } - - $result = 0; + public static function onRecacheFallback( LocalisationCache $lc, $code, array &$cache ) { + self::$onRecacheFallbackCalled = true; - // Update all MW core messages. - if( !$skipCore ) { - $result = self::updateMediawikiMessages( $verbose, $coreUrl ); + $dir = LocalisationUpdate::getDirectory(); + if ( !$dir ) { + return true; } - // Update all Extension messages. - if( !$skipExtensions ) { - if( $all ) { - global $IP; - $extFiles = array(); - - // Look in extensions/ for all available items... - // TODO: add support for $wgExtensionAssetsPath - $dirs = new RecursiveDirectoryIterator( "$IP/extensions/" ); - - // I ain't kidding... RecursiveIteratorIterator. - foreach( new RecursiveIteratorIterator( $dirs ) as $pathname => $item ) { - $filename = basename( $pathname ); - $matches = array(); - if( preg_match( '/^(.*)\.i18n\.php$/', $filename, $matches ) ) { - $group = $matches[1]; - $extFiles[$group] = $pathname; - } - } - } else { - global $wgExtensionMessagesFiles; - $extFiles = $wgExtensionMessagesFiles; - } - foreach ( $extFiles as $extension => $locFile ) { - $result += self::updateExtensionMessages( $locFile, $extension, $verbose, $extUrl ); - } + $fileName = "$dir/" . self::getFilename( $code ); + if ( is_readable( $fileName ) ) { + $data = FormatJson::decode( file_get_contents( $fileName ), true ); + $cache['messages'] = array_merge( $cache['messages'], $data ); } - self::writeHashes(); - - // And output the result! - self::myLog( "Updated {$result} messages in total" ); - self::myLog( "Done" ); - return true; } /** - * Update Extension Messages. - * - * @param $file String - * @param $extension String - * @param $verbose Boolean - * - * @return Integer: the amount of updated messages - */ - public static function updateExtensionMessages( $file, $extension, $verbose, $extUrl ) { - $match = array(); - $ok = preg_match( '~^.*/extensions/([^/]+)/(.*)$~U', $file, $match ); - if ( !$ok ) { - return null; - } - - $ext = $match[1]; - $extFile = $match[2]; - - // Create a full path. - $svnfile = str_replace( - array( '$1', '$2', '$3', '$4' ), - array( $ext, $extFile, urlencode( $ext ), urlencode( $extFile ) ), - $extUrl - ); - - // Compare the 2 files. - $result = self::compareExtensionFiles( $extension, $svnfile, $file, $verbose ); - - return $result; - } - - /** - * Update the MediaWiki Core Messages. - * - * @param $verbose Boolean - * - * @return Integer: the amount of updated messages - */ - public static function updateMediawikiMessages( $verbose, $coreUrl ) { - // Find the changed English strings (as these messages won't be updated in ANY language). - $localUrl = Language::getMessagesFileName( 'en' ); - $repoUrl = str_replace( - array( '$2', '$4' ), - array( 'languages/messages/MessagesEn.php', 'languages%2Fmessages%2FMessagesEn.php' ), - $coreUrl - ); - $changedEnglishStrings = self::compareFiles( $repoUrl, $localUrl, $verbose ); - - // Count the changes. - $changedCount = 0; - - $languages = Language::fetchLanguageNames( null, 'mwfile' ); - foreach ( array_keys( $languages ) as $code ) { - $localUrl = Language::getMessagesFileName( $code ); - // Not prefixed with $IP - $filename = Language::getFilename( 'languages/messages/Messages', $code ); - $repoUrl = str_replace( - array( '$2', '$4' ), - array( $filename, urlencode( $filename ) ), - $coreUrl - ); - - // Compare the files. - $changedCount += self::compareFiles( $repoUrl, $localUrl, $verbose, $changedEnglishStrings, false, true ); - } - - // Log some nice info. - self::myLog( "{$changedCount} MediaWiki messages are updated" ); - - return $changedCount; - } - - /** - * Removes all unneeded content from a file and returns it. - * - * @param $contents String - * - * @return String - */ - public static function cleanupFile( $contents ) { - // We don't need any PHP tags. - $contents = strtr( $contents, - array( - '<?php' => '', - '?' . '>' => '' - ) - ); - - $results = array(); - - // And we only want message arrays. - preg_match_all( '/\$messages(.*\s)*?\);/', $contents, $results ); - - // But we want them all in one string. - if( !empty( $results[0] ) && is_array( $results[0] ) ) { - $contents = implode( "\n\n", $results[0] ); - } else { - $contents = ''; - } - - // And we hate the windows vs linux linebreaks. - $contents = preg_replace( '/\r\n?/', "\n", $contents ); - - return $contents; - } - - /** - * Returns the contents of a file or false on failiure. - * - * @param $file String - * - * @return string or false - */ - public static function getFileContents( $file ) { - global $wgLocalisationUpdateRetryAttempts; - - $attempts = 0; - $filecontents = ''; - - // Use cURL to get the SVN contents. - if ( preg_match( "/^http/", $file ) ) { - while( !$filecontents && $attempts <= $wgLocalisationUpdateRetryAttempts ) { - if( $attempts > 0 ) { - $delay = 1; - self::myLog( 'Failed to download ' . $file . "; retrying in ${delay}s..." ); - sleep( $delay ); - } - - $filecontents = Http::get( $file ); - $attempts++; - } - if ( !$filecontents ) { - self::myLog( 'Cannot get the contents of ' . $file . ' (curl)' ); - return false; - } - } else {// otherwise try file_get_contents - if ( !( $filecontents = file_get_contents( $file ) ) ) { - self::myLog( 'Cannot get the contents of ' . $file ); - return false; - } - } - - return $filecontents; - } - - /** - * Returns a pair of arrays containing the messages from two files, or - * a pair of nulls if the files don't need to be checked. - * - * @param $tag String - * @param $file1 String - * @param $file2 String - * @param $verbose Boolean - * @param $alwaysGetResult Boolean - * - * @return array - */ - public static function loadFilesToCompare( $tag, $file1, $file2, $verbose, $alwaysGetResult = true ) { - $file1contents = self::getFileContents( $file1 ); - if ( $file1contents === false || $file1contents === '' ) { - self::myLog( "Failed to read $file1" ); - return array( null, null ); - } - - $file2contents = self::getFileContents( $file2 ); - if ( $file2contents === false || $file2contents === '' ) { - self::myLog( "Failed to read $file2" ); - return array( null, null ); - } - - // Only get the part we need. - $file1contents = self::cleanupFile( $file1contents ); - $file1hash = md5( $file1contents ); - - $file2contents = self::cleanupFile( $file2contents ); - $file2hash = md5( $file2contents ); - - // Check if the file has changed since our last update. - if ( !$alwaysGetResult ) { - if ( !self::checkHash( $file1, $file1hash ) && !self::checkHash( $file2, $file2hash ) ) { - self::myLog( "Skipping {$tag} since the files haven't changed since our last update", $verbose ); - return array( null, null ); - } - } - - // Get the array with messages. - $messages1 = self::parsePHP( $file1contents, 'messages' ); - if ( !is_array( $messages1 ) ) { - if ( strpos( $file1contents, '$messages' ) === false ) { - // No $messages array. This happens for some languages that only have a fallback - $messages1 = array(); - } else { - // Broken file? Report and bail - self::myLog( "Failed to parse $file1" ); - return array( null, null ); - } - } - - $messages2 = self::parsePHP( $file2contents, 'messages' ); - if ( !is_array( $messages2 ) ) { - // Broken file? Report and bail - if ( strpos( $file2contents, '$messages' ) === false ) { - // No $messages array. This happens for some languages that only have a fallback - $messages2 = array(); - } else { - self::myLog( "Failed to parse $file2" ); - return array( null, null ); - } - } - - self::saveHash( $file1, $file1hash ); - self::saveHash( $file2, $file2hash ); - - return array( $messages1, $messages2 ); - } - - /** - * Compare new and old messages lists, and optionally save the new - * messages if they've changed. - * - * @param $langcode String - * @param $old_messages Array - * @param $new_messages Array - * @param $verbose Boolean - * @param $forbiddenKeys Array - * @param $saveResults Boolean - * - * @return array|int - */ - private static function compareLanguageArrays( $langcode, $old_messages, $new_messages, $verbose, $forbiddenKeys, $saveResults ) { - // Get the currently-cached messages, if any - $cur_messages = self::readFile( $langcode ); - - // Update the messages lists with the cached messages - $old_messages = array_merge( $old_messages, $cur_messages ); - $new_messages = array_merge( $cur_messages, $new_messages ); - - // Use the old/cached version for any forbidden keys - if ( count( $forbiddenKeys ) ) { - $new_messages = array_merge( - array_diff_key( $new_messages, $forbiddenKeys ), - array_intersect_key( $old_messages, $forbiddenKeys ) - ); - } - - - if ( $saveResults ) { - // If anything has changed from the saved version, save the new version - if ( $new_messages != $cur_messages ) { - // Count added, updated, and deleted messages: - // diff( new, cur ) gives added + updated, and diff( cur, new ) - // gives deleted + updated. - $changed = array_diff_assoc( $new_messages, $cur_messages ) + - array_diff_assoc( $cur_messages, $new_messages ); - $updates = count( $changed ); - self::myLog( "{$updates} messages updated for {$langcode}.", $verbose ); - self::writeFile( $langcode, $new_messages ); - } else { - $updates = 0; - } - return $updates; - } else { - // Find all deleted or changed messages - $changedStrings = array_diff_assoc( $old_messages, $new_messages ); - return $changedStrings; - } - } - - /** - * Returns an array containing the differences between the files. - * - * @param $newfile String - * @param $oldfile String - * @param $verbose Boolean - * @param $forbiddenKeys Array - * @param $alwaysGetResult Boolean - * @param $saveResults Boolean - * - * @return array|int + * Hook: LocalisationCacheRecache */ - public static function compareFiles( $newfile, $oldfile, $verbose, array $forbiddenKeys = array(), $alwaysGetResult = true, $saveResults = false ) { - // Get the languagecode. - $langcode = Language::getCodeFromFileName( $newfile, 'Messages' ); - - list( $new_messages, $old_messages ) = self::loadFilesToCompare( - $langcode, $newfile, $oldfile, $verbose, $alwaysGetResult - ); - if ( $new_messages === null || $old_messages === null ) { - return $saveResults ? 0 : array(); - } - - return self::compareLanguageArrays( $langcode, $old_messages, $new_messages, $verbose, $forbiddenKeys, $saveResults ); - } - - /** - * - * @param $extension String - * @param $newfile String - * @param $oldfile String - * @param $verbose Boolean - * @param $alwaysGetResult Boolean - * @param $saveResults Boolean - * - * @return Integer: the amount of updated messages - */ - public static function compareExtensionFiles( $extension, $newfile, $oldfile, $verbose ) { - list( $new_messages, $old_messages ) = self::loadFilesToCompare( - $extension, $newfile, $oldfile, $verbose, false - ); - if ( $new_messages === null || $old_messages === null ) { - return 0; - } - - // Update counter. - $updates = 0; - - if ( empty( $new_messages['en'] ) ) { - $new_messages['en'] = array(); - } - - if ( empty( $old_messages['en'] ) ) { - $old_messages['en'] = array(); - } - - // Find the changed english strings. - $forbiddenKeys = self::compareLanguageArrays( 'en', $old_messages['en'], $new_messages['en'], $verbose, array(), false ); - - // Do an update for each language. - foreach ( $new_messages as $language => $messages ) { - if ( $language == 'en' ) { // Skip english. - continue; - } - - if ( !isset( $old_messages[$language] ) ) { - $old_messages[$language] = array(); + public static function onRecache( LocalisationCache $lc, $code, array &$cache ) { + $dir = LocalisationUpdate::getDirectory(); + if ( !$dir ) { + return true; + } + + $codeSequence = array_merge( array( $code ), $cache['fallbackSequence'] ); + foreach ( $codeSequence as $csCode ) { + $fileName = "$dir/" . self::getFilename( $csCode ); + if ( !self::$onRecacheFallbackCalled && is_readable( $fileName ) ) { + // We're on an old version of MW that doesn't have the hook + // needed to do things correctly. L10n will be broken here in + // certain reasonably-common situations (see bug 68781), but + // there's nothing we can do about it. + $data = FormatJson::decode( file_get_contents( $fileName ), true ); + $cache['messages'] = array_merge( $cache['messages'], $data ); } - - $updates += self::compareLanguageArrays( $language, $old_messages[$language], $messages, $verbose, $forbiddenKeys, true ); + $cache['deps'][] = new FileDependency( $fileName ); } - // And log some stuff. - self::myLog( "Updated " . $updates . " messages for the '{$extension}' extension", $verbose ); - - return $updates; - } - - /** - * Checks whether a messages file has a certain hash. - * - * TODO: Swap return values, this is insane - * - * @param $file string Filename - * @param $hash string Hash - * - * @return bool True if $file does NOT have hash $hash, false if it does - */ - public static function checkHash( $file, $hash ) { - $hashes = self::readFile( 'hashes' ); - return @$hashes[$file] !== $hash; - } - - /** - * @param $file - * @param $hash - */ - public static function saveHash( $file, $hash ) { - if ( is_null( self::$newHashes ) ) { - self::$newHashes = self::readFile( 'hashes' ); - } - - self::$newHashes[$file] = $hash; - } - - public static function writeHashes() { - self::writeFile( 'hashes', self::$newHashes ); + return true; } /** - * Logs a message. + * Returns a directory where updated translations are stored. * - * @param $log String - * @param bool $verbose - */ - public static function myLog( $log, $verbose = true ) { - if ( !$verbose ) { - return; - } - if ( isset( $_SERVER ) && array_key_exists( 'REQUEST_METHOD', $_SERVER ) ) { - wfDebug( $log . "\n" ); - } else { - print( $log . "\n" ); - } - } - - /** - * @param $php - * @param $varname - * @return bool|array - */ - public static function parsePHP( $php, $varname ) { - try { - $reader = new QuickArrayReader("<?php $php"); - return $reader->getVar( $varname ); - } catch( Exception $e ) { - self::myLog( "Failed to read file: " . $e ); - return false; - } - } - - /** - * @param $lang - * @return string - * @throws MWException + * @return string|false False if not configured. + * @since 1.1 */ - public static function filename( $lang ) { + public static function getDirectory() { global $wgLocalisationUpdateDirectory, $wgCacheDirectory; - $dir = $wgLocalisationUpdateDirectory ? + // ?: can be used once we drop support for MW 1.19 + return $wgLocalisationUpdateDirectory ? $wgLocalisationUpdateDirectory : $wgCacheDirectory; - - if ( !$dir ) { - throw new MWException( 'No cache directory configured' ); - } - - return "$dir/l10nupdate-$lang.cache"; } /** - * @param $lang - * @return mixed - */ - public static function readFile( $lang ) { - if ( !isset( self::$filecache[$lang] ) ) { - $file = self::filename( $lang ); - $contents = @file_get_contents( $file ); - - if ( $contents === false ) { - wfDebug( "Failed to read file '$file'\n" ); - $retval = array(); - } else { - $retval = unserialize( $contents ); - - if ( $retval === false ) { - wfDebug( "Corrupted data in file '$file'\n" ); - $retval = array(); - } - } - self::$filecache[$lang] = $retval; - } - - return self::$filecache[$lang]; - } - - /** - * @param $lang - * @param $var - * @throws MWException + * Returns a filename where updated translations are stored. + * + * @param string $language Language tag + * @return string + * @since 1.1 */ - public static function writeFile( $lang, $var ) { - $file = self::filename( $lang ); - - if ( !@file_put_contents( $file, serialize( $var ) ) ) { - throw new MWException( "Failed to write to file '$file'" ); - } - - self::$filecache[$lang] = $var; + public static function getFilename( $language ) { + return "l10nupdate-$language.json"; } - } |