3 * Helper class for checkLanguage.php script.
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License along
16 * with this program; if not, write to the Free Software Foundation, Inc.,
17 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
18 * http://www.gnu.org/copyleft/gpl.html
21 * @ingroup MaintenanceLanguage
25 * @ingroup MaintenanceLanguage
27 class CheckLanguageCLI {
28 protected $code = null;
30 protected $doLinks = false;
31 protected $linksPrefix = '';
32 protected $wikiCode = 'en';
33 protected $checkAll = false;
34 protected $output = 'plain';
35 protected $checks = [];
38 protected $results = [];
40 private $includeExif = false;
44 * @param array $options Options for script.
46 public function __construct( array $options ) {
47 if ( isset( $options['help'] ) ) {
52 if ( isset( $options['lang'] ) ) {
53 $this->code = $options['lang'];
55 global $wgLanguageCode;
56 $this->code = $wgLanguageCode;
59 if ( isset( $options['level'] ) ) {
60 $this->level = $options['level'];
63 $this->doLinks = isset( $options['links'] );
64 $this->includeExif = !isset( $options['noexif'] );
65 $this->checkAll = isset( $options['all'] );
67 if ( isset( $options['prefix'] ) ) {
68 $this->linksPrefix = $options['prefix'];
71 if ( isset( $options['wikilang'] ) ) {
72 $this->wikiCode = $options['wikilang'];
75 if ( isset( $options['whitelist'] ) ) {
76 $this->checks = explode( ',', $options['whitelist'] );
77 } elseif ( isset( $options['blacklist'] ) ) {
78 $this->checks = array_diff(
79 isset( $options['easy'] ) ? $this->easyChecks() : $this->defaultChecks(),
80 explode( ',', $options['blacklist'] )
82 } elseif ( isset( $options['easy'] ) ) {
83 $this->checks = $this->easyChecks();
85 $this->checks = $this->defaultChecks();
88 if ( isset( $options['output'] ) ) {
89 $this->output = $options['output'];
92 $this->L = new Languages( $this->includeExif );
96 * Get the default checks.
97 * @return array A list of the default checks.
99 protected function defaultChecks() {
101 'untranslated', 'duplicate', 'obsolete', 'variables', 'empty', 'plural',
102 'whitespace', 'xhtml', 'chars', 'links', 'unbalanced', 'namespace',
103 'projecttalk', 'magic', 'magic-old', 'magic-over', 'magic-case',
104 'special', 'special-old',
109 * Get the checks which check other things than messages.
110 * @return array A list of the non-message checks.
112 protected function nonMessageChecks() {
114 'namespace', 'projecttalk', 'magic', 'magic-old', 'magic-over',
115 'magic-case', 'special', 'special-old',
120 * Get the checks that can easily be treated by non-speakers of the language.
121 * @return array A list of the easy checks.
123 protected function easyChecks() {
125 'duplicate', 'obsolete', 'empty', 'whitespace', 'xhtml', 'chars', 'magic-old',
126 'magic-over', 'magic-case', 'special-old',
132 * @return array An array of all check names mapped to their function names.
134 protected function getChecks() {
136 'untranslated' => 'getUntranslatedMessages',
137 'duplicate' => 'getDuplicateMessages',
138 'obsolete' => 'getObsoleteMessages',
139 'variables' => 'getMessagesWithMismatchVariables',
140 'plural' => 'getMessagesWithoutPlural',
141 'empty' => 'getEmptyMessages',
142 'whitespace' => 'getMessagesWithWhitespace',
143 'xhtml' => 'getNonXHTMLMessages',
144 'chars' => 'getMessagesWithWrongChars',
145 'links' => 'getMessagesWithDubiousLinks',
146 'unbalanced' => 'getMessagesWithUnbalanced',
147 'namespace' => 'getUntranslatedNamespaces',
148 'projecttalk' => 'getProblematicProjectTalks',
149 'magic' => 'getUntranslatedMagicWords',
150 'magic-old' => 'getObsoleteMagicWords',
151 'magic-over' => 'getOverridingMagicWords',
152 'magic-case' => 'getCaseMismatchMagicWords',
153 'special' => 'getUntraslatedSpecialPages',
154 'special-old' => 'getObsoleteSpecialPages',
159 * Get total count for each check non-messages check.
160 * @return array An array of all check names mapped to a two-element array:
161 * function name to get the total count and language code or null
164 protected function getTotalCount() {
166 'namespace' => [ 'getNamespaceNames', 'en' ],
167 'projecttalk' => null,
168 'magic' => [ 'getMagicWords', 'en' ],
169 'magic-old' => [ 'getMagicWords', null ],
170 'magic-over' => [ 'getMagicWords', null ],
171 'magic-case' => [ 'getMagicWords', null ],
172 'special' => [ 'getSpecialPageAliases', 'en' ],
173 'special-old' => [ 'getSpecialPageAliases', null ],
178 * Get all check descriptions.
179 * @return array An array of all check names mapped to their descriptions.
181 protected function getDescriptions() {
183 'untranslated' => '$1 message(s) of $2 are not translated to $3, but exist in en:',
184 'duplicate' => '$1 message(s) of $2 are translated the same in en and $3:',
186 '$1 message(s) of $2 do not exist in en or are in the ignore list, but exist in $3:',
187 'variables' => '$1 message(s) of $2 in $3 don\'t match the variables used in en:',
188 'plural' => '$1 message(s) of $2 in $3 don\'t use {{plural}} while en uses:',
189 'empty' => '$1 message(s) of $2 in $3 are empty or -:',
190 'whitespace' => '$1 message(s) of $2 in $3 have trailing whitespace:',
191 'xhtml' => '$1 message(s) of $2 in $3 contain illegal XHTML:',
193 '$1 message(s) of $2 in $3 include hidden chars which should not be used in the messages:',
194 'links' => '$1 message(s) of $2 in $3 have problematic link(s):',
195 'unbalanced' => '$1 message(s) of $2 in $3 have unbalanced {[]}:',
196 'namespace' => '$1 namespace name(s) of $2 are not translated to $3, but exist in en:',
198 '$1 namespace name(s) and alias(es) in $3 are project talk namespaces without the parameter:',
199 'magic' => '$1 magic word(s) of $2 are not translated to $3, but exist in en:',
200 'magic-old' => '$1 magic word(s) of $2 do not exist in en, but exist in $3:',
201 'magic-over' => '$1 magic word(s) of $2 in $3 do not contain the original en word(s):',
203 '$1 magic word(s) of $2 in $3 change the case-sensitivity of the original en word:',
204 'special' => '$1 special page alias(es) of $2 are not translated to $3, but exist in en:',
205 'special-old' => '$1 special page alias(es) of $2 do not exist in en, but exist in $3:',
211 * @return string The help string.
213 protected function help() {
215 Run this script to check a specific language file, or all of them.
216 Command line settings are in form --parameter[=value].
218 --help: Show this help.
219 --lang: Language code (default: the installation default language).
220 --all: Check all customized languages.
221 --level: Show the following display level (default: 2):
222 * 0: Skip the checks (useful for checking syntax).
223 * 1: Show only the stub headers and number of wrong messages, without
225 * 2: Show only the headers and the message keys, without the message
227 * 3: Show both the headers and the complete messages, with both keys and
229 --links: Link the message values (default off).
230 --prefix: prefix to add to links.
231 --wikilang: For the links, what is the content language of the wiki to
232 display the output in (default en).
233 --noexif: Do not check for Exif messages (a bit hard and boring to
234 translate), if you know what they are currently not translated and want
235 to focus on other problems (default off).
236 --whitelist: Do only the following checks (form: code,code).
237 --blacklist: Do not do the following checks (form: code,code).
238 --easy: Do only the easy checks, which can be treated by non-speakers of
241 Check codes (ideally, all of them should result 0; all the checks are executed
242 by default (except language-specific check blacklists in checkLanguage.inc):
243 * untranslated: Messages which are required to translate, but are not
245 * duplicate: Messages which translation equal to fallback.
246 * obsolete: Messages which are untranslatable or do not exist, but are
248 * variables: Messages without variables which should be used, or with
249 variables which should not be used.
250 * empty: Empty messages and messages that contain only -.
251 * whitespace: Messages which have trailing whitespace.
252 * xhtml: Messages which are not well-formed XHTML (checks only few common
254 * chars: Messages with hidden characters.
255 * links: Messages which contains broken links to pages (does not find all).
256 * unbalanced: Messages which contains unequal numbers of opening {[ and
258 * namespace: Namespace names that were not translated.
259 * projecttalk: Namespace names and aliases where the project talk does not
261 * magic: Magic words that were not translated.
262 * magic-old: Magic words which do not exist.
263 * magic-over: Magic words that override the original English word.
264 * magic-case: Magic words whose translation changes the case-sensitivity of
265 the original English word.
266 * special: Special page names that were not translated.
267 * special-old: Special page names which do not exist.
273 * Execute the script.
275 public function execute() {
277 if ( $this->level > 0 ) {
278 switch ( $this->output ) {
286 throw new MWException( "Invalid output type $this->output" );
292 * Execute the checks.
294 protected function doChecks() {
295 $ignoredCodes = [ 'en', 'enRTL' ];
299 if ( $this->checkAll ) {
300 foreach ( $this->L->getLanguages() as $language ) {
301 if ( !in_array( $language, $ignoredCodes ) ) {
302 $this->results[$language] = $this->checkLanguage( $language );
306 if ( in_array( $this->code, $ignoredCodes ) ) {
307 throw new MWException( "Cannot check code $this->code." );
309 $this->results[$this->code] = $this->checkLanguage( $this->code );
313 $results = $this->results;
314 foreach ( $results as $code => $checks ) {
315 foreach ( $checks as $check => $messages ) {
316 foreach ( $messages as $key => $details ) {
317 if ( $this->isCheckBlacklisted( $check, $code, $key ) ) {
318 unset( $this->results[$code][$check][$key] );
326 * Get the check blacklist.
327 * @return array The list of checks which should not be executed.
329 protected function getCheckBlacklist() {
330 static $blacklist = null;
332 if ( $blacklist !== null ) {
336 // @codingStandardsIgnoreStart Ignore that globals should have a "wg" prefix.
337 global $checkBlacklist;
338 // @codingStandardsIgnoreEnd
340 $blacklist = $checkBlacklist;
342 Hooks::run( 'LocalisationChecksBlacklist', [ &$blacklist ] );
348 * Verify whether a check is blacklisted.
350 * @param string $check Check name
351 * @param string $code Language code
352 * @param string|bool $message Message name, or False for a whole language
353 * @return bool Whether the check is blacklisted
355 protected function isCheckBlacklisted( $check, $code, $message ) {
356 $blacklist = $this->getCheckBlacklist();
358 foreach ( $blacklist as $item ) {
359 if ( isset( $item['check'] ) && $check !== $item['check'] ) {
363 if ( isset( $item['code'] ) && !in_array( $code, $item['code'] ) ) {
367 if ( isset( $item['message'] ) &&
368 ( $message === false || !in_array( $message, $item['message'] ) )
381 * @param string $code The language code.
382 * @throws MWException
383 * @return array The results.
385 protected function checkLanguage( $code ) {
388 if ( $this->level === 0 ) {
389 $this->L->getMessages( $code );
394 $checkFunctions = $this->getChecks();
395 foreach ( $this->checks as $check ) {
396 if ( $this->isCheckBlacklisted( $check, $code, false ) ) {
397 $results[$check] = [];
401 $callback = [ $this->L, $checkFunctions[$check] ];
402 if ( !is_callable( $callback ) ) {
403 throw new MWException( "Unkown check $check." );
405 $results[$check] = call_user_func( $callback, $code );
412 * Format a message key.
413 * @param string $key The message key.
414 * @param string $code The language code.
415 * @return string The formatted message key.
417 protected function formatKey( $key, $code ) {
418 if ( $this->doLinks ) {
419 $displayKey = ucfirst( $key );
420 if ( $code == $this->wikiCode ) {
421 return "[[{$this->linksPrefix}MediaWiki:$displayKey|$key]]";
423 return "[[{$this->linksPrefix}MediaWiki:$displayKey/$code|$key]]";
431 * Output the checks results as plain text.
433 protected function outputText() {
434 foreach ( $this->results as $code => $results ) {
435 $translated = $this->L->getMessages( $code );
436 $translated = count( $translated['translated'] );
437 foreach ( $results as $check => $messages ) {
438 $count = count( $messages );
440 if ( $check == 'untranslated' ) {
441 $translatable = $this->L->getGeneralMessages();
442 $total = count( $translatable['translatable'] );
443 } elseif ( in_array( $check, $this->nonMessageChecks() ) ) {
444 $totalCount = $this->getTotalCount();
445 $totalCount = $totalCount[$check];
446 $callback = [ $this->L, $totalCount[0] ];
447 $callCode = $totalCount[1] ? $totalCount[1] : $code;
448 $total = count( call_user_func( $callback, $callCode ) );
450 $total = $translated;
452 $search = [ '$1', '$2', '$3' ];
453 $replace = [ $count, $total, $code ];
454 $descriptions = $this->getDescriptions();
455 echo "\n" . str_replace( $search, $replace, $descriptions[$check] ) . "\n";
456 if ( $this->level == 1 ) {
457 echo "[messages are hidden]\n";
459 foreach ( $messages as $key => $value ) {
460 if ( !in_array( $check, $this->nonMessageChecks() ) ) {
461 $key = $this->formatKey( $key, $code );
463 if ( $this->level == 2 || empty( $value ) ) {
466 echo "* $key: '$value'\n";
476 * Output the checks results as wiki text.
478 function outputWiki() {
480 $rows[] = '! Language !! Code !! Total !! ' .
481 implode( ' !! ', array_diff( $this->checks, $this->nonMessageChecks() ) );
482 foreach ( $this->results as $code => $results ) {
483 $detailTextForLang = "==$code==\n";
486 $detailTextForLangChecks = [];
487 foreach ( $results as $check => $messages ) {
488 if ( in_array( $check, $this->nonMessageChecks() ) ) {
491 $count = count( $messages );
494 $messageDetails = [];
495 foreach ( $messages as $key => $details ) {
496 $displayKey = $this->formatKey( $key, $code );
497 $messageDetails[] = $displayKey;
499 $detailTextForLangChecks[] = "=== $code-$check ===\n* " . implode( ', ', $messageDetails );
500 $numbers[] = "'''[[#$code-$check|$count]]'''";
506 if ( count( $detailTextForLangChecks ) ) {
507 $detailText .= $detailTextForLang . implode( "\n", $detailTextForLangChecks ) . "\n";
511 # Don't list languages without problems
514 $language = Language::fetchLanguageName( $code );
515 $rows[] = "| $language || $code || $problems || " . implode( ' || ', $numbers );
518 $tableRows = implode( "\n|-\n", $rows );
520 $version = SpecialVersion::getVersion( 'nodb' );
521 // @codingStandardsIgnoreStart Long line.
523 '''Check results are for:''' <code>$version</code>
526 {| class="sortable wikitable" border="2" cellpadding="4" cellspacing="0" style="background-color: #F9F9F9; border: 1px #AAAAAA solid; border-collapse: collapse; clear: both;"
533 // @codingStandardsIgnoreEnd
537 * Check if there are any results for the checks, in any language.
538 * @return bool True if there are any results, false if not.
540 protected function isEmpty() {
541 foreach ( $this->results as $results ) {
542 foreach ( $results as $messages ) {
543 if ( !empty( $messages ) ) {
554 * @ingroup MaintenanceLanguage
556 class CheckExtensionsCLI extends CheckLanguageCLI {
561 * @param array $options Options for script.
562 * @param string $extension The extension name (or names).
564 public function __construct( array $options, $extension ) {
565 if ( isset( $options['help'] ) ) {
570 if ( isset( $options['lang'] ) ) {
571 $this->code = $options['lang'];
573 global $wgLanguageCode;
574 $this->code = $wgLanguageCode;
577 if ( isset( $options['level'] ) ) {
578 $this->level = $options['level'];
581 $this->doLinks = isset( $options['links'] );
583 if ( isset( $options['wikilang'] ) ) {
584 $this->wikiCode = $options['wikilang'];
587 if ( isset( $options['whitelist'] ) ) {
588 $this->checks = explode( ',', $options['whitelist'] );
589 } elseif ( isset( $options['blacklist'] ) ) {
590 $this->checks = array_diff(
591 isset( $options['easy'] ) ? $this->easyChecks() : $this->defaultChecks(),
592 explode( ',', $options['blacklist'] )
594 } elseif ( isset( $options['easy'] ) ) {
595 $this->checks = $this->easyChecks();
597 $this->checks = $this->defaultChecks();
600 if ( isset( $options['output'] ) ) {
601 $this->output = $options['output'];
604 # Some additional checks not enabled by default
605 if ( isset( $options['duplicate'] ) ) {
606 $this->checks[] = 'duplicate';
609 $this->extensions = [];
610 $extensions = new PremadeMediawikiExtensionGroups();
611 $extensions->addAll();
612 if ( $extension == 'all' ) {
613 foreach ( MessageGroups::singleton()->getGroups() as $group ) {
614 if ( strpos( $group->getId(), 'ext-' ) === 0 && !$group->isMeta() ) {
615 $this->extensions[] = new ExtensionLanguages( $group );
618 } elseif ( $extension == 'wikimedia' ) {
619 $wikimedia = MessageGroups::getGroup( 'ext-0-wikimedia' );
620 foreach ( $wikimedia->wmfextensions() as $extension ) {
621 $group = MessageGroups::getGroup( $extension );
622 $this->extensions[] = new ExtensionLanguages( $group );
624 } elseif ( $extension == 'flaggedrevs' ) {
625 foreach ( MessageGroups::singleton()->getGroups() as $group ) {
626 if ( strpos( $group->getId(), 'ext-flaggedrevs-' ) === 0 && !$group->isMeta() ) {
627 $this->extensions[] = new ExtensionLanguages( $group );
631 $extensions = explode( ',', $extension );
632 foreach ( $extensions as $extension ) {
633 $group = MessageGroups::getGroup( 'ext-' . $extension );
635 $extension = new ExtensionLanguages( $group );
636 $this->extensions[] = $extension;
638 print "No such extension $extension.\n";
645 * Get the default checks.
646 * @return array A list of the default checks.
648 protected function defaultChecks() {
650 'untranslated', 'duplicate', 'obsolete', 'variables', 'empty', 'plural',
651 'whitespace', 'xhtml', 'chars', 'links', 'unbalanced',
656 * Get the checks which check other things than messages.
657 * @return array A list of the non-message checks.
659 protected function nonMessageChecks() {
664 * Get the checks that can easily be treated by non-speakers of the language.
665 * @return array A list of the easy checks.
667 protected function easyChecks() {
669 'duplicate', 'obsolete', 'empty', 'whitespace', 'xhtml', 'chars',
675 * @return string The help string.
677 protected function help() {
679 Run this script to check the status of a specific language in extensions, or
680 all of them. Command line settings are in form --parameter[=value], except for
683 * First parameter (mandatory): Extension name, multiple extension names
684 (separated by commas), "all" for all the extensions, "wikimedia" for
685 extensions used by Wikimedia or "flaggedrevs" for all FLaggedRevs
687 * lang: Language code (default: the installation default language).
688 * help: Show this help.
689 * level: Show the following display level (default: 2).
690 * links: Link the message values (default off).
691 * wikilang: For the links, what is the content language of the wiki to
692 display the output in (default en).
693 * whitelist: Do only the following checks (form: code,code).
694 * blacklist: Do not perform the following checks (form: code,code).
695 * easy: Do only the easy checks, which can be treated by non-speakers of
698 Check codes (ideally, all of them should result 0; all the checks are executed
699 by default (except language-specific check blacklists in checkLanguage.inc):
700 * untranslated: Messages which are required to translate, but are not
702 * duplicate: Messages which translation equal to fallback.
703 * obsolete: Messages which are untranslatable, but translated.
704 * variables: Messages without variables which should be used, or with
705 variables which should not be used.
706 * empty: Empty messages.
707 * whitespace: Messages which have trailing whitespace.
708 * xhtml: Messages which are not well-formed XHTML (checks only few common
710 * chars: Messages with hidden characters.
711 * links: Messages which contains broken links to pages (does not find all).
712 * unbalanced: Messages which contains unequal numbers of opening {[ and
715 Display levels (default: 2):
716 * 0: Skip the checks (useful for checking syntax).
717 * 1: Show only the stub headers and number of wrong messages, without list
719 * 2: Show only the headers and the message keys, without the message
721 * 3: Show both the headers and the complete messages, with both keys and
728 * Execute the script.
730 public function execute() {
735 * Check a language and show the results.
736 * @param string $code The language code.
737 * @throws MWException
739 protected function checkLanguage( $code ) {
740 foreach ( $this->extensions as $extension ) {
741 $this->L = $extension;
743 $this->results[$code] = parent::checkLanguage( $code );
745 if ( !$this->isEmpty() ) {
746 echo $extension->name() . ":\n";
748 if ( $this->level > 0 ) {
749 switch ( $this->output ) {
757 throw new MWException( "Invalid output type $this->output" );
767 // Blacklist some checks for some languages or some messages
768 // Possible keys of the sub arrays are: 'check', 'code' and 'message'.
772 'code' => [ 'az', 'bo', 'cdo', 'dz', 'id', 'fa', 'gan', 'gan-hans',
773 'gan-hant', 'gn', 'hak', 'hu', 'ja', 'jv', 'ka', 'kk-arab',
774 'kk-cyrl', 'kk-latn', 'km', 'kn', 'ko', 'lzh', 'mn', 'ms',
775 'my', 'sah', 'sq', 'tet', 'th', 'to', 'tr', 'vi', 'wuu', 'xmf',
776 'yo', 'yue', 'zh', 'zh-classical', 'zh-cn', 'zh-hans',
777 'zh-hant', 'zh-hk', 'zh-sg', 'zh-tw', 'zh-yue'