languages/LanguageZh.php

   1 <?php
   2 /**
   3   * @package MediaWiki
   4   * @subpackage Language
   5   */
   6 require_once( "LanguageZh_cn.php");
   7 require_once( "LanguageZh_tw.php");
   8 require_once( "LanguageZh_sg.php");
   9 require_once( "LanguageZh_hk.php");
  10
  11 /*
  12    hook to refresh the cache of conversion tables when
  13    MediaWiki:zhconversiontable* is updated
  14 */
  15 function zhOnArticleSaveComplete($article, $user, $text, $summary, $isminor, $iswatch, $section) {
  16         $titleobj = $article->getTitle();
  17         if($titleobj->getNamespace() == NS_MEDIAWIKI) {
  18                 global $wgContLang; // should be an LanguageZh.
  19                 if(get_class($wgContLang) != 'languagezh')
  20                         return true;
  21
  22                 $title = $titleobj->getDBkey();
  23                 $t = explode('/', $title, 3);
  24                 $c = count($t);
  25                 if( $c > 1 && $t[0] == 'Zhconversiontable' ) {
  26                         if(in_array($t[1], array('zh-cn', 'zh-tw', 'zh-sg', 'zh-hk'))) {
  27                                 $wgContLang->reloadTables();
  28                         }
  29                 }
  30         }
  31         return true;
  32 }
  33
  34 $wgHooks['ArticleSaveComplete'][] = 'zhOnArticleSaveComplete';
  35
  36 /* class that handles both Traditional and Simplified Chinese
  37    right now it only distinguish zh_cn and zh_tw (actuall, zh_cn and
  38    non-zh_cn), will add support for zh_sg, zh_hk, etc, later.
  39 */
  40 class LanguageZh extends LanguageZh_cn {
  41
  42         var $mZhLanguageCode=false;
  43         var $mTables=false; //the mapping tables
  44         var $mTablesLoaded = false;
  45         var $mCacheKey;
  46         var $mDoTitleConvert = true, $mDoContentConvert = true;
  47         var $mTitleDisplay='';
  48         function LanguageZh() {
  49                 global $wgDBname;
  50                 $this->mCacheKey = $wgDBname . ":zhtables";
  51         }
  52
  53         // a write lock
  54         function lockCache() {
  55                 global $wgMemc;
  56                 $success = false;
  57                 for($i=0; $i<30; $i++) {
  58                         if($success = $wgMemc->add($this->mCacheKey . "lock", 1, 10))
  59                                 break;
  60                         sleep(1);
  61                 }
  62                 return $success;
  63         }
  64
  65         function unlockCache() {
  66                 global $wgMemc;
  67                 $wgMemc->delete($this->mCacheKey . "lock");
  68         }
  69
  70         function updateTable($code, $table) {
  71                 global $wgMemc;
  72                 if(!$this->mTablesLoaded)
  73                         $this->loadTables();
  74
  75                 $this->mTables[$code] = array_merge($this->mTables[$code], $table);
  76                 if($this->lockCache()) {
  77                         $wgMemc->delete($this->mCacheKey);
  78                         $wgMemc->set($this->mCacheKey, $this->mTables, 43200);
  79                         $this->unlockCache();
  80                 }
  81         }
  82
  83         function reloadTables() {
  84                 if($this->mTables)
  85                         unset($this->mTables);
  86                 $this->mTablesLoaded = false;
  87                 $this->loadTables(false);
  88         }
  89
  90         // load conversion tables either from the cache or the disk
  91         function loadTables($fromcache=true) {
  92                 global $wgMemc;
  93                 if( $this->mTablesLoaded )
  94                         return;
  95                 $this->mTablesLoaded = true;
  96                 if($fromcache) {
  97                         $this->mTables = $wgMemc->get( $this->mCacheKey );
  98                         if( !empty( $this->mTables ) ) //all done
  99                                 return;
 100                 }
 101                 // not in cache, or we need a fresh reload.
 102                 // we will first load the tables from file
 103                 // then update them using things in MediaWiki:Zhconversiontable/*
 104                 global $wgMessageCache;
 105                 require( "includes/ZhConversion.php" );
 106                 $this->mTables = array();
 107                 $this->mTables['zh-cn'] = $zh2CN;
 108                 $this->mTables['zh-tw'] = $zh2TW;
 109                 $this->mTables['zh-sg'] = $zh2SG;
 110                 $this->mTables['zh-hk'] = $zh2HK;
 111
 112                 $cached = $this->parseCachedTable('zh-cn');
 113                 $this->mTables['zh-cn'] = array_merge($this->mTables['zh-cn'], $cached);
 114
 115                 $cached = $this->parseCachedTable('zh-tw');
 116                 $this->mTables['zh-tw'] = array_merge($this->mTables['zh-tw'], $cached);
 117
 118                 $cached = $this->parseCachedTable('zh-sg');
 119                 $this->mTables['zh-sg'] = array_merge($this->mTables['zh-cn'], $this->mTables['zh-sg'], $cached);
 120
 121                 $cached = $this->parseCachedTable('zh-hk');
 122                 $this->mTables['zh-hk'] = array_merge($this->mTables['zh-tw'], $this->mTables['zh-hk'], $cached);
 123                 if($this->lockCache()) {
 124                         $wgMemc->set($this->mCacheKey, $this->mTables, 43200);
 125                         $this->unlockCache();
 126                 }
 127         }
 128
 129
 130         /*
 131                 parse the conversion table stored in the cache
 132
 133                 the tables should be in blocks of the following form:
 134
 135                         -{
 136                                 word => word ;
 137                                 word => word ;
 138                                 ...
 139                         }-
 140
 141                 to make the tables more manageable, subpages are allowed
 142                 and will be parsed recursively if $recursive=true
 143
 144         */
 145         function parseCachedTable($code, $subpage='', $recursive=true) {
 146                 global $wgMessageCache;
 147                 static $parsed = array();
 148
 149                 if(!is_object($wgMessageCache))
 150                         return array();
 151
 152                 $key = 'zhconversiontable/'.$code;
 153                 if($subpage)
 154                         $key .= '/' . $subpage;
 155
 156                 if(array_key_exists($key, $parsed))
 157                         return array();
 158
 159
 160                 $txt = $wgMessageCache->get( $key, true, true, true );
 161
 162                 // get all subpage links of the form
 163                 // [[MediaWiki:Zhconversiontable/zh-xx/...|...]]
 164                 $linkhead = $this->getNsText(NS_MEDIAWIKI) . ':Zhconversiontable';
 165                 $subs = explode('[[', $txt);
 166                 $sublinks = array();
 167                 foreach( $subs as $sub ) {
 168                         $link = explode(']]', $sub, 2);
 169                         if(count($link) != 2)
 170                                 continue;
 171                         $b = explode('|', $link[0]);
 172                         $b = explode('/', trim($b[0]), 3);
 173                         if(count($b)==3)
 174                                 $sublink = $b[2];
 175                         else
 176                                 $sublink = '';
 177
 178                         if($b[0] == $linkhead && $b[1] == $code) {
 179                                 $sublinks[] = $sublink;
 180                         }
 181                 }
 182
 183
 184                 // parse the mappings in this page
 185                 $blocks = explode('-{', $txt);
 186                 array_shift($blocks);
 187                 $ret = array();
 188                 foreach($blocks as $block) {
 189                         $mappings = explode('}-', $block, 2);
 190                         $stripped = str_replace(array("'", '"', '*','#'), '', $mappings[0]);
 191                         $table = explode( ';', $stripped );
 192                         foreach( $table as $t ) {
 193                                 $m = explode( '=>', $t );
 194                                 if( count( $m ) != 2)
 195                                         continue;
 196                                 // trim any trailling comments starting with '//'
 197                                 $tt = explode('//', $m[1], 2);
 198                                 $ret[trim($m[0])] = trim($tt[0]);
 199                         }
 200                 }
 201                 $parsed[$key] = true;
 202
 203
 204                 // recursively parse the subpages
 205                 if($recursive) {
 206                         foreach($sublinks as $link) {
 207                                 $s = $this->parseCachedTable($code, $link, $recursive);
 208                                 $ret = array_merge($ret, $s);
 209                         }
 210                 }
 211                 return $ret;
 212         }
 213
 214         /*
 215                 get preferred language variants.
 216         */
 217         function getPreferredVariant() {
 218                 global $wgUser, $wgRequest;
 219
 220                 if($this->mZhLanguageCode)
 221                         return $this->mZhLanguageCode;
 222
 223                 // see if the preference is set in the request
 224                 $zhreq = $wgRequest->getText( 'variant' );
 225                 if( in_array( $zhreq, $this->getVariants() ) ) {
 226                         $this->mZhLanguageCode = $zhreq;
 227                         return $zhreq;
 228                 }
 229
 230                 // get language variant preference from logged in users
 231                 if( $wgUser->isLoggedIn() ) {
 232                         $this->mZhLanguageCode = $wgUser->getOption('variant');
 233                 }
 234
 235                 if( !$this->mZhLanguageCode ) {
 236                         // see if some zh- variant is set in the http header,
 237                         $this->mZhLanguageCode="zh";
 238                         if(array_key_exists('HTTP_ACCEPT_LANGUAGE', $_SERVER)) {
 239                                 $header = str_replace( '_', '-', strtolower($_SERVER["HTTP_ACCEPT_LANGUAGE"]));
 240                                 $zh = strstr($header, 'zh-');
 241                                 if($zh) {
 242                                         $this->mZhLanguageCode = substr($zh,0,5);
 243                                 }
 244                         }
 245                 }
 246                 return $this->mZhLanguageCode;
 247         }
 248
 249         # this should give much better diff info
 250         function segmentForDiff( $text ) {
 251                 return preg_replace(
 252                         "/([\\xc0-\\xff][\\x80-\\xbf]*)/e",
 253                         "' ' .\"$1\"", $text);
 254         }
 255
 256         function unsegmentForDiff( $text ) {
 257                 return preg_replace(
 258                         "/ ([\\xc0-\\xff][\\x80-\\xbf]*)/e",
 259                         "\"$1\"", $text);
 260         }
 261
 262         function autoConvert($text, $toVariant=false) {
 263                 $fname="LanguageZh::autoConvert";
 264                 wfProfileIn( $fname );
 265
 266                 if(!$this->mTablesLoaded)
 267                         $this->loadTables();
 268
 269                 if(!$toVariant)
 270                         $toVariant = $this->getPreferredVariant();
 271                 $ret = '';
 272                 switch( $toVariant ) {
 273                         case 'zh-cn': $ret = strtr($text, $this->mTables['zh-cn']);break;
 274                         case 'zh-tw': $ret = strtr($text, $this->mTables['zh-tw']);break;
 275                         case 'zh-sg': $ret = strtr($text, $this->mTables['zh-sg']);break;
 276                         case 'zh-hk': $ret = strtr($text, $this->mTables['zh-hk']);break;
 277                         default: $ret = $text;
 278                 }
 279                 wfProfileOut( $fname );
 280                 return $ret;
 281         }
 282
 283         function autoConvertToAllVariants($text) {
 284                 $fname="LanguageZh::autoConvertToAllVariants";
 285                 wfProfileIn( $fname );
 286                 if( !$this->mTablesLoaded )
 287                         $this->loadTables();
 288
 289                 $ret = array();
 290                 $ret['zh-cn'] = strtr($text, $this->mTables['zh-cn']);
 291                 $ret['zh-tw'] = strtr($text, $this->mTables['zh-tw']);
 292                 $ret['zh-sg'] = strtr(strtr($text, $this->mTables['zh-cn']), $this->mTables['zh-sg']);
 293                 $ret['zh-hk'] = strtr(strtr($text, $this->mTables['zh-tw']), $this->mTables['zh-hk']);
 294                 wfProfileOut( $fname );
 295                 return $ret;
 296         }
 297
 298         # convert text to different variants of a language. the automatic
 299         # conversion is done in autoConvert(). here we parse the text
 300         # marked with -{}-, which specifies special conversions of the
 301         # text that can not be accomplished in autoConvert()
 302         #
 303         # syntax of the markup:
 304         # -{code1:text1;code2:text2;...}-  or
 305         # -{text}- in which case no conversion should take place for text
 306         function convert( $text , $isTitle=false) {
 307                 global $wgDisableLangConversion;
 308                 if($wgDisableLangConversion)
 309                         return $text;
 310
 311                 $mw =& MagicWord::get( MAG_NOTITLECONVERT );
 312                 if( $mw->matchAndRemove( $text ) )
 313                         $this->mDoTitleConvert = false;
 314
 315                 $mw =& MagicWord::get( MAG_NOCONTENTCONVERT );
 316                 if( $mw->matchAndRemove( $text ) ) {
 317                         $this->mDoContentConvert = false;
 318                 }
 319
 320                 // no conversion if redirecting
 321                 $mw =& MagicWord::get( MAG_REDIRECT );
 322                 if( $mw->matchStart( $text ))
 323                         return $text;
 324
 325                 if( $isTitle ) {
 326                         if( !$this->mDoTitleConvert ) {
 327                                 $this->mTitleDisplay = $text;
 328                                 return $text;
 329                         }
 330                         if( !empty($this->mTitleDisplay))
 331                                 return $this->mTitleDisplay;
 332
 333                         global $wgRequest;
 334                         $isredir = $wgRequest->getText( 'redirect', 'yes' );
 335                         $action = $wgRequest->getText( 'action' );
 336                         if ( $isredir == 'no' || $action == 'edit' ) {
 337                                 return $text;
 338                         }
 339                         else {
 340                                 $this->mTitleDisplay = $this->autoConvert($text);
 341                                 return $this->mTitleDisplay;
 342                         }
 343                 }
 344
 345                 if( !$this->mDoContentConvert )
 346                         return $text;
 347
 348                 $plang = $this->getPreferredVariant();
 349                 $fallback = $this->getVariantFallback($plang);
 350                 $variants = $this->getVariants();
 351
 352                 $tarray = explode("-{", $text);
 353                 $tfirst = array_shift($tarray);
 354                 $text = $this->autoConvert($tfirst);
 355                 foreach($tarray as $txt) {
 356                         $marked = explode("}-", $txt);
 357
 358                         //strip &nbsp; since it interferes with the parsing, plus,
 359                         //all spaces should be stripped in this tag anyway.
 360                         $marked[0] = str_replace('&nbsp;', '', $marked[0]);
 361
 362                         /* see if this conversion has special meaning
 363                            # for article title:
 364                                  -{T|zh-cn:foo;zh-tw:bar}-
 365                            # convert all occurence of foo/bar in this article:
 366                                  -{A|zh-cn:foo;zh-tw:bar}-
 367                         */
 368                         $flag = '';
 369                         $choice = false;
 370                         $tt = explode("|", $marked[0], 2);
 371                         if(sizeof($tt) == 2) {
 372                                 $flag = trim($tt[0]);
 373                                 $choice = explode(";", $tt[1]);
 374                         }
 375
 376                         if(!$choice) {
 377                                 $choice = explode(";", $marked[0]);
 378                         }
 379                         $disp = '';
 380                         $carray = array();
 381                         if(!array_key_exists(1, $choice)) {
 382                                 /* a single choice */
 383                                 $disp = $choice[0];
 384
 385                                 /* fill the carray if the conversion is for the whole article*/
 386                                 if($flag == 'A') {
 387                                         foreach($variants as $v)
 388                                                 $carray[$v] = $disp;
 389                                 }
 390                         }
 391                         else {
 392                                 foreach($choice as $c) {
 393                                         $v = explode(":", $c);
 394                                         if(sizeof($v) != 2) // syntax error, skip
 395                                                 continue;
 396                                         $carray[trim($v[0])] = trim($v[1]);
 397                                 }
 398                                 if(array_key_exists($plang, $carray))
 399                                         $disp = $carray[$plang];
 400                                 else if(array_key_exists($fallback, $carray))
 401                                         $disp = $carray[$fallback];
 402                         }
 403                         if(empty($disp)) { // syntax error
 404                                 $text .= $marked[0];
 405                         }
 406                         else {
 407                                 if($flag == 'T') // for title only
 408                                         $this->mTitleDisplay = $disp;
 409                                 else {
 410                                         $text .= $disp;
 411                                         if($flag == 'A') {
 412                                                 /* modify the conversion table for this session*/
 413
 414                                                 /* fill in the missing variants, if any,
 415                                                     with fallbacks */
 416                                                 foreach($variants as $v) {
 417                                                         if(!array_key_exists($v, $carray)) {
 418                                                                 $vf = $this->getVariantFallback($v);
 419                                                                 if(array_key_exists($vf, $carray))
 420                                                                         $carray[$v] = $carray[$vf];
 421                                                         }
 422                                                 }
 423                                                 foreach($variants as $vfrom) {
 424                                                         if(!array_key_exists($vfrom, $carray))
 425                                                                 continue;
 426                                                         foreach($variants as $vto) {
 427                                                                 if($vfrom == $vto)
 428                                                                         continue;
 429                                                                 if(!array_key_exists($vto, $carray))
 430                                                                         continue;
 431                                                                 $this->mTables[$vto][$carray[$vfrom]] = $carray[$vto];
 432                                                         }
 433                                                 }
 434                                         }
 435                                 }
 436                         }
 437                         if(array_key_exists(1, $marked))
 438                                 $text .= $this->autoConvert($marked[1]);
 439                 }
 440
 441                 return $text;
 442         }
 443
 444
 445         function getVariants() {
 446                 return array("zh", "zh-cn", "zh-tw", "zh-sg", "zh-hk");
 447         }
 448
 449         function getVariantFallback($v) {
 450                 switch ($v) {
 451                 case 'zh': return 'zh-cn'; break;
 452                 case 'zh-cn': return 'zh-sg'; break;
 453                 case 'zh-sg': return 'zh-cn'; break;
 454                 case 'zh-tw': return 'zh-hk'; break;
 455                 case 'zh-hk': return 'zh-tw'; break;
 456                 }
 457                 return false;
 458         }
 459
 460         // word segmentation
 461         function stripForSearch( $string ) {
 462                 $fname="LanguageZh::stripForSearch";
 463                 wfProfileIn( $fname );
 464
 465                 // eventually this should be a word segmentation
 466                 // for now just treat each character as a word
 467                 $t = preg_replace(
 468                                 "/([\\xc0-\\xff][\\x80-\\xbf]*)/e",
 469                                 "' ' .\"$1\"", $string);
 470
 471         //always convert to zh-cn before indexing. it should be
 472                 //better to use zh-cn for search, since conversion from
 473                 //Traditional to Simplified is less ambiguous than the
 474                 //other way around
 475
 476                 $t = $this->autoConvert($t, 'zh-cn');
 477                 $t = LanguageUtf8::stripForSearch( $t );
 478                 wfProfileOut( $fname );
 479                 return $t;
 480
 481         }
 482
 483         function convertForSearchResult( $termsArray ) {
 484                 $terms = implode( '|', $termsArray );
 485                 $terms = implode( '|', $this->autoConvertToAllVariants( $terms ) );
 486                 $ret = array_unique( explode('|', $terms) );
 487                 return $ret;
 488         }
 489
 490         function findVariantLink( &$link, &$nt ) {
 491                 static $count=0; //used to limit this operation
 492                 static $cache=array();
 493                 global $wgDisableLangConversion;
 494                 $pref = $this->getPreferredVariant();
 495                 if( $count > 50 )
 496                         return;
 497                 $count++;
 498                 $variants = $this->autoConvertToAllVariants($link);
 499                 if($variants == false) //give up
 500                         return;
 501                 foreach( $variants as $v ) {
 502                         if(isset($cache[$v]))
 503                                 continue;
 504                         $cache[$v] = 1;
 505                         $varnt = Title::newFromText( $v );
 506                         if( $varnt && $varnt->getArticleID() > 0 ) {
 507                                 $nt = $varnt;
 508                                 if( !$wgDisableLangConversion && $pref != 'zh' )
 509                                         $link = $v;
 510                                 break;
 511                         }
 512                 }
 513         }
 514
 515         function getExtraHashOptions() {
 516                 global $wgUser;
 517                 $variant = $this->getPreferredVariant();
 518                 return '!' . $variant ;
 519         }
 520
 521         function getParsedTitle() {
 522                 return $this->mTitleDisplay;
 523         }
 524 }
 525 ?>