languages/LanguageZh.php

   1 <?php
   2 /**
   3   * @package MediaWiki
   4   * @subpackage Language
   5   */
   6 require_once( "LanguageZh_cn.php");
   7 require_once( "LanguageZh_tw.php");
   8 require_once( "LanguageZh_sg.php");
   9 require_once( "LanguageZh_hk.php");
  10
  11 /*
  12    hook to refresh the cache of conversion tables when
  13    MediaWiki:zhconversiontable* is updated
  14 */
  15 function zhOnArticleSaveComplete($article, $user, $text, $summary, $isminor, $iswatch, $section) {
  16         $titleobj = $article->getTitle();
  17         if($titleobj->getNamespace() == NS_MEDIAWIKI) {
  18                 global $wgContLang; // should be an LanguageZh.
  19                 if(get_class($wgContLang) != 'languagezh')
  20                         return true;
  21
  22                 $title = $titleobj->getDBkey();
  23                 $t = explode('/', $title, 3);
  24                 $c = count($t);
  25                 if( $c > 1 && $t[0] == 'Zhconversiontable' ) {
  26                         if(in_array($t[1], array('zh-cn', 'zh-tw', 'zh-sg', 'zh-hk'))) {
  27                                 $wgContLang->reloadTables();
  28                         }
  29                 }
  30         }
  31         return true;
  32 }
  33
  34 $wgHooks['ArticleSaveComplete'][] = 'zhOnArticleSaveComplete';
  35
  36 /* class that handles both Traditional and Simplified Chinese
  37    right now it only distinguish zh_cn and zh_tw (actuall, zh_cn and
  38    non-zh_cn), will add support for zh_sg, zh_hk, etc, later.
  39 */
  40 class LanguageZh extends LanguageZh_cn {
  41
  42         var $mZhLanguageCode=false;
  43         var $mTables=false; //the mapping tables
  44         var $mTablesLoaded = false;
  45         var $mCacheKey;
  46         var $mDoTitleConvert = true, $mDoContentConvert = true;
  47         var $mTitleDisplay=false;
  48         function LanguageZh() {
  49                 global $wgDBname;
  50                 $this->mCacheKey = $wgDBname . ":zhtables";
  51         }
  52
  53         // a write lock
  54         function lockCache() {
  55                 global $wgMemc;
  56                 $success = false;
  57                 for($i=0; $i<30; $i++) {
  58                         if($success = $wgMemc->add($this->mCacheKey . "lock", 1, 10))
  59                                 break;
  60                         sleep(1);
  61                 }
  62                 return $success;
  63         }
  64
  65         function unlockCache() {
  66                 global $wgMemc;
  67                 $wgMemc->delete($this->mCacheKey . "lock");
  68         }
  69
  70         function updateTable($code, $table) {
  71                 global $wgMemc;
  72                 if(!$this->mTablesLoaded)
  73                         $this->loadTables();
  74
  75                 $this->mTables[$code] = array_merge($this->mTables[$code], $table);
  76                 if($this->lockCache()) {
  77                         $wgMemc->delete($this->mCacheKey);
  78                         $wgMemc->set($this->mCacheKey, $this->mTables, 43200);
  79                         $this->unlockCache();
  80                 }
  81         }
  82
  83         function reloadTables() {
  84                 if($this->mTables)
  85                         unset($this->mTables);
  86                 $this->mTablesLoaded = false;
  87                 $this->loadTables(false);
  88         }
  89
  90         // load conversion tables either from the cache or the disk
  91         function loadTables($fromcache=true) {
  92                 global $wgMemc;
  93                 if( $this->mTablesLoaded )
  94                         return;
  95                 $this->mTablesLoaded = true;
  96                 if($fromcache) {
  97                         $this->mTables = $wgMemc->get( $this->mCacheKey );
  98                         if( !empty( $this->mTables ) ) //all done
  99                                 return;
 100                 }
 101                 // not in cache, or we need a fresh reload.
 102                 // we will first load the tables from file
 103                 // then update them using things in MediaWiki:Zhconversiontable/*
 104                 global $wgMessageCache;
 105                 require( "includes/ZhConversion.php" );
 106                 $this->mTables = array();
 107                 $this->mTables['zh-cn'] = $zh2CN;
 108                 $this->mTables['zh-tw'] = $zh2TW;
 109                 $this->mTables['zh-sg'] = $zh2SG;
 110                 $this->mTables['zh-hk'] = $zh2HK;
 111
 112                 $cached = $this->parseCachedTable('zh-cn');
 113                 $this->mTables['zh-cn'] = array_merge($this->mTables['zh-cn'], $cached);
 114
 115                 $cached = $this->parseCachedTable('zh-tw');
 116                 $this->mTables['zh-tw'] = array_merge($this->mTables['zh-tw'], $cached);
 117
 118                 $cached = $this->parseCachedTable('zh-sg');
 119                 $this->mTables['zh-sg'] = array_merge($this->mTables['zh-cn'], $this->mTables['zh-sg'], $cached);
 120
 121                 $cached = $this->parseCachedTable('zh-hk');
 122                 $this->mTables['zh-hk'] = array_merge($this->mTables['zh-tw'], $this->mTables['zh-hk'], $cached);
 123                 if($this->lockCache()) {
 124                         $wgMemc->set($this->mCacheKey, $this->mTables, 43200);
 125                         $this->unlockCache();
 126                 }
 127         }
 128
 129
 130         /*
 131                 parse the conversion table stored in the cache
 132
 133                 the tables should be in blocks of the following form:
 134
 135                         -{
 136                                 word => word ;
 137                                 word => word ;
 138                                 ...
 139                         }-
 140
 141                 to make the tables more manageable, subpages are allowed
 142                 and will be parsed recursively if $recursive=true
 143
 144         */
 145         function parseCachedTable($code, $subpage='', $recursive=true) {
 146                 global $wgMessageCache;
 147                 static $parsed = array();
 148
 149                 if(!is_object($wgMessageCache))
 150                         return array();
 151
 152                 $key = 'zhconversiontable/'.$code;
 153                 if($subpage)
 154                         $key .= '/' . $subpage;
 155
 156                 if(array_key_exists($key, $parsed))
 157                         return array();
 158
 159
 160                 $txt = $wgMessageCache->get( $key, true, true, true );
 161
 162                 // get all subpage links of the form
 163                 // [[MediaWiki:Zhconversiontable/zh-xx/...|...]]
 164                 $linkhead = $this->getNsText(NS_MEDIAWIKI) . ':Zhconversiontable';
 165                 $subs = explode('[[', $txt);
 166                 $sublinks = array();
 167                 foreach( $subs as $sub ) {
 168                         $link = explode(']]', $sub, 2);
 169                         if(count($link) != 2)
 170                                 continue;
 171                         $b = explode('|', $link[0]);
 172                         $b = explode('/', trim($b[0]), 3);
 173                         if(count($b)==3)
 174                                 $sublink = $b[2];
 175                         else
 176                                 $sublink = '';
 177
 178                         if($b[0] == $linkhead && $b[1] == $code) {
 179                                 $sublinks[] = $sublink;
 180                         }
 181                 }
 182
 183
 184                 // parse the mappings in this page
 185                 $blocks = explode('-{', $txt);
 186                 array_shift($blocks);
 187                 $ret = array();
 188                 foreach($blocks as $block) {
 189                         $mappings = explode('}-', $block, 2);
 190                         $stripped = str_replace(array("'", '"', '*','#'), '', $mappings[0]);
 191                         $table = explode( ';', $stripped );
 192                         foreach( $table as $t ) {
 193                                 $m = explode( '=>', $t );
 194                                 if( count( $m ) != 2)
 195                                         continue;
 196                                 // trim any trailling comments starting with '//'
 197                                 $tt = explode('//', $m[1], 2);
 198                                 $ret[trim($m[0])] = trim($tt[0]);
 199                         }
 200                 }
 201                 $parsed[$key] = true;
 202
 203
 204                 // recursively parse the subpages
 205                 if($recursive) {
 206                         foreach($sublinks as $link) {
 207                                 $s = $this->parseCachedTable($code, $link, $recursive);
 208                                 $ret = array_merge($ret, $s);
 209                         }
 210                 }
 211                 return $ret;
 212         }
 213
 214         /*
 215                 get preferred language variants.
 216         */
 217         function getPreferredVariant() {
 218                 global $wgUser, $wgRequest;
 219
 220                 if($this->mZhLanguageCode)
 221                         return $this->mZhLanguageCode;
 222
 223                 // see if the preference is set in the request
 224                 $zhreq = $wgRequest->getText( 'variant' );
 225                 if( in_array( $zhreq, $this->getVariants() ) ) {
 226                         $this->mZhLanguageCode = $zhreq;
 227                         return $zhreq;
 228                 }
 229
 230                 // get language variant preference from logged in users
 231                 if( $wgUser->isLoggedIn() ) {
 232                         $this->mZhLanguageCode = $wgUser->getOption('variant');
 233                 }
 234
 235                 if( !$this->mZhLanguageCode ) {
 236                         // see if some zh- variant is set in the http header,
 237                         $this->mZhLanguageCode="zh";
 238                         if(array_key_exists('HTTP_ACCEPT_LANGUAGE', $_SERVER)) {
 239                                 $header = str_replace( '_', '-', strtolower($_SERVER["HTTP_ACCEPT_LANGUAGE"]));
 240                                 $zh = strstr($header, 'zh-');
 241                                 if($zh) {
 242                                         $this->mZhLanguageCode = substr($zh,0,5);
 243                                 }
 244                         }
 245                 }
 246                 return $this->mZhLanguageCode;
 247         }
 248
 249         # this should give much better diff info
 250         function segmentForDiff( $text ) {
 251                 return preg_replace(
 252                         "/([\\xc0-\\xff][\\x80-\\xbf]*)/e",
 253                         "' ' .\"$1\"", $text);
 254         }
 255
 256         function unsegmentForDiff( $text ) {
 257                 return preg_replace(
 258                         "/ ([\\xc0-\\xff][\\x80-\\xbf]*)/e",
 259                         "\"$1\"", $text);
 260         }
 261
 262         function autoConvert($text, $toVariant=false) {
 263                 $fname="LanguageZh::autoConvert";
 264                 wfProfileIn( $fname );
 265
 266                 if(!$this->mTablesLoaded)
 267                         $this->loadTables();
 268
 269                 if(!$toVariant)
 270                         $toVariant = $this->getPreferredVariant();
 271                 $ret = '';
 272                 switch( $toVariant ) {
 273                         case 'zh-cn': $ret = strtr($text, $this->mTables['zh-cn']);break;
 274                         case 'zh-tw': $ret = strtr($text, $this->mTables['zh-tw']);break;
 275                         case 'zh-sg': $ret = strtr($text, $this->mTables['zh-sg']);break;
 276                         case 'zh-hk': $ret = strtr($text, $this->mTables['zh-hk']);break;
 277                         default: $ret = $text;
 278                 }
 279                 wfProfileOut( $fname );
 280                 return $ret;
 281         }
 282
 283         function autoConvertToAllVariants($text) {
 284                 $fname="LanguageZh::autoConvertToAllVariants";
 285                 wfProfileIn( $fname );
 286                 if( !$this->mTablesLoaded )
 287                         $this->loadTables();
 288
 289                 $ret = array();
 290                 $ret['zh-cn'] = strtr($text, $this->mTables['zh-cn']);
 291                 $ret['zh-tw'] = strtr($text, $this->mTables['zh-tw']);
 292                 $ret['zh-sg'] = strtr(strtr($text, $this->mTables['zh-cn']), $this->mTables['zh-sg']);
 293                 $ret['zh-hk'] = strtr(strtr($text, $this->mTables['zh-tw']), $this->mTables['zh-hk']);
 294                 wfProfileOut( $fname );
 295                 return $ret;
 296         }
 297
 298         # convert text to different variants of a language. the automatic
 299         # conversion is done in autoConvert(). here we parse the text
 300         # marked with -{}-, which specifies special conversions of the
 301         # text that can not be accomplished in autoConvert()
 302         #
 303         # syntax of the markup:
 304         # -{code1:text1;code2:text2;...}-  or
 305         # -{text}- in which case no conversion should take place for text
 306         function convert( $text , $isTitle=false) {
 307                 global $wgDisableLangConversion;
 308                 if($wgDisableLangConversion)
 309                         return $text;
 310
 311                 $mw =& MagicWord::get( MAG_NOTITLECONVERT );
 312                 if( $mw->matchAndRemove( $text ) )
 313                         $this->mDoTitleConvert = false;
 314
 315                 $mw =& MagicWord::get( MAG_NOCONTENTCONVERT );
 316                 if( $mw->matchAndRemove( $text ) ) {
 317                         $this->mDoContentConvert = false;
 318                 }
 319
 320                 // no conversion if redirecting
 321                 $mw =& MagicWord::get( MAG_REDIRECT );
 322                 if( $mw->matchStart( $text ))
 323                         return $text;
 324
 325                 if( $isTitle ) {
 326                         if( !$this->mDoTitleConvert )
 327                                 return $text;
 328
 329                         if( $this->mTitleDisplay != false )
 330                                 return $this->mTitleDisplay;
 331
 332                         global $wgRequest;
 333                         $isredir = $wgRequest->getText( 'redirect', 'yes' );
 334                         $action = $wgRequest->getText( 'action' );
 335                         if ( $isredir == 'no' || $action == 'edit' ) {
 336                                 return $text;
 337                         }
 338                         else {
 339                                 return $this->autoConvert($text);
 340                         }
 341                 }
 342
 343                 if( !$this->mDoContentConvert )
 344                         return $text;
 345
 346                 $plang = $this->getPreferredVariant();
 347                 $fallback = $this->getVariantFallback($plang);
 348
 349                 $tarray = explode("-{", $text);
 350                 $tfirst = array_shift($tarray);
 351                 $text = $this->autoConvert($tfirst);
 352                 foreach($tarray as $txt) {
 353                         $marked = explode("}-", $txt);
 354
 355                         //strip &nbsp; since it interferes with the parsing, plus,
 356                         //all spaces should be stripped in this tag anyway.
 357                         $marked[0] = str_replace('&nbsp;', '', $marked[0]);
 358
 359                         $choice = explode(";", $marked[0]);
 360                         /* see if this conversion is specifically for the
 361                           article title. the format is
 362                                 -{T|zh-cn:foo;zh-tw:bar}-
 363                         */
 364                         $fortitle = false;
 365                         $tt = explode("|", $marked[0], 2);
 366                         if(sizeof($tt) == 2 && trim($tt[0]) == 'T') {
 367                                 $choice = explode(";", $tt[1]);
 368                                 $fortitle = true;
 369                         }
 370                         else {
 371                                 $choice = explode(";", $marked[0]);
 372                         }
 373                         $disp = '';
 374                         if(!array_key_exists(1, $choice)) {
 375                                 /* a single choice */
 376                                 $disp = $choice[0];
 377                         } else {
 378                                 $choice1=false;
 379                                 $choice2=false;
 380                                 foreach($choice as $c) {
 381                                         $v = explode(":", $c);
 382                                         if(!array_key_exists(1, $v)) {
 383                                                 //syntax error in the markup, give up
 384                                                 break;
 385                                         }
 386                                         $code = trim($v[0]);
 387                                         $content = trim($v[1]);
 388                                         if($code == $plang) {
 389                                                 $choice1 = $content;
 390                                                 break;
 391                                         }
 392                                         if($code == $fallback)
 393                                                 $choice2 = $content;
 394                                 }
 395                                 if ( $choice1 )
 396                                         $disp = $choice1;
 397                                 elseif ( $choice2 )
 398                                         $disp = $choice2;
 399                                 else
 400                                         $disp = $marked[0];
 401                         }
 402
 403                         if($fortitle)
 404                                 $this->mTitleDisplay = $disp;
 405                         else
 406                                 $text .= $disp;
 407                         if(array_key_exists(1, $marked))
 408                                 $text .= $this->autoConvert($marked[1]);
 409                 }
 410
 411                 return $text;
 412         }
 413
 414
 415         function getVariants() {
 416                 return array("zh", "zh-cn", "zh-tw", "zh-sg", "zh-hk");
 417         }
 418
 419         function getVariantFallback($v) {
 420                 switch ($v) {
 421                 case 'zh': return 'zh-cn'; break;
 422                 case 'zh-cn': return 'zh-sg'; break;
 423                 case 'zh-sg': return 'zh-cn'; break;
 424                 case 'zh-tw': return 'zh-hk'; break;
 425                 case 'zh-hk': return 'zh-tw'; break;
 426                 }
 427                 return false;
 428         }
 429
 430         // word segmentation
 431         function stripForSearch( $string ) {
 432                 $fname="LanguageZh::stripForSearch";
 433                 wfProfileIn( $fname );
 434
 435                 // eventually this should be a word segmentation
 436                 // for now just treat each character as a word
 437                 $t = preg_replace(
 438                                 "/([\\xc0-\\xff][\\x80-\\xbf]*)/e",
 439                                 "' ' .\"$1\"", $string);
 440
 441         //always convert to zh-cn before indexing. it should be
 442                 //better to use zh-cn for search, since conversion from
 443                 //Traditional to Simplified is less ambiguous than the
 444                 //other way around
 445
 446                 $t = $this->autoConvert($t, 'zh-cn');
 447                 $t = LanguageUtf8::stripForSearch( $t );
 448                 wfProfileOut( $fname );
 449                 return $t;
 450
 451         }
 452
 453         function convertForSearchResult( $termsArray ) {
 454                 $terms = implode( '|', $termsArray );
 455                 $terms = implode( '|', $this->autoConvertToAllVariants( $terms ) );
 456                 $ret = array_unique( explode('|', $terms) );
 457                 return $ret;
 458         }
 459
 460         function findVariantLink( &$link, &$nt ) {
 461                 static $count=0; //used to limit this operation
 462                 static $cache=array();
 463                 global $wgDisableLangConversion;
 464                 $pref = $this->getPreferredVariant();
 465                 if( $count > 50 )
 466                         return;
 467                 $count++;
 468                 $variants = $this->autoConvertToAllVariants($link);
 469                 if($variants == false) //give up
 470                         return;
 471                 foreach( $variants as $v ) {
 472                         if(isset($cache[$v]))
 473                                 continue;
 474                         $cache[$v] = 1;
 475                         $varnt = Title::newFromText( $v );
 476                         if( $varnt && $varnt->getArticleID() > 0 ) {
 477                                 $nt = $varnt;
 478                                 if( !$wgDisableLangConversion && $pref != 'zh' )
 479                                         $link = $v;
 480                                 break;
 481                         }
 482                 }
 483         }
 484
 485         function getExtraHashOptions() {
 486                 global $wgUser;
 487                 $variant = $this->getPreferredVariant();
 488                 return '!' . $variant ;
 489         }
 490
 491 }
 492 ?>