includes/Sanitizer.php

   1 <?php
   2 /**
   3  * HTML sanitizer for %MediaWiki.
   4  *
   5  * Copyright © 2002-2005 Brion Vibber <brion@pobox.com> et al
   6  * http://www.mediawiki.org/
   7  *
   8  * This program is free software; you can redistribute it and/or modify
   9  * it under the terms of the GNU General Public License as published by
  10  * the Free Software Foundation; either version 2 of the License, or
  11  * (at your option) any later version.
  12  *
  13  * This program is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  16  * GNU General Public License for more details.
  17  *
  18  * You should have received a copy of the GNU General Public License along
  19  * with this program; if not, write to the Free Software Foundation, Inc.,
  20  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  21  * http://www.gnu.org/copyleft/gpl.html
  22  *
  23  * @file
  24  * @ingroup Parser
  25  */
  26
  27 /**
  28  * HTML sanitizer for MediaWiki
  29  * @ingroup Parser
  30  */
  31 class Sanitizer {
  32         /**
  33          * Regular expression to match various types of character references in
  34          * Sanitizer::normalizeCharReferences and Sanitizer::decodeCharReferences
  35          */
  36         const CHAR_REFS_REGEX =
  37                 '/&([A-Za-z0-9\x80-\xff]+);
  38                  |&\#([0-9]+);
  39                  |&\#[xX]([0-9A-Fa-f]+);
  40                  |(&)/x';
  41
  42         /**
  43          * Blacklist for evil uris like javascript:
  44          * WARNING: DO NOT use this in any place that actually requires blacklisting
  45          * for security reasons. There are NUMEROUS[1] ways to bypass blacklisting, the
  46          * only way to be secure from javascript: uri based xss vectors is to whitelist
  47          * things that you know are safe and deny everything else.
  48          * [1]: http://ha.ckers.org/xss.html
  49          */
  50         const EVIL_URI_PATTERN = '!(^|\s|\*/\s*)(javascript|vbscript)([^\w]|$)!i';
  51         const XMLNS_ATTRIBUTE_PATTERN = "/^xmlns:[:A-Z_a-z-.0-9]+$/";
  52
  53         /**
  54          * List of all named character entities defined in HTML 4.01
  55          * http://www.w3.org/TR/html4/sgml/entities.html
  56          * As well as &apos; which is only defined starting in XHTML1.
  57          * @private
  58          */
  59         static $htmlEntities = array(
  60                 'Aacute'   => 193,
  61                 'aacute'   => 225,
  62                 'Acirc'    => 194,
  63                 'acirc'    => 226,
  64                 'acute'    => 180,
  65                 'AElig'    => 198,
  66                 'aelig'    => 230,
  67                 'Agrave'   => 192,
  68                 'agrave'   => 224,
  69                 'alefsym'  => 8501,
  70                 'Alpha'    => 913,
  71                 'alpha'    => 945,
  72                 'amp'      => 38,
  73                 'and'      => 8743,
  74                 'ang'      => 8736,
  75                 'apos'     => 39, // New in XHTML & HTML 5; avoid in output for compatibility with IE.
  76                 'Aring'    => 197,
  77                 'aring'    => 229,
  78                 'asymp'    => 8776,
  79                 'Atilde'   => 195,
  80                 'atilde'   => 227,
  81                 'Auml'     => 196,
  82                 'auml'     => 228,
  83                 'bdquo'    => 8222,
  84                 'Beta'     => 914,
  85                 'beta'     => 946,
  86                 'brvbar'   => 166,
  87                 'bull'     => 8226,
  88                 'cap'      => 8745,
  89                 'Ccedil'   => 199,
  90                 'ccedil'   => 231,
  91                 'cedil'    => 184,
  92                 'cent'     => 162,
  93                 'Chi'      => 935,
  94                 'chi'      => 967,
  95                 'circ'     => 710,
  96                 'clubs'    => 9827,
  97                 'cong'     => 8773,
  98                 'copy'     => 169,
  99                 'crarr'    => 8629,
 100                 'cup'      => 8746,
 101                 'curren'   => 164,
 102                 'dagger'   => 8224,
 103                 'Dagger'   => 8225,
 104                 'darr'     => 8595,
 105                 'dArr'     => 8659,
 106                 'deg'      => 176,
 107                 'Delta'    => 916,
 108                 'delta'    => 948,
 109                 'diams'    => 9830,
 110                 'divide'   => 247,
 111                 'Eacute'   => 201,
 112                 'eacute'   => 233,
 113                 'Ecirc'    => 202,
 114                 'ecirc'    => 234,
 115                 'Egrave'   => 200,
 116                 'egrave'   => 232,
 117                 'empty'    => 8709,
 118                 'emsp'     => 8195,
 119                 'ensp'     => 8194,
 120                 'Epsilon'  => 917,
 121                 'epsilon'  => 949,
 122                 'equiv'    => 8801,
 123                 'Eta'      => 919,
 124                 'eta'      => 951,
 125                 'ETH'      => 208,
 126                 'eth'      => 240,
 127                 'Euml'     => 203,
 128                 'euml'     => 235,
 129                 'euro'     => 8364,
 130                 'exist'    => 8707,
 131                 'fnof'     => 402,
 132                 'forall'   => 8704,
 133                 'frac12'   => 189,
 134                 'frac14'   => 188,
 135                 'frac34'   => 190,
 136                 'frasl'    => 8260,
 137                 'Gamma'    => 915,
 138                 'gamma'    => 947,
 139                 'ge'       => 8805,
 140                 'gt'       => 62,
 141                 'harr'     => 8596,
 142                 'hArr'     => 8660,
 143                 'hearts'   => 9829,
 144                 'hellip'   => 8230,
 145                 'Iacute'   => 205,
 146                 'iacute'   => 237,
 147                 'Icirc'    => 206,
 148                 'icirc'    => 238,
 149                 'iexcl'    => 161,
 150                 'Igrave'   => 204,
 151                 'igrave'   => 236,
 152                 'image'    => 8465,
 153                 'infin'    => 8734,
 154                 'int'      => 8747,
 155                 'Iota'     => 921,
 156                 'iota'     => 953,
 157                 'iquest'   => 191,
 158                 'isin'     => 8712,
 159                 'Iuml'     => 207,
 160                 'iuml'     => 239,
 161                 'Kappa'    => 922,
 162                 'kappa'    => 954,
 163                 'Lambda'   => 923,
 164                 'lambda'   => 955,
 165                 'lang'     => 9001,
 166                 'laquo'    => 171,
 167                 'larr'     => 8592,
 168                 'lArr'     => 8656,
 169                 'lceil'    => 8968,
 170                 'ldquo'    => 8220,
 171                 'le'       => 8804,
 172                 'lfloor'   => 8970,
 173                 'lowast'   => 8727,
 174                 'loz'      => 9674,
 175                 'lrm'      => 8206,
 176                 'lsaquo'   => 8249,
 177                 'lsquo'    => 8216,
 178                 'lt'       => 60,
 179                 'macr'     => 175,
 180                 'mdash'    => 8212,
 181                 'micro'    => 181,
 182                 'middot'   => 183,
 183                 'minus'    => 8722,
 184                 'Mu'       => 924,
 185                 'mu'       => 956,
 186                 'nabla'    => 8711,
 187                 'nbsp'     => 160,
 188                 'ndash'    => 8211,
 189                 'ne'       => 8800,
 190                 'ni'       => 8715,
 191                 'not'      => 172,
 192                 'notin'    => 8713,
 193                 'nsub'     => 8836,
 194                 'Ntilde'   => 209,
 195                 'ntilde'   => 241,
 196                 'Nu'       => 925,
 197                 'nu'       => 957,
 198                 'Oacute'   => 211,
 199                 'oacute'   => 243,
 200                 'Ocirc'    => 212,
 201                 'ocirc'    => 244,
 202                 'OElig'    => 338,
 203                 'oelig'    => 339,
 204                 'Ograve'   => 210,
 205                 'ograve'   => 242,
 206                 'oline'    => 8254,
 207                 'Omega'    => 937,
 208                 'omega'    => 969,
 209                 'Omicron'  => 927,
 210                 'omicron'  => 959,
 211                 'oplus'    => 8853,
 212                 'or'       => 8744,
 213                 'ordf'     => 170,
 214                 'ordm'     => 186,
 215                 'Oslash'   => 216,
 216                 'oslash'   => 248,
 217                 'Otilde'   => 213,
 218                 'otilde'   => 245,
 219                 'otimes'   => 8855,
 220                 'Ouml'     => 214,
 221                 'ouml'     => 246,
 222                 'para'     => 182,
 223                 'part'     => 8706,
 224                 'permil'   => 8240,
 225                 'perp'     => 8869,
 226                 'Phi'      => 934,
 227                 'phi'      => 966,
 228                 'Pi'       => 928,
 229                 'pi'       => 960,
 230                 'piv'      => 982,
 231                 'plusmn'   => 177,
 232                 'pound'    => 163,
 233                 'prime'    => 8242,
 234                 'Prime'    => 8243,
 235                 'prod'     => 8719,
 236                 'prop'     => 8733,
 237                 'Psi'      => 936,
 238                 'psi'      => 968,
 239                 'quot'     => 34,
 240                 'radic'    => 8730,
 241                 'rang'     => 9002,
 242                 'raquo'    => 187,
 243                 'rarr'     => 8594,
 244                 'rArr'     => 8658,
 245                 'rceil'    => 8969,
 246                 'rdquo'    => 8221,
 247                 'real'     => 8476,
 248                 'reg'      => 174,
 249                 'rfloor'   => 8971,
 250                 'Rho'      => 929,
 251                 'rho'      => 961,
 252                 'rlm'      => 8207,
 253                 'rsaquo'   => 8250,
 254                 'rsquo'    => 8217,
 255                 'sbquo'    => 8218,
 256                 'Scaron'   => 352,
 257                 'scaron'   => 353,
 258                 'sdot'     => 8901,
 259                 'sect'     => 167,
 260                 'shy'      => 173,
 261                 'Sigma'    => 931,
 262                 'sigma'    => 963,
 263                 'sigmaf'   => 962,
 264                 'sim'      => 8764,
 265                 'spades'   => 9824,
 266                 'sub'      => 8834,
 267                 'sube'     => 8838,
 268                 'sum'      => 8721,
 269                 'sup'      => 8835,
 270                 'sup1'     => 185,
 271                 'sup2'     => 178,
 272                 'sup3'     => 179,
 273                 'supe'     => 8839,
 274                 'szlig'    => 223,
 275                 'Tau'      => 932,
 276                 'tau'      => 964,
 277                 'there4'   => 8756,
 278                 'Theta'    => 920,
 279                 'theta'    => 952,
 280                 'thetasym' => 977,
 281                 'thinsp'   => 8201,
 282                 'THORN'    => 222,
 283                 'thorn'    => 254,
 284                 'tilde'    => 732,
 285                 'times'    => 215,
 286                 'trade'    => 8482,
 287                 'Uacute'   => 218,
 288                 'uacute'   => 250,
 289                 'uarr'     => 8593,
 290                 'uArr'     => 8657,
 291                 'Ucirc'    => 219,
 292                 'ucirc'    => 251,
 293                 'Ugrave'   => 217,
 294                 'ugrave'   => 249,
 295                 'uml'      => 168,
 296                 'upsih'    => 978,
 297                 'Upsilon'  => 933,
 298                 'upsilon'  => 965,
 299                 'Uuml'     => 220,
 300                 'uuml'     => 252,
 301                 'weierp'   => 8472,
 302                 'Xi'       => 926,
 303                 'xi'       => 958,
 304                 'Yacute'   => 221,
 305                 'yacute'   => 253,
 306                 'yen'      => 165,
 307                 'Yuml'     => 376,
 308                 'yuml'     => 255,
 309                 'Zeta'     => 918,
 310                 'zeta'     => 950,
 311                 'zwj'      => 8205,
 312                 'zwnj'     => 8204
 313         );
 314
 315         /**
 316          * Character entity aliases accepted by MediaWiki
 317          */
 318         static $htmlEntityAliases = array(
 319                 'רלמ' => 'rlm',
 320                 'رلم' => 'rlm',
 321         );
 322
 323         /**
 324          * Lazy-initialised attributes regex, see getAttribsRegex()
 325          */
 326         static $attribsRegex;
 327
 328         /**
 329          * Regular expression to match HTML/XML attribute pairs within a tag.
 330          * Allows some... latitude.
 331          * Used in Sanitizer::fixTagAttributes and Sanitizer::decodeTagAttributes
 332          */
 333         static function getAttribsRegex() {
 334                 if ( self::$attribsRegex === null ) {
 335                         $attribFirst = '[:A-Z_a-z0-9]';
 336                         $attrib = '[:A-Z_a-z-.0-9]';
 337                         $space = '[\x09\x0a\x0d\x20]';
 338                         self::$attribsRegex =
 339                                 "/(?:^|$space)({$attribFirst}{$attrib}*)
 340                                   ($space*=$space*
 341                                         (?:
 342                                          # The attribute value: quoted or alone
 343                                           \"([^<\"]*)\"
 344                                          | '([^<']*)'
 345                                          |  ([a-zA-Z0-9!#$%&()*,\\-.\\/:;<>?@[\\]^_`{|}~]+)
 346                                          |  (\#[0-9a-fA-F]+) # Technically wrong, but lots of
 347                                                                                  # colors are specified like this.
 348                                                                                  # We'll be normalizing it.
 349                                         )
 350                                 )?(?=$space|\$)/sx";
 351                 }
 352                 return self::$attribsRegex;
 353         }
 354
 355         /**
 356          * Cleans up HTML, removes dangerous tags and attributes, and
 357          * removes HTML comments
 358          * @private
 359          * @param $text String
 360          * @param $processCallback Callback to do any variable or parameter replacements in HTML attribute values
 361          * @param array $args for the processing callback
 362          * @param array $extratags for any extra tags to include
 363          * @param array $removetags for any tags (default or extra) to exclude
 364          * @return string
 365          */
 366         static function removeHTMLtags( $text, $processCallback = null, $args = array(), $extratags = array(), $removetags = array() ) {
 367                 global $wgUseTidy, $wgAllowMicrodataAttributes, $wgAllowImageTag;
 368
 369                 static $htmlpairsStatic, $htmlsingle, $htmlsingleonly, $htmlnest, $tabletags,
 370                         $htmllist, $listtags, $htmlsingleallowed, $htmlelementsStatic, $staticInitialised;
 371
 372                 wfProfileIn( __METHOD__ );
 373
 374                 // Base our staticInitialised variable off of the global config state so that if the globals
 375                 // are changed (like in the screwed up test system) we will re-initialise the settings.
 376                 $globalContext = implode( '-', compact( 'wgAllowMicrodataAttributes', 'wgAllowImageTag' ) );
 377                 if ( !$staticInitialised || $staticInitialised != $globalContext ) {
 378
 379                         $htmlpairsStatic = array( # Tags that must be closed
 380                                 'b', 'bdi', 'del', 'i', 'ins', 'u', 'font', 'big', 'small', 'sub', 'sup', 'h1',
 381                                 'h2', 'h3', 'h4', 'h5', 'h6', 'cite', 'code', 'em', 's',
 382                                 'strike', 'strong', 'tt', 'var', 'div', 'center',
 383                                 'blockquote', 'ol', 'ul', 'dl', 'table', 'caption', 'pre',
 384                                 'ruby', 'rt', 'rb', 'rp', 'p', 'span', 'abbr', 'dfn',
 385                                 'kbd', 'samp', 'data', 'time', 'mark'
 386                         );
 387                         $htmlsingle = array(
 388                                 'br', 'hr', 'li', 'dt', 'dd'
 389                         );
 390                         $htmlsingleonly = array( # Elements that cannot have close tags
 391                                 'br', 'hr'
 392                         );
 393                         if ( $wgAllowMicrodataAttributes ) {
 394                                 $htmlsingle[] = $htmlsingleonly[] = 'meta';
 395                                 $htmlsingle[] = $htmlsingleonly[] = 'link';
 396                         }
 397                         $htmlnest = array( # Tags that can be nested--??
 398                                 'table', 'tr', 'td', 'th', 'div', 'blockquote', 'ol', 'ul',
 399                                 'li', 'dl', 'dt', 'dd', 'font', 'big', 'small', 'sub', 'sup', 'span',
 400                                 'var', 'kbd', 'samp'
 401                         );
 402                         $tabletags = array( # Can only appear inside table, we will close them
 403                                 'td', 'th', 'tr',
 404                         );
 405                         $htmllist = array( # Tags used by list
 406                                 'ul', 'ol',
 407                         );
 408                         $listtags = array( # Tags that can appear in a list
 409                                 'li',
 410                         );
 411
 412                         if ( $wgAllowImageTag ) {
 413                                 $htmlsingle[] = 'img';
 414                                 $htmlsingleonly[] = 'img';
 415                         }
 416
 417                         $htmlsingleallowed = array_unique( array_merge( $htmlsingle, $tabletags ) );
 418                         $htmlelementsStatic = array_unique( array_merge( $htmlsingle, $htmlpairsStatic, $htmlnest ) );
 419
 420                         # Convert them all to hashtables for faster lookup
 421                         $vars = array( 'htmlpairsStatic', 'htmlsingle', 'htmlsingleonly', 'htmlnest', 'tabletags',
 422                                 'htmllist', 'listtags', 'htmlsingleallowed', 'htmlelementsStatic' );
 423                         foreach ( $vars as $var ) {
 424                                 $$var = array_flip( $$var );
 425                         }
 426                         $staticInitialised = $globalContext;
 427                 }
 428                 # Populate $htmlpairs and $htmlelements with the $extratags and $removetags arrays
 429                 $extratags = array_flip( $extratags );
 430                 $removetags = array_flip( $removetags );
 431                 $htmlpairs = array_merge( $extratags, $htmlpairsStatic );
 432                 $htmlelements = array_diff_key( array_merge( $extratags, $htmlelementsStatic ), $removetags );
 433
 434                 # Remove HTML comments
 435                 $text = Sanitizer::removeHTMLcomments( $text );
 436                 $bits = explode( '<', $text );
 437                 $text = str_replace( '>', '&gt;', array_shift( $bits ) );
 438                 if ( !$wgUseTidy ) {
 439                         $tagstack = $tablestack = array();
 440                         foreach ( $bits as $x ) {
 441                                 $regs = array();
 442                                 # $slash: Does the current element start with a '/'?
 443                                 # $t: Current element name
 444                                 # $params: String between element name and >
 445                                 # $brace: Ending '>' or '/>'
 446                                 # $rest: Everything until the next element of $bits
 447                                 if ( preg_match( '!^(/?)(\\w+)([^>]*?)(/{0,1}>)([^<]*)$!', $x, $regs ) ) {
 448                                         list( /* $qbar */, $slash, $t, $params, $brace, $rest ) = $regs;
 449                                 } else {
 450                                         $slash = $t = $params = $brace = $rest = null;
 451                                 }
 452
 453                                 $badtag = false;
 454                                 if ( isset( $htmlelements[$t = strtolower( $t )] ) ) {
 455                                         # Check our stack
 456                                         if ( $slash && isset( $htmlsingleonly[$t] ) ) {
 457                                                 $badtag = true;
 458                                         } elseif ( $slash ) {
 459                                                 # Closing a tag... is it the one we just opened?
 460                                                 $ot = @array_pop( $tagstack );
 461                                                 if ( $ot != $t ) {
 462                                                         if ( isset( $htmlsingleallowed[$ot] ) ) {
 463                                                                 # Pop all elements with an optional close tag
 464                                                                 # and see if we find a match below them
 465                                                                 $optstack = array();
 466                                                                 array_push( $optstack, $ot );
 467                                                                 wfSuppressWarnings();
 468                                                                 $ot = array_pop( $tagstack );
 469                                                                 wfRestoreWarnings();
 470                                                                 while ( $ot != $t && isset( $htmlsingleallowed[$ot] ) ) {
 471                                                                         array_push( $optstack, $ot );
 472                                                                         wfSuppressWarnings();
 473                                                                         $ot = array_pop( $tagstack );
 474                                                                         wfRestoreWarnings();
 475                                                                 }
 476                                                                 if ( $t != $ot ) {
 477                                                                         # No match. Push the optional elements back again
 478                                                                         $badtag = true;
 479                                                                         wfSuppressWarnings();
 480                                                                         $ot = array_pop( $optstack );
 481                                                                         wfRestoreWarnings();
 482                                                                         while ( $ot ) {
 483                                                                                 array_push( $tagstack, $ot );
 484                                                                                 wfSuppressWarnings();
 485                                                                                 $ot = array_pop( $optstack );
 486                                                                                 wfRestoreWarnings();
 487                                                                         }
 488                                                                 }
 489                                                         } else {
 490                                                                 @array_push( $tagstack, $ot );
 491                                                                 # <li> can be nested in <ul> or <ol>, skip those cases:
 492                                                                 if ( !isset( $htmllist[$ot] ) || !isset( $listtags[$t] ) ) {
 493                                                                         $badtag = true;
 494                                                                 }
 495                                                         }
 496                                                 } else {
 497                                                         if ( $t == 'table' ) {
 498                                                                 $tagstack = array_pop( $tablestack );
 499                                                         }
 500                                                 }
 501                                                 $newparams = '';
 502                                         } else {
 503                                                 # Keep track for later
 504                                                 if ( isset( $tabletags[$t] ) &&
 505                                                 !in_array( 'table', $tagstack ) ) {
 506                                                         $badtag = true;
 507                                                 } elseif ( in_array( $t, $tagstack ) &&
 508                                                 !isset( $htmlnest[$t] ) ) {
 509                                                         $badtag = true;
 510                                                 # Is it a self closed htmlpair ? (bug 5487)
 511                                                 } elseif ( $brace == '/>' &&
 512                                                 isset( $htmlpairs[$t] ) ) {
 513                                                         $badtag = true;
 514                                                 } elseif ( isset( $htmlsingleonly[$t] ) ) {
 515                                                         # Hack to force empty tag for unclosable elements
 516                                                         $brace = '/>';
 517                                                 } elseif ( isset( $htmlsingle[$t] ) ) {
 518                                                         # Hack to not close $htmlsingle tags
 519                                                         $brace = null;
 520                                                         # Still need to push this optionally-closed tag to
 521                                                         # the tag stack so that we can match end tags
 522                                                         # instead of marking them as bad.
 523                                                         array_push( $tagstack, $t );
 524                                                 } elseif ( isset( $tabletags[$t] )
 525                                                 && in_array( $t, $tagstack ) ) {
 526                                                         // New table tag but forgot to close the previous one
 527                                                         $text .= "</$t>";
 528                                                 } else {
 529                                                         if ( $t == 'table' ) {
 530                                                                 array_push( $tablestack, $tagstack );
 531                                                                 $tagstack = array();
 532                                                         }
 533                                                         array_push( $tagstack, $t );
 534                                                 }
 535
 536                                                 # Replace any variables or template parameters with
 537                                                 # plaintext results.
 538                                                 if ( is_callable( $processCallback ) ) {
 539                                                         call_user_func_array( $processCallback, array( &$params, $args ) );
 540                                                 }
 541
 542                                                 if ( !Sanitizer::validateTag( $params, $t ) ) {
 543                                                         $badtag = true;
 544                                                 }
 545
 546                                                 # Strip non-approved attributes from the tag
 547                                                 $newparams = Sanitizer::fixTagAttributes( $params, $t );
 548                                         }
 549                                         if ( !$badtag ) {
 550                                                 $rest = str_replace( '>', '&gt;', $rest );
 551                                                 $close = ( $brace == '/>' && !$slash ) ? ' /' : '';
 552                                                 $text .= "<$slash$t$newparams$close>$rest";
 553                                                 continue;
 554                                         }
 555                                 }
 556                                 $text .= '&lt;' . str_replace( '>', '&gt;', $x );
 557                         }
 558                         # Close off any remaining tags
 559                         while ( is_array( $tagstack ) && ( $t = array_pop( $tagstack ) ) ) {
 560                                 $text .= "</$t>\n";
 561                                 if ( $t == 'table' ) {
 562                                         $tagstack = array_pop( $tablestack );
 563                                 }
 564                         }
 565                 } else {
 566                         # this might be possible using tidy itself
 567                         foreach ( $bits as $x ) {
 568                                 preg_match( '/^(\\/?)(\\w+)([^>]*?)(\\/{0,1}>)([^<]*)$/',
 569                                 $x, $regs );
 570                                 @list( /* $qbar */, $slash, $t, $params, $brace, $rest ) = $regs;
 571                                 $badtag = false;
 572                                 if ( isset( $htmlelements[$t = strtolower( $t )] ) ) {
 573                                         if ( is_callable( $processCallback ) ) {
 574                                                 call_user_func_array( $processCallback, array( &$params, $args ) );
 575                                         }
 576
 577                                         if ( !Sanitizer::validateTag( $params, $t ) ) {
 578                                                 $badtag = true;
 579                                         }
 580
 581                                         $newparams = Sanitizer::fixTagAttributes( $params, $t );
 582                                         if ( !$badtag ) {
 583                                                 $rest = str_replace( '>', '&gt;', $rest );
 584                                                 $text .= "<$slash$t$newparams$brace$rest";
 585                                                 continue;
 586                                         }
 587                                 }
 588                                 $text .= '&lt;' . str_replace( '>', '&gt;', $x );
 589                         }
 590                 }
 591                 wfProfileOut( __METHOD__ );
 592                 return $text;
 593         }
 594
 595         /**
 596          * Remove '<!--', '-->', and everything between.
 597          * To avoid leaving blank lines, when a comment is both preceded
 598          * and followed by a newline (ignoring spaces), trim leading and
 599          * trailing spaces and one of the newlines.
 600          *
 601          * @private
 602          * @param $text String
 603          * @return string
 604          */
 605         static function removeHTMLcomments( $text ) {
 606                 wfProfileIn( __METHOD__ );
 607                 while ( ( $start = strpos( $text, '<!--' ) ) !== false ) {
 608                         $end = strpos( $text, '-->', $start + 4 );
 609                         if ( $end === false ) {
 610                                 # Unterminated comment; bail out
 611                                 break;
 612                         }
 613
 614                         $end += 3;
 615
 616                         # Trim space and newline if the comment is both
 617                         # preceded and followed by a newline
 618                         $spaceStart = max( $start - 1, 0 );
 619                         $spaceLen = $end - $spaceStart;
 620                         while ( substr( $text, $spaceStart, 1 ) === ' ' && $spaceStart > 0 ) {
 621                                 $spaceStart--;
 622                                 $spaceLen++;
 623                         }
 624                         while ( substr( $text, $spaceStart + $spaceLen, 1 ) === ' ' ) {
 625                                 $spaceLen++;
 626                         }
 627                         if ( substr( $text, $spaceStart, 1 ) === "\n" and substr( $text, $spaceStart + $spaceLen, 1 ) === "\n" ) {
 628                                 # Remove the comment, leading and trailing
 629                                 # spaces, and leave only one newline.
 630                                 $text = substr_replace( $text, "\n", $spaceStart, $spaceLen + 1 );
 631                         }
 632                         else {
 633                                 # Remove just the comment.
 634                                 $text = substr_replace( $text, '', $start, $end - $start );
 635                         }
 636                 }
 637                 wfProfileOut( __METHOD__ );
 638                 return $text;
 639         }
 640
 641         /**
 642          * Takes attribute names and values for a tag and the tag name and
 643          * validates that the tag is allowed to be present.
 644          * This DOES NOT validate the attributes, nor does it validate the
 645          * tags themselves. This method only handles the special circumstances
 646          * where we may want to allow a tag within content but ONLY when it has
 647          * specific attributes set.
 648          *
 649          * @param $params
 650          * @param $element
 651          * @return bool
 652          */
 653         static function validateTag( $params, $element ) {
 654                 $params = Sanitizer::decodeTagAttributes( $params );
 655
 656                 if ( $element == 'meta' || $element == 'link' ) {
 657                         if ( !isset( $params['itemprop'] ) ) {
 658                                 // <meta> and <link> must have an itemprop="" otherwise they are not valid or safe in content
 659                                 return false;
 660                         }
 661                         if ( $element == 'meta' && !isset( $params['content'] ) ) {
 662                                 // <meta> must have a content="" for the itemprop
 663                                 return false;
 664                         }
 665                         if ( $element == 'link' && !isset( $params['href'] ) ) {
 666                                 // <link> must have an associated href=""
 667                                 return false;
 668                         }
 669                 }
 670
 671                 return true;
 672         }
 673
 674         /**
 675          * Take an array of attribute names and values and normalize or discard
 676          * illegal values for the given element type.
 677          *
 678          * - Discards attributes not on a whitelist for the given element
 679          * - Unsafe style attributes are discarded
 680          * - Invalid id attributes are re-encoded
 681          *
 682          * @param $attribs Array
 683          * @param $element String
 684          * @return Array
 685          *
 686          * @todo Check for legal values where the DTD limits things.
 687          * @todo Check for unique id attribute :P
 688          */
 689         static function validateTagAttributes( $attribs, $element ) {
 690                 return Sanitizer::validateAttributes( $attribs,
 691                         Sanitizer::attributeWhitelist( $element ) );
 692         }
 693
 694         /**
 695          * Take an array of attribute names and values and normalize or discard
 696          * illegal values for the given whitelist.
 697          *
 698          * - Discards attributes not the given whitelist
 699          * - Unsafe style attributes are discarded
 700          * - Invalid id attributes are re-encoded
 701          *
 702          * @param $attribs Array
 703          * @param array $whitelist list of allowed attribute names
 704          * @return Array
 705          *
 706          * @todo Check for legal values where the DTD limits things.
 707          * @todo Check for unique id attribute :P
 708          */
 709         static function validateAttributes( $attribs, $whitelist ) {
 710                 global $wgAllowRdfaAttributes, $wgAllowMicrodataAttributes;
 711
 712                 $whitelist = array_flip( $whitelist );
 713                 $hrefExp = '/^(' . wfUrlProtocols() . ')[^\s]+$/';
 714
 715                 $out = array();
 716                 foreach ( $attribs as $attribute => $value ) {
 717                         #allow XML namespace declaration if RDFa is enabled
 718                         if ( $wgAllowRdfaAttributes && preg_match( self::XMLNS_ATTRIBUTE_PATTERN, $attribute ) ) {
 719                                 if ( !preg_match( self::EVIL_URI_PATTERN, $value ) ) {
 720                                         $out[$attribute] = $value;
 721                                 }
 722
 723                                 continue;
 724                         }
 725
 726                         # Allow any attribute beginning with "data-"
 727                         if ( !preg_match( '/^data-/i', $attribute ) && !isset( $whitelist[$attribute] ) ) {
 728                                 continue;
 729                         }
 730
 731                         # Strip javascript "expression" from stylesheets.
 732                         # http://msdn.microsoft.com/workshop/author/dhtml/overview/recalc.asp
 733                         if ( $attribute == 'style' ) {
 734                                 $value = Sanitizer::checkCss( $value );
 735                         }
 736
 737                         if ( $attribute === 'id' ) {
 738                                 $value = Sanitizer::escapeId( $value, 'noninitial' );
 739                         }
 740
 741                         # WAI-ARIA
 742                         # http://www.w3.org/TR/wai-aria/
 743                         # http://www.whatwg.org/html/elements.html#wai-aria
 744                         # For now we only support role="presentation" until we work out what roles should be
 745                         # usable by content and we ensure that our code explicitly rejects patterns that
 746                         # violate HTML5's ARIA restrictions.
 747                         if ( $attribute === 'role' && $value !== 'presentation' ) {
 748                                 continue;
 749                         }
 750
 751                         //RDFa and microdata properties allow URLs, URIs and/or CURIs. check them for sanity
 752                         if ( $attribute === 'rel' || $attribute === 'rev' ||
 753                                 $attribute === 'about' || $attribute === 'property' || $attribute === 'resource' || #RDFa
 754                                 $attribute === 'datatype' || $attribute === 'typeof' ||                             #RDFa
 755                                 $attribute === 'itemid' || $attribute === 'itemprop' || $attribute === 'itemref' || #HTML5 microdata
 756                                 $attribute === 'itemscope' || $attribute === 'itemtype' ) {                         #HTML5 microdata
 757
 758                                 //Paranoia. Allow "simple" values but suppress javascript
 759                                 if ( preg_match( self::EVIL_URI_PATTERN, $value ) ) {
 760                                         continue;
 761                                 }
 762                         }
 763
 764                         # NOTE: even though elements using href/src are not allowed directly, supply
 765                         #       validation code that can be used by tag hook handlers, etc
 766                         if ( $attribute === 'href' || $attribute === 'src' ) {
 767                                 if ( !preg_match( $hrefExp, $value ) ) {
 768                                         continue; //drop any href or src attributes not using an allowed protocol.
 769                                                   //NOTE: this also drops all relative URLs
 770                                 }
 771                         }
 772
 773                         // If this attribute was previously set, override it.
 774                         // Output should only have one attribute of each name.
 775                         $out[$attribute] = $value;
 776                 }
 777
 778                 if ( $wgAllowMicrodataAttributes ) {
 779                         # itemtype, itemid, itemref don't make sense without itemscope
 780                         if ( !array_key_exists( 'itemscope', $out ) ) {
 781                                 unset( $out['itemtype'] );
 782                                 unset( $out['itemid'] );
 783                                 unset( $out['itemref'] );
 784                         }
 785                         # TODO: Strip itemprop if we aren't descendants of an itemscope or pointed to by an itemref.
 786                 }
 787                 return $out;
 788         }
 789
 790         /**
 791          * Merge two sets of HTML attributes.  Conflicting items in the second set
 792          * will override those in the first, except for 'class' attributes which
 793          * will be combined (if they're both strings).
 794          *
 795          * @todo implement merging for other attributes such as style
 796          * @param $a Array
 797          * @param $b Array
 798          * @return array
 799          */
 800         static function mergeAttributes( $a, $b ) {
 801                 $out = array_merge( $a, $b );
 802                 if ( isset( $a['class'] ) && isset( $b['class'] )
 803                         && is_string( $a['class'] ) && is_string( $b['class'] )
 804                         && $a['class'] !== $b['class']
 805                 ) {
 806                         $classes = preg_split( '/\s+/', "{$a['class']} {$b['class']}",
 807                                 -1, PREG_SPLIT_NO_EMPTY );
 808                         $out['class'] = implode( ' ', array_unique( $classes ) );
 809                 }
 810                 return $out;
 811         }
 812
 813         /**
 814          * Pick apart some CSS and check it for forbidden or unsafe structures.
 815          * Returns a sanitized string. This sanitized string will have
 816          * character references and escape sequences decoded and comments
 817          * stripped (unless it is itself one valid comment, in which case the value
 818          * will be passed through). If the input is just too evil, only a comment
 819          * complaining about evilness will be returned.
 820          *
 821          * Currently URL references, 'expression', 'tps' are forbidden.
 822          *
 823          * NOTE: Despite the fact that character references are decoded, the
 824          * returned string may contain character references given certain
 825          * clever input strings. These character references must
 826          * be escaped before the return value is embedded in HTML.
 827          *
 828          * @param $value String
 829          * @return String
 830          */
 831         static function checkCss( $value ) {
 832                 // Decode character references like &#123;
 833                 $value = Sanitizer::decodeCharReferences( $value );
 834
 835                 // Decode escape sequences and line continuation
 836                 // See the grammar in the CSS 2 spec, appendix D.
 837                 // This has to be done AFTER decoding character references.
 838                 // This means it isn't possible for this function to return
 839                 // unsanitized escape sequences. It is possible to manufacture
 840                 // input that contains character references that decode to
 841                 // escape sequences that decode to character references, but
 842                 // it's OK for the return value to contain character references
 843                 // because the caller is supposed to escape those anyway.
 844                 static $decodeRegex;
 845                 if ( !$decodeRegex ) {
 846                         $space = '[\\x20\\t\\r\\n\\f]';
 847                         $nl = '(?:\\n|\\r\\n|\\r|\\f)';
 848                         $backslash = '\\\\';
 849                         $decodeRegex = "/ $backslash
 850                                 (?:
 851                                         ($nl) |  # 1. Line continuation
 852                                         ([0-9A-Fa-f]{1,6})$space? |  # 2. character number
 853                                         (.) | # 3. backslash cancelling special meaning
 854                                         () | # 4. backslash at end of string
 855                                 )/xu";
 856                 }
 857                 $value = preg_replace_callback( $decodeRegex,
 858                         array( __CLASS__, 'cssDecodeCallback' ), $value );
 859
 860                 // Let the value through if it's nothing but a single comment, to
 861                 // allow other functions which may reject it to pass some error
 862                 // message through.
 863                 if ( !preg_match( '! ^ \s* /\* [^*\\/]* \*/ \s* $ !x', $value ) ) {
 864                         // Remove any comments; IE gets token splitting wrong
 865                         // This must be done AFTER decoding character references and
 866                         // escape sequences, because those steps can introduce comments
 867                         // This step cannot introduce character references or escape
 868                         // sequences, because it replaces comments with spaces rather
 869                         // than removing them completely.
 870                         $value = StringUtils::delimiterReplace( '/*', '*/', ' ', $value );
 871
 872                         // Remove anything after a comment-start token, to guard against
 873                         // incorrect client implementations.
 874                         $commentPos = strpos( $value, '/*' );
 875                         if ( $commentPos !== false ) {
 876                                 $value = substr( $value, 0, $commentPos );
 877                         }
 878                 }
 879
 880                 // Reject problematic keywords and control characters
 881                 if ( preg_match( '/[\000-\010\016-\037\177]/', $value ) ) {
 882                         return '/* invalid control char */';
 883                 } elseif ( preg_match( '! expression | filter\s*: | accelerator\s*: | url\s*\( | image\s*\( | image-set\s*\( !ix', $value ) ) {
 884                         return '/* insecure input */';
 885                 }
 886                 return $value;
 887         }
 888
 889         /**
 890          * @param $matches array
 891          * @return String
 892          */
 893         static function cssDecodeCallback( $matches ) {
 894                 if ( $matches[1] !== '' ) {
 895                         // Line continuation
 896                         return '';
 897                 } elseif ( $matches[2] !== '' ) {
 898                         $char = codepointToUtf8( hexdec( $matches[2] ) );
 899                 } elseif ( $matches[3] !== '' ) {
 900                         $char = $matches[3];
 901                 } else {
 902                         $char = '\\';
 903                 }
 904                 if ( $char == "\n" || $char == '"' || $char == "'" || $char == '\\' ) {
 905                         // These characters need to be escaped in strings
 906                         // Clean up the escape sequence to avoid parsing errors by clients
 907                         return '\\' . dechex( ord( $char ) ) . ' ';
 908                 } else {
 909                         // Decode unnecessary escape
 910                         return $char;
 911                 }
 912         }
 913
 914         /**
 915          * Take a tag soup fragment listing an HTML element's attributes
 916          * and normalize it to well-formed XML, discarding unwanted attributes.
 917          * Output is safe for further wikitext processing, with escaping of
 918          * values that could trigger problems.
 919          *
 920          * - Normalizes attribute names to lowercase
 921          * - Discards attributes not on a whitelist for the given element
 922          * - Turns broken or invalid entities into plaintext
 923          * - Double-quotes all attribute values
 924          * - Attributes without values are given the name as attribute
 925          * - Double attributes are discarded
 926          * - Unsafe style attributes are discarded
 927          * - Prepends space if there are attributes.
 928          *
 929          * @param $text String
 930          * @param $element String
 931          * @return String
 932          */
 933         static function fixTagAttributes( $text, $element ) {
 934                 if ( trim( $text ) == '' ) {
 935                         return '';
 936                 }
 937
 938                 $decoded = Sanitizer::decodeTagAttributes( $text );
 939                 $stripped = Sanitizer::validateTagAttributes( $decoded, $element );
 940
 941                 return Sanitizer::safeEncodeTagAttributes( $stripped );
 942         }
 943
 944         /**
 945          * Encode an attribute value for HTML output.
 946          * @param $text String
 947          * @return HTML-encoded text fragment
 948          */
 949         static function encodeAttribute( $text ) {
 950                 $encValue = htmlspecialchars( $text, ENT_QUOTES );
 951
 952                 // Whitespace is normalized during attribute decoding,
 953                 // so if we've been passed non-spaces we must encode them
 954                 // ahead of time or they won't be preserved.
 955                 $encValue = strtr( $encValue, array(
 956                         "\n" => '&#10;',
 957                         "\r" => '&#13;',
 958                         "\t" => '&#9;',
 959                 ) );
 960
 961                 return $encValue;
 962         }
 963
 964         /**
 965          * Encode an attribute value for HTML tags, with extra armoring
 966          * against further wiki processing.
 967          * @param $text String
 968          * @return HTML-encoded text fragment
 969          */
 970         static function safeEncodeAttribute( $text ) {
 971                 $encValue = Sanitizer::encodeAttribute( $text );
 972
 973                 # Templates and links may be expanded in later parsing,
 974                 # creating invalid or dangerous output. Suppress this.
 975                 $encValue = strtr( $encValue, array(
 976                         '<'    => '&lt;',   // This should never happen,
 977                         '>'    => '&gt;',   // we've received invalid input
 978                         '"'    => '&quot;', // which should have been escaped.
 979                         '{'    => '&#123;',
 980                         '['    => '&#91;',
 981                         "''"   => '&#39;&#39;',
 982                         'ISBN' => '&#73;SBN',
 983                         'RFC'  => '&#82;FC',
 984                         'PMID' => '&#80;MID',
 985                         '|'    => '&#124;',
 986                         '__'   => '&#95;_',
 987                 ) );
 988
 989                 # Stupid hack
 990                 $encValue = preg_replace_callback(
 991                         '/((?i)' . wfUrlProtocols() . ')/',
 992                         array( 'Sanitizer', 'armorLinksCallback' ),
 993                         $encValue );
 994                 return $encValue;
 995         }
 996
 997         /**
 998          * Given a value, escape it so that it can be used in an id attribute and
 999          * return it.  This will use HTML5 validation if $wgExperimentalHtmlIds is
1000          * true, allowing anything but ASCII whitespace.  Otherwise it will use
1001          * HTML 4 rules, which means a narrow subset of ASCII, with bad characters
1002          * escaped with lots of dots.
1003          *
1004          * To ensure we don't have to bother escaping anything, we also strip ', ",
1005          * & even if $wgExperimentalIds is true.  TODO: Is this the best tactic?
1006          * We also strip # because it upsets IE, and % because it could be
1007          * ambiguous if it's part of something that looks like a percent escape
1008          * (which don't work reliably in fragments cross-browser).
1009          *
1010          * @see http://www.w3.org/TR/html401/types.html#type-name Valid characters
1011          *                                                          in the id and
1012          *                                                          name attributes
1013          * @see http://www.w3.org/TR/html401/struct/links.html#h-12.2.3 Anchors with the id attribute
1014          * @see http://www.whatwg.org/html/elements.html#the-id-attribute
1015          *   HTML5 definition of id attribute
1016          *
1017          * @param string $id id to escape
1018          * @param $options Mixed: string or array of strings (default is array()):
1019          *   'noninitial': This is a non-initial fragment of an id, not a full id,
1020          *       so don't pay attention if the first character isn't valid at the
1021          *       beginning of an id.  Only matters if $wgExperimentalHtmlIds is
1022          *       false.
1023          *   'legacy': Behave the way the old HTML 4-based ID escaping worked even
1024          *       if $wgExperimentalHtmlIds is used, so we can generate extra
1025          *       anchors and links won't break.
1026          * @return String
1027          */
1028         static function escapeId( $id, $options = array() ) {
1029                 global $wgExperimentalHtmlIds;
1030                 $options = (array)$options;
1031
1032                 if ( $wgExperimentalHtmlIds && !in_array( 'legacy', $options ) ) {
1033                         $id = Sanitizer::decodeCharReferences( $id );
1034                         $id = preg_replace( '/[ \t\n\r\f_\'"&#%]+/', '_', $id );
1035                         $id = trim( $id, '_' );
1036                         if ( $id === '' ) {
1037                                 # Must have been all whitespace to start with.
1038                                 return '_';
1039                         } else {
1040                                 return $id;
1041                         }
1042                 }
1043
1044                 # HTML4-style escaping
1045                 static $replace = array(
1046                         '%3A' => ':',
1047                         '%' => '.'
1048                 );
1049
1050                 $id = urlencode( Sanitizer::decodeCharReferences( strtr( $id, ' ', '_' ) ) );
1051                 $id = str_replace( array_keys( $replace ), array_values( $replace ), $id );
1052
1053                 if ( !preg_match( '/^[a-zA-Z]/', $id )
1054                 && !in_array( 'noninitial', $options ) ) {
1055                         // Initial character must be a letter!
1056                         $id = "x$id";
1057                 }
1058                 return $id;
1059         }
1060
1061         /**
1062          * Given a value, escape it so that it can be used as a CSS class and
1063          * return it.
1064          *
1065          * @todo For extra validity, input should be validated UTF-8.
1066          *
1067          * @see http://www.w3.org/TR/CSS21/syndata.html Valid characters/format
1068          *
1069          * @param $class String
1070          * @return String
1071          */
1072         static function escapeClass( $class ) {
1073                 // Convert ugly stuff to underscores and kill underscores in ugly places
1074                 return rtrim( preg_replace(
1075                         array( '/(^[0-9\\-])|[\\x00-\\x20!"#$%&\'()*+,.\\/:;<=>?@[\\]^`{|}~]|\\xC2\\xA0/', '/_+/' ),
1076                         '_',
1077                         $class ), '_' );
1078         }
1079
1080         /**
1081          * Given HTML input, escape with htmlspecialchars but un-escape entities.
1082          * This allows (generally harmless) entities like &#160; to survive.
1083          *
1084          * @param string $html to escape
1085          * @return String: escaped input
1086          */
1087         static function escapeHtmlAllowEntities( $html ) {
1088                 $html = Sanitizer::decodeCharReferences( $html );
1089                 # It seems wise to escape ' as well as ", as a matter of course.  Can't
1090                 # hurt.
1091                 $html = htmlspecialchars( $html, ENT_QUOTES );
1092                 return $html;
1093         }
1094
1095         /**
1096          * Regex replace callback for armoring links against further processing.
1097          * @param $matches Array
1098          * @return string
1099          */
1100         private static function armorLinksCallback( $matches ) {
1101                 return str_replace( ':', '&#58;', $matches[1] );
1102         }
1103
1104         /**
1105          * Return an associative array of attribute names and values from
1106          * a partial tag string. Attribute names are forces to lowercase,
1107          * character references are decoded to UTF-8 text.
1108          *
1109          * @param $text String
1110          * @return Array
1111          */
1112         public static function decodeTagAttributes( $text ) {
1113                 if ( trim( $text ) == '' ) {
1114                         return array();
1115                 }
1116
1117                 $attribs = array();
1118                 $pairs = array();
1119                 if ( !preg_match_all(
1120                         self::getAttribsRegex(),
1121                         $text,
1122                         $pairs,
1123                         PREG_SET_ORDER ) ) {
1124                         return $attribs;
1125                 }
1126
1127                 foreach ( $pairs as $set ) {
1128                         $attribute = strtolower( $set[1] );
1129                         $value = Sanitizer::getTagAttributeCallback( $set );
1130
1131                         // Normalize whitespace
1132                         $value = preg_replace( '/[\t\r\n ]+/', ' ', $value );
1133                         $value = trim( $value );
1134
1135                         // Decode character references
1136                         $attribs[$attribute] = Sanitizer::decodeCharReferences( $value );
1137                 }
1138                 return $attribs;
1139         }
1140
1141         /**
1142          * Build a partial tag string from an associative array of attribute
1143          * names and values as returned by decodeTagAttributes.
1144          *
1145          * @param $assoc_array Array
1146          * @return String
1147          */
1148         public static function safeEncodeTagAttributes( $assoc_array ) {
1149                 $attribs = array();
1150                 foreach ( $assoc_array as $attribute => $value ) {
1151                         $encAttribute = htmlspecialchars( $attribute );
1152                         $encValue = Sanitizer::safeEncodeAttribute( $value );
1153
1154                         $attribs[] = "$encAttribute=\"$encValue\"";
1155                 }
1156                 return count( $attribs ) ? ' ' . implode( ' ', $attribs ) : '';
1157         }
1158
1159         /**
1160          * Pick the appropriate attribute value from a match set from the
1161          * attribs regex matches.
1162          *
1163          * @param $set Array
1164          * @throws MWException
1165          * @return String
1166          */
1167         private static function getTagAttributeCallback( $set ) {
1168                 if ( isset( $set[6] ) ) {
1169                         # Illegal #XXXXXX color with no quotes.
1170                         return $set[6];
1171                 } elseif ( isset( $set[5] ) ) {
1172                         # No quotes.
1173                         return $set[5];
1174                 } elseif ( isset( $set[4] ) ) {
1175                         # Single-quoted
1176                         return $set[4];
1177                 } elseif ( isset( $set[3] ) ) {
1178                         # Double-quoted
1179                         return $set[3];
1180                 } elseif ( !isset( $set[2] ) ) {
1181                         # In XHTML, attributes must have a value.
1182                         # For 'reduced' form, return explicitly the attribute name here.
1183                         return $set[1];
1184                 } else {
1185                         throw new MWException( "Tag conditions not met. This should never happen and is a bug." );
1186                 }
1187         }
1188
1189         /**
1190          * Normalize whitespace and character references in an XML source-
1191          * encoded text for an attribute value.
1192          *
1193          * See http://www.w3.org/TR/REC-xml/#AVNormalize for background,
1194          * but note that we're not returning the value, but are returning
1195          * XML source fragments that will be slapped into output.
1196          *
1197          * @param $text String
1198          * @return String
1199          */
1200         private static function normalizeAttributeValue( $text ) {
1201                 return str_replace( '"', '&quot;',
1202                         self::normalizeWhitespace(
1203                                 Sanitizer::normalizeCharReferences( $text ) ) );
1204         }
1205
1206         /**
1207          * @param $text string
1208          * @return mixed
1209          */
1210         private static function normalizeWhitespace( $text ) {
1211                 return preg_replace(
1212                         '/\r\n|[\x20\x0d\x0a\x09]/',
1213                         ' ',
1214                         $text );
1215         }
1216
1217         /**
1218          * Normalizes whitespace in a section name, such as might be returned
1219          * by Parser::stripSectionName(), for use in the id's that are used for
1220          * section links.
1221          *
1222          * @param $section String
1223          * @return String
1224          */
1225         static function normalizeSectionNameWhitespace( $section ) {
1226                 return trim( preg_replace( '/[ _]+/', ' ', $section ) );
1227         }
1228
1229         /**
1230          * Ensure that any entities and character references are legal
1231          * for XML and XHTML specifically. Any stray bits will be
1232          * &amp;-escaped to result in a valid text fragment.
1233          *
1234          * a. named char refs can only be &lt; &gt; &amp; &quot;, others are
1235          *   numericized (this way we're well-formed even without a DTD)
1236          * b. any numeric char refs must be legal chars, not invalid or forbidden
1237          * c. use lower cased "&#x", not "&#X"
1238          * d. fix or reject non-valid attributes
1239          *
1240          * @param $text String
1241          * @return String
1242          * @private
1243          */
1244         static function normalizeCharReferences( $text ) {
1245                 return preg_replace_callback(
1246                         self::CHAR_REFS_REGEX,
1247                         array( 'Sanitizer', 'normalizeCharReferencesCallback' ),
1248                         $text );
1249         }
1250         /**
1251          * @param $matches String
1252          * @return String
1253          */
1254         static function normalizeCharReferencesCallback( $matches ) {
1255                 $ret = null;
1256                 if ( $matches[1] != '' ) {
1257                         $ret = Sanitizer::normalizeEntity( $matches[1] );
1258                 } elseif ( $matches[2] != '' ) {
1259                         $ret = Sanitizer::decCharReference( $matches[2] );
1260                 } elseif ( $matches[3] != '' ) {
1261                         $ret = Sanitizer::hexCharReference( $matches[3] );
1262                 }
1263                 if ( is_null( $ret ) ) {
1264                         return htmlspecialchars( $matches[0] );
1265                 } else {
1266                         return $ret;
1267                 }
1268         }
1269
1270         /**
1271          * If the named entity is defined in the HTML 4.0/XHTML 1.0 DTD,
1272          * return the equivalent numeric entity reference (except for the core &lt;
1273          * &gt; &amp; &quot;). If the entity is a MediaWiki-specific alias, returns
1274          * the HTML equivalent. Otherwise, returns HTML-escaped text of
1275          * pseudo-entity source (eg &amp;foo;)
1276          *
1277          * @param $name String
1278          * @return String
1279          */
1280         static function normalizeEntity( $name ) {
1281                 if ( isset( self::$htmlEntityAliases[$name] ) ) {
1282                         return '&' . self::$htmlEntityAliases[$name] . ';';
1283                 } elseif ( in_array( $name,
1284                 array( 'lt', 'gt', 'amp', 'quot' ) ) ) {
1285                         return "&$name;";
1286                 } elseif ( isset( self::$htmlEntities[$name] ) ) {
1287                         return '&#' . self::$htmlEntities[$name] . ';';
1288                 } else {
1289                         return "&amp;$name;";
1290                 }
1291         }
1292
1293         /**
1294          * @param $codepoint
1295          * @return null|string
1296          */
1297         static function decCharReference( $codepoint ) {
1298                 $point = intval( $codepoint );
1299                 if ( Sanitizer::validateCodepoint( $point ) ) {
1300                         return sprintf( '&#%d;', $point );
1301                 } else {
1302                         return null;
1303                 }
1304         }
1305
1306         /**
1307          * @param $codepoint
1308          * @return null|string
1309          */
1310         static function hexCharReference( $codepoint ) {
1311                 $point = hexdec( $codepoint );
1312                 if ( Sanitizer::validateCodepoint( $point ) ) {
1313                         return sprintf( '&#x%x;', $point );
1314                 } else {
1315                         return null;
1316                 }
1317         }
1318
1319         /**
1320          * Returns true if a given Unicode codepoint is a valid character in XML.
1321          * @param $codepoint Integer
1322          * @return Boolean
1323          */
1324         private static function validateCodepoint( $codepoint ) {
1325                 return $codepoint == 0x09
1326                         || $codepoint == 0x0a
1327                         || $codepoint == 0x0d
1328                         || ( $codepoint >= 0x20 && $codepoint <= 0xd7ff )
1329                         || ( $codepoint >= 0xe000 && $codepoint <= 0xfffd )
1330                         || ( $codepoint >= 0x10000 && $codepoint <= 0x10ffff );
1331         }
1332
1333         /**
1334          * Decode any character references, numeric or named entities,
1335          * in the text and return a UTF-8 string.
1336          *
1337          * @param $text String
1338          * @return String
1339          */
1340         public static function decodeCharReferences( $text ) {
1341                 return preg_replace_callback(
1342                         self::CHAR_REFS_REGEX,
1343                         array( 'Sanitizer', 'decodeCharReferencesCallback' ),
1344                         $text );
1345         }
1346
1347         /**
1348          * Decode any character references, numeric or named entities,
1349          * in the next and normalize the resulting string. (bug 14952)
1350          *
1351          * This is useful for page titles, not for text to be displayed,
1352          * MediaWiki allows HTML entities to escape normalization as a feature.
1353          *
1354          * @param string $text (already normalized, containing entities)
1355          * @return String (still normalized, without entities)
1356          */
1357         public static function decodeCharReferencesAndNormalize( $text ) {
1358                 global $wgContLang;
1359                 $text = preg_replace_callback(
1360                         self::CHAR_REFS_REGEX,
1361                         array( 'Sanitizer', 'decodeCharReferencesCallback' ),
1362                         $text, /* limit */ -1, $count );
1363
1364                 if ( $count ) {
1365                         return $wgContLang->normalize( $text );
1366                 } else {
1367                         return $text;
1368                 }
1369         }
1370
1371         /**
1372          * @param $matches String
1373          * @return String
1374          */
1375         static function decodeCharReferencesCallback( $matches ) {
1376                 if ( $matches[1] != '' ) {
1377                         return Sanitizer::decodeEntity( $matches[1] );
1378                 } elseif ( $matches[2] != '' ) {
1379                         return Sanitizer::decodeChar( intval( $matches[2] ) );
1380                 } elseif ( $matches[3] != '' ) {
1381                         return Sanitizer::decodeChar( hexdec( $matches[3] ) );
1382                 }
1383                 # Last case should be an ampersand by itself
1384                 return $matches[0];
1385         }
1386
1387         /**
1388          * Return UTF-8 string for a codepoint if that is a valid
1389          * character reference, otherwise U+FFFD REPLACEMENT CHARACTER.
1390          * @param $codepoint Integer
1391          * @return String
1392          * @private
1393          */
1394         static function decodeChar( $codepoint ) {
1395                 if ( Sanitizer::validateCodepoint( $codepoint ) ) {
1396                         return codepointToUtf8( $codepoint );
1397                 } else {
1398                         return UTF8_REPLACEMENT;
1399                 }
1400         }
1401
1402         /**
1403          * If the named entity is defined in the HTML 4.0/XHTML 1.0 DTD,
1404          * return the UTF-8 encoding of that character. Otherwise, returns
1405          * pseudo-entity source (eg "&foo;")
1406          *
1407          * @param $name String
1408          * @return String
1409          */
1410         static function decodeEntity( $name ) {
1411                 if ( isset( self::$htmlEntityAliases[$name] ) ) {
1412                         $name = self::$htmlEntityAliases[$name];
1413                 }
1414                 if ( isset( self::$htmlEntities[$name] ) ) {
1415                         return codepointToUtf8( self::$htmlEntities[$name] );
1416                 } else {
1417                         return "&$name;";
1418                 }
1419         }
1420
1421         /**
1422          * Fetch the whitelist of acceptable attributes for a given element name.
1423          *
1424          * @param $element String
1425          * @return Array
1426          */
1427         static function attributeWhitelist( $element ) {
1428                 $list = Sanitizer::setupAttributeWhitelist();
1429                 return isset( $list[$element] )
1430                         ? $list[$element]
1431                         : array();
1432         }
1433
1434         /**
1435          * Foreach array key (an allowed HTML element), return an array
1436          * of allowed attributes
1437          * @return Array
1438          */
1439         static function setupAttributeWhitelist() {
1440                 global $wgAllowRdfaAttributes, $wgAllowMicrodataAttributes;
1441
1442                 static $whitelist, $staticInitialised;
1443                 $globalContext = implode( '-', compact( 'wgAllowRdfaAttributes', 'wgAllowMicrodataAttributes' ) );
1444
1445                 if ( isset( $whitelist ) && $staticInitialised == $globalContext ) {
1446                         return $whitelist;
1447                 }
1448
1449                 $common = array(
1450                         # HTML
1451                         'id',
1452                         'class',
1453                         'style',
1454                         'lang',
1455                         'dir',
1456                         'title',
1457
1458                         # WAI-ARIA
1459                         'role',
1460                 );
1461
1462                 if ( $wgAllowRdfaAttributes ) {
1463                         #RDFa attributes as specified in section 9 of http://www.w3.org/TR/2008/REC-rdfa-syntax-20081014
1464                         $common = array_merge( $common, array(
1465                                 'about', 'property', 'resource', 'datatype', 'typeof',
1466                         ) );
1467                 }
1468
1469                 if ( $wgAllowMicrodataAttributes ) {
1470                         # add HTML5 microdata tags as specified by http://www.whatwg.org/html/microdata.html#the-microdata-model
1471                         $common = array_merge( $common, array(
1472                                 'itemid', 'itemprop', 'itemref', 'itemscope', 'itemtype'
1473                         ) );
1474                 }
1475
1476                 $block = array_merge( $common, array( 'align' ) );
1477                 $tablealign = array( 'align', 'char', 'charoff', 'valign' );
1478                 $tablecell = array(
1479                         'abbr',
1480                         'axis',
1481                         'headers',
1482                         'scope',
1483                         'rowspan',
1484                         'colspan',
1485                         'nowrap', # deprecated
1486                         'width', # deprecated
1487                         'height', # deprecated
1488                         'bgcolor', # deprecated
1489                 );
1490
1491                 # Numbers refer to sections in HTML 4.01 standard describing the element.
1492                 # See: http://www.w3.org/TR/html4/
1493                 $whitelist = array(
1494                         # 7.5.4
1495                         'div'        => $block,
1496                         'center'     => $common, # deprecated
1497                         'span'       => $block, # ??
1498
1499                         # 7.5.5
1500                         'h1'         => $block,
1501                         'h2'         => $block,
1502                         'h3'         => $block,
1503                         'h4'         => $block,
1504                         'h5'         => $block,
1505                         'h6'         => $block,
1506
1507                         # 7.5.6
1508                         # address
1509
1510                         # 8.2.4
1511                         # bdo
1512
1513                         # 9.2.1
1514                         'em'         => $common,
1515                         'strong'     => $common,
1516                         'cite'       => $common,
1517                         'dfn'        => $common,
1518                         'code'       => $common,
1519                         'samp'       => $common,
1520                         'kbd'        => $common,
1521                         'var'        => $common,
1522                         'abbr'       => $common,
1523                         # acronym
1524
1525                         # 9.2.2
1526                         'blockquote' => array_merge( $common, array( 'cite' ) ),
1527                         # q
1528
1529                         # 9.2.3
1530                         'sub'        => $common,
1531                         'sup'        => $common,
1532
1533                         # 9.3.1
1534                         'p'          => $block,
1535
1536                         # 9.3.2
1537                         'br'         => array( 'id', 'class', 'title', 'style', 'clear' ),
1538
1539                         # 9.3.4
1540                         'pre'        => array_merge( $common, array( 'width' ) ),
1541
1542                         # 9.4
1543                         'ins'        => array_merge( $common, array( 'cite', 'datetime' ) ),
1544                         'del'        => array_merge( $common, array( 'cite', 'datetime' ) ),
1545
1546                         # 10.2
1547                         'ul'         => array_merge( $common, array( 'type' ) ),
1548                         'ol'         => array_merge( $common, array( 'type', 'start' ) ),
1549                         'li'         => array_merge( $common, array( 'type', 'value' ) ),
1550
1551                         # 10.3
1552                         'dl'         => $common,
1553                         'dd'         => $common,
1554                         'dt'         => $common,
1555
1556                         # 11.2.1
1557                         'table'      => array_merge( $common,
1558                                                                 array( 'summary', 'width', 'border', 'frame',
1559                                                                                 'rules', 'cellspacing', 'cellpadding',
1560                                                                                 'align', 'bgcolor',
1561                                                                 ) ),
1562
1563                         # 11.2.2
1564                         'caption'    => array_merge( $common, array( 'align' ) ),
1565
1566                         # 11.2.3
1567                         'thead'      => array_merge( $common, $tablealign ),
1568                         'tfoot'      => array_merge( $common, $tablealign ),
1569                         'tbody'      => array_merge( $common, $tablealign ),
1570
1571                         # 11.2.4
1572                         'colgroup'   => array_merge( $common, array( 'span', 'width' ), $tablealign ),
1573                         'col'        => array_merge( $common, array( 'span', 'width' ), $tablealign ),
1574
1575                         # 11.2.5
1576                         'tr'         => array_merge( $common, array( 'bgcolor' ), $tablealign ),
1577
1578                         # 11.2.6
1579                         'td'         => array_merge( $common, $tablecell, $tablealign ),
1580                         'th'         => array_merge( $common, $tablecell, $tablealign ),
1581
1582                         # 12.2 # NOTE: <a> is not allowed directly, but the attrib whitelist is used from the Parser object
1583                         'a'          => array_merge( $common, array( 'href', 'rel', 'rev' ) ), # rel/rev esp. for RDFa
1584
1585                         # 13.2
1586                         # Not usually allowed, but may be used for extension-style hooks
1587                         # such as <math> when it is rasterized, or if $wgAllowImageTag is
1588                         # true
1589                         'img'        => array_merge( $common, array( 'alt', 'src', 'width', 'height' ) ),
1590
1591                         # 15.2.1
1592                         'tt'         => $common,
1593                         'b'          => $common,
1594                         'i'          => $common,
1595                         'big'        => $common,
1596                         'small'      => $common,
1597                         'strike'     => $common,
1598                         's'          => $common,
1599                         'u'          => $common,
1600
1601                         # 15.2.2
1602                         'font'       => array_merge( $common, array( 'size', 'color', 'face' ) ),
1603                         # basefont
1604
1605                         # 15.3
1606                         'hr'         => array_merge( $common, array( 'noshade', 'size', 'width' ) ),
1607
1608                         # HTML Ruby annotation text module, simple ruby only.
1609                         # http://www.whatwg.org/html/text-level-semantics.html#the-ruby-element
1610                         'ruby'       => $common,
1611                         # rbc
1612                         # rtc
1613                         'rb'         => $common,
1614                         'rt'         => $common, #array_merge( $common, array( 'rbspan' ) ),
1615                         'rp'         => $common,
1616
1617                         # MathML root element, where used for extensions
1618                         # 'title' may not be 100% valid here; it's XHTML
1619                         # http://www.w3.org/TR/REC-MathML/
1620                         'math'       => array( 'class', 'style', 'id', 'title' ),
1621
1622                         # HTML 5 section 4.6
1623                         'bdi' => $common,
1624
1625                         # HTML5 elements, defined by:
1626                         # http://www.whatwg.org/html/
1627                         'data' => array_merge( $common, array( 'value' ) ),
1628                         'time' => array_merge( $common, array( 'datetime' ) ),
1629                         'mark' => $common,
1630
1631                         // meta and link are only permitted by removeHTMLtags when Microdata
1632                         // is enabled so we don't bother adding a conditional to hide these
1633                         // Also meta and link are only valid in WikiText as Microdata elements
1634                         // (ie: validateTag rejects tags missing the attributes needed for Microdata)
1635                         // So we don't bother including $common attributes that have no purpose.
1636                         'meta' => array( 'itemprop', 'content' ),
1637                         'link' => array( 'itemprop', 'href' ),
1638                 );
1639
1640                 $staticInitialised = $globalContext;
1641
1642                 return $whitelist;
1643         }
1644
1645         /**
1646          * Take a fragment of (potentially invalid) HTML and return
1647          * a version with any tags removed, encoded as plain text.
1648          *
1649          * Warning: this return value must be further escaped for literal
1650          * inclusion in HTML output as of 1.10!
1651          *
1652          * @param string $text HTML fragment
1653          * @return String
1654          */
1655         static function stripAllTags( $text ) {
1656                 # Actual <tags>
1657                 $text = StringUtils::delimiterReplace( '<', '>', '', $text );
1658
1659                 # Normalize &entities and whitespace
1660                 $text = self::decodeCharReferences( $text );
1661                 $text = self::normalizeWhitespace( $text );
1662
1663                 return $text;
1664         }
1665
1666         /**
1667          * Hack up a private DOCTYPE with HTML's standard entity declarations.
1668          * PHP 4 seemed to know these if you gave it an HTML doctype, but
1669          * PHP 5.1 doesn't.
1670          *
1671          * Use for passing XHTML fragments to PHP's XML parsing functions
1672          *
1673          * @return String
1674          */
1675         static function hackDocType() {
1676                 $out = "<!DOCTYPE html [\n";
1677                 foreach ( self::$htmlEntities as $entity => $codepoint ) {
1678                         $out .= "<!ENTITY $entity \"&#$codepoint;\">";
1679                 }
1680                 $out .= "]>\n";
1681                 return $out;
1682         }
1683
1684         /**
1685          * @param $url string
1686          * @return mixed|string
1687          */
1688         static function cleanUrl( $url ) {
1689                 # Normalize any HTML entities in input. They will be
1690                 # re-escaped by makeExternalLink().
1691                 $url = Sanitizer::decodeCharReferences( $url );
1692
1693                 # Escape any control characters introduced by the above step
1694                 $url = preg_replace_callback( '/[\][<>"\\x00-\\x20\\x7F\|]/',
1695                         array( __CLASS__, 'cleanUrlCallback' ), $url );
1696
1697                 # Validate hostname portion
1698                 $matches = array();
1699                 if ( preg_match( '!^([^:]+:)(//[^/]+)?(.*)$!iD', $url, $matches ) ) {
1700                         list( /* $whole */, $protocol, $host, $rest ) = $matches;
1701
1702                         // Characters that will be ignored in IDNs.
1703                         // http://tools.ietf.org/html/3454#section-3.1
1704                         // Strip them before further processing so blacklists and such work.
1705                         $strip = "/
1706                                 \\s|          # general whitespace
1707                                 \xc2\xad|     # 00ad SOFT HYPHEN
1708                                 \xe1\xa0\x86| # 1806 MONGOLIAN TODO SOFT HYPHEN
1709                                 \xe2\x80\x8b| # 200b ZERO WIDTH SPACE
1710                                 \xe2\x81\xa0| # 2060 WORD JOINER
1711                                 \xef\xbb\xbf| # feff ZERO WIDTH NO-BREAK SPACE
1712                                 \xcd\x8f|     # 034f COMBINING GRAPHEME JOINER
1713                                 \xe1\xa0\x8b| # 180b MONGOLIAN FREE VARIATION SELECTOR ONE
1714                                 \xe1\xa0\x8c| # 180c MONGOLIAN FREE VARIATION SELECTOR TWO
1715                                 \xe1\xa0\x8d| # 180d MONGOLIAN FREE VARIATION SELECTOR THREE
1716                                 \xe2\x80\x8c| # 200c ZERO WIDTH NON-JOINER
1717                                 \xe2\x80\x8d| # 200d ZERO WIDTH JOINER
1718                                 [\xef\xb8\x80-\xef\xb8\x8f] # fe00-fe0f VARIATION SELECTOR-1-16
1719                                 /xuD";
1720
1721                         $host = preg_replace( $strip, '', $host );
1722
1723                         // @todo FIXME: Validate hostnames here
1724
1725                         return $protocol . $host . $rest;
1726                 } else {
1727                         return $url;
1728                 }
1729         }
1730
1731         /**
1732          * @param $matches array
1733          * @return string
1734          */
1735         static function cleanUrlCallback( $matches ) {
1736                 return urlencode( $matches[0] );
1737         }
1738
1739         /**
1740          * Does a string look like an e-mail address?
1741          *
1742          * This validates an email address using an HTML5 specification found at:
1743          * http://www.whatwg.org/html/states-of-the-type-attribute.html#valid-e-mail-address
1744          * Which as of 2011-01-24 says:
1745          *
1746          *   A valid e-mail address is a string that matches the ABNF production
1747          *   1*( atext / "." ) "@" ldh-str *( "." ldh-str ) where atext is defined
1748          *   in RFC 5322 section 3.2.3, and ldh-str is defined in RFC 1034 section
1749          *   3.5.
1750          *
1751          * This function is an implementation of the specification as requested in
1752          * bug 22449.
1753          *
1754          * Client-side forms will use the same standard validation rules via JS or
1755          * HTML 5 validation; additional restrictions can be enforced server-side
1756          * by extensions via the 'isValidEmailAddr' hook.
1757          *
1758          * Note that this validation doesn't 100% match RFC 2822, but is believed
1759          * to be liberal enough for wide use. Some invalid addresses will still
1760          * pass validation here.
1761          *
1762          * @since 1.18
1763          *
1764          * @param string $addr E-mail address
1765          * @return Bool
1766          */
1767         public static function validateEmail( $addr ) {
1768                 $result = null;
1769                 if ( !wfRunHooks( 'isValidEmailAddr', array( $addr, &$result ) ) ) {
1770                         return $result;
1771                 }
1772
1773                 // Please note strings below are enclosed in brackets [], this make the
1774                 // hyphen "-" a range indicator. Hence it is double backslashed below.
1775                 // See bug 26948
1776                 $rfc5322_atext = "a-z0-9!#$%&'*+\\-\/=?^_`{|}~";
1777                 $rfc1034_ldh_str = "a-z0-9\\-";
1778
1779                 $HTML5_email_regexp = "/
1780                 ^                      # start of string
1781                 [$rfc5322_atext\\.]+    # user part which is liberal :p
1782                 @                      # 'apostrophe'
1783                 [$rfc1034_ldh_str]+       # First domain part
1784                 (\\.[$rfc1034_ldh_str]+)*  # Following part prefixed with a dot
1785                 $                      # End of string
1786                 /ix"; // case Insensitive, eXtended
1787
1788                 return (bool) preg_match( $HTML5_email_regexp, $addr );
1789         }
1790 }