tests/HTMLPurifier/EncoderTest.php

   1 <?php
   2
   3 class HTMLPurifier_EncoderTest extends HTMLPurifier_Harness
   4 {
   5
   6     protected $_entity_lookup;
   7
   8     public function setUp()
   9     {
  10         $this->_entity_lookup = HTMLPurifier_EntityLookup::instance();
  11         parent::setUp();
  12     }
  13
  14     public function assertCleanUTF8($string, $expect = null)
  15     {
  16         if ($expect === null) $expect = $string;
  17         $this->assertIdentical(HTMLPurifier_Encoder::cleanUTF8($string), $expect, 'iconv: %s');
  18         $this->assertIdentical(HTMLPurifier_Encoder::cleanUTF8($string, true), $expect, 'PHP: %s');
  19     }
  20
  21     public function test_cleanUTF8()
  22     {
  23         $this->assertCleanUTF8('Normal string.');
  24         $this->assertCleanUTF8("Test\tAllowed\nControl\rCharacters");
  25         $this->assertCleanUTF8("null byte: \0", 'null byte: ');
  26         $this->assertCleanUTF8("あ（い）う（え）お\0", "あ（い）う（え）お"); // test for issue #122
  27         $this->assertCleanUTF8("\1\2\3\4\5\6\7", '');
  28         $this->assertCleanUTF8("\x7F", ''); // one byte invalid SGML char
  29         $this->assertCleanUTF8("\xC2\x80", ''); // two byte invalid SGML
  30         $this->assertCleanUTF8("\xF3\xBF\xBF\xBF"); // valid four byte
  31         $this->assertCleanUTF8("\xDF\xFF", ''); // malformed UTF8
  32         // invalid codepoints
  33         $this->assertCleanUTF8("\xED\xB0\x80", '');
  34     }
  35
  36     public function test_convertToUTF8_noConvert()
  37     {
  38         // UTF-8 means that we don't touch it
  39         $this->assertIdentical(
  40             HTMLPurifier_Encoder::convertToUTF8("\xF6", $this->config, $this->context),
  41             "\xF6", // this is invalid
  42             'Expected identical [Binary: F6]'
  43         );
  44     }
  45
  46     public function test_convertToUTF8_spuriousEncoding()
  47     {
  48         if (!HTMLPurifier_Encoder::iconvAvailable()) return;
  49         $this->config->set('Core.Encoding', 'utf99');
  50         $this->expectError('Invalid encoding utf99');
  51         $this->assertIdentical(
  52             HTMLPurifier_Encoder::convertToUTF8("\xF6", $this->config, $this->context),
  53             ''
  54         );
  55     }
  56
  57     public function test_convertToUTF8_iso8859_1()
  58     {
  59         $this->config->set('Core.Encoding', 'ISO-8859-1');
  60         $this->assertIdentical(
  61             HTMLPurifier_Encoder::convertToUTF8("\xF6", $this->config, $this->context),
  62             "\xC3\xB6"
  63         );
  64     }
  65
  66     public function test_convertToUTF8_withoutIconv()
  67     {
  68         $this->config->set('Core.Encoding', 'ISO-8859-1');
  69         $this->config->set('Test.ForceNoIconv', true);
  70         $this->assertIdentical(
  71             HTMLPurifier_Encoder::convertToUTF8("\xF6", $this->config, $this->context),
  72             "\xC3\xB6"
  73         );
  74
  75     }
  76
  77     public function getZhongWen()
  78     {
  79         return "\xE4\xB8\xAD\xE6\x96\x87 (Chinese)";
  80     }
  81
  82     public function test_convertFromUTF8_utf8()
  83     {
  84         // UTF-8 means that we don't touch it
  85         $this->assertIdentical(
  86             HTMLPurifier_Encoder::convertFromUTF8("\xC3\xB6", $this->config, $this->context),
  87             "\xC3\xB6"
  88         );
  89     }
  90
  91     public function test_convertFromUTF8_iso8859_1()
  92     {
  93         $this->config->set('Core.Encoding', 'ISO-8859-1');
  94         $this->assertIdentical(
  95             HTMLPurifier_Encoder::convertFromUTF8("\xC3\xB6", $this->config, $this->context),
  96             "\xF6",
  97             'Expected identical [Binary: F6]'
  98         );
  99     }
 100
 101     public function test_convertFromUTF8_iconvNoChars()
 102     {
 103         if (!HTMLPurifier_Encoder::iconvAvailable()) return;
 104         $this->config->set('Core.Encoding', 'ISO-8859-1');
 105         $this->assertIdentical(
 106             HTMLPurifier_Encoder::convertFromUTF8($this->getZhongWen(), $this->config, $this->context),
 107             " (Chinese)"
 108         );
 109     }
 110
 111     public function test_convertFromUTF8_phpNormal()
 112     {
 113         // Plain PHP implementation has slightly different behavior
 114         $this->config->set('Core.Encoding', 'ISO-8859-1');
 115         $this->config->set('Test.ForceNoIconv', true);
 116         $this->assertIdentical(
 117             HTMLPurifier_Encoder::convertFromUTF8("\xC3\xB6", $this->config, $this->context),
 118             "\xF6",
 119             'Expected identical [Binary: F6]'
 120         );
 121     }
 122
 123     public function test_convertFromUTF8_phpNoChars()
 124     {
 125         $this->config->set('Core.Encoding', 'ISO-8859-1');
 126         $this->config->set('Test.ForceNoIconv', true);
 127         $this->assertIdentical(
 128             HTMLPurifier_Encoder::convertFromUTF8($this->getZhongWen(), $this->config, $this->context),
 129             "?? (Chinese)"
 130         );
 131     }
 132
 133     public function test_convertFromUTF8_withProtection()
 134     {
 135         // Preserve the characters!
 136         $this->config->set('Core.Encoding', 'ISO-8859-1');
 137         $this->config->set('Core.EscapeNonASCIICharacters', true);
 138         $this->assertIdentical(
 139             HTMLPurifier_Encoder::convertFromUTF8($this->getZhongWen(), $this->config, $this->context),
 140             "&#20013;&#25991; (Chinese)"
 141         );
 142     }
 143
 144     public function test_convertFromUTF8_withProtectionButUtf8()
 145     {
 146         // Preserve the characters!
 147         $this->config->set('Core.EscapeNonASCIICharacters', true);
 148         $this->assertIdentical(
 149             HTMLPurifier_Encoder::convertFromUTF8($this->getZhongWen(), $this->config, $this->context),
 150             "&#20013;&#25991; (Chinese)"
 151         );
 152     }
 153
 154     public function test_convertToASCIIDumbLossless()
 155     {
 156         // Uppercase thorn letter
 157         $this->assertIdentical(
 158             HTMLPurifier_Encoder::convertToASCIIDumbLossless("\xC3\x9Eorn"),
 159             "&#222;orn"
 160         );
 161
 162         $this->assertIdentical(
 163             HTMLPurifier_Encoder::convertToASCIIDumbLossless("an"),
 164             "an"
 165         );
 166
 167         // test up to four bytes
 168         $this->assertIdentical(
 169             HTMLPurifier_Encoder::convertToASCIIDumbLossless("\xF3\xA0\x80\xA0"),
 170             "&#917536;"
 171         );
 172
 173     }
 174
 175     public function assertASCIISupportCheck($enc, $ret)
 176     {
 177         $test = HTMLPurifier_Encoder::testEncodingSupportsASCII($enc, true);
 178         if ($test === false) return;
 179         $this->assertIdentical(
 180             HTMLPurifier_Encoder::testEncodingSupportsASCII($enc),
 181             $ret
 182         );
 183         $this->assertIdentical(
 184             HTMLPurifier_Encoder::testEncodingSupportsASCII($enc, true),
 185             $ret
 186         );
 187     }
 188
 189     public function test_testEncodingSupportsASCII()
 190     {
 191         if (HTMLPurifier_Encoder::iconvAvailable()) {
 192             $this->assertASCIISupportCheck('Shift_JIS', array("\xC2\xA5" => '\\', "\xE2\x80\xBE" => '~'));
 193             $this->assertASCIISupportCheck('JOHAB', array("\xE2\x82\xA9" => '\\'));
 194         }
 195         $this->assertASCIISupportCheck('ISO-8859-1', array());
 196         $this->assertASCIISupportCheck('dontexist', array()); // canary
 197     }
 198
 199     public function testShiftJIS()
 200     {
 201         if (!HTMLPurifier_Encoder::iconvAvailable()) return;
 202         $this->config->set('Core.Encoding', 'Shift_JIS');
 203         // This actually looks like a Yen, but we're going to treat it differently
 204         $this->assertIdentical(
 205             HTMLPurifier_Encoder::convertFromUTF8('\\~', $this->config, $this->context),
 206             '\\~'
 207         );
 208         $this->assertIdentical(
 209             HTMLPurifier_Encoder::convertToUTF8('\\~', $this->config, $this->context),
 210             '\\~'
 211         );
 212     }
 213
 214     public function testIconvTruncateBug()
 215     {
 216         if (!HTMLPurifier_Encoder::iconvAvailable()) return;
 217         if (HTMLPurifier_Encoder::testIconvTruncateBug() !== HTMLPurifier_Encoder::ICONV_TRUNCATES) return;
 218         $this->config->set('Core.Encoding', 'ISO-8859-1');
 219         $this->assertIdentical(
 220             HTMLPurifier_Encoder::convertFromUTF8("\xE4\xB8\xAD" . str_repeat('a', 10000), $this->config, $this->context),
 221             str_repeat('a', 10000)
 222         );
 223     }
 224
 225     public function testIconvChunking()
 226     {
 227         if (!HTMLPurifier_Encoder::iconvAvailable()) return;
 228         if (HTMLPurifier_Encoder::testIconvTruncateBug() !== HTMLPurifier_Encoder::ICONV_TRUNCATES) return;
 229         $this->assertIdentical(HTMLPurifier_Encoder::iconv('utf-8', 'iso-8859-1//IGNORE', "a\xF3\xA0\x80\xA0b", 4), 'ab');
 230         $this->assertIdentical(HTMLPurifier_Encoder::iconv('utf-8', 'iso-8859-1//IGNORE', "aa\xE4\xB8\xADb", 4), 'aab');
 231         $this->assertIdentical(HTMLPurifier_Encoder::iconv('utf-8', 'iso-8859-1//IGNORE', "aaa\xCE\xB1b", 4), 'aaab');
 232         $this->assertIdentical(HTMLPurifier_Encoder::iconv('utf-8', 'iso-8859-1//IGNORE', "aaaa\xF3\xA0\x80\xA0b", 4), 'aaaab');
 233         $this->assertIdentical(HTMLPurifier_Encoder::iconv('utf-8', 'iso-8859-1//IGNORE', "aaaa\xE4\xB8\xADb", 4), 'aaaab');
 234         $this->assertIdentical(HTMLPurifier_Encoder::iconv('utf-8', 'iso-8859-1//IGNORE', "aaaa\xCE\xB1b", 4), 'aaaab');
 235     }
 236
 237 }
 238
 239 // vim: et sw=4 sts=4