tests/utf8_processor.lua

   1 local lpeg = require("lpeg")
   2
   3 local string = string
   4
   5 local floor = require("math").floor
   6
   7 local _ENV = nil
   8
   9 local function encode_utf(codepoint)
  10         if codepoint > 0x10FFFF then
  11                 error("Codepoint > 10FFFF cannot be encoded")
  12         elseif codepoint > 0xFFFF then
  13                 -- Surrogate pair needed
  14                 codepoint = codepoint - 0x10000
  15                 local first, second = floor(codepoint / 0x0400) + 0xD800, codepoint % 0x0400 + 0xDC00
  16                 return ("\\u%.4X\\u%.4X"):format(first, second)
  17         else
  18                 return ("\\u%.4X"):format(codepoint)
  19         end
  20 end
  21
  22 -- decode a two-byte UTF-8 sequence
  23 local function f2 (s)
  24         local c1, c2 = string.byte(s, 1, 2)
  25         return encode_utf(c1 * 64 + c2 - 12416)
  26 end
  27
  28 -- decode a three-byte UTF-8 sequence
  29 local function f3 (s)
  30         local c1, c2, c3 = string.byte(s, 1, 3)
  31         return encode_utf((c1 * 64 + c2) * 64 + c3 - 925824)
  32 end
  33
  34 -- decode a four-byte UTF-8 sequence
  35 local function f4 (s)
  36         local c1, c2, c3, c4 = string.byte(s, 1, 4)
  37         return encode_utf(((c1 * 64 + c2) * 64 + c3) * 64 + c4 - 63447168)
  38 end
  39
  40 local cont = lpeg.R("\128\191")   -- continuation byte
  41
  42 local utf8 = lpeg.R("\0\127") -- Do nothing here
  43         + lpeg.R("\194\223") * cont / f2
  44         + lpeg.R("\224\239") * cont * cont / f3
  45         + lpeg.R("\240\244") * cont * cont * cont / f4
  46
  47 local utf8_decode_pattern = lpeg.Cs(utf8^0) * -1
  48
  49
  50 local function process(s)
  51         return utf8_decode_pattern:match(s)
  52 end
  53
  54 return {
  55     process = process
  56 }