usr/src/cmd/iconv/charmap.c

   1 /*
   2  * This file and its contents are supplied under the terms of the
   3  * Common Development and Distribution License ("CDDL"), version 1.0.
   4  * You may only use this file in accordance with the terms of version
   5  * 1.0 of the CDDL.
   6  *
   7  * A full copy of the text of the CDDL should have accompanied this
   8  * source.  A copy of the CDDL is also available via the Internet at
   9  * http://www.illumos.org/license/CDDL.
  10  */
  11
  12 /*
  13  * Copyright 2011 Nexenta Systems, Inc.  All rights reserved.
  14  */
  15
  16 /*
  17  * CHARMAP file handling for iconv.
  18  */
  19
  20 #include <stdio.h>
  21 #include <stdlib.h>
  22 #include <string.h>
  23 #include <errno.h>
  24 #include <limits.h>
  25 #include <unistd.h>
  26 #include <alloca.h>
  27 #include <sys/avl.h>
  28 #include <stddef.h>
  29 #include <unistd.h>
  30 #include "charmap.h"
  31 #include "parser.tab.h"
  32 #include <assert.h>
  33
  34 enum cmap_pass cmap_pass;
  35 static avl_tree_t       cmap_sym;
  36 static avl_tree_t       cmap_mbs;
  37
  38 typedef struct charmap {
  39         const char *cm_name;
  40         struct charmap *cm_alias_of;
  41         avl_node_t cm_avl_sym;
  42         avl_node_t cm_avl_mbs;
  43         int cm_warned;
  44         int cm_frmbs_len;
  45         int cm_tombs_len;
  46         char cm_frmbs[MB_LEN_MAX + 1]; /* input */
  47         char cm_tombs[MB_LEN_MAX + 1]; /* output */
  48 } charmap_t;
  49
  50 static void add_charmap_impl_fr(char *sym, char *mbs, int mbs_len, int nodups);
  51 static void add_charmap_impl_to(char *sym, char *mbs, int mbs_len, int nodups);
  52
  53 /*
  54  * Array of POSIX specific portable characters.
  55  */
  56 static const struct {
  57         char    *name;
  58         int     ch;
  59 } portable_chars[] = {
  60         { "NUL",                 '\0' },
  61         { "alert",              '\a' },
  62         { "backspace",          '\b' },
  63         { "tab",                '\t' },
  64         { "carriage-return",    '\r' },
  65         { "newline",            '\n' },
  66         { "vertical-tab",       '\v' },
  67         { "form-feed",          '\f' },
  68         { "space",              ' ' },
  69         { "exclamation-mark",   '!' },
  70         { "quotation-mark",     '"' },
  71         { "number-sign",        '#' },
  72         { "dollar-sign",        '$' },
  73         { "percent-sign",       '%' },
  74         { "ampersand",          '&' },
  75         { "apostrophe",         '\'' },
  76         { "left-parenthesis",   '(' },
  77         { "right-parenthesis",  '(' },
  78         { "asterisk",           '*' },
  79         { "plus-sign",          '+' },
  80         { "comma",               ','},
  81         { "hyphen-minus",       '-' },
  82         { "hyphen",             '-' },
  83         { "full-stop",          '.' },
  84         { "period",             '.' },
  85         { "slash",              '/' },
  86         { "solidus",            '/' },
  87         { "zero",               '0' },
  88         { "one",                '1' },
  89         { "two",                '2' },
  90         { "three",              '3' },
  91         { "four",               '4' },
  92         { "five",               '5' },
  93         { "six",                '6' },
  94         { "seven",              '7' },
  95         { "eight",              '8' },
  96         { "nine",               '9' },
  97         { "colon",              ':' },
  98         { "semicolon",          ';' },
  99         { "less-than-sign",     '<' },
 100         { "equals-sign",        '=' },
 101         { "greater-than-sign",  '>' },
 102         { "question-mark",      '?' },
 103         { "commercial-at",      '@' },
 104         { "left-square-bracket", '[' },
 105         { "backslash",          '\\' },
 106         { "reverse-solidus",    '\\' },
 107         { "right-square-bracket", ']' },
 108         { "circumflex",         '^' },
 109         { "circumflex-accent",  '^' },
 110         { "low-line",           '_' },
 111         { "underscore",         '_' },
 112         { "grave-accent",       '`' },
 113         { "left-brace",         '{' },
 114         { "left-curly-bracket", '{' },
 115         { "vertical-line",      '|' },
 116         { "right-brace",        '}' },
 117         { "right-curly-bracket", '}' },
 118         { "tilde",              '~' },
 119         { "A", 'A' },
 120         { "B", 'B' },
 121         { "C", 'C' },
 122         { "D", 'D' },
 123         { "E", 'E' },
 124         { "F", 'F' },
 125         { "G", 'G' },
 126         { "H", 'H' },
 127         { "I", 'I' },
 128         { "J", 'J' },
 129         { "K", 'K' },
 130         { "L", 'L' },
 131         { "M", 'M' },
 132         { "N", 'N' },
 133         { "O", 'O' },
 134         { "P", 'P' },
 135         { "Q", 'Q' },
 136         { "R", 'R' },
 137         { "S", 'S' },
 138         { "T", 'T' },
 139         { "U", 'U' },
 140         { "V", 'V' },
 141         { "W", 'W' },
 142         { "X", 'X' },
 143         { "Y", 'Y' },
 144         { "Z", 'Z' },
 145         { "a", 'a' },
 146         { "b", 'b' },
 147         { "c", 'c' },
 148         { "d", 'd' },
 149         { "e", 'e' },
 150         { "f", 'f' },
 151         { "g", 'g' },
 152         { "h", 'h' },
 153         { "i", 'i' },
 154         { "j", 'j' },
 155         { "k", 'k' },
 156         { "l", 'l' },
 157         { "m", 'm' },
 158         { "n", 'n' },
 159         { "o", 'o' },
 160         { "p", 'p' },
 161         { "q", 'q' },
 162         { "r", 'r' },
 163         { "s", 's' },
 164         { "t", 't' },
 165         { "u", 'u' },
 166         { "v", 'v' },
 167         { "w", 'w' },
 168         { "x", 'x' },
 169         { "y", 'y' },
 170         { "z", 'z' },
 171         { NULL, 0 }
 172 };
 173
 174 static int
 175 cmap_compare_sym(const void *n1, const void *n2)
 176 {
 177         const charmap_t *c1 = n1;
 178         const charmap_t *c2 = n2;
 179         int rv;
 180
 181         rv = strcmp(c1->cm_name, c2->cm_name);
 182         return ((rv < 0) ? -1 : (rv > 0) ? 1 : 0);
 183 }
 184
 185 /*
 186  * In order for partial match searches to work,
 187  * we need these sorted by mbs contents.
 188  */
 189 static int
 190 cmap_compare_mbs(const void *n1, const void *n2)
 191 {
 192         const charmap_t *c1 = n1;
 193         const charmap_t *c2 = n2;
 194         int len, rv;
 195
 196         len = c1->cm_frmbs_len;
 197         if (len < c2->cm_frmbs_len)
 198                 len = c2->cm_frmbs_len;
 199         rv = memcmp(c1->cm_frmbs, c2->cm_frmbs, len);
 200         if (rv < 0)
 201                 return (-1);
 202         if (rv > 0)
 203                 return (1);
 204         /* they match through length */
 205         if (c1->cm_frmbs_len < c2->cm_frmbs_len)
 206                 return (-1);
 207         if (c2->cm_frmbs_len < c1->cm_frmbs_len)
 208                 return (1);
 209         return (0);
 210 }
 211
 212 void
 213 charmap_init(char *to_map, char *from_map)
 214 {
 215         avl_create(&cmap_sym, cmap_compare_sym, sizeof (charmap_t),
 216             offsetof(charmap_t, cm_avl_sym));
 217
 218         avl_create(&cmap_mbs, cmap_compare_mbs, sizeof (charmap_t),
 219             offsetof(charmap_t, cm_avl_mbs));
 220
 221         cmap_pass = CMAP_PASS_FROM;
 222         reset_scanner(from_map);
 223         (void) yyparse();
 224         add_charmap_posix();
 225
 226         cmap_pass = CMAP_PASS_TO;
 227         reset_scanner(to_map);
 228         (void) yyparse();
 229 }
 230
 231 void
 232 charmap_dump()
 233 {
 234         charmap_t *cm;
 235         int i;
 236
 237         cm = avl_first(&cmap_mbs);
 238         while (cm != NULL) {
 239                 (void) printf("name=\"%s\"\n", cm->cm_name);
 240
 241                 (void) printf("\timbs=\"");
 242                 for (i = 0; i < cm->cm_frmbs_len; i++)
 243                         (void) printf("\\x%02x", cm->cm_frmbs[i] & 0xFF);
 244                 (void) printf("\"\n");
 245
 246                 (void) printf("\tombs=\"");
 247                 for (i = 0; i < cm->cm_tombs_len; i++)
 248                         (void) printf("\\x%02x", cm->cm_tombs[i] & 0xFF);
 249                 (void) printf("\"\n");
 250
 251                 cm = AVL_NEXT(&cmap_mbs, cm);
 252         }
 253 }
 254
 255 /*
 256  * We parse two charmap files:  First the "from" map, where we build
 257  * cmap_mbs and cmap_sym which we'll later use to translate the input
 258  * stream (mbs encodings) to symbols.  Second, we parse the "to" map,
 259  * where we fill in the tombs members of entries in cmap_sym, (which
 260  * must alread exist) used later to write the output encoding.
 261  */
 262 static void
 263 add_charmap_impl(char *sym, char *mbs, int mbs_len, int nodups)
 264 {
 265
 266         /*
 267          * While parsing both the "from" and "to" cmaps,
 268          * require both the symbol and encoding.
 269          */
 270         if (sym == NULL || mbs == NULL) {
 271                 errf(_("invalid charmap entry"));
 272                 return;
 273         }
 274
 275         switch (cmap_pass) {
 276         case CMAP_PASS_FROM:
 277                 add_charmap_impl_fr(sym, mbs, mbs_len, nodups);
 278                 break;
 279         case CMAP_PASS_TO:
 280                 add_charmap_impl_to(sym, mbs, mbs_len, nodups);
 281                 break;
 282         default:
 283                 abort();
 284                 break;
 285         }
 286 }
 287
 288 static void
 289 add_charmap_impl_fr(char *sym, char *mbs, int mbs_len, int nodups)
 290 {
 291         charmap_t       *m, *n, *s;
 292         avl_index_t     where_sym, where_mbs;
 293
 294         if ((n = calloc(1, sizeof (*n))) == NULL) {
 295                 errf(_("out of memory"));
 296                 return;
 297         }
 298         n->cm_name = sym;
 299
 300         assert(0 < mbs_len && mbs_len <= MB_LEN_MAX);
 301         (void) memcpy(n->cm_frmbs, mbs, mbs_len);
 302         n->cm_frmbs_len = mbs_len;
 303
 304         m = avl_find(&cmap_mbs, n, &where_mbs);
 305         s = avl_find(&cmap_sym, n, &where_sym);
 306
 307         /*
 308          * If we found the symbol, this is a dup.
 309          */
 310         if (s != NULL) {
 311                 if (nodups) {
 312                         warn(_("%s: duplicate character symbol"), sym);
 313                 }
 314                 free(n);
 315                 return;
 316         }
 317
 318         /*
 319          * If we found the mbs, the new one is an alias,
 320          * which we'll add _only_ to the symbol AVL.
 321          */
 322         if (m != NULL) {
 323                 /* The new one is an alias of the original. */
 324                 n->cm_alias_of = m;
 325                 avl_insert(&cmap_sym, n, where_sym);
 326                 return;
 327         }
 328
 329         avl_insert(&cmap_sym, n, where_sym);
 330         avl_insert(&cmap_mbs, n, where_mbs);
 331 }
 332
 333 static void
 334 add_charmap_impl_to(char *sym, char *mbs, int mbs_len, int nodups)
 335 {
 336         charmap_t       srch = {0};
 337         charmap_t       *m;
 338
 339         assert(0 < mbs_len && mbs_len <= MB_LEN_MAX);
 340
 341         srch.cm_name = sym;
 342
 343         m = avl_find(&cmap_sym, &srch, NULL);
 344         if (m == NULL) {
 345                 if (sflag == 0)
 346                         warn(_("%s: symbol not found"), sym);
 347                 return;
 348         }
 349         if (m->cm_alias_of != NULL) {
 350                 m = m->cm_alias_of;
 351
 352                 /* don't warn for dups with aliases */
 353                 if (m->cm_tombs_len != 0)
 354                         return;
 355         }
 356
 357         if (m->cm_tombs_len != 0) {
 358                 if (nodups) {
 359                         warn(_("%s: duplicate encoding for"), sym);
 360                 }
 361                 return;
 362         }
 363
 364         (void) memcpy(m->cm_tombs, mbs, mbs_len);
 365         m->cm_tombs_len = mbs_len;
 366 }
 367
 368 void
 369 add_charmap(char *sym, char *mbs)
 370 {
 371         /* mbs[0] is the length */
 372         int mbs_len = *mbs++;
 373         assert(0 < mbs_len && mbs_len <= MB_LEN_MAX);
 374         add_charmap_impl(sym, mbs, mbs_len, 1);
 375 }
 376
 377
 378 /*
 379  * This is called by the parser with start/end symbol strings (ssym, esym),
 380  * which are allocated in the scanner (T_SYMBOL) and free'd here.
 381  */
 382 void
 383 add_charmap_range(char *ssym, char *esym, char *mbs)
 384 {
 385         int     ls, le;
 386         int     si;
 387         int     sn, en;
 388         int     i;
 389         int     mbs_len;
 390         char    tmbs[MB_LEN_MAX+1];
 391         char    *mb_last;
 392
 393         static const char *digits = "0123456789";
 394
 395         /* mbs[0] is the length */
 396         mbs_len = *mbs++;
 397         assert(0 < mbs_len && mbs_len <= MB_LEN_MAX);
 398         (void) memcpy(tmbs, mbs, mbs_len);
 399         mb_last = tmbs + mbs_len - 1;
 400
 401         ls = strlen(ssym);
 402         le = strlen(esym);
 403
 404         if (((si = strcspn(ssym, digits)) == 0) || (si == ls) ||
 405             (strncmp(ssym, esym, si) != 0) ||
 406             (strspn(ssym + si, digits) != (ls - si)) ||
 407             (strspn(esym + si, digits) != (le - si)) ||
 408             ((sn = atoi(ssym + si)) > ((en = atoi(esym + si))))) {
 409                 errf(_("malformed charmap range"));
 410                 return;
 411         }
 412
 413         ssym[si] = 0;
 414         for (i = sn; i <= en; i++) {
 415                 char *nn;
 416                 (void) asprintf(&nn, "%s%0*u", ssym, ls - si, i);
 417                 if (nn == NULL) {
 418                         errf(_("out of memory"));
 419                         return;
 420                 }
 421
 422                 add_charmap_impl(nn, tmbs, mbs_len, 1);
 423                 (*mb_last)++;
 424         }
 425         free(ssym);
 426         free(esym);
 427 }
 428
 429 void
 430 add_charmap_char(char *name, int c)
 431 {
 432         char mbs[MB_LEN_MAX+1];
 433
 434         mbs[0] = c;
 435         mbs[1] = '\0';
 436         add_charmap_impl(name, mbs, 1, 0);
 437 }
 438
 439 /*
 440  * POSIX insists that certain entries be present, even when not in the
 441  * orginal charmap file.
 442  */
 443 void
 444 add_charmap_posix(void)
 445 {
 446         int     i;
 447
 448         for (i = 0; portable_chars[i].name; i++) {
 449                 add_charmap_char(portable_chars[i].name, portable_chars[i].ch);
 450         }
 451 }
 452
 453 /*
 454  * This is called with a buffer of (typically) MB_LEN_MAX bytes,
 455  * which is potentially a multi-byte symbol, but often contains
 456  * extra bytes. Find and return the longest match in the charmap.
 457  */
 458 static charmap_t *
 459 find_mbs(const char *mbs, int len)
 460 {
 461         charmap_t srch = {0};
 462         charmap_t *cm = NULL;
 463
 464         while (len > 0) {
 465                 (void) memcpy(srch.cm_frmbs, mbs, len);
 466                 srch.cm_frmbs_len = len;
 467                 cm = avl_find(&cmap_mbs, &srch, NULL);
 468                 if (cm != NULL)
 469                         break;
 470                 len--;
 471         }
 472
 473         return (cm);
 474 }
 475
 476 /*
 477  * Return true if this sequence matches the initial part
 478  * of any sequence known in this charmap.
 479  */
 480 static boolean_t
 481 find_mbs_partial(const char *mbs, int len)
 482 {
 483         charmap_t srch = {0};
 484         charmap_t *cm;
 485         avl_index_t where;
 486
 487         (void) memcpy(srch.cm_frmbs, mbs, len);
 488         srch.cm_frmbs_len = len;
 489         cm = avl_find(&cmap_mbs, &srch, &where);
 490         if (cm != NULL) {
 491                 /* full match - not expected, but OK */
 492                 return (B_TRUE);
 493         }
 494         cm = avl_nearest(&cmap_mbs, where, AVL_AFTER);
 495         if (cm != NULL && 0 == memcmp(cm->cm_frmbs, mbs, len))
 496                 return (B_TRUE);
 497
 498         return (B_FALSE);
 499 }
 500
 501 /*
 502  * Do like iconv(3), but with charmaps.
 503  */
 504 size_t
 505 cm_iconv(const char **iptr, size_t *ileft, char **optr, size_t *oleft)
 506 {
 507         charmap_t *cm;
 508         int mbs_len;
 509
 510         /* Ignore state reset requests. */
 511         if (iptr == NULL || *iptr == NULL)
 512                 return (0);
 513
 514         if (*oleft < MB_LEN_MAX) {
 515                 errno = E2BIG;
 516                 return ((size_t)-1);
 517         }
 518
 519         while (*ileft > 0 && *oleft >= MB_LEN_MAX) {
 520                 mbs_len = MB_LEN_MAX;
 521                 if (mbs_len > *ileft)
 522                         mbs_len = *ileft;
 523                 cm = find_mbs(*iptr, mbs_len);
 524                 if (cm == NULL) {
 525                         if (mbs_len < MB_LEN_MAX &&
 526                             find_mbs_partial(*iptr, mbs_len)) {
 527                                 /* incomplete sequence */
 528                                 errno = EINVAL;
 529                         } else {
 530                                 errno = EILSEQ;
 531                         }
 532                         return ((size_t)-1);
 533                 }
 534                 assert(cm->cm_frmbs_len > 0);
 535                 if (cm->cm_tombs_len == 0) {
 536                         if (sflag == 0 && cm->cm_warned == 0) {
 537                                 cm->cm_warned = 1;
 538                                 warn(_("To-map does not encode <%s>\n"),
 539                                     cm->cm_name);
 540                         }
 541                         if (cflag == 0) {
 542                                 errno = EILSEQ;
 543                                 return ((size_t)-1);
 544                         }
 545                         /* just skip this input seq. */
 546                         *iptr  += cm->cm_frmbs_len;
 547                         *ileft -= cm->cm_frmbs_len;
 548                         continue;
 549                 }
 550
 551                 *iptr  += cm->cm_frmbs_len;
 552                 *ileft -= cm->cm_frmbs_len;
 553                 (void) memcpy(*optr, cm->cm_tombs, cm->cm_tombs_len);
 554                 *optr  += cm->cm_tombs_len;
 555                 *oleft -= cm->cm_tombs_len;
 556         }
 557
 558         return (0);
 559 }