extensions/spellcheck/hunspell/src/hashmgr.cpp

   1 /******* BEGIN LICENSE BLOCK *******
   2  * Version: MPL 1.1/GPL 2.0/LGPL 2.1
   3  *
   4  * The contents of this file are subject to the Mozilla Public License Version
   5  * 1.1 (the "License"); you may not use this file except in compliance with
   6  * the License. You may obtain a copy of the License at
   7  * http://www.mozilla.org/MPL/
   8  *
   9  * Software distributed under the License is distributed on an "AS IS" basis,
  10  * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
  11  * for the specific language governing rights and limitations under the
  12  * License.
  13  *
  14  * The Initial Developers of the Original Code are Kevin Hendricks (MySpell)
  15  * and László Németh (Hunspell). Portions created by the Initial Developers
  16  * are Copyright (C) 2002-2005 the Initial Developers. All Rights Reserved.
  17  *
  18  * Contributor(s): Kevin Hendricks (kevin.hendricks@sympatico.ca)
  19  *                 David Einstein (deinst@world.std.com)
  20  *                 László Németh (nemethl@gyorsposta.hu)
  21  *                 Davide Prina
  22  *                 Giuseppe Modugno
  23  *                 Gianluca Turconi
  24  *                 Simon Brouwer
  25  *                 Noll Janos
  26  *                 Biro Arpad
  27  *                 Goldman Eleonora
  28  *                 Sarlos Tamas
  29  *                 Bencsath Boldizsar
  30  *                 Halacsy Peter
  31  *                 Dvornik Laszlo
  32  *                 Gefferth Andras
  33  *                 Nagy Viktor
  34  *                 Varga Daniel
  35  *                 Chris Halls
  36  *                 Rene Engelhard
  37  *                 Bram Moolenaar
  38  *                 Dafydd Jones
  39  *                 Harri Pitkanen
  40  *                 Andras Timar
  41  *                 Tor Lillqvist
  42  *
  43  * Alternatively, the contents of this file may be used under the terms of
  44  * either the GNU General Public License Version 2 or later (the "GPL"), or
  45  * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
  46  * in which case the provisions of the GPL or the LGPL are applicable instead
  47  * of those above. If you wish to allow use of your version of this file only
  48  * under the terms of either the GPL or the LGPL, and not to allow others to
  49  * use your version of this file under the terms of the MPL, indicate your
  50  * decision by deleting the provisions above and replace them with the notice
  51  * and other provisions required by the GPL or the LGPL. If you do not delete
  52  * the provisions above, a recipient may use your version of this file under
  53  * the terms of any one of the MPL, the GPL or the LGPL.
  54  *
  55  ******* END LICENSE BLOCK *******/
  56
  57 #ifndef MOZILLA_CLIENT
  58 #include <cstdlib>
  59 #include <cstring>
  60 #include <cstdio>
  61 #include <cctype>
  62 #else
  63 #include <stdlib.h>
  64 #include <string.h>
  65 #include <stdio.h>
  66 #include <ctype.h>
  67 #endif
  68
  69 #include "atypes.hxx"
  70 #include "csutil.hxx"
  71 #include "hashmgr.hxx"
  72
  73 #ifdef MOZILLA_CLIENT
  74 #ifdef __SUNPRO_CC // for SunONE Studio compiler
  75 using namespace std;
  76 #endif
  77 #else
  78 #ifndef W32
  79 using namespace std;
  80 #endif
  81 #endif
  82
  83 // build a hash table from a munched word list
  84
  85 HashMgr::HashMgr(const char * tpath, const char * apath, const char * key)
  86 {
  87   tablesize = 0;
  88   tableptr = NULL;
  89   flag_mode = FLAG_CHAR;
  90   complexprefixes = 0;
  91   utf8 = 0;
  92   langnum = 0;
  93   lang = NULL;
  94   enc = NULL;
  95   csconv = 0;
  96   ignorechars = NULL;
  97   ignorechars_utf16 = NULL;
  98   ignorechars_utf16_len = 0;
  99   numaliasf = 0;
 100   aliasf = NULL;
 101   numaliasm = 0;
 102   aliasm = NULL;
 103   forbiddenword = FORBIDDENWORD; // forbidden word signing flag
 104   load_config(apath, key);
 105   int ec = load_tables(tpath, key);
 106   if (ec) {
 107     /* error condition - what should we do here */
 108     HUNSPELL_WARNING(stderr, "Hash Manager Error : %d\n",ec);
 109     if (tableptr) {
 110       free(tableptr);
 111       tableptr = NULL;
 112     }
 113     tablesize = 0;
 114   }
 115 }
 116
 117
 118 HashMgr::~HashMgr()
 119 {
 120   if (tableptr) {
 121     // now pass through hash table freeing up everything
 122     // go through column by column of the table
 123     for (int i=0; i < tablesize; i++) {
 124       struct hentry * pt = tableptr[i];
 125       struct hentry * nt = NULL;
 126       while(pt) {
 127         nt = pt->next;
 128         if (pt->astr && (!aliasf || TESTAFF(pt->astr, ONLYUPCASEFLAG, pt->alen))) free(pt->astr);
 129         free(pt);
 130         pt = nt;
 131       }
 132     }
 133     free(tableptr);
 134   }
 135   tablesize = 0;
 136
 137   if (aliasf) {
 138     for (int j = 0; j < (numaliasf); j++) free(aliasf[j]);
 139     free(aliasf);
 140     aliasf = NULL;
 141     if (aliasflen) {
 142       free(aliasflen);
 143       aliasflen = NULL;
 144     }
 145   }
 146   if (aliasm) {
 147     for (int j = 0; j < (numaliasm); j++) free(aliasm[j]);
 148     free(aliasm);
 149     aliasm = NULL;
 150   }
 151
 152 #ifndef OPENOFFICEORG
 153 #ifndef MOZILLA_CLIENT
 154   if (utf8) free_utf_tbl();
 155 #endif
 156 #endif
 157
 158   if (enc) free(enc);
 159   if (lang) free(lang);
 160
 161   if (ignorechars) free(ignorechars);
 162   if (ignorechars_utf16) free(ignorechars_utf16);
 163 }
 164
 165 // lookup a root word in the hashtable
 166
 167 struct hentry * HashMgr::lookup(const char *word) const
 168 {
 169     struct hentry * dp;
 170     if (tableptr) {
 171        dp = tableptr[hash(word)];
 172        if (!dp) return NULL;
 173        for (  ;  dp != NULL;  dp = dp->next) {
 174           if (strcmp(word,&(dp->word)) == 0) return dp;
 175        }
 176     }
 177     return NULL;
 178 }
 179
 180 // add a word to the hash table (private)
 181 int HashMgr::add_word(const char * word, int wbl, int wcl, unsigned short * aff,
 182     int al, const char * desc, bool onlyupcase)
 183 {
 184     bool upcasehomonym = false;
 185     int descl = desc ? (aliasm ? sizeof(short) : strlen(desc) + 1) : 0;
 186     // variable-length hash record with word and optional fields
 187     struct hentry* hp =
 188         (struct hentry *) malloc (sizeof(struct hentry) + wbl + descl);
 189     if (!hp) return 1;
 190     char * hpw = &(hp->word);
 191     strcpy(hpw, word);
 192     if (ignorechars != NULL) {
 193       if (utf8) {
 194         remove_ignored_chars_utf(hpw, ignorechars_utf16, ignorechars_utf16_len);
 195       } else {
 196         remove_ignored_chars(hpw, ignorechars);
 197       }
 198     }
 199     if (complexprefixes) {
 200         if (utf8) reverseword_utf(hpw); else reverseword(hpw);
 201     }
 202
 203     int i = hash(hpw);
 204
 205     hp->blen = (unsigned char) wbl;
 206     hp->clen = (unsigned char) wcl;
 207     hp->alen = (short) al;
 208     hp->astr = aff;
 209     hp->next = NULL;
 210     hp->next_homonym = NULL;
 211
 212     // store the description string or its pointer
 213     if (desc) {
 214         hp->var = H_OPT;
 215         if (aliasm) {
 216             hp->var += H_OPT_ALIASM;
 217             store_pointer(hpw + wbl + 1, get_aliasm(atoi(desc)));
 218         } else {
 219             strcpy(hpw + wbl + 1, desc);
 220             if (complexprefixes) {
 221                 if (utf8) reverseword_utf(HENTRY_DATA(hp));
 222                 else reverseword(HENTRY_DATA(hp));
 223             }
 224         }
 225         if (strstr(HENTRY_DATA(hp), MORPH_PHON)) hp->var += H_OPT_PHON;
 226     } else hp->var = 0;
 227
 228        struct hentry * dp = tableptr[i];
 229        if (!dp) {
 230          tableptr[i] = hp;
 231          return 0;
 232        }
 233        while (dp->next != NULL) {
 234          if ((!dp->next_homonym) && (strcmp(&(hp->word), &(dp->word)) == 0)) {
 235             // remove hidden onlyupcase homonym
 236             if (!onlyupcase) {
 237                 if ((dp->astr) && TESTAFF(dp->astr, ONLYUPCASEFLAG, dp->alen)) {
 238                     free(dp->astr);
 239                     dp->astr = hp->astr;
 240                     dp->alen = hp->alen;
 241                     free(hp);
 242                     return 0;
 243                 } else {
 244                     dp->next_homonym = hp;
 245                 }
 246             } else {
 247                 upcasehomonym = true;
 248             }
 249          }
 250          dp=dp->next;
 251        }
 252        if (strcmp(&(hp->word), &(dp->word)) == 0) {
 253             // remove hidden onlyupcase homonym
 254             if (!onlyupcase) {
 255                 if ((dp->astr) && TESTAFF(dp->astr, ONLYUPCASEFLAG, dp->alen)) {
 256                     free(dp->astr);
 257                     dp->astr = hp->astr;
 258                     dp->alen = hp->alen;
 259                     free(hp);
 260                     return 0;
 261                 } else {
 262                     dp->next_homonym = hp;
 263                 }
 264             } else {
 265                 upcasehomonym = true;
 266             }
 267        }
 268        if (!upcasehomonym) {
 269             dp->next = hp;
 270        } else {
 271             // remove hidden onlyupcase homonym
 272             if (hp->astr) free(hp->astr);
 273             free(hp);
 274        }
 275     return 0;
 276 }
 277
 278 int HashMgr::add_hidden_capitalized_word(char * word, int wbl, int wcl,
 279     unsigned short * flags, int al, char * dp, int captype)
 280 {
 281     // add inner capitalized forms to handle the following allcap forms:
 282     // Mixed caps: OpenOffice.org -> OPENOFFICE.ORG
 283     // Allcaps with suffixes: CIA's -> CIA'S
 284     if (((captype == HUHCAP) || (captype == HUHINITCAP) ||
 285       ((captype == ALLCAP) && (flags != NULL))) &&
 286       !((flags != NULL) && TESTAFF(flags, forbiddenword, al))) {
 287           unsigned short * flags2 = (unsigned short *) malloc (sizeof(unsigned short) * (al+1));
 288           if (!flags2) return 1;
 289           if (al) memcpy(flags2, flags, al * sizeof(unsigned short));
 290           flags2[al] = ONLYUPCASEFLAG;
 291           if (utf8) {
 292               char st[BUFSIZE];
 293               w_char w[BUFSIZE];
 294               int wlen = u8_u16(w, BUFSIZE, word);
 295               mkallsmall_utf(w, wlen, langnum);
 296               mkallcap_utf(w, 1, langnum);
 297               u16_u8(st, BUFSIZE, w, wlen);
 298               return add_word(st,wbl,wcl,flags2,al+1,dp, true);
 299            } else {
 300                mkallsmall(word, csconv);
 301                mkinitcap(word, csconv);
 302                return add_word(word,wbl,wcl,flags2,al+1,dp, true);
 303            }
 304     }
 305     return 0;
 306 }
 307
 308 // detect captype and modify word length for UTF-8 encoding
 309 int HashMgr::get_clen_and_captype(const char * word, int wbl, int * captype) {
 310     int len;
 311     if (utf8) {
 312       w_char dest_utf[BUFSIZE];
 313       len = u8_u16(dest_utf, BUFSIZE, word);
 314       *captype = get_captype_utf8(dest_utf, len, langnum);
 315     } else {
 316       len = wbl;
 317       *captype = get_captype((char *) word, len, csconv);
 318     }
 319     return len;
 320 }
 321
 322 // remove word (personal dictionary function for standalone applications)
 323 int HashMgr::remove(const char * word)
 324 {
 325     struct hentry * dp = lookup(word);
 326     while (dp) {
 327         if (dp->alen == 0 || !TESTAFF(dp->astr, forbiddenword, dp->alen)) {
 328             unsigned short * flags =
 329                 (unsigned short *) malloc(sizeof(short *) * (dp->alen + 1));
 330             if (!flags) return 1;
 331             for (int i = 0; i < dp->alen; i++) flags[i] = dp->astr[i];
 332             flags[dp->alen] = forbiddenword;
 333             dp->astr = flags;
 334             dp->alen++;
 335             flag_qsort(flags, 0, dp->alen);
 336         }
 337         dp = dp->next_homonym;
 338     }
 339     return 0;
 340 }
 341
 342 /* remove forbidden flag to add a personal word to the hash */
 343 int HashMgr::remove_forbidden_flag(const char * word) {
 344     struct hentry * dp = lookup(word);
 345     if (!dp) return 1;
 346     while (dp) {
 347          if (dp->astr && TESTAFF(dp->astr, forbiddenword, dp->alen)) {
 348             if (dp->alen == 1) dp->alen = 0; // XXX forbidden words of personal dic.
 349             else {
 350                 unsigned short * flags2 =
 351                     (unsigned short *) malloc(sizeof(short *) * (dp->alen - 1));
 352                 if (!flags2) return 1;
 353                 int i, j = 0;
 354                 for (i = 0; i < dp->alen; i++) {
 355                     if (dp->astr[i] != forbiddenword) flags2[j++] = dp->astr[i];
 356                 }
 357                 dp->alen--;
 358                 dp->astr = flags2; // XXX allowed forbidden words
 359             }
 360          }
 361          dp = dp->next_homonym;
 362        }
 363    return 0;
 364 }
 365
 366 // add a custom dic. word to the hash table (public)
 367 int HashMgr::add(const char * word)
 368 {
 369     unsigned short * flags = NULL;
 370     int al = 0;
 371     if (remove_forbidden_flag(word)) {
 372         int captype;
 373         int wbl = strlen(word);
 374         int wcl = get_clen_and_captype(word, wbl, &captype);
 375         add_word(word, wbl, wcl, flags, al, NULL, false);
 376         return add_hidden_capitalized_word((char *) word, wbl, wcl, flags, al, NULL, captype);
 377     }
 378     return 0;
 379 }
 380
 381 int HashMgr::add_with_affix(const char * word, const char * example)
 382 {
 383     // detect captype and modify word length for UTF-8 encoding
 384     struct hentry * dp = lookup(example);
 385     remove_forbidden_flag(word);
 386     if (dp && dp->astr) {
 387         int captype;
 388         int wbl = strlen(word);
 389         int wcl = get_clen_and_captype(word, wbl, &captype);
 390         if (aliasf) {
 391             add_word(word, wbl, wcl, dp->astr, dp->alen, NULL, false);
 392         } else {
 393             unsigned short * flags = (unsigned short *) malloc (dp->alen * sizeof(short));
 394             if (flags) {
 395                 memcpy((void *) flags, (void *) dp->astr, dp->alen * sizeof(short));
 396                 add_word(word, wbl, wcl, flags, dp->alen, NULL, false);
 397             } else return 1;
 398         }
 399         return add_hidden_capitalized_word((char *) word, wbl, wcl, dp->astr, dp->alen, NULL, captype);
 400     }
 401     return 1;
 402 }
 403
 404 // walk the hash table entry by entry - null at end
 405 // initialize: col=-1; hp = NULL; hp = walk_hashtable(&col, hp);
 406 struct hentry * HashMgr::walk_hashtable(int &col, struct hentry * hp) const
 407 {
 408   if (hp && hp->next != NULL) return hp->next;
 409   for (col++; col < tablesize; col++) {
 410     if (tableptr[col]) return tableptr[col];
 411   }
 412   // null at end and reset to start
 413   col = -1;
 414   return NULL;
 415 }
 416
 417 // load a munched word list and build a hash table on the fly
 418 int HashMgr::load_tables(const char * tpath, const char * key)
 419 {
 420   int al;
 421   char * ap;
 422   char * dp;
 423   char * dp2;
 424   unsigned short * flags;
 425   char * ts;
 426
 427   // open dictionary file
 428   FileMgr * dict = new FileMgr(tpath, key);
 429   if (dict == NULL) return 1;
 430
 431   // first read the first line of file to get hash table size */
 432   if (!(ts = dict->getline())) {
 433     HUNSPELL_WARNING(stderr, "error: empty dic file\n");
 434     delete dict;
 435     return 2;
 436   }
 437   mychomp(ts);
 438
 439   /* remove byte order mark */
 440   if (strncmp(ts,"\xEF\xBB\xBF",3) == 0) {
 441     memmove(ts, ts+3, strlen(ts+3)+1);
 442     HUNSPELL_WARNING(stderr, "warning: dic file begins with byte order mark: possible incompatibility with old Hunspell versions\n");
 443   }
 444
 445   tablesize = atoi(ts);
 446   if (tablesize == 0) {
 447     HUNSPELL_WARNING(stderr, "error: line 1: missing or bad word count in the dic file\n");
 448     delete dict;
 449     return 4;
 450   }
 451   tablesize = tablesize + 5 + USERWORD;
 452   if ((tablesize %2) == 0) tablesize++;
 453
 454   // allocate the hash table
 455   tableptr = (struct hentry **) malloc(tablesize * sizeof(struct hentry *));
 456   if (! tableptr) {
 457     delete dict;
 458     return 3;
 459   }
 460   for (int i=0; i<tablesize; i++) tableptr[i] = NULL;
 461
 462   // loop through all words on much list and add to hash
 463   // table and create word and affix strings
 464
 465   while ((ts = dict->getline())) {
 466     mychomp(ts);
 467     // split each line into word and morphological description
 468     dp = ts;
 469     while ((dp = strchr(dp, ':'))) {
 470         if ((dp > ts + 3) && (*(dp - 3) == ' ' || *(dp - 3) == '\t')) {
 471             for (dp -= 4; dp >= ts && (*dp == ' ' || *dp == '\t'); dp--);
 472             if (dp < ts) { // missing word
 473                 dp = NULL;
 474             } else {
 475                 *(dp + 1) = '\0';
 476                 dp = dp + 2;
 477             }
 478             break;
 479         }
 480         dp++;
 481     }
 482
 483     // tabulator is the old morphological field separator
 484     dp2 = strchr(ts, '\t');
 485     if (dp2 && (!dp || dp2 < dp)) {
 486         *dp2 = '\0';
 487         dp = dp2 + 1;
 488     }
 489
 490     // split each line into word and affix char strings
 491     // "\/" signs slash in words (not affix separator)
 492     // "/" at beginning of the line is word character (not affix separator)
 493     ap = strchr(ts,'/');
 494     while (ap) {
 495         if (ap == ts) {
 496             ap++;
 497             continue;
 498         } else if (*(ap - 1) != '\\') break;
 499         // replace "\/" with "/"
 500         for (char * sp = ap - 1; *sp; *sp = *(sp + 1), sp++);
 501         ap = strchr(ap,'/');
 502     }
 503
 504     if (ap) {
 505       *ap = '\0';
 506       if (aliasf) {
 507         int index = atoi(ap + 1);
 508         al = get_aliasf(index, &flags, dict);
 509         if (!al) {
 510             HUNSPELL_WARNING(stderr, "error: line %d: bad flag vector alias\n", dict->getlinenum());
 511             *ap = '\0';
 512         }
 513       } else {
 514         al = decode_flags(&flags, ap + 1, dict);
 515         flag_qsort(flags, 0, al);
 516       }
 517     } else {
 518       al = 0;
 519       ap = NULL;
 520       flags = NULL;
 521     }
 522
 523     int captype;
 524     int wbl = strlen(ts);
 525     int wcl = get_clen_and_captype(ts, wbl, &captype);
 526     // add the word and its index plus its capitalized form optionally
 527     if (add_word(ts,wbl,wcl,flags,al,dp, false) ||
 528         add_hidden_capitalized_word(ts, wbl, wcl, flags, al, dp, captype)) {
 529         delete dict;
 530         return 5;
 531     }
 532   }
 533
 534   delete dict;
 535   return 0;
 536 }
 537
 538 // the hash function is a simple load and rotate
 539 // algorithm borrowed
 540
 541 int HashMgr::hash(const char * word) const
 542 {
 543     long  hv = 0;
 544     for (int i=0; i < 4  &&  *word != 0; i++)
 545         hv = (hv << 8) | (*word++);
 546     while (*word != 0) {
 547       ROTATE(hv,ROTATE_LEN);
 548       hv ^= (*word++);
 549     }
 550     return (unsigned long) hv % tablesize;
 551 }
 552
 553 int HashMgr::decode_flags(unsigned short ** result, char * flags, FileMgr * af) {
 554     int len;
 555     switch (flag_mode) {
 556       case FLAG_LONG: { // two-character flags (1x2yZz -> 1x 2y Zz)
 557         len = strlen(flags);
 558         if (len%2 == 1) HUNSPELL_WARNING(stderr, "error: line %d: bad flagvector\n", af->getlinenum());
 559         len /= 2;
 560         *result = (unsigned short *) malloc(len * sizeof(short));
 561         if (!*result) return -1;
 562         for (int i = 0; i < len; i++) {
 563             (*result)[i] = (((unsigned short) flags[i * 2]) << 8) + (unsigned short) flags[i * 2 + 1];
 564         }
 565         break;
 566       }
 567       case FLAG_NUM: { // decimal numbers separated by comma (4521,23,233 -> 4521 23 233)
 568         int i;
 569         len = 1;
 570         char * src = flags;
 571         unsigned short * dest;
 572         char * p;
 573         for (p = flags; *p; p++) {
 574           if (*p == ',') len++;
 575         }
 576         *result = (unsigned short *) malloc(len * sizeof(short));
 577         if (!*result) return -1;
 578         dest = *result;
 579         for (p = flags; *p; p++) {
 580           if (*p == ',') {
 581             i = atoi(src);
 582             if (i >= DEFAULTFLAGS) HUNSPELL_WARNING(stderr, "error: line %d: flag id %d is too large (max: %d)\n",
 583               af->getlinenum(), i, DEFAULTFLAGS - 1);
 584             *dest = (unsigned short) i;
 585             if (*dest == 0) HUNSPELL_WARNING(stderr, "error: line %d: 0 is wrong flag id\n", af->getlinenum());
 586             src = p + 1;
 587             dest++;
 588           }
 589         }
 590         i = atoi(src);
 591         if (i >= DEFAULTFLAGS) HUNSPELL_WARNING(stderr, "error: line %d: flag id %d is too large (max: %d)\n",
 592           af->getlinenum(), i, DEFAULTFLAGS - 1);
 593         *dest = (unsigned short) i;
 594         if (*dest == 0) HUNSPELL_WARNING(stderr, "error: line %d: 0 is wrong flag id\n", af->getlinenum());
 595         break;
 596       }
 597       case FLAG_UNI: { // UTF-8 characters
 598         w_char w[BUFSIZE/2];
 599         len = u8_u16(w, BUFSIZE/2, flags);
 600         *result = (unsigned short *) malloc(len * sizeof(short));
 601         if (!*result) return -1;
 602         memcpy(*result, w, len * sizeof(short));
 603         break;
 604       }
 605       default: { // Ispell's one-character flags (erfg -> e r f g)
 606         unsigned short * dest;
 607         len = strlen(flags);
 608         *result = (unsigned short *) malloc(len * sizeof(short));
 609         if (!*result) return -1;
 610         dest = *result;
 611         for (unsigned char * p = (unsigned char *) flags; *p; p++) {
 612           *dest = (unsigned short) *p;
 613           dest++;
 614         }
 615       }
 616     }
 617     return len;
 618 }
 619
 620 unsigned short HashMgr::decode_flag(const char * f) {
 621     unsigned short s = 0;
 622     int i;
 623     switch (flag_mode) {
 624       case FLAG_LONG:
 625         s = ((unsigned short) f[0] << 8) + (unsigned short) f[1];
 626         break;
 627       case FLAG_NUM:
 628         i = atoi(f);
 629         if (i >= DEFAULTFLAGS) HUNSPELL_WARNING(stderr, "error: flag id %d is too large (max: %d)\n", i, DEFAULTFLAGS - 1);
 630         s = (unsigned short) i;
 631         break;
 632       case FLAG_UNI:
 633         u8_u16((w_char *) &s, 1, f);
 634         break;
 635       default:
 636         s = (unsigned short) *((unsigned char *)f);
 637     }
 638     if (s == 0) HUNSPELL_WARNING(stderr, "error: 0 is wrong flag id\n");
 639     return s;
 640 }
 641
 642 char * HashMgr::encode_flag(unsigned short f) {
 643     unsigned char ch[10];
 644     if (f==0) return mystrdup("(NULL)");
 645     if (flag_mode == FLAG_LONG) {
 646         ch[0] = (unsigned char) (f >> 8);
 647         ch[1] = (unsigned char) (f - ((f >> 8) << 8));
 648         ch[2] = '\0';
 649     } else if (flag_mode == FLAG_NUM) {
 650         sprintf((char *) ch, "%d", f);
 651     } else if (flag_mode == FLAG_UNI) {
 652         u16_u8((char *) &ch, 10, (w_char *) &f, 1);
 653     } else {
 654         ch[0] = (unsigned char) (f);
 655         ch[1] = '\0';
 656     }
 657     return mystrdup((char *) ch);
 658 }
 659
 660 // read in aff file and set flag mode
 661 int  HashMgr::load_config(const char * affpath, const char * key)
 662 {
 663   char * line; // io buffers
 664   int firstline = 1;
 665
 666   // open the affix file
 667   FileMgr * afflst = new FileMgr(affpath, key);
 668   if (!afflst) {
 669     HUNSPELL_WARNING(stderr, "Error - could not open affix description file %s\n",affpath);
 670     return 1;
 671   }
 672
 673     // read in each line ignoring any that do not
 674     // start with a known line type indicator
 675
 676     while ((line = afflst->getline())) {
 677         mychomp(line);
 678
 679        /* remove byte order mark */
 680        if (firstline) {
 681          firstline = 0;
 682          if (strncmp(line,"\xEF\xBB\xBF",3) == 0) memmove(line, line+3, strlen(line+3)+1);
 683        }
 684
 685         /* parse in the try string */
 686         if ((strncmp(line,"FLAG",4) == 0) && isspace(line[4])) {
 687             if (flag_mode != FLAG_CHAR) {
 688                 HUNSPELL_WARNING(stderr, "error: line %d: multiple definitions of the FLAG affix file parameter\n", afflst->getlinenum());
 689             }
 690             if (strstr(line, "long")) flag_mode = FLAG_LONG;
 691             if (strstr(line, "num")) flag_mode = FLAG_NUM;
 692             if (strstr(line, "UTF-8")) flag_mode = FLAG_UNI;
 693             if (flag_mode == FLAG_CHAR) {
 694                 HUNSPELL_WARNING(stderr, "error: line %d: FLAG needs `num', `long' or `UTF-8' parameter\n", afflst->getlinenum());
 695             }
 696         }
 697         if (strncmp(line,"FORBIDDENWORD",13) == 0) {
 698           char * st = NULL;
 699           if (parse_string(line, &st, afflst->getlinenum())) {
 700              delete afflst;
 701              return 1;
 702           }
 703           forbiddenword = decode_flag(st);
 704           free(st);
 705         }
 706         if (strncmp(line, "SET", 3) == 0) {
 707           if (parse_string(line, &enc, afflst->getlinenum())) {
 708              delete afflst;
 709              return 1;
 710           }
 711           if (strcmp(enc, "UTF-8") == 0) {
 712             utf8 = 1;
 713 #ifndef OPENOFFICEORG
 714 #ifndef MOZILLA_CLIENT
 715             initialize_utf_tbl();
 716 #endif
 717 #endif
 718           } else csconv = get_current_cs(enc);
 719         }
 720         if (strncmp(line, "LANG", 4) == 0) {
 721           if (parse_string(line, &lang, afflst->getlinenum())) {
 722              delete afflst;
 723              return 1;
 724           }
 725           langnum = get_lang_num(lang);
 726         }
 727
 728        /* parse in the ignored characters (for example, Arabic optional diacritics characters */
 729        if (strncmp(line,"IGNORE",6) == 0) {
 730           if (parse_array(line, &ignorechars, &ignorechars_utf16,
 731                  &ignorechars_utf16_len, utf8, afflst->getlinenum())) {
 732              delete afflst;
 733              return 1;
 734           }
 735        }
 736
 737        if ((strncmp(line,"AF",2) == 0) && isspace(line[2])) {
 738           if (parse_aliasf(line, afflst)) {
 739              delete afflst;
 740              return 1;
 741           }
 742        }
 743
 744        if ((strncmp(line,"AM",2) == 0) && isspace(line[2])) {
 745           if (parse_aliasm(line, afflst)) {
 746              delete afflst;
 747              return 1;
 748           }
 749        }
 750
 751        if (strncmp(line,"COMPLEXPREFIXES",15) == 0) complexprefixes = 1;
 752        if (((strncmp(line,"SFX",3) == 0) || (strncmp(line,"PFX",3) == 0)) && isspace(line[3])) break;
 753     }
 754     if (csconv == NULL) csconv = get_current_cs(SPELL_ENCODING);
 755     delete afflst;
 756     return 0;
 757 }
 758
 759 /* parse in the ALIAS table */
 760 int  HashMgr::parse_aliasf(char * line, FileMgr * af)
 761 {
 762    if (numaliasf != 0) {
 763       HUNSPELL_WARNING(stderr, "error: line %d: multiple table definitions\n", af->getlinenum());
 764       return 1;
 765    }
 766    char * tp = line;
 767    char * piece;
 768    int i = 0;
 769    int np = 0;
 770    piece = mystrsep(&tp, 0);
 771    while (piece) {
 772        if (*piece != '\0') {
 773           switch(i) {
 774              case 0: { np++; break; }
 775              case 1: {
 776                        numaliasf = atoi(piece);
 777                        if (numaliasf < 1) {
 778                           numaliasf = 0;
 779                           aliasf = NULL;
 780                           aliasflen = NULL;
 781                           HUNSPELL_WARNING(stderr, "error: line %d: bad entry number\n", af->getlinenum());
 782                           return 1;
 783                        }
 784                        aliasf = (unsigned short **) malloc(numaliasf * sizeof(unsigned short *));
 785                        aliasflen = (unsigned short *) malloc(numaliasf * sizeof(short));
 786                        if (!aliasf || !aliasflen) {
 787                           numaliasf = 0;
 788                           if (aliasf) free(aliasf);
 789                           if (aliasflen) free(aliasflen);
 790                           aliasf = NULL;
 791                           aliasflen = NULL;
 792                           return 1;
 793                        }
 794                        np++;
 795                        break;
 796                      }
 797              default: break;
 798           }
 799           i++;
 800        }
 801        piece = mystrsep(&tp, 0);
 802    }
 803    if (np != 2) {
 804       numaliasf = 0;
 805       free(aliasf);
 806       free(aliasflen);
 807       aliasf = NULL;
 808       aliasflen = NULL;
 809       HUNSPELL_WARNING(stderr, "error: line %d: missing data\n", af->getlinenum());
 810       return 1;
 811    }
 812
 813    /* now parse the numaliasf lines to read in the remainder of the table */
 814    char * nl;
 815    for (int j=0; j < numaliasf; j++) {
 816         if (!(nl = af->getline())) return 1;
 817         mychomp(nl);
 818         tp = nl;
 819         i = 0;
 820         aliasf[j] = NULL;
 821         aliasflen[j] = 0;
 822         piece = mystrsep(&tp, 0);
 823         while (piece) {
 824            if (*piece != '\0') {
 825                switch(i) {
 826                   case 0: {
 827                              if (strncmp(piece,"AF",2) != 0) {
 828                                  numaliasf = 0;
 829                                  free(aliasf);
 830                                  free(aliasflen);
 831                                  aliasf = NULL;
 832                                  aliasflen = NULL;
 833                                  HUNSPELL_WARNING(stderr, "error: line %d: table is corrupt\n", af->getlinenum());
 834                                  return 1;
 835                              }
 836                              break;
 837                           }
 838                   case 1: {
 839                             aliasflen[j] = (unsigned short) decode_flags(&(aliasf[j]), piece, af);
 840                             flag_qsort(aliasf[j], 0, aliasflen[j]);
 841                             break;
 842                           }
 843                   default: break;
 844                }
 845                i++;
 846            }
 847            piece = mystrsep(&tp, 0);
 848         }
 849         if (!aliasf[j]) {
 850              free(aliasf);
 851              free(aliasflen);
 852              aliasf = NULL;
 853              aliasflen = NULL;
 854              numaliasf = 0;
 855              HUNSPELL_WARNING(stderr, "error: line %d: table is corrupt\n", af->getlinenum());
 856              return 1;
 857         }
 858    }
 859    return 0;
 860 }
 861
 862 int HashMgr::is_aliasf() {
 863     return (aliasf != NULL);
 864 }
 865
 866 int HashMgr::get_aliasf(int index, unsigned short ** fvec, FileMgr * af) {
 867     if ((index > 0) && (index <= numaliasf)) {
 868         *fvec = aliasf[index - 1];
 869         return aliasflen[index - 1];
 870     }
 871     HUNSPELL_WARNING(stderr, "error: line %d: bad flag alias index: %d\n", af->getlinenum(), index);
 872     *fvec = NULL;
 873     return 0;
 874 }
 875
 876 /* parse morph alias definitions */
 877 int  HashMgr::parse_aliasm(char * line, FileMgr * af)
 878 {
 879    if (numaliasm != 0) {
 880       HUNSPELL_WARNING(stderr, "error: line %d: multiple table definitions\n", af->getlinenum());
 881       return 1;
 882    }
 883    char * tp = line;
 884    char * piece;
 885    int i = 0;
 886    int np = 0;
 887    piece = mystrsep(&tp, 0);
 888    while (piece) {
 889        if (*piece != '\0') {
 890           switch(i) {
 891              case 0: { np++; break; }
 892              case 1: {
 893                        numaliasm = atoi(piece);
 894                        if (numaliasm < 1) {
 895                           HUNSPELL_WARNING(stderr, "error: line %d: bad entry number\n", af->getlinenum());
 896                           return 1;
 897                        }
 898                        aliasm = (char **) malloc(numaliasm * sizeof(char *));
 899                        if (!aliasm) {
 900                           numaliasm = 0;
 901                           return 1;
 902                        }
 903                        np++;
 904                        break;
 905                      }
 906              default: break;
 907           }
 908           i++;
 909        }
 910        piece = mystrsep(&tp, 0);
 911    }
 912    if (np != 2) {
 913       numaliasm = 0;
 914       free(aliasm);
 915       aliasm = NULL;
 916       HUNSPELL_WARNING(stderr, "error: line %d: missing data\n", af->getlinenum());
 917       return 1;
 918    }
 919
 920    /* now parse the numaliasm lines to read in the remainder of the table */
 921    char * nl = line;
 922    for (int j=0; j < numaliasm; j++) {
 923         if (!(nl = af->getline())) return 1;
 924         mychomp(nl);
 925         tp = nl;
 926         i = 0;
 927         aliasm[j] = NULL;
 928         piece = mystrsep(&tp, ' ');
 929         while (piece) {
 930            if (*piece != '\0') {
 931                switch(i) {
 932                   case 0: {
 933                              if (strncmp(piece,"AM",2) != 0) {
 934                                  HUNSPELL_WARNING(stderr, "error: line %d: table is corrupt\n", af->getlinenum());
 935                                  numaliasm = 0;
 936                                  free(aliasm);
 937                                  aliasm = NULL;
 938                                  return 1;
 939                              }
 940                              break;
 941                           }
 942                   case 1: {
 943                             // add the remaining of the line
 944                             if (*tp) {
 945                                 *(tp - 1) = ' ';
 946                                 tp = tp + strlen(tp);
 947                             }
 948                             if (complexprefixes) {
 949                                 if (utf8) reverseword_utf(piece);
 950                                     else reverseword(piece);
 951                             }
 952                             aliasm[j] = mystrdup(piece);
 953                             if (!aliasm[j]) {
 954                                  numaliasm = 0;
 955                                  free(aliasm);
 956                                  aliasm = NULL;
 957                                  return 1;
 958                             }
 959                             break; }
 960                   default: break;
 961                }
 962                i++;
 963            }
 964            piece = mystrsep(&tp, ' ');
 965         }
 966         if (!aliasm[j]) {
 967              numaliasm = 0;
 968              free(aliasm);
 969              aliasm = NULL;
 970              HUNSPELL_WARNING(stderr, "error: line %d: table is corrupt\n", af->getlinenum());
 971              return 1;
 972         }
 973    }
 974    return 0;
 975 }
 976
 977 int HashMgr::is_aliasm() {
 978     return (aliasm != NULL);
 979 }
 980
 981 char * HashMgr::get_aliasm(int index) {
 982     if ((index > 0) && (index <= numaliasm)) return aliasm[index - 1];
 983     HUNSPELL_WARNING(stderr, "error: bad morph. alias index: %d\n", index);
 984     return NULL;
 985 }