src/tagger.cc

   1 /*
   2  * Copyright (C) 2004 Jesus Gimenez, Lluis Marquez and Senen Moya
   3  *
   4  * This library is free software; you can redistribute it and/or
   5  * modify it under the terms of the GNU Lesser General Public
   6  * License as published by the Free Software Foundation; either
   7  * version 2.1 of the License, or (at your option) any later version.
   8  *
   9  * This library is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  12  * Lesser General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU Lesser General Public
  15  * License along with this library; if not, write to the Free Software
  16  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  17  */
  18
  19 #include "tagger.h"
  20 #include "strategies.h"
  21 #include "hash.h"
  22 #include "list.h"
  23 #include "weight.h"
  24 #include "dict.h"
  25 #include "swindow.h"
  26 #include "marks.h"
  27 #include "common.h"
  28 #include "nodo.h"
  29
  30 #include <sstream>
  31 #include <string.h>
  32 #include <stdio.h>
  33
  34
  35 /***************************************************************/
  36
  37 struct tms  tbuffStartUp,tbuffEndStartUp;
  38 clock_t startUpTime,endStartUpTime;
  39 double  sysFexTime=0, usrFexTime=0,realFexTime=0;
  40 double  sysSVMTime=0, usrSVMTime=0,realSVMTime=0;
  41
  42 /***************************************************************/
  43
  44 extern int verbose;
  45
  46
  47 /***************************************************************/
  48
  49 hash_t<weight_node_t*> *tagger::taggerCreateBiasHash(const std::string& name)
  50 {
  51   hash_t<weight_node_t*> *bias = new hash_t<weight_node_t*>;
  52   int i=0;
  53   char c=' ';
  54   weight_node_t *w;
  55   FILE *f;
  56
  57   bias->hash_init(40);
  58
  59   if ((f = fopen(name.c_str(), "rt"))== NULL)
  60   {
  61     std::cerr << "Error opening file: "<<name<<std::endl;
  62     exit(0);
  63   }
  64
  65   while (!feof(f))
  66   {
  67     c = fgetc(f);
  68     if (c!='#')
  69     {
  70       w = new weight_node_t;
  71       std::string weight;
  72       i=0;
  73       w->pos = "";
  74       while ((c!='\n') && (!feof(f)))
  75       {
  76         if (c!=' ' && c!='\n' && c!='\t' && i==1)  weight.push_back(c);
  77         else if (c!=' ' && c!='\n' && c!='\t' && i==0)
  78         {
  79           if (c!=':') w->pos += c;
  80           else i=1;
  81         }
  82         c = fgetc(f);
  83       }
  84       w->data = (long double)0;
  85       std::istringstream iss(weight);
  86       iss >> w->data;
  87       bias->hash_insert(w->pos,w);
  88     } //end if
  89     else while((c=fgetc(f))!='\n');
  90   }
  91   fclose(f);
  92   return bias;
  93 }
  94
  95 /***************************************************************/
  96
  97 tagger::tagger(const std::string& model) : stk(), sw(0)
  98 {
  99   std::string name;
 100   flow = "LR";
 101   taggerShowScoresFlag = false;
 102   taggerShowCommentsFlag = true;
 103   taggerNumLaps = 1;
 104   taggerKFilter = 0;
 105   taggerUFilter = 0;
 106   taggerStrategy = STRA_1P_DEFAULT; //modstrat //0;
 107   taggerWinIndex = -1;
 108   taggerWinLength = -1;
 109   taggerModelName = model;
 110   taggerBackupDict ="";
 111 }
 112
 113 /***************************************************************/
 114
 115 void tagger::taggerLoadModels(models_t *model, int taggerNumModel)
 116 {
 117   std::string flow2,flow1;
 118   std::ostringstream name;
 119
 120   //Cargamos la lista de "features" para palabras conocidas
 121   name << taggerModelName << ".A" << taggerNumModel;
 122   if (verbose)  std::cerr << std::endl << "Loading FEATURES FOR KNOWN WORDS from < "<<name.str()<<" >"<< std::endl;
 123   createFeatureList(name.str(),&model->featureList);
 124   //Cargamos la lista de "features" para palabras desconocidas
 125   name.str("");
 126   name << taggerModelName << ".A" << taggerNumModel << ".UNK";
 127   if (verbose)  std::cerr << std::endl << "Loading FEATURES FOR UNKNOWN WORDS from < "<<name.str()<<" >"<< std::endl;
 128   createFeatureList(name.str(),&model->featureListUnk);
 129
 130   if (flow == "LRL")
 131   {
 132     flow1 = "LR"; flow2 = "RL";
 133
 134     name.str("");
 135     name << flow1 << " (Right-to-Left)";
 136     if (verbose) std::cerr << std::endl <<"READING MODELS < direction =  "<<name.str()<<" >"<<std::endl;
 137
 138     name.str("");
 139     name << taggerModelName << ".M"<<taggerNumModel<<"."<<flow2<<".MRG";
 140     if (verbose) std::cerr<<"-. Loading MERGED MODEL FOR KNOWN WORDS from < "<<name.str()<<" >"<< std::endl;
 141     model->wr2 = new weightRepository(name.str(),taggerKFilter);
 142
 143     name.str("");
 144     name << taggerModelName << ".UNK.M"<<taggerNumModel<<"."<<flow2<<".MRG";
 145     if (verbose) std::cerr <<"-. Loading MERGED MODEL FOR UNKKNOWN WORDS from < "<<name.str()<<" >"<<std::endl<<std::endl;
 146     model->wrUnk2 = new weightRepository(name.str(),taggerUFilter);
 147   }
 148   else flow1 =flow;
 149
 150   name.str("");
 151   if (flow1 =="RL") name <<flow1<<" (Right-to-Left)";
 152   else name << flow1 << " (Left-to-Right)";
 153
 154   if (verbose) std::cerr<<std::endl<<"READING MODELS < direction =  "<<name.str()<<" >"<<std::endl;
 155
 156   name.str("");
 157   name << taggerModelName << ".M"<<taggerNumModel<<"."<<flow1<<".MRG";
 158   if (verbose) std::cerr<<"-. Loading MERGED MODEL FOR KNOWN WORDS from < "<<name.str()<<" >"<<std::endl;
 159   model->wr = new weightRepository(name.str(),taggerKFilter);
 160
 161   name.str("");
 162   name << taggerModelName << ".UNK.M"<<taggerNumModel<<"."<<flow1<<".MRG";
 163   if (verbose) std::cerr<<"-. Loading MERGED MODEL FOR UNKNOWN WORDS from < "<<name.str()<<" >"<<std::endl;
 164   model->wrUnk = new weightRepository(name.str(),taggerUFilter);
 165
 166 }
 167
 168 /***************************************************************/
 169
 170 void tagger::taggerLoadModelsForTagging()
 171 {
 172   startUpTime = times(&tbuffStartUp);
 173
 174   int modelsNeeded=1;
 175
 176   std::string name = taggerModelName + ".DICT";
 177   if (!taggerBackupDict.empty())
 178   {
 179     if (verbose) std::cerr << "Loading DICTIONARY from < "<<name<<" > with BACKUP DICTIONARY from < "<<taggerBackupDict<<" >"<< std::endl;
 180     d  = new dictionary(name,taggerBackupDict);
 181   }
 182   else
 183   {
 184     if (verbose) std::cerr<<"Loading DICTIONARY from < "<<name<<" >"<<std::endl;
 185     d  = new dictionary(name);
 186   }
 187
 188   name = taggerModelName + ".UNKP";
 189   if (verbose) std::cerr << "Loading UNKNOWN WORDS POS from < "<<name<<" >"<<std::endl;
 190
 191   if ( taggerStrategy == STRA_2P_RELABELING //modstrat 1
 192        || taggerStrategy == STRA_1P_ROBUST_UNK /*modstrat 4*/ )  modelsNeeded = 2;
 193
 194   taggerModelList = new models_t[modelsNeeded];
 195   taggerModelRunning = &taggerModelList[0];
 196
 197   if (taggerStrategy == STRA_1P_DEFAULT ) //modstrat 0)
 198     taggerLoadModels(taggerModelRunning,0);
 199   else if (taggerStrategy == STRA_1P_UNSUPERVISED ) //modstrat 2)
 200     taggerLoadModels(taggerModelRunning,3);
 201   else if (taggerStrategy ==  STRA_1P_ROBUST_UNK ) //modstrat 4)
 202     {
 203         taggerLoadModels(taggerModelRunning,0);
 204         taggerLoadModels(&taggerModelList[1],2);
 205     }
 206   else if (taggerStrategy == STRA_1P_VERY_ROBUST_UNK ) //modstrat 5)
 207     taggerLoadModels(taggerModelRunning,4);
 208      else  if (taggerStrategy == STRA_2P_RELABELING )//modstrat 1)
 209     {
 210         taggerLoadModels(taggerModelRunning,2);
 211         taggerLoadModels(&taggerModelList[1],1);
 212         taggerNumLaps = 2;
 213     }
 214   else
 215     {
 216       std::cerr<<"Execution error: Strategy "<<taggerStrategy<<" doesn't exist!!"<<std::endl<<std::endl;
 217       exit(0);
 218     }
 219
 220   endStartUpTime = times(&tbuffEndStartUp);
 221 }
 222
 223 void tagger::taggerInit(std::istream& input, std::ostream& output)
 224 {
 225   //  int modelsNeeded=1;
 226   std::string name;
 227
 228   if (sw != 0) delete sw;
 229   //Mirar si existe fichero .WIN
 230   if (taggerWinIndex==-1 && taggerWinLength==-1)
 231   {
 232     name = taggerModelName + ".WIN";
 233     FILE *f =  fopen (name.c_str(),"r");
 234     if ( f == NULL ) sw = new swindow(input, &output, d);
 235     else
 236     {
 237       fscanf(f,"%d %d",&taggerWinLength,&taggerWinIndex);
 238       fclose(f);
 239       sw = new swindow(input,taggerWinLength,taggerWinIndex, &output, d);
 240     }
 241   }
 242   else if (taggerWinIndex==-1) sw = new swindow (input,taggerWinLength, &output, d);
 243   else sw = new swindow (input,taggerWinLength,taggerWinIndex, &output, d);
 244 }
 245
 246 void tagger::taggerInit()
 247 {
 248   if (sw != NULL) delete sw;
 249   sw = new swindow(5, d);
 250 }
 251
 252 /***************************************************************/
 253
 254 tagger::~tagger()
 255 {
 256   int modelsNeeded=1;
 257
 258   if (taggerStrategy == STRA_2P_RELABELING ) //modstrat 1)
 259     modelsNeeded = 2;
 260
 261   delete d;
 262   delete sw;
 263
 264   for (int i=0;i<modelsNeeded;i++)
 265   {
 266
 267     delete taggerModelList[i].wr;
 268     delete taggerModelList[i].wrUnk;
 269     destroyFeatureList(&taggerModelList[i].featureList);
 270     destroyFeatureList(&taggerModelList[i].featureListUnk);
 271
 272     if (flow =="LRL")
 273     {
 274       delete  taggerModelList[i].wr2;
 275       delete  taggerModelList[i].wrUnk2;
 276     }
 277   }
 278   delete[] taggerModelList;
 279 }
 280
 281 /***************************************************************/
 282 /***************************************************************/
 283
 284
 285 void tagger::taggerShowComments()
 286 {
 287   taggerShowCommentsFlag = true;
 288 }
 289
 290 /***************************************************************/
 291
 292 void tagger::taggerShowNoComments()
 293 {
 294   taggerShowCommentsFlag = false;
 295 }
 296
 297
 298 /***************************************************************/
 299
 300
 301 void tagger::taggerActiveShowScoresFlag()
 302 {
 303   taggerShowScoresFlag = true;
 304 }
 305
 306 /***************************************************************/
 307
 308 void tagger::taggerDesactiveShowScoresFlag()
 309 {
 310   this->taggerShowScoresFlag = false;
 311 }
 312
 313 /***************************************************************/
 314
 315 void tagger::taggerPutFlow(const std::string& inFlow)
 316 {
 317   flow = inFlow;
 318 }
 319
 320 /***************************************************************/
 321
 322 void tagger::taggerPutStrategy(int num)
 323 {
 324    taggerStrategy = num;
 325 }
 326
 327 /***************************************************************/
 328
 329 void tagger::taggerPutWinLength(int l)
 330 {
 331   taggerWinLength = l;
 332 }
 333
 334 /***************************************************************/
 335
 336 void tagger::taggerPutWinIndex(int i)
 337 {
 338   taggerWinIndex = i;
 339 }
 340
 341 /***************************************************************/
 342
 343 void tagger::taggerPutBackupDictionary(const std::string& dictName)
 344 {
 345   taggerBackupDict = dictName;
 346 }
 347
 348 /***************************************************************/
 349
 350 void tagger::taggerPutKWeightFilter(float kfilter)
 351 {
 352   taggerKFilter = kfilter;
 353 }
 354
 355 /***************************************************************/
 356
 357 void tagger::taggerPutUWeightFilter(float ufilter)
 358 {
 359   taggerUFilter = ufilter;
 360 }
 361
 362 /***************************************************************/
 363 /***************************************************************/
 364 /***************************************************************/
 365 /***************************************************************/
 366
 367 int tagger::taggerRightSenseSpecialForUnknown()
 368 {
 369   int cont=1;
 370
 371   while(sw->previous());
 372   nodo *elem = sw->getIndex();
 373
 374   if (sw->winExistUnkWord(1,d)==-1)
 375     taggerModelRunning=&taggerModelList[1];
 376   else taggerModelRunning=&taggerModelList[0];
 377
 378   taggerGenerateScore(elem,1);
 379
 380   while(sw->next())
 381     {
 382       elem = sw->getIndex();
 383
 384       if (sw->winExistUnkWord(1,d)==-1)
 385   taggerModelRunning=&taggerModelList[1];
 386       else taggerModelRunning=&taggerModelList[0];
 387
 388       taggerGenerateScore(elem,1);
 389       cont++;
 390     }
 391
 392   if (flow == "LRL") sw->winMaterializePOSValues(1);
 393
 394   return cont;
 395 }
 396
 397 /***************************************************************/
 398
 399 int tagger::taggerLeftSenseSpecialForUnknown()
 400 {
 401   int cont=1;
 402   while(sw->next());
 403   nodo *elem = sw->getIndex();
 404   if (sw->winExistUnkWord(2,d)==-1)
 405     taggerModelRunning=&taggerModelList[1];
 406   else taggerModelRunning=&taggerModelList[0];
 407
 408   taggerGenerateScore(elem,2);
 409
 410   while(sw->previous())
 411     {
 412       elem = sw->getIndex();
 413
 414       if (sw->winExistUnkWord(2,d)==-1)
 415   taggerModelRunning=&taggerModelList[1];
 416       else taggerModelRunning=&taggerModelList[0];
 417
 418       taggerGenerateScore(elem,2);
 419       cont++;
 420     }
 421
 422   if (flow=="LRL") sw->winMaterializePOSValues(0);
 423   return cont;
 424 }
 425
 426 /***************************************************************/
 427
 428 int tagger::taggerRightSense()
 429 {
 430   int cont=1;
 431
 432   while(sw->previous());
 433   nodo *elem = sw->getIndex();
 434   if (elem == 0)
 435   {
 436     std::cerr << "tagger::taggerRightSense: ERROR index null at beginning" << std::endl;
 437     return -1;
 438   }
 439   taggerGenerateScore(elem,1);
 440
 441   while(sw->next())
 442   {
 443     elem = sw->getIndex();
 444     taggerGenerateScore(elem,1);
 445     cont++;
 446   }
 447
 448   if (flow =="LRL") sw->winMaterializePOSValues(1);
 449
 450   return cont;
 451 }
 452
 453 /***************************************************************/
 454
 455 int tagger::taggerLeftSense()
 456 {
 457   int cont=1;
 458   while(sw->next());
 459   nodo *elem = sw->getIndex();
 460   taggerGenerateScore(elem,2);
 461
 462   while(sw->previous())
 463   {
 464     elem = sw->getIndex();
 465     taggerGenerateScore(elem,2);
 466     cont++;
 467   }
 468
 469   if (flow =="LRL") sw->winMaterializePOSValues(0);
 470   return cont;
 471 }
 472
 473 /***************************************************************/
 474
 475 void tagger::taggerRun()
 476 {
 477   int contWords=0,contSentences=0;
 478
 479   struct  tms     tbuff1,tbuff2;
 480   clock_t start,end;
 481   start = times(&tbuff1);
 482
 483   switch(taggerStrategy)
 484   {
 485     case STRA_1P_DEFAULT/*modstrat 0*/: taggerDoNormal(&contWords,&contSentences); break;
 486     case STRA_2P_RELABELING/*modstrat 1*/: taggerDoNTimes(&contWords,&contSentences,taggerNumLaps); break;
 487     case STRA_1P_UNSUPERVISED/*modstrat 2*/: taggerDoNormal(&contWords,&contSentences); break;
 488     case STRA_1P_SENTENCE_LEVEL/*modstrat 3*/: /*taggerDoNTimes(&contWords,&contSentences,taggerNumLaps);*/ break;
 489     case STRA_1P_ROBUST_UNK/*modstrat 4*/: taggerDoSpecialForUnknown(&contWords,&contSentences); break;
 490     case STRA_1P_VERY_ROBUST_UNK/*modstrat 5*/: taggerDoNormal(&contWords,&contSentences); break;
 491     case STRA_1P_ROBUST_SENTENCE_LEVEL: break;
 492   }
 493   end = times(&tbuff2);
 494
 495
 496   if (verbose)
 497     { taggerShowVerbose(contSentences,1);
 498
 499     std::cerr<<"* -------------------------------------------------------------------"<<std::endl;
 500     showTime("Start Up Time",
 501        ((double)(endStartUpTime-startUpTime))/CLOCKS_PER_SECOND,
 502        ((double)tbuffEndStartUp.tms_utime-(double)tbuffStartUp.tms_utime)/CLOCKS_PER_SECOND,
 503        ((double)tbuffEndStartUp.tms_stime-(double)tbuffStartUp.tms_stime)/CLOCKS_PER_SECOND);
 504     std::cerr<<"* -------------------------------------------------------------------"<<std::endl;
 505     showTime("Features Extraction Time",realFexTime,usrFexTime,sysFexTime);
 506     showTime("SVM Time",realSVMTime,usrSVMTime,sysSVMTime);
 507     showTime("Process Time",((double)(end-start))/CLOCKS_PER_SECOND - realFexTime - realSVMTime,
 508        ((double)tbuff2.tms_utime-(double)tbuff1.tms_utime)/CLOCKS_PER_SECOND - usrFexTime -usrSVMTime,
 509        ((double)tbuff2.tms_stime-(double)tbuff1.tms_stime)/CLOCKS_PER_SECOND - sysFexTime -sysSVMTime);
 510     std::cerr<<"* -------------------------------------------------------------------"<<std::endl;
 511     std::cerr<<"[ Tagging Time = Feature Extraction Time + SVM Time + Process Time ]"<<std::endl;
 512     showTime("Tagging Time",((double)(end-start))/CLOCKS_PER_SECOND,
 513        ((double)tbuff2.tms_utime-(double)tbuff1.tms_utime)/CLOCKS_PER_SECOND,
 514        ((double)tbuff2.tms_stime-(double)tbuff1.tms_stime)/CLOCKS_PER_SECOND);
 515     std::cerr<<"* -------------------------------------------------------------------"<<std::endl;
 516     std::cerr<<"[ Overall Time = Start up Time + Tagging Time ]"<<std::endl;
 517     showTime("Overall Time",((double)(end-start+endStartUpTime-startUpTime))/CLOCKS_PER_SECOND,
 518        ((double)tbuff2.tms_utime-(double)tbuff1.tms_utime+
 519         (double)tbuffEndStartUp.tms_utime-(double)tbuffStartUp.tms_utime)/CLOCKS_PER_SECOND,
 520        ((double)tbuff2.tms_stime-(double)tbuff1.tms_stime+
 521         (double)tbuffEndStartUp.tms_stime-(double)tbuffStartUp.tms_stime)/CLOCKS_PER_SECOND);
 522         std::cerr<<"* -------------------------------------------------------------------"<<std::endl;
 523     taggerStadistics(contWords,contSentences,
 524          ((double)(end-start))/CLOCKS_PER_SECOND,
 525          ((double)tbuff2.tms_utime-(double)tbuff1.tms_utime)/CLOCKS_PER_SECOND,
 526          ((double)tbuff2.tms_stime-(double)tbuff1.tms_stime)/CLOCKS_PER_SECOND);
 527     }
 528 }
 529
 530 /***************************************************************/
 531
 532 void tagger::taggerDoNormal(int *numWords, int *numSentences)
 533 {
 534   int contWordsLR=0,contWordsRL=0,contSentences=0,ret = 1;
 535
 536   while ((ret>=0))
 537   {
 538     if (verbose) taggerShowVerbose(contSentences,0);
 539
 540     if ((flow=="LRL") || (flow =="LR"))
 541       contWordsLR = contWordsLR+taggerRightSense();
 542     if ((flow =="LRL") || (flow == "RL"))
 543       contWordsRL = contWordsRL+taggerLeftSense();
 544     contSentences++;
 545     sw->show(taggerShowScoresFlag, taggerShowCommentsFlag);
 546     sw->deleteList();
 547     ret = sw->iniGeneric(d);
 548   }
 549   if (contWordsRL==0) *numWords=contWordsLR/taggerNumLaps;
 550   else *numWords=contWordsRL/taggerNumLaps;
 551   *numSentences = contSentences;
 552 }
 553
 554 /***************************************************************/
 555
 556 void tagger::taggerDoSpecialForUnknown(int *numWords, int *numSentences)
 557 {
 558   int contWordsLR=0,contWordsRL=0,contSentences=0,ret = 1;
 559
 560   while ((ret>=0))
 561   {
 562     if (verbose) taggerShowVerbose(contSentences,0);
 563
 564     if ((flow == "LRL") || (flow =="LR"))
 565       contWordsLR = contWordsLR+taggerRightSenseSpecialForUnknown();
 566     if ((flow == "LRL") || (flow =="RL"))
 567       contWordsRL = contWordsRL+taggerLeftSenseSpecialForUnknown();
 568
 569     contSentences++;
 570     sw->show(taggerShowScoresFlag, taggerShowCommentsFlag);
 571     sw->deleteList();
 572     ret = sw->iniGeneric(d);
 573   }
 574   if (contWordsRL==0) *numWords=contWordsLR/taggerNumLaps;
 575   else *numWords=contWordsRL/taggerNumLaps;
 576   *numSentences = contSentences;
 577 }
 578
 579 /***************************************************************/
 580
 581 void tagger::taggerDoNTimes(int *numWords, int *numSentences,int laps)
 582 {
 583   int contWordsLR=0,contWordsRL=0,contSentences=0,ret = 1;
 584
 585   while ((ret>=0))
 586   {
 587     if (verbose) taggerShowVerbose(contSentences,0);
 588
 589     for (int pasadas=0;pasadas<laps;pasadas++)
 590     {
 591       taggerModelRunning = &taggerModelList[pasadas];
 592       if ((flow == "LRL") || (flow == "LR"))
 593         contWordsLR = contWordsLR+taggerRightSense();
 594       if (flow == "LRL" && pasadas>0)
 595         sw->winMaterializePOSValues(2);
 596       if ((flow == "LRL") || (flow =="RL"))
 597         contWordsRL = contWordsRL+taggerLeftSense();
 598     }
 599
 600     contSentences++;
 601     sw->show(taggerShowScoresFlag, taggerShowCommentsFlag);
 602     sw->deleteList();
 603     ret = sw->iniGeneric(d);
 604   }
 605   if (contWordsRL==0) *numWords=contWordsLR/taggerNumLaps;
 606   else *numWords=contWordsRL/taggerNumLaps;
 607   *numSentences = contSentences;
 608 }
 609
 610 /***************************************************************/
 611
 612 void tagger::taggerGenerateScore(nodo *elem,int direction)
 613 {
 614
 615   struct  tms tbuffStartFex,tbuffEndFex;
 616   clock_t startFexTime,endFexTime;
 617   struct  tms tbuffStartSVM,tbuffEndSVM;
 618   clock_t startSVMTime,endSVMTime;
 619
 620   weight_node_t *weight;
 621   weightRepository *weightRep;
 622   int numMaybe,max=0;
 623   int is_unk=FALSE;
 624   simpleList<nodo_feature_list*> *featureList;
 625
 626   startFexTime = times(&tbuffStartFex);
 627
 628   dataDict* i = d->getElement(elem->wrd);
 629   if ((long)i!=HASH_FAIL)
 630   {
 631     featureList = &taggerModelRunning->featureList;
 632     numMaybe = d->getElementNumMaybe(i);
 633     weight = taggerCreateWeightNodeArray(numMaybe,i);
 634     if ((flow =="LRL") && (direction==2))
 635     {
 636       weightRep = taggerModelRunning->wr2;  //wr2;
 637       //bias = taggerModelRunning->bias2;  //taggerBias2;
 638     }
 639     else
 640     {
 641       weightRep = taggerModelRunning->wr; //wr;
 642       //bias = taggerModelRunning->bias; //taggerBias;
 643     }
 644   }
 645   else
 646   {
 647     featureList = &taggerModelRunning->featureListUnk;
 648     weight = taggerCreateWeightUnkArray(&numMaybe);
 649     is_unk = TRUE;
 650
 651     if (flow =="LRL" && (direction==2))
 652     {
 653       weightRep = taggerModelRunning->wrUnk2;  //wrUnk2;
 654       //bias =  taggerModelRunning->biasUnk2; //taggerBiasUnk2;
 655     }
 656     else
 657     { weightRep = taggerModelRunning->wrUnk; //wrUnk;
 658       //bias =taggerModelRunning->biasUnk; //taggerBiasUnk;
 659     }
 660   }
 661
 662
 663   if (numMaybe>1)
 664   {
 665     bool ret = true;
 666     while (ret)
 667     {
 668       nodo_feature_list* aux= *featureList->getIndex();
 669       if (aux->mark == SLASTW)  sw->winPushSwnFeature(stk);
 670       else if (aux->mark == WMARK)  sw->winPushWordFeature((void *)aux,d,stk,direction);
 671       else if (aux->mark == KMARK)  sw->winPushAmbiguityFeature((void *)aux,d,stk,direction);
 672       else if (aux->mark == MMARK)  sw->winPushMaybeFeature((void *)aux,d,stk,direction);
 673       else if (aux->mark == PMARK)  sw->winPushPosFeature((void *)aux,d,stk,direction);
 674       else if (aux->mark == MFTMARK)  sw->winPushMFTFeature((void *)aux,d,stk,direction);
 675       else if (is_unk==TRUE)
 676       {
 677         int *param;
 678         if (!aux->l.isEmpty())
 679         {
 680           param = *aux->l.getIndex();
 681         }
 682         if (aux->mark == PREFIX_MARK)  sw->winPushPrefixFeature(elem->wrd, stk, *param);
 683         else if (aux->mark == SUFFIX_MARK) sw->winPushSuffixFeature(elem->wrd, stk, *param);
 684         else if (aux->mark == CHAR_A_MARK) sw->winPushLetterFeature(elem->wrd, stk, COUNTING_FROM_BEGIN, *param);
 685         else if (aux->mark == CHAR_Z_MARK) sw->winPushLetterFeature(elem->wrd, stk, COUNTING_FROM_END, *param);
 686         else if (aux->mark == LENGTH_MARK) sw->winPushLenghtFeature(elem->wrd,stk);
 687         else if (aux->mark == START_CAPITAL_MARK) sw->winPushStartWithCapFeature(elem->wrd,stk);
 688         else if (aux->mark == START_LOWER_MARK)  sw->winPushStartWithLowerFeature(elem->wrd,stk);
 689         else if (aux->mark == START_NUMBER_MARK) sw->winPushStartWithNumberFeature(elem->wrd,stk);
 690         else if (aux->mark == ALL_UPPER_MARK) sw->winPushAllUpFeature(elem->wrd,stk);
 691         else if (aux->mark == ALL_LOWER_MARK) sw->winPushAllLowFeature(elem->wrd,stk);
 692         else if (aux->mark == CONTAIN_CAP_MARK) sw->winPushContainCapFeature(elem->wrd, stk);
 693         else if (aux->mark == CONTAIN_CAPS_MARK) sw->winPushContainCapsFeature(elem->wrd, stk);
 694         else if (aux->mark == CONTAIN_COMMA_MARK) sw->winPushContainCommaFeature(elem->wrd, stk);
 695         else if (aux->mark == CONTAIN_NUMBER_MARK) sw->winPushContainNumFeature(elem->wrd, stk);
 696         else if (aux->mark == CONTAIN_PERIOD_MARK) sw->winPushContainPeriodFeature(elem->wrd, stk);
 697         else if (aux->mark == MULTIWORD_MARK) sw->winPushMultiwordFeature(elem->wrd, stk);
 698       }
 699       ret = featureList->next();
 700     }
 701     featureList->setFirst();
 702
 703     endFexTime = times(&tbuffEndFex);
 704     realFexTime = realFexTime + ((double)(endFexTime-startFexTime))/CLOCKS_PER_SECOND;
 705     usrFexTime = usrFexTime + (((double)tbuffEndFex.tms_utime-(double)tbuffStartFex.tms_utime)/CLOCKS_PER_SECOND);
 706     sysFexTime = sysFexTime + (((double)tbuffEndFex.tms_stime-(double)tbuffStartFex.tms_stime)/CLOCKS_PER_SECOND);
 707
 708     startSVMTime = times(&tbuffStartSVM);
 709
 710     elem->strScores = taggerSumWeight(weightRep,weight,numMaybe,&max);
 711     //std::cerr << "tagger::taggerGenerateScore got elem strScores: '" << elem->strScores << "'" << std::endl;
 712
 713     endSVMTime = times(&tbuffEndSVM);
 714     realSVMTime = realSVMTime + ((double)(endSVMTime-startSVMTime))/CLOCKS_PER_SECOND;
 715     usrSVMTime = usrSVMTime + (((double)tbuffEndSVM.tms_utime-(double)tbuffStartSVM.tms_utime)/CLOCKS_PER_SECOND);
 716     sysSVMTime = sysSVMTime + (((double)tbuffEndSVM.tms_stime-(double)tbuffStartSVM.tms_stime)/CLOCKS_PER_SECOND);
 717   }
 718
 719   elem->pos = weight[max].pos;
 720   elem->weight = weight[max].data;
 721
 722   if (flow =="LRL")
 723   {
 724     weight_node_t* score = new weight_node_t();
 725     score->data = weight[max].data;
 726     score->pos = weight[max].pos;
 727     elem->stackScores.push(score);
 728   }
 729
 730   delete[] weight;
 731 }
 732
 733
 734
 735 /***************************************************************/
 736
 737 /* Returns an array ready to be filled with maybe informations */
 738 weight_node_t *tagger::taggerCreateWeightNodeArray(int numMaybe,dataDict* index)
 739 {
 740   int j = numMaybe;
 741   weight_node_t *weight = new weight_node_t[numMaybe];
 742   simpleList<infoDict*> *list = &d->getElementMaybe(index);
 743
 744   bool ret=true;
 745   while (ret && numMaybe > 0)
 746     {
 747       infoDict *pInfoDict = *list->getIndex();
 748       j--;
 749       weight[j].pos = pInfoDict->pos;
 750       weight[j].data = 0;
 751       ret=list->next();
 752     }
 753
 754   list->setFirst();
 755   return weight;
 756 }
 757
 758 /***************************************************************/
 759
 760 std::string tagger::taggerSumWeight(weightRepository* wRep, weight_node_t* weight, int numMaybe, int* max)
 761 {
 762 //   weight_node_t *aux;
 763   long double w,b = 0;
 764   std::string feature;
 765   int putBias=1;
 766
 767   while (!stk.empty())
 768   {
 769     *max=0;
 770     feature = stk.top();
 771     stk.pop();
 772     for (int j=0; j<numMaybe;j++)
 773     {
 774       if (putBias)
 775         {
 776           b = wRep->wrGetWeight("BIASES",weight[j].pos);
 777           weight[j].data = weight[j].data - b;
 778         }
 779       w = wRep->wrGetWeight(feature,weight[j].pos);
 780       weight[j].data=weight[j].data+w;
 781       if (((float)weight[*max].data)<((float)weight[j].data)) *max=j;
 782     }
 783     putBias=0;
 784   }
 785   std::ostringstream tmp;
 786   if ( true )
 787   {
 788     for (int i=0; i<numMaybe; i++)
 789     {
 790       if ( i == 0 )
 791       {
 792         tmp << weight[i].pos << "_" << (float)weight[i].data;
 793       }
 794       else
 795       {
 796         tmp << " " << weight[i].pos << "_" << (float)weight[i].data;
 797       }
 798     }
 799   }
 800   //std::cerr << "tagger::taggerSumWeight generated string '" << tmp.str() << "'" << std::endl;
 801   return tmp.str();
 802 }
 803
 804 /***************************************************************/
 805 /***************************************************************/
 806
 807 weight_node_t *tagger::taggerCreateWeightUnkArray(int *numMaybe)
 808 {
 809   int i=0;
 810   char c=' ';
 811   FILE *f;
 812   std::string name = taggerModelName + ".UNKP";
 813
 814   *numMaybe = 0;
 815
 816   if ((f = fopen(name.c_str(), "rt"))== NULL)
 817     {
 818       std::cerr << "Error opening file: "<<name<<std::endl;
 819       exit(0);
 820     }
 821
 822   // first read: count the number of POS
 823   while (!feof(f))
 824     { if (fgetc(f)=='\n') (*numMaybe)++;
 825     }
 826
 827   // second read, fill in the weight nodes
 828   fseek(f,0,SEEK_SET);
 829
 830   weight_node_t *weight = new weight_node_t[*numMaybe];
 831   while (!feof(f) && (i<*numMaybe))
 832     { weight[i].pos = "";
 833     weight[i].data=0;
 834     c = fgetc(f);
 835     while ((c!='\n') && (!feof(f)))
 836       {
 837   if (c!=' ' && c!='\n' && c!='\t') weight[i].pos += c;
 838   c = fgetc(f);
 839       }
 840     i++;
 841     }
 842   fclose(f);
 843   return weight;
 844 }
 845
 846 /***************************************************************/
 847 /***************************************************************/
 848
 849 void tagger::taggerStadistics(int numWords, int numSentences, double /*realTime*/, double usrTime, double sysTime)
 850 {
 851   std::ostringstream message;
 852   float media=0;
 853   if ((sysTime+usrTime)!=0)  media = (float) (((double) numWords)/(sysTime+usrTime));
 854
 855   message << std::endl << numSentences <<" sentences were tagged." << std::endl;
 856   message << numWords << " words were tagged." << std::endl;
 857   message << media << " words/second were tagged." << std::endl;
 858   std::cerr << message.str();
 859 }
 860
 861 /***************************************************************/
 862
 863 void tagger::taggerShowVerbose(int num,int isEnd)
 864 {
 865   if (isEnd) { std::cerr<<"."<<num<<" sentences [DONE]"<<std::endl<<std::endl; return; }
 866   else if (num%100==0) std::cerr <<num;
 867   else if (num%10==0) std::cerr << ".";
 868 }
 869
 870