lib/tagger.cc

   1 /*
   2  * Copyright (C) 2004 Jesus Gimenez, Lluis Marquez and Senen Moya
   3  *
   4  * This library is free software; you can redistribute it and/or
   5  * modify it under the terms of the GNU Lesser General Public
   6  * License as published by the Free Software Foundation; either
   7  * version 2.1 of the License, or (at your option) any later version.
   8  *
   9  * This library is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  12  * Lesser General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU Lesser General Public
  15  * License along with this library; if not, write to the Free Software
  16  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  17  */
  18
  19 #include <unistd.h>
  20 #include <stdio.h>
  21 #include <time.h>
  22 #include <sys/times.h>
  23 #include <stdlib.h>
  24 #include <string.h>
  25 #include "marks.h"
  26 #include "strategies.h"
  27 #include "hash.h"
  28 #include "list.h"
  29 #include "weight.h"
  30 #include "dict.h"
  31 #include "stack.h"
  32 #include "swindow.h"
  33 #include "tagger.h"
  34 #include "common.h"
  35
  36 /***************************************************************/
  37
  38 struct tms  tbuffStartUp,tbuffEndStartUp;
  39 clock_t startUpTime,endStartUpTime;
  40 double  sysFexTime=0, usrFexTime=0,realFexTime=0;
  41 double  sysSVMTime=0, usrSVMTime=0,realSVMTime=0;
  42
  43 /***************************************************************/
  44
  45 extern int verbose;
  46 int NUM_UNK_POS=0;
  47
  48
  49 /***************************************************************/
  50
  51 hash_t *tagger::taggerCreateBiasHash(char *name)
  52 {
  53   hash_t *bias = new hash_t;
  54   int i=0;
  55   char c=' ',weight[20];
  56   weight_node_t *w;
  57   FILE *f;
  58
  59   hash_init(bias,40);
  60
  61   if ((f = fopen(name, "rt"))== NULL)
  62     {
  63       fprintf(stderr, "Error opening file: %s",name);
  64       exit(0);
  65     }
  66
  67   while (!feof(f))
  68     {
  69       c = fgetc(f);
  70       if (c!='#')
  71         {
  72           w = new weight_node_t;
  73           strcpy(weight,"");
  74           i=0;
  75           strcpy(w->pos,"");
  76           while ((c!='\n') && (!feof(f)))
  77             {
  78               if (c!=' ' && c!='\n' && c!='\t' && i==1)  sprintf(weight,"%s%c",weight,c);
  79               else if (c!=' ' && c!='\n' && c!='\t' && i==0)
  80                 {
  81                   if (c!=':') sprintf(w->pos,"%s%c",w->pos,c);
  82                   else i=1;
  83                 }
  84               c = fgetc(f);
  85             }
  86           w->data = (long double)0;
  87           w->data = atof (weight);
  88           hash_insert(bias,w->pos,(int)w);
  89         } //end if
  90       else while(c=fgetc(f)!='\n');
  91     }
  92   fclose(f);
  93   return bias;
  94 }
  95
  96 /***************************************************************/
  97
  98 tagger::tagger(char *model)
  99 {
 100   char name[150]="";
 101   strcpy(flow,"LR");
 102   taggerNumLaps = 1;
 103   taggerKFilter = 0;
 104   taggerUFilter = 0;
 105   taggerStrategy = STRA_1P_DEFAULT; //modstrat //0;
 106   taggerWinIndex = -1;
 107   taggerWinLength = -1;
 108   strcpy(taggerModelName,model);
 109   strcpy (taggerBackupDict,"");
 110   stk = new stack_t;
 111   init_stack(stk);
 112 }
 113
 114 /***************************************************************/
 115
 116 void tagger::taggerLoadModels(models_t *model, int taggerNumModel)
 117 {
 118   char name[150],flow2[5],flow1[5];
 119
 120   //Cargamos la lista de "features" para palabras conocidas
 121   sprintf(name,"%s.A%d",taggerModelName,taggerNumModel);
 122   if (verbose)  fprintf(stderr,"\nLoading FEATURES FOR KNOWN WORDS from < %s >\n",name);
 123   createFeatureList(name,&model->featureList);
 124   //Cargamos la lista de "features" para palabras desconocidas
 125   sprintf(name,"%s.A%d.UNK",taggerModelName,taggerNumModel);
 126   if (verbose)  fprintf(stderr,"\nLoading FEATURES FOR UNKNOWN WORDS from < %s >\n",name);
 127   createFeatureList(name,&model->featureListUnk);
 128
 129   if (strcmp(flow,"LRL")==0)
 130     {
 131       strcpy(flow1,"LR");       strcpy(flow2,"RL");
 132
 133       sprintf(name,"%s (Right-to-Left)",flow1);
 134       if (verbose) fprintf(stderr,"\nREADING MODELS < direction =  %s >\n",name);
 135
 136       sprintf(name,"%s.M%d.%s.MRG",taggerModelName,taggerNumModel,flow2);
 137       if (verbose) fprintf(stderr,"-. Loading MERGED MODEL FOR KNOWN WORDS from < %s >\n",name);
 138       model->wr2 = new weightRepository(name,taggerKFilter);
 139
 140       sprintf(name,"%s.UNK.M%d.%s.MRG",taggerModelName,taggerNumModel,flow2);
 141       if (verbose) fprintf(stderr,"-. Loading MERGED MODEL FOR UNKKNOWN WORDS from < %s >\n\n",name);
 142       model->wrUnk2 = new weightRepository(name,taggerUFilter);
 143     }
 144   else strcpy(flow1,flow);
 145
 146   if (strcmp(flow1,"RL")==0) sprintf(name,"%s (Right-to-Left)",flow1);
 147   else sprintf(name,"%s (Left-to-Right)",flow1);
 148
 149   if (verbose) fprintf(stderr,"\nREADING MODELS < direction =  %s >\n",name);
 150
 151   sprintf(name,"%s.M%d.%s.MRG",taggerModelName,taggerNumModel,flow1);
 152   if (verbose) fprintf(stderr,"-. Loading MERGED MODEL FOR KNOWN WORDS from < %s >\n",name);
 153   model->wr = new weightRepository(name,taggerKFilter);
 154
 155   sprintf(name,"%s.UNK.M%d.%s.MRG",taggerModelName,taggerNumModel,flow1);
 156   if (verbose) fprintf(stderr,"-. Loading MERGED MODEL FOR UNKNOWN WORDS from < %s >\n",name);
 157   model->wrUnk = new weightRepository(name,taggerUFilter);
 158
 159 }
 160
 161 /***************************************************************/
 162
 163 void tagger::taggerLoadModelsForTagging()
 164 {
 165   startUpTime = times(&tbuffStartUp);
 166
 167   int modelsNeeded=1;
 168   char name[150];
 169
 170   sprintf(name,"%s.DICT",taggerModelName);
 171   if (strcmp(taggerBackupDict,"")!=0)
 172     {
 173       if (verbose) fprintf(stderr,"Loading DICTIONARY from < %s > with BACKUP DICTIONARY from < %s >\n",name,taggerBackupDict);
 174       d  = new dictionary(name,taggerBackupDict);
 175     }
 176   else
 177     {
 178       if (verbose) fprintf(stderr,"Loading DICTIONARY from < %s >\n",name);
 179       d  = new dictionary(name);
 180     }
 181
 182   sprintf(name,"%s.UNKP",taggerModelName);
 183   if (verbose) fprintf(stderr,"Loading UNKNOWN WORDS POS from < %s >\n",name);
 184   weightUnk = taggerCreateWeightUnkArray(name);
 185
 186   if ( taggerStrategy == STRA_2P_RELABELING //modstrat 1
 187        || taggerStrategy == STRA_1P_ROBUST_UNK /*modstrat 4*/ )  modelsNeeded = 2;
 188
 189   taggerModelList = new models_t[modelsNeeded];
 190   taggerModelRunning = &taggerModelList[0];
 191
 192   if (taggerStrategy == STRA_1P_DEFAULT ) //modstrat 0)
 193         taggerLoadModels(taggerModelRunning,0);
 194   else if (taggerStrategy == STRA_1P_UNSUPERVISED ) //modstrat 2)
 195         taggerLoadModels(taggerModelRunning,3);
 196   else if (taggerStrategy ==  STRA_1P_ROBUST_UNK ) //modstrat 4)
 197     {
 198         taggerLoadModels(taggerModelRunning,0);
 199         taggerLoadModels(&taggerModelList[1],2);
 200     }
 201   else if (taggerStrategy == STRA_1P_VERY_ROBUST_UNK ) //modstrat 5)
 202         taggerLoadModels(taggerModelRunning,4);
 203            else  if (taggerStrategy == STRA_2P_RELABELING )//modstrat 1)
 204     {
 205         taggerLoadModels(taggerModelRunning,2);
 206         taggerLoadModels(&taggerModelList[1],1);
 207         taggerNumLaps = 2;
 208     }
 209   else
 210     {
 211       fprintf(stderr,"Execution error: Strategy %d doesn't exist!!\n\n",taggerStrategy);
 212       exit(0);
 213     }
 214
 215   endStartUpTime = times(&tbuffEndStartUp);
 216 }
 217
 218 void tagger::taggerInit()
 219 {
 220
 221   //  int modelsNeeded=1;
 222   char name[150];
 223
 224   //Mirar si existe fichero .WIN
 225   if (taggerWinIndex==-1 && taggerWinLength==-1)
 226     {
 227       sprintf(name,"%s.WIN",taggerModelName);
 228       FILE *f =  fopen (name,"r");
 229       if ( f == NULL ) sw = new swindow(stdin);
 230       else
 231         {
 232           fscanf(f,"%d %d",&taggerWinLength,&taggerWinIndex);
 233           fclose(f);
 234           sw = new swindow (stdin,taggerWinLength,taggerWinIndex);
 235         }
 236     }
 237   else if (taggerWinIndex==-1) sw = new swindow (stdin,taggerWinLength);
 238   else sw = new swindow (stdin,taggerWinLength,taggerWinIndex);
 239 }
 240
 241
 242 /***************************************************************/
 243
 244 tagger::~tagger()
 245 {
 246   int modelsNeeded=1;
 247
 248   if (taggerStrategy == STRA_2P_RELABELING ) //modstrat 1)
 249     modelsNeeded = 2;
 250
 251   delete stk;
 252   delete d;
 253   delete sw;
 254   delete[] weightUnk; //Mod
 255
 256   for (int i=0;i<modelsNeeded;i++)
 257     {
 258
 259       delete taggerModelList[i].wr;
 260       delete taggerModelList[i].wrUnk;
 261
 262       if (strcmp(flow,"LRL")==0)
 263         {
 264           delete  taggerModelList[i].wr2;
 265           delete  taggerModelList[i].wrUnk2;
 266         }
 267     }
 268 }
 269
 270 /***************************************************************/
 271 /***************************************************************/
 272
 273 void tagger::taggerPutFlow(char *inFlow)
 274 {
 275   strcpy(flow,inFlow);
 276 }
 277
 278 /***************************************************************/
 279
 280 void tagger::taggerPutStrategy(int num)
 281 {
 282    taggerStrategy = num;
 283 }
 284
 285 /***************************************************************/
 286
 287 void tagger::taggerPutWinLength(int l)
 288 {
 289   taggerWinLength = l;
 290 }
 291
 292 /***************************************************************/
 293
 294 void tagger::taggerPutWinIndex(int i)
 295 {
 296   taggerWinIndex = i;
 297 }
 298
 299 /***************************************************************/
 300
 301 void tagger::taggerPutBackupDictionary(char *dictName)
 302 {
 303   strcpy (taggerBackupDict,dictName);
 304 }
 305
 306 /***************************************************************/
 307
 308 void tagger::taggerPutKWeightFilter(float kfilter)
 309 {
 310   taggerKFilter = kfilter;
 311 }
 312
 313 /***************************************************************/
 314
 315 void tagger::taggerPutUWeightFilter(float ufilter)
 316 {
 317   taggerUFilter = ufilter;
 318 }
 319
 320 /***************************************************************/
 321 /***************************************************************/
 322 /***************************************************************/
 323 /***************************************************************/
 324
 325 int tagger::taggerRightSenseSpecialForUnknown()
 326 {
 327   int cont=1;
 328
 329   while(sw->previous()==0);
 330   nodo *elem = sw->getIndex();
 331
 332   if (sw->winExistUnkWord(1,d)==-1)
 333     taggerModelRunning=&taggerModelList[1];
 334   else taggerModelRunning=&taggerModelList[0];
 335
 336   taggerGenerateScore(elem,1);
 337
 338   while(sw->next()==0)
 339     {
 340       elem = sw->getIndex();
 341
 342       if (sw->winExistUnkWord(1,d)==-1)
 343         taggerModelRunning=&taggerModelList[1];
 344       else taggerModelRunning=&taggerModelList[0];
 345
 346       taggerGenerateScore(elem,1);
 347       cont++;
 348     }
 349
 350   if (strcmp(flow,"LRL")==0) sw->winMaterializePOSValues(1);
 351
 352   return cont;
 353 }
 354
 355 /***************************************************************/
 356
 357 int tagger::taggerLeftSenseSpecialForUnknown()
 358 {
 359   int cont=1;
 360   while(sw->next()==0);
 361   nodo *elem = sw->getIndex();
 362   if (sw->winExistUnkWord(2,d)==-1)
 363     taggerModelRunning=&taggerModelList[1];
 364   else taggerModelRunning=&taggerModelList[0];
 365
 366   taggerGenerateScore(elem,2);
 367
 368   while(sw->previous()==0)
 369     {
 370       elem = sw->getIndex();
 371
 372       if (sw->winExistUnkWord(2,d)==-1)
 373         taggerModelRunning=&taggerModelList[1];
 374       else taggerModelRunning=&taggerModelList[0];
 375
 376       taggerGenerateScore(elem,2);
 377       cont++;
 378     }
 379
 380   if (strcmp(flow,"LRL")==0) sw->winMaterializePOSValues(0);
 381   return cont;
 382 }
 383
 384 /***************************************************************/
 385
 386 int tagger::taggerRightSense()
 387 {
 388   int cont=1;
 389
 390   while(sw->previous()==0);
 391   nodo *elem = sw->getIndex();
 392   taggerGenerateScore(elem,1);
 393
 394   while(sw->next()==0)
 395     {
 396       elem = sw->getIndex();
 397       taggerGenerateScore(elem,1);
 398       cont++;
 399     }
 400
 401   if (strcmp(flow,"LRL")==0) sw->winMaterializePOSValues(1);
 402
 403   return cont;
 404 }
 405
 406 /***************************************************************/
 407
 408 int tagger::taggerLeftSense()
 409 {
 410   int cont=1;
 411   while(sw->next()==0);
 412   nodo *elem = sw->getIndex();
 413   taggerGenerateScore(elem,2);
 414
 415   while(sw->previous()==0)
 416     {
 417       elem = sw->getIndex();
 418       taggerGenerateScore(elem,2);
 419       cont++;
 420     }
 421
 422   if (strcmp(flow,"LRL")==0) sw->winMaterializePOSValues(0);
 423   return cont;
 424 }
 425
 426 /***************************************************************/
 427
 428 void tagger::taggerRun()
 429 {
 430   int contWords=0,contSentences=0;
 431
 432   struct  tms     tbuff1,tbuff2;
 433   clock_t start,end;
 434   start = times(&tbuff1);
 435
 436   switch(taggerStrategy)
 437     {
 438     case STRA_1P_DEFAULT/*modstrat 0*/: taggerDoNormal(&contWords,&contSentences); break;
 439     case STRA_2P_RELABELING/*modstrat 1*/: taggerDoNTimes(&contWords,&contSentences,taggerNumLaps); break;
 440     case STRA_1P_UNSUPERVISED/*modstrat 2*/: taggerDoNormal(&contWords,&contSentences); break;
 441     case STRA_1P_SENTENCE_LEVEL/*modstrat 3*/: /*taggerDoNTimes(&contWords,&contSentences,taggerNumLaps);*/ break;
 442     case STRA_1P_ROBUST_UNK/*modstrat 4*/: taggerDoSpecialForUnknown(&contWords,&contSentences); break;
 443     case STRA_1P_VERY_ROBUST_UNK/*modstrat 5*/: taggerDoNormal(&contWords,&contSentences); break;
 444     case STRA_1P_ROBUST_SENTENCE_LEVEL: break;
 445     }
 446   end = times(&tbuff2);
 447
 448
 449   if (verbose)
 450     { taggerShowVerbose(contSentences,1);
 451
 452     fprintf(stderr,"* -------------------------------------------------------------------\n");
 453     showTime("Start Up Time",
 454              ((double)(endStartUpTime-startUpTime))/CLOCKS_PER_SECOND,
 455              ((double)tbuffEndStartUp.tms_utime-(double)tbuffStartUp.tms_utime)/CLOCKS_PER_SECOND,
 456              ((double)tbuffEndStartUp.tms_stime-(double)tbuffStartUp.tms_stime)/CLOCKS_PER_SECOND);
 457     fprintf(stderr,"* -------------------------------------------------------------------\n");
 458     showTime("Features Extraction Time",realFexTime,usrFexTime,sysFexTime);
 459     showTime("SVM Time",realSVMTime,usrSVMTime,sysSVMTime);
 460     showTime("Process Time",((double)(end-start))/CLOCKS_PER_SECOND - realFexTime - realSVMTime,
 461              ((double)tbuff2.tms_utime-(double)tbuff1.tms_utime)/CLOCKS_PER_SECOND - usrFexTime -usrSVMTime,
 462              ((double)tbuff2.tms_stime-(double)tbuff1.tms_stime)/CLOCKS_PER_SECOND - sysFexTime -sysSVMTime);
 463     fprintf(stderr,"* -------------------------------------------------------------------\n");
 464     fprintf(stderr,"[ Tagging Time = Feature Extraction Time + SVM Time + Process Time ]\n");
 465     showTime("Tagging Time",((double)(end-start))/CLOCKS_PER_SECOND,
 466              ((double)tbuff2.tms_utime-(double)tbuff1.tms_utime)/CLOCKS_PER_SECOND,
 467              ((double)tbuff2.tms_stime-(double)tbuff1.tms_stime)/CLOCKS_PER_SECOND);
 468     fprintf(stderr,"* -------------------------------------------------------------------\n");
 469     fprintf(stderr,"[ Overall Time = Start up Time + Tagging Time ]\n");
 470     showTime("Overall Time",((double)(end-start+endStartUpTime-startUpTime))/CLOCKS_PER_SECOND,
 471              ((double)tbuff2.tms_utime-(double)tbuff1.tms_utime+
 472               (double)tbuffEndStartUp.tms_utime-(double)tbuffStartUp.tms_utime)/CLOCKS_PER_SECOND,
 473              ((double)tbuff2.tms_stime-(double)tbuff1.tms_stime+
 474               (double)tbuffEndStartUp.tms_stime-(double)tbuffStartUp.tms_stime)/CLOCKS_PER_SECOND);
 475     fprintf(stderr,"* -------------------------------------------------------------------\n");
 476     taggerStadistics(contWords,contSentences,
 477                      ((double)(end-start))/CLOCKS_PER_SECOND,
 478                      ((double)tbuff2.tms_utime-(double)tbuff1.tms_utime)/CLOCKS_PER_SECOND,
 479                      ((double)tbuff2.tms_stime-(double)tbuff1.tms_stime)/CLOCKS_PER_SECOND);
 480     }
 481 }
 482
 483 /***************************************************************/
 484
 485 void tagger::taggerDoNormal(int *numWords, int *numSentences)
 486 {
 487   int contWordsLR=0,contWordsRL=0,contSentences=0,ret = 1;
 488
 489   while ((ret>=0))
 490     {
 491       if (verbose) taggerShowVerbose(contSentences,0);
 492
 493       if ((strcmp(flow,"LRL")==0) || (strcmp(flow,"LR")==0))
 494         contWordsLR = contWordsLR+taggerRightSense();
 495       if ((strcmp(flow,"LRL")==0) || (strcmp(flow,"RL")==0))
 496         contWordsRL = contWordsRL+taggerLeftSense();
 497       contSentences++;
 498       sw->show();
 499       sw->deleteList();
 500       ret = sw->iniGeneric();
 501     }
 502   if (contWordsRL==0) *numWords=contWordsLR/taggerNumLaps;
 503   else *numWords=contWordsRL/taggerNumLaps;
 504   *numSentences = contSentences;
 505 }
 506
 507 /***************************************************************/
 508
 509 void tagger::taggerDoSpecialForUnknown(int *numWords, int *numSentences)
 510 {
 511   int contWordsLR=0,contWordsRL=0,contSentences=0,ret = 1;
 512
 513   while ((ret>=0))
 514     {
 515       if (verbose) taggerShowVerbose(contSentences,0);
 516
 517       if ((strcmp(flow,"LRL")==0) || (strcmp(flow,"LR")==0))
 518         contWordsLR = contWordsLR+taggerRightSenseSpecialForUnknown();
 519       if ((strcmp(flow,"LRL")==0) || (strcmp(flow,"RL")==0))
 520         contWordsRL = contWordsRL+taggerLeftSenseSpecialForUnknown();
 521
 522       contSentences++;
 523       sw->show();
 524       sw->deleteList();
 525       ret = sw->iniGeneric();
 526     }
 527   if (contWordsRL==0) *numWords=contWordsLR/taggerNumLaps;
 528   else *numWords=contWordsRL/taggerNumLaps;
 529   *numSentences = contSentences;
 530 }
 531
 532 /***************************************************************/
 533
 534 void tagger::taggerDoNTimes(int *numWords, int *numSentences,int laps)
 535 {
 536   int contWordsLR=0,contWordsRL=0,contSentences=0,ret = 1;
 537
 538   while ((ret>=0))
 539     {
 540
 541       if (verbose) taggerShowVerbose(contSentences,0);
 542
 543       for (int pasadas=0;pasadas<laps;pasadas++)
 544         {
 545
 546           taggerModelRunning = &taggerModelList[pasadas];
 547           if ((strcmp(flow,"LRL")==0) || (strcmp(flow,"LR")==0))
 548             contWordsLR = contWordsLR+taggerRightSense();
 549
 550           if (strcmp(flow,"LRL")==0 && pasadas>0)
 551             sw->winMaterializePOSValues(2);
 552           if ((strcmp(flow,"LRL")==0) || (strcmp(flow,"RL")==0))
 553             contWordsRL = contWordsRL+taggerLeftSense();
 554
 555         }
 556
 557       contSentences++;
 558       sw->show();
 559       sw->deleteList();
 560       ret = sw->iniGeneric();
 561     }
 562   if (contWordsRL==0) *numWords=contWordsLR/taggerNumLaps;
 563   else *numWords=contWordsRL/taggerNumLaps;
 564   *numSentences = contSentences;
 565 }
 566
 567 /***************************************************************/
 568 /***************************************************************/
 569
 570 void tagger::taggerGenerateScore(nodo *elem,int direction)
 571 {
 572
 573   struct  tms tbuffStartFex,tbuffEndFex;
 574   clock_t startFexTime,endFexTime;
 575   struct  tms tbuffStartSVM,tbuffEndSVM;
 576   clock_t startSVMTime,endSVMTime;
 577
 578   weight_node_t *weight;
 579   nodo_feature_list *aux;
 580   weightRepository *weightRep;
 581   hash_t *bias;
 582   int i,numMaybe,ret=1,max=0;
 583   int is_unk=FALSE;
 584   simpleList *featureList;
 585
 586   startFexTime = times(&tbuffStartFex);
 587
 588   i = d->getElement(elem->wrd);
 589   if (i!=HASH_FAIL)
 590     {
 591       featureList = &taggerModelRunning->featureList;
 592       numMaybe = d->getElementNumMaybe(i);
 593       weight = taggerCreateWeightNodeArray(numMaybe,i);
 594       if ((strcmp(flow,"LRL")==0) && (direction==2))
 595         {
 596           weightRep = taggerModelRunning->wr2;  //wr2;
 597           //bias = taggerModelRunning->bias2;  //taggerBias2;
 598         }
 599       else
 600         {
 601           weightRep = taggerModelRunning->wr; //wr;
 602           //bias = taggerModelRunning->bias; //taggerBias;
 603         }
 604     }
 605   else
 606     {
 607       numMaybe = NUM_UNK_POS;
 608       weight = taggerInitializeWeightNodeArray(numMaybe,weightUnk);
 609       featureList = &taggerModelRunning->featureListUnk;
 610       is_unk = TRUE;
 611
 612       if ((strcmp(flow,"LRL")==0) && (direction==2))
 613         {
 614                 weightRep = taggerModelRunning->wrUnk2;  //wrUnk2;
 615                 //bias =        taggerModelRunning->biasUnk2; //taggerBiasUnk2;
 616         }
 617       else
 618         {       weightRep = taggerModelRunning->wrUnk; //wrUnk;
 619                 //bias =taggerModelRunning->biasUnk; //taggerBiasUnk;
 620         }
 621     }
 622
 623
 624   if (numMaybe>1)
 625     {
 626       while (ret>=0)
 627         {
 628           aux = (nodo_feature_list *) featureList->getIndex();
 629           if (strcmp(aux->mark,SLASTW)==0)  sw->winPushSwnFeature(stk);
 630           else if (strcmp(aux->mark,WMARK)==0)  sw->winPushWordFeature((void *)aux,d,stk,direction);
 631           else if (strcmp(aux->mark,KMARK)==0)  sw->winPushAmbiguityFeature((void *)aux,d,stk,direction);
 632           else if (strcmp(aux->mark,MMARK)==0)  sw->winPushMaybeFeature((void *)aux,d,stk,direction);
 633           else if (strcmp(aux->mark,PMARK)==0)  sw->winPushPosFeature((void *)aux,d,stk,direction);
 634           else if (strcmp(aux->mark,MFTMARK)==0)  sw->winPushMFTFeature((void *)aux,d,stk,direction);
 635           else if (is_unk==TRUE)
 636             {
 637               int *param;
 638               if (aux->n>0)
 639                 {
 640                   param = (int *) aux->l.getIndex();
 641                 }
 642               if (strcmp(aux->mark,PREFIX_MARK)==0)  sw->winPushPrefixFeature(elem->wrd, stk, *param);
 643               else if (strcmp(aux->mark,SUFFIX_MARK)==0) sw->winPushSuffixFeature(elem->wrd, stk, *param);
 644               else if (strcmp(aux->mark,CHAR_A_MARK)==0) sw->winPushLetterFeature(elem->wrd, stk, *param, COUNTING_FROM_BEGIN);
 645               else if (strcmp(aux->mark,CHAR_Z_MARK)==0) sw->winPushLetterFeature(elem->wrd, stk, *param, COUNTING_FROM_END);
 646               else if (strcmp(aux->mark,LENGTH_MARK)==0) sw->winPushLenghtFeature(elem->wrd,stk);
 647               else if (strcmp(aux->mark,START_CAPITAL_MARK)==0) sw->winPushStartWithCapFeature(elem->wrd,stk);
 648               else if (strcmp(aux->mark,START_LOWER_MARK)==0)  sw->winPushStartWithLowerFeature(elem->wrd,stk);
 649               else if (strcmp(aux->mark,START_NUMBER_MARK)==0) sw->winPushStartWithNumberFeature(elem->wrd,stk);
 650               else if (strcmp(aux->mark,ALL_UPPER_MARK)==0) sw->winPushAllUpFeature(elem->wrd,stk);
 651               else if (strcmp(aux->mark,ALL_LOWER_MARK)==0) sw->winPushAllLowFeature(elem->wrd,stk);
 652               else if (strcmp(aux->mark,CONTAIN_CAP_MARK)==0) sw->winPushContainCapFeature(elem->wrd, stk);
 653               else if (strcmp(aux->mark,CONTAIN_CAPS_MARK)==0) sw->winPushContainCapsFeature(elem->wrd, stk);
 654               else if (strcmp(aux->mark,CONTAIN_COMMA_MARK)==0) sw->winPushContainCommaFeature(elem->wrd, stk);
 655               else if (strcmp(aux->mark,CONTAIN_NUMBER_MARK)==0) sw->winPushContainNumFeature(elem->wrd, stk);
 656               else if (strcmp(aux->mark,CONTAIN_PERIOD_MARK)==0) sw->winPushContainPeriodFeature(elem->wrd, stk);
 657               else if (strcmp(aux->mark,MULTIWORD_MARK)==0) sw->winPushMultiwordFeature(elem->wrd, stk);
 658             }
 659           ret = featureList->next();
 660         }
 661       featureList->setFirst();
 662
 663       endFexTime = times(&tbuffEndFex);
 664       realFexTime = realFexTime + ((double)(endFexTime-startFexTime))/CLOCKS_PER_SECOND;
 665       usrFexTime = usrFexTime + (((double)tbuffEndFex.tms_utime-(double)tbuffStartFex.tms_utime)/CLOCKS_PER_SECOND);
 666       sysFexTime = sysFexTime + (((double)tbuffEndFex.tms_stime-(double)tbuffStartFex.tms_stime)/CLOCKS_PER_SECOND);
 667
 668       startSVMTime = times(&tbuffStartSVM);
 669
 670       taggerSumWeight(weightRep,bias,weight,numMaybe,&max);
 671
 672       endSVMTime = times(&tbuffEndSVM);
 673       realSVMTime = realSVMTime + ((double)(endSVMTime-startSVMTime))/CLOCKS_PER_SECOND;
 674       usrSVMTime = usrSVMTime + (((double)tbuffEndSVM.tms_utime-(double)tbuffStartSVM.tms_utime)/CLOCKS_PER_SECOND);
 675       sysSVMTime = sysSVMTime + (((double)tbuffEndSVM.tms_stime-(double)tbuffStartSVM.tms_stime)/CLOCKS_PER_SECOND);
 676     }
 677
 678   strcpy(elem->pos,weight[max].pos);
 679   elem->weight = weight[max].data;
 680
 681   if (strcmp(flow,"LRL")==0)
 682     {
 683         weight_node_t *score = new weight_node_t;
 684         score->data = weight[max].data;
 685         strcpy(score->pos,weight[max].pos);
 686         push(elem->stackScores,score);
 687     }
 688
 689   if (i!=HASH_FAIL) delete[] weight; //mod delete[] instead of delete
 690 }
 691
 692 /***************************************************************/
 693
 694 weight_node_t *tagger::taggerCreateWeightNodeArray(int numMaybe,int index)
 695 {
 696   int ret=1,j = numMaybe;
 697   weight_node_t *weight = new weight_node_t[numMaybe];
 698   simpleList *list = (simpleList *) d->getElementMaybe(index);
 699
 700   while (ret>=0 && numMaybe > 0)
 701     {
 702       infoDict *pInfoDict = (infoDict *) list->getIndex();
 703       j--;
 704       sprintf(weight[j].pos,"%s",pInfoDict->txt);
 705       weight[j].data = 0;
 706       ret=list->next();
 707     }
 708
 709   list->setFirst();
 710   return weight;
 711 }
 712
 713 /***************************************************************/
 714
 715 weight_node_t *tagger::taggerInitializeWeightNodeArray(int numMaybe,weight_node_t *w)
 716 {
 717   for (int i=0;i<numMaybe;i++) w[i].data=0;
 718   return w;
 719 }
 720
 721 /***************************************************************/
 722
 723 void tagger::taggerSumWeight(weightRepository *wRep,hash_t *bias,weight_node_t *weight, int numMaybe, int *max)
 724 {
 725   weight_node_t *aux;
 726   long double w,b = 0;
 727   char *feature;
 728   int putBias=1;
 729
 730   while (!empty(stk))
 731     {
 732       *max=0;
 733       feature = (char *) pop(stk);
 734       for (int j=0; j<numMaybe;j++)
 735         {
 736           if (putBias)
 737             {
 738               b = wRep->wrGetWeight("BIASES",weight[j].pos);
 739               weight[j].data = weight[j].data - b;
 740             }
 741           w = wRep->wrGetWeight(feature,weight[j].pos);
 742           weight[j].data=weight[j].data+w;
 743           if (((float)weight[*max].data)<((float)weight[j].data)) *max=j;
 744         }
 745       delete[] feature; //mod delete[] instead of delete
 746       putBias=0;
 747     }
 748 }
 749
 750 /***************************************************************/
 751 /***************************************************************/
 752
 753 weight_node_t *tagger::taggerCreateWeightUnkArray(char *name)
 754 {
 755   NUM_UNK_POS=0;
 756   int i=0;
 757   char c=' ';
 758   FILE *f;
 759
 760   if ((f = fopen(name, "rt"))== NULL)
 761     {
 762       fprintf(stderr, "Error opening file: %s",name);
 763       exit(0);
 764     }
 765
 766   while (!feof(f))
 767     { if (fgetc(f)=='\n') NUM_UNK_POS++;
 768     }
 769
 770   fseek(f,0,SEEK_SET);
 771
 772   weight_node_t *weight = new weight_node_t[NUM_UNK_POS];
 773   while (!feof(f) && (i<NUM_UNK_POS))
 774     {   strcpy(weight[i].pos,"");
 775     weight[i].data=0;
 776     c = fgetc(f);
 777     while ((c!='\n') && (!feof(f)))
 778       {
 779         if (c!=' ' && c!='\n' && c!='\t') sprintf(weight[i].pos,"%s%c",weight[i].pos,c);
 780         c = fgetc(f);
 781       }
 782     i++;
 783     }
 784   fclose(f);
 785   return weight;
 786 }
 787
 788 /***************************************************************/
 789 /***************************************************************/
 790
 791 void tagger::taggerStadistics(int numWords, int numSentences,  double realTime,double usrTime, double sysTime)
 792 {
 793   char message[200]="";
 794   float media=0;
 795   if (time!=0)  media = (float) (((double) numWords)/(sysTime+usrTime));
 796
 797   sprintf(message,"%s\n%d sentences were tagged.",message,numSentences);
 798   sprintf(message,"%s\n%d words were tagged.",message,numWords);
 799   sprintf(message,"%s\n%f words/second were tagged.\n",message,media);
 800   fwrite(message,strlen(message),1,stderr);
 801 }
 802
 803 /***************************************************************/
 804
 805 void tagger::taggerShowVerbose(int num,int isEnd)
 806 {
 807   if (isEnd) { fprintf(stderr,".%d sentences [DONE]\n\n",num); return; }
 808   else if (num%100==0) fprintf(stderr,"%d",num);
 809   else if (num%10==0) fprintf(stderr,".");
 810 }
 811
 812