utils/segm-test.cpp

   1 #include "wfst.h"
   2 #include "sentence.h"
   3 #include <fstream>
   4 #include <iostream>
   5 #include <algorithm>
   6 #include <iterator>
   7
   8 using namespace std;
   9
  10 int main(int argc,char **argv)
  11 {
  12   dic_init();
  13
  14   cerr << "Loading... ";
  15   warch.load("wordlist.wl");
  16
  17   File f("ngram","rt");
  18   get_ngram().read(f);
  19   cerr << "done" << endl;
  20
  21   get_sarch().set_blocked(true);
  22
  23   string s;
  24   while (getline(cin,s)) {
  25     if (s.empty()) continue;
  26
  27     vector<string> ss;
  28     sentences_split(s,ss);
  29     for (int ii = 0;ii < ss.size();ii ++) {
  30       Sentence st(ss[ii]);
  31       st.standardize();
  32       st.tokenize();
  33       Lattice words;
  34       Segmentation seg;
  35       Segmentor segtor;
  36       words.construct(st);
  37       segtor.init(words,0,4);
  38       while (segtor.step(seg)) {
  39         int ngram_length=2;
  40         seg.prob = 0;
  41         VocabIndex *vi = new VocabIndex[ngram_length];
  42         vi[ngram_length] = Vocab_None;
  43         for (int i = ngram_length-1;i < seg.size();i ++) {
  44           for (int j = 0;j < ngram_length-1;j++)
  45             vi[j] = seg[i-1-j].node.node->get_id();
  46           seg.prob += -get_ngram().wordProb(seg[i].node.node->get_id(),vi);
  47         }
  48
  49         cout << seg << " " << seg.prob << endl;
  50         //cout << seg << endl;
  51       }
  52       segtor.done();
  53     }
  54   }
  55
  56   return 0;
  57 }