tests/tokenize-test.cpp

   1 #include "tokenize.h"
   2 #include <algorithm>
   3 #include <iostream>
   4 #include <iterator>
   5 #include <spell.h>
   6 #include <dictionary.h>
   7
   8 using namespace std;
   9
  10 int main()
  11 {
  12   string str;
  13
  14   dic_init();
  15
  16   while (getline(cin,str)) {
  17     /*
  18     Tokens output;
  19     tokenize(str,output);
  20     //copy(output.begin(),output.end(),ostream_iterator<string>(cout,"\n"));
  21     for (int i = 0;i < output.size();i ++)
  22       if (output[i].is_token)
  23         cout << output[i].value << endl;
  24     */
  25     Sentence st(str);
  26     st.standardize();
  27     st.tokenize();
  28     cout << st << endl;
  29   }
  30   return 0;
  31 }
  32