utils/tokenize-test.cpp

   1 #include "tokenize.h"
   2 #include <algorithm>
   3 #include <iostream>
   4 #include <iterator>
   5 #include <spell.h>
   6 #include <dictionary.h>
   7
   8 using namespace std;
   9
  10 int main(int argc, char **argv)
  11 {
  12   bool revert = argc == 2 && string(argv[1]) == "-r";
  13   string str;
  14
  15   dic_init();
  16
  17   int count = 0;
  18   while (getline(cin,str)) {
  19     if (++count % 200 == 0) cerr << count << endl;
  20     if (revert) {
  21       Sentence st(cin);
  22       cout << st << endl;
  23     }
  24     else {
  25       Sentence st(str);
  26       st.standardize();
  27       st.tokenize();
  28       cout << st << endl;
  29     }
  30   }
  31   return 0;
  32 }