src/aoi.cxx

   1 /*
   2     Copyright 2013 Karel Matas
   3
   4     This program is free software: you can redistribute it and/or modify
   5     it under the terms of the GNU General Public License as published by
   6     the Free Software Foundation, either version 3 of the License, or
   7     (at your option) any later version.
   8
   9     This program is distributed in the hope that it will be useful,
  10     but WITHOUT ANY WARRANTY; without even the implied warranty of
  11     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12     GNU General Public License for more details.
  13
  14     You should have received a copy of the GNU General Public License
  15     along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16 */
  17 #include <cmath>
  18 #include <set>
  19 #include <FL/fl_ask.H>
  20 #include "aoi.hxx"
  21 #include "utils.hxx"
  22
  23 namespace aoi {
  24
  25 using std::set;
  26 using aoi_ui::TextStyle;
  27 using utils::to_string;
  28
  29 const char *MANAGEDB_ITEM_KANJIDIC        = "Kanjidic";
  30 const char *MANAGEDB_ITEM_JMDICT                  = "JMdict";
  31 const char *MANAGEDB_ITEM_COMPONENTS  = "Components";
  32 const char *TEMP_JMDICT               = "gztemp.jmdict";
  33 const char *TEMP_KANJIDIC             = "gztemp.kanjidic";
  34 const char *TEMP_COMPONENTS           = "gztemp.components";
  35 const char *LOG_FILE                  = "aoi.log";
  36 const char SENSE_SEARCH_CHAR          = ':';
  37
  38 App* App::instance_ = nullptr;
  39
  40 App::App ()
  41 {
  42   remove(LOG_FILE);
  43   logger_.filename(LOG_FILE);
  44   logger_.loglevel(Logger::MSG_DEBUG);
  45
  46   cfg_  = new aoi_config::Config();
  47   ui_   = new aoi_ui::GUI(this);
  48   rmn_  = new Romanization();
  49
  50   ui_->progress(0,"Opening database...");
  51
  52   try {
  53     db_ = new SQLite3::SQLite3( get_config("db/file_main").c_str() );
  54   }
  55   catch (SQLite3::CantOpenDatabase &e){
  56     log_e( "Aoi: Can't open database '"  + string(e.what()) + "'.");
  57   }
  58
  59   std::stringstream ss;
  60     ss << "ATTACH DATABASE '" << get_config("db/file_user") << "'as user;";
  61   query(ss.str().c_str());
  62
  63   ui_->progress( 30, "Checking tables..." );
  64   check_tables();
  65   ui_->progress( 60, "Checking indexes..." );
  66   check_indexes();
  67
  68   load_config();
  69   init_dicview();
  70
  71   #ifdef DEBUG
  72 //    ui_->progress(90, "DEBUG query..." );
  73 //    parse_dic_input("ana*");
  74 //    cb_kanji_search();
  75 //    ui_->cb_toggle_group(nullptr);
  76   #endif
  77
  78   ui_->progress( 98, "Initializing fonts..." );
  79   ui_->fontname_kanji( get_config("font/kanji") );
  80   ui_->help_file( get_config("sources/help_index") );
  81
  82   ui_->progress_hide();
  83
  84
  85   auto q = query("select val from aoi where key='jmdict_version'");
  86   string jmdict_version = (q.empty()) ? "NONE":q[0];
  87   q = query("select val from aoi where key='kanjidic_version'");
  88   string kanjidic_version = q.empty() ? "NONE":q[0];
  89   if ( jmdict_version == "NONE" || kanjidic_version == "NONE" )
  90     cb_manage_db();
  91
  92
  93   #ifdef DEBUG
  94     logger_.loglevel(Logger::MSG_DEBUG);
  95   #endif
  96
  97   instance_ = this;
  98 }
  99
 100
 101 App::~App ()
 102 {
 103   delete db_;
 104   delete rmn_;
 105   delete ui_;
 106   delete cfg_;
 107 }
 108
 109
 110 void App::init_dicview ()
 111 {
 112   // initialize textstyles
 113   TextStyle style_default(FL_HELVETICA,1.2);
 114   TextStyle style_reading(aoi_ui::FONT_KANJI,1.5);
 115   TextStyle style_reading_freq = style_reading;
 116   style_reading_freq.color = get_color("frequent");
 117   TextStyle style_kanji(aoi_ui::FONT_KANJI,1.7);
 118   TextStyle style_kanji_freq = style_kanji;
 119   style_kanji_freq.color = get_color("frequent");
 120   TextStyle style_inf(FL_HELVETICA_ITALIC,0.8,get_color("pos"));
 121   style_inf.offset_y = 3;
 122   TextStyle style_pos = style_inf;
 123   TextStyle style_misc = style_pos;
 124   style_misc.color = get_color("misc");
 125   TextStyle style_field = style_pos;
 126   style_field.color = get_color("field");
 127   TextStyle style_dial = style_pos;
 128
 129   // register TextStyles
 130   ui_->register_tag_dicview( "default",       style_default );
 131   ui_->register_tag_dicview( "reading",       style_reading );
 132   ui_->register_tag_dicview( "reading_freq",  style_reading_freq );
 133   ui_->register_tag_dicview( "kanji",         style_kanji );
 134   ui_->register_tag_dicview( "kanji_freq",    style_kanji_freq );
 135   ui_->register_tag_dicview( "kinf",          style_inf );
 136   ui_->register_tag_dicview( "rinf",          style_inf );
 137   ui_->register_tag_dicview( "pos",           style_pos );
 138   ui_->register_tag_dicview( "misc",          style_misc );
 139   ui_->register_tag_dicview( "field",         style_field );
 140   ui_->register_tag_dicview( "dial",          style_dial );
 141 }
 142
 143
 144 vector<string> App::query ( const char *q, bool log_query, bool replace_separator )
 145 {
 146   if ( !db_ ){
 147     log_e("App::query(): Database does not exist.");
 148     return {};
 149   }
 150   ui_->cursor_wait();
 151   vector<string> result;
 152   try {
 153     if ( log_query )
 154       log_d(string(q));
 155     result = db_->query(q);
 156   }
 157   catch (SQLite3::DatabaseError &e){
 158     log_e("App: DatabaseError:: " + string(e.what()) + string("\nQuery: ")
 159       + string(e.query()) );
 160     ui_->cursor_default();
 161   }
 162   string msg = std::to_string(db_->result_rows()) + " results";
 163   log( "App::query(): " + msg);
 164   if ( replace_separator )
 165     for ( string &s: result )
 166       utils::replace_all(s, parsers::SEPARATOR_SQL, ", ");
 167   ui_->cursor_default();
 168   return result;
 169 }
 170
 171
 172 void App::cb_set_components ()
 173 {
 174   if ( curr_components_.empty() )
 175     return;
 176
 177   using aoi_ui::ComponentView;
 178
 179   // prepare cells
 180   vector<string> included = ui_->components_include();
 181   vector<string> excluded = ui_->components_exclude();
 182
 183   vector<ComponentView::Cell> v;
 184   for ( string &s: included )
 185     v.push_back( ComponentView::Cell(s,ComponentView::CELL_SELECTED_1) );
 186   for ( string &s: excluded )
 187     v.push_back( ComponentView::Cell(s,ComponentView::CELL_SELECTED_2) );
 188
 189   // sort by occurences
 190   if ( !ui_->sort_components_by_strokes() ){
 191     for ( auto mi = curr_components_.rbegin(); mi!=curr_components_.rend(); ++mi ){
 192       if ( !utils::is_in( included, mi->second )
 193             && !utils::is_in( excluded, mi->second ) )
 194         v.push_back( ComponentView::Cell(mi->second) );
 195     }
 196   }
 197   // sort by strokes
 198   else {
 199     vector<string> comps_by_strokes;
 200     for ( auto mi: curr_components_ ){
 201       if ( mi.first < get_config<int>("knj/min_compo_count") )
 202         continue;
 203       comps_by_strokes.push_back(mi.second);
 204     }
 205
 206     struct SortByStrokes {
 207       map<string,int> comps;
 208       SortByStrokes( const map<string,int> &c ): comps(c){};
 209       bool operator() (const string &c1, const string &c2 ){
 210         return ( this->comps[c1] < this->comps[c2] );
 211       }
 212     } sbc(components_);
 213
 214     std::sort( comps_by_strokes.begin(), comps_by_strokes.end(), sbc );
 215
 216     int prev_strokes=0;
 217     for ( string &s: comps_by_strokes ){
 218       int curr_strokes = components_[s];
 219       if ( prev_strokes != curr_strokes )
 220         v.push_back( ComponentView::Cell( std::to_string(curr_strokes),
 221                                           ComponentView::CELL_LABEL) );
 222       v.push_back( ComponentView::Cell(s) );
 223       prev_strokes = curr_strokes;
 224     }
 225   }
 226
 227   ui_->set_components( v );
 228 }
 229
 230
 231 void App::cb_kanji_search ()
 232 {
 233
 234   if ( components_.empty() ){
 235     log("Loading components...");
 236     vector<string> res = query("select component, strokes from components");
 237     for ( size_t i=0; i<res.size(); i=i+2 ){
 238       components_[res[i]] = std::stoi(res[i+1]);
 239     }
 240   }
 241
 242   // strokes
 243   std::pair<int,int> strokes = utils::parse_range( utils::strip( ui_->strokes() ) );
 244   // jlpt
 245   std::pair<int,int> jlpt = utils::parse_range( utils::strip( ui_->jlpt() ) );
 246   // grade
 247   std::pair<int,int> grade = utils::parse_range( utils::strip( ui_->grade() ) );
 248
 249
 250
 251   // skip
 252   vector<string> skip = utils::split_string( ui_->skip() );
 253   std::stringstream sskip;
 254   if ( skip.size() > 0 ){
 255     string skip1 = utils::strip(skip[0].c_str());
 256     sskip << " S.skip1=" << skip1;
 257   }
 258   if ( skip.size() > 1 ){
 259     std::pair<int,int> skip2 = utils::parse_range( utils::strip(skip[1].c_str()) );
 260     sskip << " and S.skip2>=" << skip2.first << " and S.skip2<=" << skip2.second;
 261   }
 262   if ( skip.size() > 2 ){
 263     std::pair<int,int> skip3 = utils::parse_range( utils::strip(skip[2].c_str()) );
 264     sskip << " and S.skip3>=" << skip3.first << " and S.skip3<=" << skip3.second;
 265   }
 266   if ( !sskip.str().empty() )
 267     sskip << " and ";
 268
 269   // ordering
 270   string order_by;
 271   switch ( ui_->sort_mode() ){
 272     case SORT_FREQ_ASC:
 273       order_by = "(case freq when 0 then 9999 else freq end) asc";
 274       break;
 275     case SORT_FREQ_DESC:
 276       order_by = "(case freq when 0 then 9999 else freq end) desc";
 277       break;
 278     case SORT_STROKES_ASC:
 279       order_by = "strokes asc";
 280       break;
 281     case SORT_STROKES_DESC:
 282       order_by = "strokes desc";
 283       break;
 284   }
 285
 286   vector<string> components_include = ui_->components_include();
 287   vector<string> components_exclude = ui_->components_exclude();
 288   std::stringstream comps;
 289   for ( auto c: components_include )
 290     comps << " and components like '%" << c << "%'";
 291   for ( auto c: components_exclude )
 292     comps << " and components not like '%" << c << "%'";
 293   printf("%s\n",comps.str().c_str());
 294
 295   string jis208 = "";
 296   if ( get_config<bool>("knj/jis208_only") )
 297     jis208 = " flags glob '*jis208*' and ";
 298
 299   // build query
 300   std::stringstream ss;
 301   ss  <<  "select distinct "
 302       <<    "K.kanji,freq,components,flags "
 303       <<  " from k_kanji as K"
 304       <<    (sskip.str().empty() ? " where ":", k_skip as S where K.kanji=S.kanji and")
 305       <<    jis208
 306       <<    sskip.str()
 307       <<    " strokes>=" << strokes.first << " and strokes<=" << strokes.second
 308       <<    " and jlpt>=" << jlpt.first << " and jlpt<=" << jlpt.second
 309       <<    " and grade>=" << grade.first << " and grade<=" << grade.second
 310       <<    comps.str()
 311       <<  " order by " << order_by;
 312
 313
 314   // perform query
 315   vector<string> q = query( ss.str().c_str(), true, false );
 316   vector<aoi_ui::KanjiView::Cell> data;
 317   vector<string> components;
 318   set<string> flags;
 319   for ( size_t i=0; i<q.size(); i+=4 ){
 320     string kanji = q[i];
 321     int freq = std::stoi(q[i+1]);
 322     data.push_back(
 323       aoi_ui::KanjiView::Cell(
 324         kanji,
 325         (freq>0)? get_color("frequent"):-1
 326       )
 327     );
 328     components.push_back(q[i+2]);
 329     for ( string &s: utils::split_string( q[i+3], parsers::SEPARATOR_SQL) )
 330       flags.insert(s);
 331   }
 332
 333   log_d("Groups: " + utils::to_string(flags));
 334
 335   ui_->set_kanjiview( data );
 336   char b[32];
 337   sprintf( b, "%d results", db_->result_rows() );
 338   ui_->set_kanji_results( b );
 339
 340   utils::Histogram<string> histogram;
 341   for ( string &s: components )
 342     histogram.add( utils::str_to_chars(s.c_str()) );
 343   curr_components_ = histogram.sorted();
 344
 345   cb_set_components();
 346 }
 347
 348
 349 void App::alert ( const string &msg, const string &desc )
 350 {
 351   std::stringstream ss;
 352   ss << msg;
 353   if ( !desc.empty() )
 354     ss << "\n\nDetails:\n" << desc;
 355   log_e(ss);
 356   ui_->alert(ss.str());
 357 }
 358
 359
 360 int App::run ( int argc, char **argv )
 361 {
 362   return ui_->run(argc,argv);
 363 }
 364
 365
 366 void App::parse_kanjidic ( const char *fname, bool delete_source )
 367 {
 368   if ( !utils::file_exists( fname ) ){
 369     log_e("App::parse_kanjidic(): File does not exist: " + string(fname));
 370     return;
 371   }
 372   ui_->progress(0, "Loading kanjidic...");
 373   log("Loading kanjidic...");
 374
 375   const char *SEP = parsers::SEPARATOR_SQL;
 376
 377   int n_kanji = 0;
 378   try {
 379     ui_->progress(1, "Creating parser...");
 380     std::stringstream msg;
 381     msg << "Decompressing file: " << fname << " -> " << TEMP_KANJIDIC;
 382     log_d(msg);
 383
 384     ui_->cursor_wait();
 385     utils::gzip_decompress_file( fname, TEMP_KANJIDIC);
 386     parsers::KanjidicParser p(TEMP_KANJIDIC);
 387     auto kanji = p.get_entry();
 388
 389     query(  "DROP TABLE IF EXISTS k_kanji;"
 390             "DROP TABLE IF EXISTS components;"
 391             "DROP TABLE IF EXISTS k_skip;"
 392             "DELETE FROM aoi WHERE key='kanjidic_version';" );
 393
 394     check_tables();
 395
 396     std::stringstream ss;
 397     ss << "BEGIN TRANSACTION;\n";
 398     ss << "INSERT INTO aoi (key,val) VALUES ('kanjidic_version','"
 399       << p.get_version() << "');\n";
 400     while ( kanji.kanji() != "" ){
 401       n_kanji++;
 402       if ( n_kanji % 100 == 0)
 403         // percent = 100 * n_kanji/13108
 404         ui_->progress( n_kanji/14, std::to_string(n_kanji)+string(" kanji loaded"));
 405       ss << "INSERT INTO k_kanji "
 406         << "(kanji,ucs,onyomi,kunyomi,meaning,nanori,flags,jlpt,grade,freq,strokes,"
 407         << "rad_classic,rad_nelson,components)"
 408         << " VALUES('"
 409         << kanji.kanji() << "','"
 410         << kanji.ucs() << "','"
 411         << to_string(kanji.onyomi(),SEP) << "','"
 412         << to_string(kanji.kunyomi(),SEP) << "','"
 413         << SQLite3::escape(to_string(kanji.meaning(),SEP)) << "','"
 414         << to_string(kanji.nanori(),SEP) << "','"
 415         << to_string(kanji.flags(),SEP) << "',"
 416         << kanji.jlpt() << ","
 417         << kanji.grade() << ","
 418         << kanji.freq() << ","
 419         << kanji.strokes() << ","
 420         << kanji.rad_classic() << ","
 421         << kanji.rad_nelson() << ","
 422         << "''"
 423         << ");\n";
 424       for ( SKIP &s: kanji.skip() ) {
 425         ss << "INSERT INTO k_skip (kanji,skip1,skip2,skip3,misclass) VALUES('"
 426           << kanji.kanji() << "'," << s.s1 << "," << s.s2 << "," << s.s3
 427           << ",'" << s.misclass << "');\n";
 428       }
 429       kanji = p.get_entry();
 430     }
 431     ss << "END TRANSACTION;\n";
 432     log("Writing to db ...");
 433
 434 //    #ifdef DEBUG
 435 //      log_d("Writing file 'script.kanjidic.sql'...");
 436 //      std::ofstream f ("script.kanjidic.sql");
 437 //      f << ss.str();
 438 //      f.close();
 439 //    #endif
 440     ui_->progress(95, "Writing to database...");
 441     query(ss.str().c_str(),false);
 442   }
 443   catch ( utils::ParsingError &e ){
 444     ui_->cursor_default();
 445     ui_->progress_hide();
 446     std::string msg = "App::parse_kanjidic(): ParsingError: ";
 447     msg += e.what();
 448     alert(msg);
 449     return;
 450   }
 451   ui_->progress(98, "Checking indexes...");
 452   check_indexes();
 453   ui_->progress_hide();
 454   remove( TEMP_KANJIDIC );
 455   if ( delete_source )
 456     remove( fname );
 457   ui_->cursor_default();
 458 }
 459
 460
 461 void App::parse_jmdict ( const char *fname, bool delete_source )
 462 {
 463   if ( !utils::file_exists( fname ) ){
 464     log_e("App::parse_jmdict(): File does not exist: " + string(fname));
 465     return;
 466   }
 467
 468   ui_->progress( 0, "Loading JMdict..." );
 469   log("Loading JMdict...");
 470   std::stringstream ss;
 471   ss << "BEGIN TRANSACTION;\n";
 472
 473   // tables
 474   for ( auto &mi: aoi_config::db_tables.at("main") ){
 475     ss << "DROP TABLE IF EXISTS " << mi.first << ";\n"
 476     << "CREATE TABLE " << mi.first << " ( ";
 477     for ( size_t i=0; i<mi.second.size(); i++ )
 478       ss << mi.second[i].name << " " << mi.second[i].type
 479         << ((i==mi.second.size()-1) ? "":",");
 480     ss << ");\n";
 481   }
 482
 483   // create parser
 484   int n_entries = 0;
 485   try {
 486     ui_->progress(1, "Creating parser...");
 487
 488     ui_->cursor_wait();
 489     utils::gzip_decompress_file( fname, TEMP_JMDICT);
 490     parsers::JmdictParser jmp(TEMP_JMDICT);
 491     const char *SEP = parsers::SEPARATOR_SQL;
 492
 493     // version
 494     ss << "INSERT INTO aoi (key,val) VALUES ('jmdict_version', '"
 495       << jmp.get_version() << "');\n";
 496     log("jmdict_version: "+jmp.get_version());
 497
 498     // get entities
 499     for ( std::pair<string,string> elt: jmp.get_entities() ){
 500       ss << "INSERT INTO entities (abbr,desc) VALUES ('" << SQLite3::escape(elt.first)
 501         << "','" << SQLite3::escape(elt.second) << "');\n";
 502     }
 503
 504     int n_entries = 1;
 505     parsers::JmdictEntry entry = jmp.get_entry();
 506     while ( entry.did != -1 ) {
 507       if ( n_entries % 1000 == 0 ){
 508         // entries is about 180 000 -> percent = 100*n/180000 ~ n/2000
 509         ui_->progress( n_entries/2000, std::to_string(n_entries)+string(" entries loaded") );
 510         n_entries++;
 511       }
 512
 513
 514       // READING
 515       for ( parsers::ReadingElement &rele: entry.r_ele ){
 516         ss << "INSERT INTO d_reading (rid,did,reading,inf,nokanji,freq) VALUES ("
 517           << rele.rid << "," << entry.did << ",'" << rele.reading << "','"
 518           << to_string(rele.inf,SEP) << "',"
 519           << rele.nokanji << "," << rele.freq << ");\n";
 520         // XXX: d_re_restr
 521       }
 522       // KANJI
 523       for ( parsers::KanjiElement &kele: entry.k_ele ){
 524         ss << "INSERT INTO d_kanji (kid,did,kanji,inf,freq) VALUES ("
 525           << kele.kid << "," << entry.did << ",'" << kele.kanji << "','"
 526           << to_string(kele.inf,SEP) << "'," << kele.freq << ");\n";
 527       }
 528       // SENSE
 529       for ( parsers::SenseElement &sele: entry.s_ele ){
 530         ss << "INSERT INTO d_sense (sid,did,gloss,xref,ant,inf,pos,field,misc,dial) VALUES ("
 531           << sele.sid << "," << entry.did << ",'"
 532           << SQLite3::escape(to_string(sele.gloss,SEP)) << "','"
 533           << to_string(sele.xref,SEP) << "','"
 534           << to_string(sele.ant,SEP) << "','"
 535           << SQLite3::escape(to_string(sele.s_inf,SEP)) << "','"
 536           << to_string(sele.pos,SEP) << "','"
 537           << to_string(sele.field,SEP) << "','"
 538           << to_string(sele.misc,SEP) << "','"
 539           << to_string(sele.dial,SEP) << "'"
 540         << ");\n";
 541         // d_stagk, d_stagr
 542       }
 543       // tbl: header
 544       n_entries++;
 545       entry = jmp.get_entry();
 546     }
 547   }
 548   catch ( utils::ParsingError &e ){
 549     ui_->progress_hide();
 550     ui_->cursor_default();
 551     std::string msg = "App::parse_jmdict(): ParsingError: ";
 552     msg += e.what();
 553     alert(msg);
 554     return;
 555   }
 556
 557   log(std::to_string(n_entries) + " entries processed.");
 558
 559   ss << "END TRANSACTION;\n";
 560   // no need for vacuum here - check_index() will perform it eventually (see below)
 561
 562 //  #ifdef DEBUG
 563 //    log_d("Writing file 'script.jmdict.sql'...");
 564 //    std::ofstream f ("script.jmdict.sql");
 565 //    f << ss.str();
 566 //    f.close();
 567 //  #endif
 568
 569   ui_->progress( 95, "Creating database... ");
 570   log("Creating database...");
 571   try {
 572     // don't log query (ifdef DEBUG then SQL script is already created)
 573     query(ss.str().c_str(),false);
 574   }
 575   catch ( SQLite3::DatabaseError &e ){
 576     string msg = "App::parse_jmdict(): DatabaseError: ";
 577     msg += e.what();
 578     alert(msg);
 579     return;
 580   }
 581   ui_->progress( 99, "Checking indexes... ");
 582   log("Creating database...");
 583   check_indexes();
 584   ui_->progress_hide();
 585   log("Deleting temporary file...");
 586   remove(TEMP_JMDICT);
 587   if ( delete_source )
 588     remove( fname );
 589   ui_->cursor_default();
 590 }
 591
 592
 593 void App::check_tables ()
 594 {
 595   for ( auto &dbit: aoi_config::db_tables) {
 596     // no need for vacuum here, it will be done by check_indexes()
 597     log("Checking tables in database: " + string(dbit.first));
 598     std::stringstream q;
 599     q << "SELECT name FROM " << dbit.first << "." << "sqlite_master WHERE type='table'";
 600     vector<string> existing = query(q.str().c_str());
 601     // CREATE TABLE name ( column1 TYPE, column2 TYPE, ... )
 602     for ( auto &table: dbit.second ){
 603       if ( utils::is_in(existing, string(table.first)) )
 604         continue;
 605       log("Creating table " + string(table.first));
 606       vector<string> v;
 607       for ( auto &column: table.second ){
 608         std::stringstream sstr;
 609         sstr << column.name << " " << column.type;
 610         v.push_back(sstr.str());
 611       }
 612       std::stringstream ss;
 613       ss << "CREATE TABLE " << dbit.first << "." << table.first
 614         << " (" << to_string(v,",") << ");\n";
 615       query(ss.str().c_str());
 616     }
 617     log("Check done.");
 618   }
 619 }
 620
 621
 622 void App::check_indexes ()
 623 {
 624   bool do_vacuum = false;
 625   log("Checking indexes...");
 626   for ( auto &dbit: aoi_config::db_tables ) {
 627     std::stringstream q;
 628     q << "SELECT name FROM " << dbit.first << "." << "sqlite_master WHERE type='index'";
 629     vector<string> existing = query(q.str().c_str());
 630     // CREATE INDEX idx_table_column ON table ( column ASC )
 631     for ( auto &mi: dbit.second ){  // tables
 632       for ( auto &c: mi.second ){ // columns
 633         std::stringstream idx_name;
 634         idx_name << "idx_" << mi.first << "_" << c.name;
 635         if ( c.index && !utils::is_in( existing, idx_name.str() ) ){
 636           log(string("Creating index ") + idx_name.str());
 637           std::stringstream ss;
 638           ss << "CREATE INDEX " << idx_name.str() << " ON " << mi.first
 639             << "(" << c.name << " " << c.sort << ")";
 640           query(ss.str().c_str());
 641           do_vacuum = true;
 642         }
 643       }
 644     }
 645   }
 646   if ( do_vacuum )
 647     query("VACUUM");
 648   log("Check done.");
 649 }
 650
 651
 652 void App::cb_dic_input ()
 653 {
 654   parse_dic_input( ui_->get_dic_input() );
 655   ui_->reset_filters();
 656 }
 657
 658
 659 void App::on_dic_selected ( int id )
 660 {
 661   log_d("App::on_dic_selected()");
 662 }
 663
 664 void App::cb_edit_word ( int id )
 665 {
 666   std::stringstream ss;
 667   ss << "App::edit_word( " << id << " )";
 668   log_d(ss);
 669   ui_->edit_word();
 670 }
 671
 672
 673 void App::cb_popup_kanji ( const string &kanji )
 674 {
 675   std::stringstream ss;
 676   ss << "App::on_kanji_clicked()" << kanji;
 677   Kanji k = db_get_kanji(kanji);
 678   ui_->popup_kanji( k );
 679   // XXX: this should be somewhere else (it is not logical here)
 680   ui_->highlight_components( k.components() );
 681   log(ss);
 682 }
 683
 684
 685 void App::set_listview ( const vector<string> &v )
 686 {
 687   // TODO: nokanji
 688   if ( !listview_items_.empty() ) listview_items_.clear();
 689   vector<string> d;
 690   vector<int> cell_ids;
 691   size_t i = 0;
 692   while ( i < v.size() ) {
 693     int cell_id = std::stoi(v[i]);  // jmdict id
 694     cell_ids.push_back(cell_id);
 695     cell_ids.push_back(cell_id);
 696     cell_ids.push_back(cell_id);
 697     // pos
 698     set<string> pos;
 699     for ( string &elt: utils::split_string( v[i+1], ",") )
 700       if ( elt.size() > 0 )
 701         pos.insert(utils::strip(elt.c_str()));
 702     d.push_back( v[i+2] );            // reading
 703     d.push_back( v[i+3] );            // kanji
 704     d.push_back( v[i+4] );            // sense
 705     listview_items_.push_back( {cell_id, pos, v[i+2], v[i+3], v[i+4]} );
 706     i += 5;
 707   }
 708   char buff[32];
 709   sprintf( buff, "%d results", db_->result_rows() );
 710   ui_->set_dic_results( buff );
 711   ui_->set_listview(d,cell_ids);
 712 }
 713
 714
 715 void App::parse_dic_input ( const char *str )
 716 {
 717   log_d(string("App::parse_dic_input: \"") +string(str) + string("\""));
 718
 719   string stripped = utils::strip(str);
 720
 721   if ( stripped.empty() )
 722     return;
 723
 724   const char *s = stripped.c_str();
 725
 726   string qq;
 727   if ( s[0] != SENSE_SEARCH_CHAR ){
 728     DictionaryInputParser p;
 729     qq = p.parse(s);
 730
 731     if ( p.warning() ){
 732       int res = fl_choice(
 733         "Too broad search.\n"\
 734         "Your computer may become unresponsible for a long time.\n"\
 735         "Proceed?",
 736         "Proceed",
 737         "Cancel",
 738         0
 739         );
 740         if ( res == 1 ) // Cancel
 741           return;
 742     }
 743
 744     if ( !strchr(s,'*') && !strchr(s,'?') && !strchr(s,'[') && !strchr(s,'{')
 745         && !strchr(s,'(') )
 746       qq += "*";
 747   }
 748
 749   std::stringstream q;
 750   if ( s[0] == SENSE_SEARCH_CHAR ){
 751     q << "select did,"
 752       << "group_concat(pos) as pos,"
 753       << q_reading("d_sense.did")
 754       << q_kanji("d_sense.did")
 755       << q_sense()
 756       << " from d_sense where gloss glob '*" << stripped.substr(1) << "*'"
 757       << " group by did";
 758   }
 759   else if ( rmn_->contains_kanji( qq.c_str() ) ) {
 760     q << "select d_kanji.did as did,"
 761       << "(select group_concat(pos) from d_sense where d_sense.did = d_kanji.did) as pos,"
 762       << q_kanji()
 763       << q_reading("d_kanji.did")
 764       << q_sense("d_kanji.did")
 765       << "from d_kanji where "
 766         << "kanji glob '" << rmn_->romaji_to_hiragana(qq.c_str()) << "' "
 767         << " or kanji glob '" << rmn_->romaji_to_katakana(qq.c_str()) << "' "
 768       << " group by did order by d_kanji.freq desc, d_kanji.kanji asc";
 769   }
 770   else {
 771     q << "select d_reading.did as did,"
 772       << "(select group_concat(pos) from d_sense where d_sense.did = d_reading.did) as pos,"
 773       << q_reading()
 774       << q_kanji("d_reading.did")
 775       << q_sense("d_reading.did")
 776       << "from d_reading where "
 777         << "reading glob '" << rmn_->romaji_to_hiragana(qq.c_str())
 778         << "' or reading glob '"<< rmn_->romaji_to_katakana(qq.c_str()) << "' "
 779       << " group by did order by d_reading.freq desc, d_reading.reading asc";
 780   }
 781   set_listview(query(q.str().c_str()));
 782 }
 783
 784
 785
 786 Kanji App::db_get_kanji ( const string &kanji )
 787 {
 788   log("App::db_get_kanji()");
 789   std::stringstream q;
 790   q << "select kanji,strokes,ucs, rad_classic, rad_nelson, "
 791           << "jlpt, grade, freq, onyomi, kunyomi, nanori, meaning,flags,components "
 792     << "from k_kanji where kanji='" << kanji << "';";
 793
 794   vector<string> res = query(q.str().c_str());
 795   Kanji kk(res[0]);
 796   kk.strokes(std::stoi(res[1]));
 797   kk.ucs(res[2]);
 798   kk.rad_classic(std::stoi(res[3]));
 799   kk.rad_nelson( (res[4].empty()) ? -1:std::stoi(res[4]));
 800   kk.jlpt(std::stoi(res[5]));
 801   kk.grade(std::stoi(res[6]));
 802   kk.freq(std::stoi(res[7]));
 803   kk.onyomi( utils::split_string(res[8],parsers::SEPARATOR_SQL) );
 804   kk.kunyomi( utils::split_string(res[9],parsers::SEPARATOR_SQL) );
 805   kk.nanori( utils::split_string(res[10],parsers::SEPARATOR_SQL) );
 806   kk.meaning( utils::split_string(res[11],parsers::SEPARATOR_SQL) );
 807   kk.flags( utils::split_string(res[12],parsers::SEPARATOR_SQL) );
 808   kk.components( res[13] );
 809
 810   string qq = "select skip1,skip2,skip3,misclass from k_skip where kanji='"+kanji+"';";
 811   vector<string> res2 = query(qq.c_str());
 812   if ( res2.size() % 4 != 0 ){
 813     std::stringstream ss;
 814     ss << "Wrong SKIP count. Kanji: " << kanji
 815       << "Query result size: " << res2.size()
 816       << " (should be 4,8 or 12).  SKIP not loaded.";
 817     log_e(ss);
 818     return kk;
 819   }
 820   for ( size_t i=0; i < res2.size(); i+=4 )
 821     kk.skip( res2[i], res2[i+1], res2[i+2], res2[i+3] );
 822   return kk;
 823 }
 824
 825
 826 void App::cb_file ( MANAGEDB_FILE_TYPE filetype, bool download )
 827 {
 828   string url;
 829   const char *item = nullptr;
 830   switch ( filetype ){
 831     case MANAGEDB_JMDICT:
 832       url = get_config("sources/url_jmdict");
 833       item = MANAGEDB_ITEM_JMDICT;
 834       break;
 835     case MANAGEDB_KANJIDIC:
 836       url = get_config("sources/url_kanjidic");
 837       item = MANAGEDB_ITEM_KANJIDIC;
 838       break;
 839     case MANAGEDB_COMPONENTS:
 840       url = get_config("sources/url_components");
 841       item = MANAGEDB_ITEM_COMPONENTS;
 842       break;
 843     default:
 844       alert("Unknown MANAGEDB_FILE_TYPE.");
 845       return;
 846   }
 847   string fname;
 848
 849   if ( download ){
 850     fname = ui_->download_dialog( url );
 851     log_d("Download finished...");
 852   }
 853   else
 854     fname = ui_->ask_file();
 855
 856   if ( fname.empty() )
 857     return;
 858
 859   log("Selected file: " + fname );
 860
 861   switch ( filetype ){
 862     case MANAGEDB_JMDICT:
 863       parse_jmdict(fname.c_str(), download);
 864       break;
 865     case MANAGEDB_KANJIDIC:
 866       parse_kanjidic(fname.c_str(), download);
 867       break;
 868     case MANAGEDB_COMPONENTS:
 869     {
 870       utils::gzip_decompress_file( fname.c_str(), TEMP_COMPONENTS);
 871       try {
 872         db_->script( TEMP_COMPONENTS );
 873       }
 874       catch ( SQLite3::DatabaseError &e ) {
 875         alert(e.what(), e.query() );
 876       }
 877       catch ( std::exception &e ) {
 878         alert( e.what() );
 879       }
 880       break;
 881     }
 882   }
 883
 884   auto label = ui()->dlg_manage_db()->item(item);
 885         if ( label )
 886                 label->version("Changed",FL_BLUE);
 887         else
 888                 log_w("Can't find item " + string(item) + " in ManageDBDialog.");
 889
 890   if ( filetype == MANAGEDB_COMPONENTS ){
 891     remove(TEMP_COMPONENTS);
 892     remove(fname.c_str());
 893   }
 894 }
 895
 896 void App::cb_filter_listview ()
 897 {
 898   vector<string> pos = ui_->listview_filters();
 899   log_d("App::cb_filter_listview(): pos: " + utils::to_string(pos));
 900   bool filter_expr  = ( utils::is_in( pos, string("expr") ) );
 901   bool filter_noun  = ( utils::is_in( pos, string("noun") ) );
 902   bool filter_verb  = ( utils::is_in( pos, string("verb") ) );
 903   bool filter_adj   = ( utils::is_in( pos, string("adj")  ) );
 904   vector<string> data;
 905   vector<int> ids;
 906   size_t n = 0;
 907   for ( auto &elt: listview_items_ ){
 908     auto start = elt.pos.begin();
 909     auto end = elt.pos.end();
 910     if ( filter_expr && std::find( start, end, "exp") == end )
 911       continue;
 912     if ( filter_noun && std::find( start, end, "n" ) == end )
 913       continue;
 914     if ( filter_adj && std::find_if( start, end,
 915       [](const string &s){ return strncmp(s.c_str(),"adj",3)==0;} ) == end )
 916       continue;
 917     if ( filter_verb && std::find_if( start, end,
 918       [](const string &s){ return strncmp(s.c_str(),"v",1)==0;} ) == end )
 919       continue;
 920     ids.push_back( elt.did );
 921     ids.push_back( elt.did );
 922     ids.push_back( elt.did );
 923     data.push_back ( elt.reading );
 924     data.push_back ( elt.kanji );
 925     data.push_back ( elt.sense );
 926     n++;
 927   }
 928   ui_->set_listview( data, ids );
 929   std::stringstream ss;
 930   ss << n << " results";
 931   if (  listview_items_.size() != n )
 932     ss << " (" <<  listview_items_.size()-n << " hidden)";
 933   log(ss.str());
 934   ui_->set_dic_results( ss.str() );
 935 }
 936
 937
 938 void App::cb_dicview_rightclick ( int did )
 939 {
 940   string q = "select group_concat(kanji,'') from d_kanji where did="+std::to_string(did);
 941   vector<string> res = query( q.c_str() );
 942   set<string> kanji;
 943   for ( string &c: utils::str_to_chars(res[0].c_str()) )
 944     if ( App::get()->rmn()->is_kanji(c.c_str()) )
 945       kanji.insert(c);
 946   ui_->menu_copy( vector<string>(kanji.begin(),kanji.end()) );
 947 }
 948
 949
 950 void App::apply_config ()
 951 {
 952   ui_->font_base_size( get_config<int>("font/base_size"));
 953   logger_.loglevel( get_config("log/level") );
 954   ui_->init_colors();
 955   Fl::check();
 956 }
 957
 958
 959 void App::cb_manage_db ()
 960 {
 961   auto q = query("select val from aoi where key='jmdict_version'");
 962   string jmdict_version = (q.empty()) ? "NONE":q[0];
 963   q = query("select val from aoi where key='kanjidic_version'");
 964   string kanjidic_version = q.empty() ? "NONE":q[0];
 965   q = query("select val from aoi where key='components_version'");
 966   string components_version = q.empty() ? "NONE":q[0];
 967   auto *d = ui_->dlg_manage_db();
 968   Fl_Color col;
 969   if ( !d->item(MANAGEDB_ITEM_JMDICT) ){
 970     col = ( jmdict_version == "NONE" ) ? FL_RED:FL_BLUE;
 971     d->add_item( MANAGEDB_ITEM_JMDICT, jmdict_version,
 972       "Japanese-English dictionary. Needed component.",
 973       col, scb_local_file_jmdict, scb_download_file_jmdict,
 974       (void*)this );
 975   }
 976   if ( !d->item(MANAGEDB_ITEM_KANJIDIC) ){
 977     col = ( kanjidic_version == "NONE" ) ? FL_RED:FL_BLUE;
 978     d->add_item( MANAGEDB_ITEM_KANJIDIC, kanjidic_version,
 979       "Kanji dictionary. Needed component.",
 980       col, scb_local_file_kanjidic, scb_download_file_kanjidic,
 981       (void*)this  );
 982   }
 983   if ( !d->item(MANAGEDB_ITEM_COMPONENTS) ){
 984     col = ( components_version == "NONE" ) ? FL_RED:FL_BLUE;
 985     d->add_item( MANAGEDB_ITEM_COMPONENTS, components_version,
 986       "Graphical components of kanji.",
 987       col, scb_local_file_components, scb_download_file_components,
 988       (void*)this );
 989   }
 990   d->show();
 991 }
 992
 993
 994 void App::load_config ()
 995 {
 996   log("Loading config...");
 997   vector<string> res = query("select key,val from config;");
 998   for ( size_t i=0; i < res.size(); i+=2 ){
 999     set_config( res[i], res[i+1] );
1000   }
1001   apply_config();
1002 }
1003
1004
1005 void App::save_config ( const std::map<string,aoi_config::Config::Value> &newmap)
1006 {
1007   log("Saving config...");
1008
1009   typedef std::pair<string,aoi_config::Config::Value> cfg_pair;
1010
1011   // merge new and old config
1012   for ( const cfg_pair &p: newmap )
1013     set_config( p.first, p.second.val );
1014   apply_config();
1015
1016   // prepare SQL script
1017   std::stringstream ss;
1018   ss << "BEGIN TRANSACTION;\n";
1019   ss << "DROP TABLE IF EXISTS user.config;\n";
1020   ss << "CREATE TABLE user.config (key TEXT, val TEXT);\n";
1021   for ( const cfg_pair &p: get_config_map() )
1022     ss << "INSERT INTO user.config (key,val) VALUES('"
1023       << p.first << "','" << p.second.val << "');\n";
1024   ss << "END TRANSACTION;\n";
1025
1026   query(ss.str().c_str());
1027
1028   // apply new fonts and colors
1029   init_dicview();
1030
1031   // TODO:
1032   //  check new keys
1033   //  check keys in oldconfig not present in newmap
1034   //  set oldkeys
1035   //  set newkeys
1036   //  mnoziny...
1037 }
1038
1039 } // namespace aoi