src/aoi.cxx

   1 /*
   2     Copyright 2013 Karel Matas
   3
   4     This program is free software: you can redistribute it and/or modify
   5     it under the terms of the GNU General Public License as published by
   6     the Free Software Foundation, either version 3 of the License, or
   7     (at your option) any later version.
   8
   9     This program is distributed in the hope that it will be useful,
  10     but WITHOUT ANY WARRANTY; without even the implied warranty of
  11     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12     GNU General Public License for more details.
  13
  14     You should have received a copy of the GNU General Public License
  15     along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16 */
  17 #include <cmath>
  18 #include <set>
  19 #include <FL/fl_ask.H>
  20 #include "aoi.hxx"
  21 #include "utils.hxx"
  22
  23 namespace aoi {
  24
  25 using std::set;
  26 using aoi_ui::TextStyle;
  27 using utils::to_string;
  28
  29 const char *MANAGEDB_ITEM_KANJIDIC        = "Kanjidic";
  30 const char *MANAGEDB_ITEM_JMDICT                  = "JMdict";
  31 const char *MANAGEDB_ITEM_COMPONENTS  = "Components";
  32 const char *TEMP_JMDICT               = "gztemp.jmdict";
  33 const char *TEMP_KANJIDIC             = "gztemp.kanjidic";
  34 const char *TEMP_COMPONENTS           = "gztemp.components";
  35 const char *LOG_FILE                  = "aoi.log";
  36 const char SENSE_SEARCH_CHAR          = ':';
  37
  38 App* App::instance_ = nullptr;
  39
  40 App::App ()
  41 {
  42   remove(LOG_FILE);
  43   logger_.filename(LOG_FILE);
  44   logger_.loglevel(Logger::MSG_DEBUG);
  45
  46   cfg_  = new aoi_config::Config();
  47   ui_   = new aoi_ui::GUI(this);
  48   rmn_  = new Romanization();
  49
  50   ui_->progress(0,"Opening database...");
  51
  52   try {
  53     db_ = new SQLite3::SQLite3( get_config("db/file_main").c_str() );
  54   }
  55   catch (SQLite3::CantOpenDatabase &e){
  56     log_e( "Aoi: Can't open database '"  + string(e.what()) + "'.");
  57   }
  58
  59   ui_->progress( 30, "Checking tables..." );
  60   check_tables();
  61   ui_->progress( 60, "Checking indexes..." );
  62   check_indexes();
  63
  64   load_config();
  65   init_dicview();
  66
  67   #ifdef DEBUG
  68 //    ui_->progress(90, "DEBUG query..." );
  69 //    parse_dic_input("ana*");
  70 //    cb_kanji_search();
  71 //    ui_->cb_toggle_group(nullptr);
  72   #endif
  73
  74   ui_->progress( 98, "Initializing fonts..." );
  75   ui_->fontname_kanji( get_config("font/kanji") );
  76   ui_->help_file( get_config("sources/help_index") );
  77
  78   ui_->progress_hide();
  79
  80
  81   auto q = query("select val from aoi where key='jmdict_version'");
  82   string jmdict_version = (q.empty()) ? "NONE":q[0];
  83   q = query("select val from aoi where key='kanjidic_version'");
  84   string kanjidic_version = q.empty() ? "NONE":q[0];
  85   if ( jmdict_version == "NONE" || kanjidic_version == "NONE" )
  86     cb_manage_db();
  87
  88
  89   #ifdef DEBUG
  90     logger_.loglevel(Logger::MSG_DEBUG);
  91   #endif
  92
  93   instance_ = this;
  94 }
  95
  96
  97 App::~App ()
  98 {
  99   delete db_;
 100   delete rmn_;
 101   delete ui_;
 102   delete cfg_;
 103 }
 104
 105
 106 void App::init_dicview ()
 107 {
 108   // initialize textstyles
 109   TextStyle style_default(FL_HELVETICA,1.2);
 110   TextStyle style_reading(aoi_ui::FONT_KANJI,1.5);
 111   TextStyle style_reading_freq = style_reading;
 112   style_reading_freq.color = get_color("frequent");
 113   TextStyle style_kanji(aoi_ui::FONT_KANJI,1.7);
 114   TextStyle style_kanji_freq = style_kanji;
 115   style_kanji_freq.color = get_color("frequent");
 116   TextStyle style_inf(FL_HELVETICA_ITALIC,0.8,get_color("pos"));
 117   style_inf.offset_y = 3;
 118   TextStyle style_pos = style_inf;
 119   TextStyle style_misc = style_pos;
 120   style_misc.color = get_color("misc");
 121   TextStyle style_field = style_pos;
 122   style_field.color = get_color("field");
 123   TextStyle style_dial = style_pos;
 124
 125   // register TextStyles
 126   ui_->register_tag_dicview( "default",       style_default );
 127   ui_->register_tag_dicview( "reading",       style_reading );
 128   ui_->register_tag_dicview( "reading_freq",  style_reading_freq );
 129   ui_->register_tag_dicview( "kanji",         style_kanji );
 130   ui_->register_tag_dicview( "kanji_freq",    style_kanji_freq );
 131   ui_->register_tag_dicview( "kinf",          style_inf );
 132   ui_->register_tag_dicview( "rinf",          style_inf );
 133   ui_->register_tag_dicview( "pos",           style_pos );
 134   ui_->register_tag_dicview( "misc",          style_misc );
 135   ui_->register_tag_dicview( "field",         style_field );
 136   ui_->register_tag_dicview( "dial",          style_dial );
 137 }
 138
 139
 140 vector<string> App::query ( const char *q, bool log_query, bool replace_separator )
 141 {
 142   if ( !db_ ){
 143     log_e("App::query(): Database does not exist.");
 144     return {};
 145   }
 146   ui_->cursor_wait();
 147   vector<string> result;
 148   try {
 149     if ( log_query )
 150       log_d(string(q));
 151     result = db_->query(q);
 152   }
 153   catch (SQLite3::DatabaseError &e){
 154     log_e("App: DatabaseError:: " + string(e.what()) + string("\nQuery: ")
 155       + string(e.query()) );
 156     ui_->cursor_default();
 157   }
 158   string msg = std::to_string(db_->result_rows()) + " results";
 159   log( "App::query(): " + msg);
 160   if ( replace_separator )
 161     for ( string &s: result )
 162       utils::replace_all(s, parsers::SEPARATOR_SQL, ", ");
 163   ui_->cursor_default();
 164   return result;
 165 }
 166
 167
 168 void App::cb_set_components ()
 169 {
 170   if ( curr_components_.empty() )
 171     return;
 172
 173   using aoi_ui::ComponentView;
 174
 175   // prepare cells
 176   vector<string> included = ui_->components_include();
 177   vector<string> excluded = ui_->components_exclude();
 178
 179   vector<ComponentView::Cell> v;
 180   for ( string &s: included )
 181     v.push_back( ComponentView::Cell(s,ComponentView::CELL_SELECTED_1) );
 182   for ( string &s: excluded )
 183     v.push_back( ComponentView::Cell(s,ComponentView::CELL_SELECTED_2) );
 184
 185   // sort by occurences
 186   if ( !ui_->sort_components_by_strokes() ){
 187     for ( auto mi = curr_components_.rbegin(); mi!=curr_components_.rend(); ++mi ){
 188       if ( !utils::is_in( included, mi->second )
 189             && !utils::is_in( excluded, mi->second ) )
 190         v.push_back( ComponentView::Cell(mi->second) );
 191     }
 192   }
 193   // sort by strokes
 194   else {
 195     vector<string> comps_by_strokes;
 196     for ( auto mi: curr_components_ ){
 197       if ( mi.first < get_config<int>("knj/min_compo_count") )
 198         continue;
 199       comps_by_strokes.push_back(mi.second);
 200     }
 201
 202     struct SortByStrokes {
 203       map<string,int> comps;
 204       SortByStrokes( const map<string,int> &c ): comps(c){};
 205       bool operator() (const string &c1, const string &c2 ){
 206         return ( this->comps[c1] < this->comps[c2] );
 207       }
 208     } sbc(components_);
 209
 210     std::sort( comps_by_strokes.begin(), comps_by_strokes.end(), sbc );
 211
 212     int prev_strokes=0;
 213     for ( string &s: comps_by_strokes ){
 214       int curr_strokes = components_[s];
 215       if ( prev_strokes != curr_strokes )
 216         v.push_back( ComponentView::Cell( std::to_string(curr_strokes),
 217                                           ComponentView::CELL_LABEL) );
 218       v.push_back( ComponentView::Cell(s) );
 219       prev_strokes = curr_strokes;
 220     }
 221   }
 222
 223   ui_->set_components( v );
 224 }
 225
 226
 227 void App::cb_kanji_search ()
 228 {
 229
 230   if ( components_.empty() ){
 231     log("Loading components...");
 232     vector<string> res = query("select component, strokes from components");
 233     for ( size_t i=0; i<res.size(); i=i+2 ){
 234       components_[res[i]] = std::stoi(res[i+1]);
 235     }
 236   }
 237
 238   // strokes
 239   std::pair<int,int> strokes = utils::parse_range( utils::strip( ui_->strokes() ) );
 240   // jlpt
 241   std::pair<int,int> jlpt = utils::parse_range( utils::strip( ui_->jlpt() ) );
 242   // grade
 243   std::pair<int,int> grade = utils::parse_range( utils::strip( ui_->grade() ) );
 244
 245
 246
 247   // skip
 248   vector<string> skip = utils::split_string( ui_->skip() );
 249   std::stringstream sskip;
 250   if ( skip.size() > 0 ){
 251     string skip1 = utils::strip(skip[0].c_str());
 252     sskip << " S.skip1=" << skip1;
 253   }
 254   if ( skip.size() > 1 ){
 255     std::pair<int,int> skip2 = utils::parse_range( utils::strip(skip[1].c_str()) );
 256     sskip << " and S.skip2>=" << skip2.first << " and S.skip2<=" << skip2.second;
 257   }
 258   if ( skip.size() > 2 ){
 259     std::pair<int,int> skip3 = utils::parse_range( utils::strip(skip[2].c_str()) );
 260     sskip << " and S.skip3>=" << skip3.first << " and S.skip3<=" << skip3.second;
 261   }
 262   if ( !sskip.str().empty() )
 263     sskip << " and ";
 264
 265   // ordering
 266   string order_by;
 267   switch ( ui_->sort_mode() ){
 268     case SORT_FREQ_ASC:
 269       order_by = "(case freq when 0 then 9999 else freq end) asc";
 270       break;
 271     case SORT_FREQ_DESC:
 272       order_by = "(case freq when 0 then 9999 else freq end) desc";
 273       break;
 274     case SORT_STROKES_ASC:
 275       order_by = "strokes asc";
 276       break;
 277     case SORT_STROKES_DESC:
 278       order_by = "strokes desc";
 279       break;
 280   }
 281
 282   vector<string> components_include = ui_->components_include();
 283   vector<string> components_exclude = ui_->components_exclude();
 284   std::stringstream comps;
 285   for ( auto c: components_include )
 286     comps << " and components like '%" << c << "%'";
 287   for ( auto c: components_exclude )
 288     comps << " and components not like '%" << c << "%'";
 289   printf("%s\n",comps.str().c_str());
 290
 291   string jis208 = "";
 292   if ( get_config<bool>("knj/jis208_only") )
 293     jis208 = " flags glob '*jis208*' and ";
 294
 295   // build query
 296   std::stringstream ss;
 297   ss  <<  "select distinct "
 298       <<    "K.kanji,freq,components,flags "
 299       <<  " from k_kanji as K, k_skip as S where K.kanji=S.kanji and "
 300       <<    jis208
 301       <<    sskip.str()
 302       <<    " strokes>=" << strokes.first << " and strokes<=" << strokes.second
 303       <<    " and jlpt>=" << jlpt.first << " and jlpt<=" << jlpt.second
 304       <<    " and grade>=" << grade.first << " and grade<=" << grade.second
 305       <<    comps.str()
 306       <<  " order by " << order_by;
 307
 308
 309   // perform query
 310   vector<string> q = query( ss.str().c_str(), true, false );
 311   vector<aoi_ui::KanjiView::Cell> data;
 312   vector<string> components;
 313   set<string> flags;
 314   for ( size_t i=0; i<q.size(); i+=4 ){
 315     string kanji = q[i];
 316     int freq = std::stoi(q[i+1]);
 317     data.push_back(
 318       aoi_ui::KanjiView::Cell(
 319         kanji,
 320         (freq>0)? get_color("frequent"):-1
 321       )
 322     );
 323     components.push_back(q[i+2]);
 324     for ( string &s: utils::split_string( q[i+3], parsers::SEPARATOR_SQL) )
 325       flags.insert(s);
 326   }
 327
 328   log_d("Groups: " + utils::to_string(flags));
 329
 330   ui_->set_kanjiview( data );
 331   char b[32];
 332   sprintf( b, "%d results", db_->result_rows() );
 333   ui_->set_kanji_results( b );
 334
 335   utils::Histogram<string> histogram;
 336   for ( string &s: components )
 337     histogram.add( utils::str_to_chars(s.c_str()) );
 338   curr_components_ = histogram.sorted();
 339
 340   cb_set_components();
 341 }
 342
 343
 344 void App::alert ( const string &msg, const string &desc )
 345 {
 346   std::stringstream ss;
 347   ss << msg;
 348   if ( !desc.empty() )
 349     ss << "\n\nDetails:\n" << desc;
 350   log_e(ss);
 351   ui_->alert(ss.str());
 352 }
 353
 354
 355 int App::run ( int argc, char **argv )
 356 {
 357   return ui_->run(argc,argv);
 358 }
 359
 360
 361 void App::parse_kanjidic ( const char *fname, bool delete_source )
 362 {
 363   if ( !utils::file_exists( fname ) ){
 364     log_e("App::parse_kanjidic(): File does not exist: " + string(fname));
 365     return;
 366   }
 367   ui_->progress(0, "Loading kanjidic...");
 368   log("Loading kanjidic...");
 369
 370   const char *SEP = parsers::SEPARATOR_SQL;
 371
 372   int n_kanji = 0;
 373   try {
 374     ui_->progress(1, "Creating parser...");
 375     std::stringstream msg;
 376     msg << "Decompressing file: " << fname << " -> " << TEMP_KANJIDIC;
 377     log_d(msg);
 378
 379     ui_->cursor_wait();
 380     utils::gzip_decompress_file( fname, TEMP_KANJIDIC);
 381     parsers::KanjidicParser p(TEMP_KANJIDIC);
 382     auto kanji = p.get_entry();
 383
 384     query(  "DROP TABLE IF EXISTS k_kanji;"
 385             "DROP TABLE IF EXISTS components;"
 386             "DROP TABLE IF EXISTS k_skip;"
 387             "DELETE FROM aoi WHERE key='kanjidic_version';" );
 388
 389     check_tables();
 390
 391     std::stringstream ss;
 392     ss << "BEGIN TRANSACTION;\n";
 393     ss << "INSERT INTO aoi (key,val) VALUES ('kanjidic_version','"
 394       << p.get_version() << "');\n";
 395     while ( kanji.kanji() != "" ){
 396       n_kanji++;
 397       if ( n_kanji % 100 == 0)
 398         // percent = 100 * n_kanji/13108
 399         ui_->progress( n_kanji/14, std::to_string(n_kanji)+string(" kanji loaded"));
 400       ss << "INSERT INTO k_kanji "
 401         << "(kanji,ucs,onyomi,kunyomi,meaning,nanori,flags,jlpt,grade,freq,strokes,"
 402         << "rad_classic,rad_nelson,components)"
 403         << " VALUES('"
 404         << kanji.kanji() << "','"
 405         << kanji.ucs() << "','"
 406         << to_string(kanji.onyomi(),SEP) << "','"
 407         << to_string(kanji.kunyomi(),SEP) << "','"
 408         << SQLite3::escape(to_string(kanji.meaning(),SEP)) << "','"
 409         << to_string(kanji.nanori(),SEP) << "','"
 410         << to_string(kanji.flags(),SEP) << "',"
 411         << kanji.jlpt() << ","
 412         << kanji.grade() << ","
 413         << kanji.freq() << ","
 414         << kanji.strokes() << ","
 415         << kanji.rad_classic() << ","
 416         << kanji.rad_nelson() << ","
 417         << "''"
 418         << ");\n";
 419       for ( SKIP &s: kanji.skip() ) {
 420         ss << "INSERT INTO k_skip (kanji,skip1,skip2,skip3,misclass) VALUES('"
 421           << kanji.kanji() << "'," << s.s1 << "," << s.s2 << "," << s.s3
 422           << ",'" << s.misclass << "');\n";
 423       }
 424       kanji = p.get_entry();
 425     }
 426     ss << "END TRANSACTION;\n";
 427     log("Writing to db ...");
 428
 429 //    #ifdef DEBUG
 430 //      log_d("Writing file 'script.kanjidic.sql'...");
 431 //      std::ofstream f ("script.kanjidic.sql");
 432 //      f << ss.str();
 433 //      f.close();
 434 //    #endif
 435     ui_->progress(95, "Writing to database...");
 436     query(ss.str().c_str(),false);
 437   }
 438   catch ( utils::ParsingError &e ){
 439     ui_->cursor_default();
 440     ui_->progress_hide();
 441     std::string msg = "App::parse_kanjidic(): ParsingError: ";
 442     msg += e.what();
 443     alert(msg);
 444     return;
 445   }
 446   ui_->progress(98, "Checking indexes...");
 447   check_indexes();
 448   ui_->progress_hide();
 449   remove( TEMP_KANJIDIC );
 450   if ( delete_source )
 451     remove( fname );
 452   ui_->cursor_default();
 453 }
 454
 455
 456 void App::parse_jmdict ( const char *fname, bool delete_source )
 457 {
 458   if ( !utils::file_exists( fname ) ){
 459     log_e("App::parse_jmdict(): File does not exist: " + string(fname));
 460     return;
 461   }
 462
 463   ui_->progress( 0, "Loading JMdict..." );
 464   log("Loading JMdict...");
 465   std::stringstream ss;
 466   ss << "BEGIN TRANSACTION;\n";
 467
 468   // tables
 469   for ( auto &mi: aoi_config::db_tables ){
 470     ss << "DROP TABLE IF EXISTS " << mi.first << ";\n"
 471     << "CREATE TABLE " << mi.first << " ( ";
 472     for ( size_t i=0; i<mi.second.size(); i++ )
 473       ss << mi.second[i].name << " " << mi.second[i].type
 474         << ((i==mi.second.size()-1) ? "":",");
 475     ss << ");\n";
 476   }
 477
 478   // create parser
 479   int n_entries = 0;
 480   try {
 481     ui_->progress(1, "Creating parser...");
 482
 483     ui_->cursor_wait();
 484     utils::gzip_decompress_file( fname, TEMP_JMDICT);
 485     parsers::JmdictParser jmp(TEMP_JMDICT);
 486     const char *SEP = parsers::SEPARATOR_SQL;
 487
 488     // version
 489     ss << "INSERT INTO aoi (key,val) VALUES ('jmdict_version', '"
 490       << jmp.get_version() << "');\n";
 491     log("jmdict_version: "+jmp.get_version());
 492
 493     // get entities
 494     for ( std::pair<string,string> elt: jmp.get_entities() ){
 495       ss << "INSERT INTO entities (abbr,desc) VALUES ('" << SQLite3::escape(elt.first)
 496         << "','" << SQLite3::escape(elt.second) << "');\n";
 497     }
 498
 499     int n_entries = 1;
 500     parsers::JmdictEntry entry = jmp.get_entry();
 501     while ( entry.did != -1 ) {
 502       if ( n_entries % 1000 == 0 ){
 503         // entries is about 180 000 -> percent = 100*n/180000 ~ n/2000
 504         ui_->progress( n_entries/2000, std::to_string(n_entries)+string(" entries loaded") );
 505         n_entries++;
 506       }
 507
 508
 509       // READING
 510       for ( parsers::ReadingElement &rele: entry.r_ele ){
 511         ss << "INSERT INTO d_reading (rid,did,reading,inf,nokanji,freq) VALUES ("
 512           << rele.rid << "," << entry.did << ",'" << rele.reading << "','"
 513           << to_string(rele.inf,SEP) << "',"
 514           << rele.nokanji << "," << rele.freq << ");\n";
 515         // XXX: d_re_restr
 516       }
 517       // KANJI
 518       for ( parsers::KanjiElement &kele: entry.k_ele ){
 519         ss << "INSERT INTO d_kanji (kid,did,kanji,inf,freq) VALUES ("
 520           << kele.kid << "," << entry.did << ",'" << kele.kanji << "','"
 521           << to_string(kele.inf,SEP) << "'," << kele.freq << ");\n";
 522       }
 523       // SENSE
 524       for ( parsers::SenseElement &sele: entry.s_ele ){
 525         ss << "INSERT INTO d_sense (sid,did,gloss,xref,ant,inf,pos,field,misc,dial) VALUES ("
 526           << sele.sid << "," << entry.did << ",'"
 527           << SQLite3::escape(to_string(sele.gloss,SEP)) << "','"
 528           << to_string(sele.xref,SEP) << "','"
 529           << to_string(sele.ant,SEP) << "','"
 530           << SQLite3::escape(to_string(sele.s_inf,SEP)) << "','"
 531           << to_string(sele.pos,SEP) << "','"
 532           << to_string(sele.field,SEP) << "','"
 533           << to_string(sele.misc,SEP) << "','"
 534           << to_string(sele.dial,SEP) << "'"
 535         << ");\n";
 536         // d_stagk, d_stagr
 537       }
 538       // tbl: header
 539       n_entries++;
 540       entry = jmp.get_entry();
 541     }
 542   }
 543   catch ( utils::ParsingError &e ){
 544     ui_->progress_hide();
 545     ui_->cursor_default();
 546     std::string msg = "App::parse_jmdict(): ParsingError: ";
 547     msg += e.what();
 548     alert(msg);
 549     return;
 550   }
 551
 552   log(std::to_string(n_entries) + " entries processed.");
 553
 554   ss << "END TRANSACTION;\n";
 555   // no need for vacuum here - check_index() will perform it eventually (see below)
 556
 557 //  #ifdef DEBUG
 558 //    log_d("Writing file 'script.jmdict.sql'...");
 559 //    std::ofstream f ("script.jmdict.sql");
 560 //    f << ss.str();
 561 //    f.close();
 562 //  #endif
 563
 564   ui_->progress( 95, "Creating database... ");
 565   log("Creating database...");
 566   try {
 567     // don't log query (ifdef DEBUG then SQL script is already created)
 568     query(ss.str().c_str(),false);
 569   }
 570   catch ( SQLite3::DatabaseError &e ){
 571     string msg = "App::parse_jmdict(): DatabaseError: ";
 572     msg += e.what();
 573     alert(msg);
 574     return;
 575   }
 576   ui_->progress( 99, "Checking indexes... ");
 577   log("Creating database...");
 578   check_indexes();
 579   ui_->progress_hide();
 580   log("Deleting temporary file...");
 581   remove(TEMP_JMDICT);
 582   if ( delete_source )
 583     remove( fname );
 584   ui_->cursor_default();
 585 }
 586
 587
 588 void App::check_tables ()
 589 {
 590   // no need for vacuum here, it will be done by check_indexes()
 591   log("Checking tables");
 592   vector<string> existing = query("SELECT name FROM sqlite_master WHERE type='table'");
 593   // CREATE TABLE name ( column1 TYPE, column2 TYPE, ... )
 594   for ( auto &table: aoi_config::db_tables ){
 595     if ( utils::is_in(existing, string(table.first)) )
 596       continue;
 597     log("Creating table " + string(table.first));
 598     vector<string> v;
 599     for ( auto &column: table.second ){
 600       std::stringstream sstr;
 601       sstr << column.name << " " << column.type;
 602       v.push_back(sstr.str());
 603     }
 604     std::stringstream ss;
 605     ss << "CREATE TABLE " << table.first << " (" << to_string(v,",") << ");\n";
 606     query(ss.str().c_str());
 607   }
 608   log("Check done.");
 609 }
 610
 611
 612 void App::check_indexes ()
 613 {
 614   log("Checking indexes...");
 615   bool do_vacuum = false;
 616   vector<string> existing = query("SELECT name FROM sqlite_master WHERE type='index'");
 617   // CREATE INDEX idx_table_column ON table ( column ASC )
 618   for ( auto &mi: aoi_config::db_tables ){  // tables
 619     for ( auto &c: mi.second ){ // columns
 620       std::stringstream idx_name;
 621       idx_name << "idx_" << mi.first << "_" << c.name;
 622       if ( c.index && !utils::is_in( existing, idx_name.str() ) ){
 623         log(string("Creating index ") + idx_name.str());
 624         std::stringstream ss;
 625         ss << "CREATE INDEX " << idx_name.str() << " ON " << mi.first
 626           << "(" << c.name << " " << c.sort << ")";
 627         query(ss.str().c_str());
 628         do_vacuum = true;
 629       }
 630     }
 631   }
 632   if ( do_vacuum ){
 633     query("VACUUM");
 634   }
 635   log("Check done.");
 636 }
 637
 638
 639 void App::cb_dic_input ()
 640 {
 641   parse_dic_input( ui_->get_dic_input() );
 642   ui_->reset_filters();
 643 }
 644
 645
 646 void App::on_dic_selected ( int id )
 647 {
 648   log_d("App::on_dic_selected()");
 649 }
 650
 651 void App::cb_edit_word ( int id )
 652 {
 653   std::stringstream ss;
 654   ss << "App::edit_word( " << id << " )";
 655   log_d(ss);
 656   ui_->edit_word();
 657 }
 658
 659
 660 void App::cb_popup_kanji ( const string &kanji )
 661 {
 662   std::stringstream ss;
 663   ss << "App::on_kanji_clicked()" << kanji;
 664   Kanji k = db_get_kanji(kanji);
 665   ui_->popup_kanji( k );
 666   // XXX: this should be somewhere else (it is not logical here)
 667   ui_->highlight_components( k.components() );
 668   log(ss);
 669 }
 670
 671
 672 void App::set_listview ( const vector<string> &v )
 673 {
 674   // TODO: nokanji
 675   if ( !listview_items_.empty() ) listview_items_.clear();
 676   vector<string> d;
 677   vector<int> cell_ids;
 678   size_t i = 0;
 679   while ( i < v.size() ) {
 680     int cell_id = std::stoi(v[i]);  // jmdict id
 681     cell_ids.push_back(cell_id);
 682     cell_ids.push_back(cell_id);
 683     cell_ids.push_back(cell_id);
 684     // pos
 685     set<string> pos;
 686     for ( string &elt: utils::split_string( v[i+1], ",") )
 687       if ( elt.size() > 0 )
 688         pos.insert(utils::strip(elt.c_str()));
 689     d.push_back( v[i+2] );            // reading
 690     d.push_back( v[i+3] );            // kanji
 691     d.push_back( v[i+4] );            // sense
 692     listview_items_.push_back( {cell_id, pos, v[i+2], v[i+3], v[i+4]} );
 693     i += 5;
 694   }
 695   char buff[32];
 696   sprintf( buff, "%d results", db_->result_rows() );
 697   ui_->set_dic_results( buff );
 698   ui_->set_listview(d,cell_ids);
 699 }
 700
 701
 702 void App::parse_dic_input ( const char *str )
 703 {
 704   log_d(string("App::parse_dic_input: \"") +string(str) + string("\""));
 705
 706   string stripped = utils::strip(str);
 707
 708   if ( stripped.empty() )
 709     return;
 710
 711   const char *s = stripped.c_str();
 712
 713   string qq;
 714   if ( s[0] != SENSE_SEARCH_CHAR ){
 715     DictionaryInputParser p;
 716     qq = p.parse(s);
 717
 718     if ( p.warning() ){
 719       int res = fl_choice(
 720         "Too broad search.\n"\
 721         "Your computer may become unresponsible for a long time.\n"\
 722         "Proceed?",
 723         "Proceed",
 724         "Cancel",
 725         0
 726         );
 727         if ( res == 1 ) // Cancel
 728           return;
 729     }
 730
 731     if ( !strchr(s,'*') && !strchr(s,'?') && !strchr(s,'[') && !strchr(s,'{')
 732         && !strchr(s,'(') )
 733       qq += "*";
 734   }
 735
 736   std::stringstream q;
 737   if ( s[0] == SENSE_SEARCH_CHAR ){
 738     q << "select did,"
 739       << "group_concat(pos) as pos,"
 740       << q_reading("d_sense.did")
 741       << q_kanji("d_sense.did")
 742       << q_sense()
 743       << " from d_sense where gloss glob '*" << stripped.substr(1) << "*'"
 744       << " group by did";
 745   }
 746   else if ( rmn_->contains_kanji( qq.c_str() ) ) {
 747     q << "select d_kanji.did as did,"
 748       << "(select group_concat(pos) from d_sense where d_sense.did = d_kanji.did) as pos,"
 749       << q_kanji()
 750       << q_reading("d_kanji.did")
 751       << q_sense("d_kanji.did")
 752       << "from d_kanji where "
 753         << "kanji glob '" << rmn_->romaji_to_hiragana(qq.c_str()) << "' "
 754         << " or kanji glob '" << rmn_->romaji_to_katakana(qq.c_str()) << "' "
 755       << " group by did order by d_kanji.freq desc, d_kanji.kanji asc";
 756   }
 757   else {
 758     q << "select d_reading.did as did,"
 759       << "(select group_concat(pos) from d_sense where d_sense.did = d_reading.did) as pos,"
 760       << q_reading()
 761       << q_kanji("d_reading.did")
 762       << q_sense("d_reading.did")
 763       << "from d_reading where "
 764         << "reading glob '" << rmn_->romaji_to_hiragana(qq.c_str())
 765         << "' or reading glob '"<< rmn_->romaji_to_katakana(qq.c_str()) << "' "
 766       << " group by did order by d_reading.freq desc, d_reading.reading asc";
 767   }
 768   set_listview(query(q.str().c_str()));
 769 }
 770
 771
 772
 773 Kanji App::db_get_kanji ( const string &kanji )
 774 {
 775   log("App::db_get_kanji()");
 776   std::stringstream q;
 777   q << "select kanji,strokes,ucs, rad_classic, rad_nelson, "
 778           << "jlpt, grade, freq, onyomi, kunyomi, nanori, meaning,flags,components "
 779     << "from k_kanji where kanji='" << kanji << "';";
 780
 781   vector<string> res = query(q.str().c_str());
 782   Kanji kk(res[0]);
 783   kk.strokes(std::stoi(res[1]));
 784   kk.ucs(res[2]);
 785   kk.rad_classic(std::stoi(res[3]));
 786   kk.rad_nelson( (res[4].empty()) ? -1:std::stoi(res[4]));
 787   kk.jlpt(std::stoi(res[5]));
 788   kk.grade(std::stoi(res[6]));
 789   kk.freq(std::stoi(res[7]));
 790   kk.onyomi( utils::split_string(res[8],parsers::SEPARATOR_SQL) );
 791   kk.kunyomi( utils::split_string(res[9],parsers::SEPARATOR_SQL) );
 792   kk.nanori( utils::split_string(res[10],parsers::SEPARATOR_SQL) );
 793   kk.meaning( utils::split_string(res[11],parsers::SEPARATOR_SQL) );
 794   kk.flags( utils::split_string(res[12],parsers::SEPARATOR_SQL) );
 795   kk.components( res[13] );
 796
 797   string qq = "select skip1,skip2,skip3,misclass from k_skip where kanji='"+kanji+"';";
 798   vector<string> res2 = query(qq.c_str());
 799   if ( res2.size() % 4 != 0 ){
 800     std::stringstream ss;
 801     ss << "Wrong SKIP count. Kanji: " << kanji
 802       << "Query result size: " << res2.size()
 803       << " (should be 4,8 or 12).  SKIP not loaded.";
 804     log_e(ss);
 805     return kk;
 806   }
 807   for ( size_t i=0; i < res2.size(); i+=4 )
 808     kk.skip( res2[i], res2[i+1], res2[i+2], res2[i+3] );
 809   return kk;
 810 }
 811
 812
 813 void App::cb_file ( MANAGEDB_FILE_TYPE filetype, bool download )
 814 {
 815   string url;
 816   const char *item = nullptr;
 817   switch ( filetype ){
 818     case MANAGEDB_JMDICT:
 819       url = get_config("sources/url_jmdict");
 820       item = MANAGEDB_ITEM_JMDICT;
 821       break;
 822     case MANAGEDB_KANJIDIC:
 823       url = get_config("sources/url_kanjidic");
 824       item = MANAGEDB_ITEM_KANJIDIC;
 825       break;
 826     case MANAGEDB_COMPONENTS:
 827       url = get_config("sources/url_components");
 828       item = MANAGEDB_ITEM_COMPONENTS;
 829       break;
 830     default:
 831       alert("Unknown MANAGEDB_FILE_TYPE.");
 832       return;
 833   }
 834   string fname;
 835
 836   if ( download ){
 837     fname = ui_->download_dialog( url );
 838     log_d("Download finished...");
 839   }
 840   else
 841     fname = ui_->open_file_dialog();
 842
 843   if ( fname.empty() )
 844     return;
 845
 846   log("Selected file: " + fname );
 847
 848   switch ( filetype ){
 849     case MANAGEDB_JMDICT:
 850       parse_jmdict(fname.c_str(), download);
 851       break;
 852     case MANAGEDB_KANJIDIC:
 853       parse_kanjidic(fname.c_str(), download);
 854       break;
 855     case MANAGEDB_COMPONENTS:
 856     {
 857       utils::gzip_decompress_file( fname.c_str(), TEMP_COMPONENTS);
 858       try {
 859         db_->script( TEMP_COMPONENTS );
 860       }
 861       catch ( SQLite3::DatabaseError &e ) {
 862         alert(e.what(), e.query() );
 863       }
 864       catch ( std::exception &e ) {
 865         alert( e.what() );
 866       }
 867       break;
 868     }
 869   }
 870
 871   auto label = ui()->dlg_manage_db()->item(item);
 872         if ( label )
 873                 label->version("Changed",FL_BLUE);
 874         else
 875                 log_w("Can't find item " + string(item) + " in ManageDBDialog.");
 876
 877   if ( filetype == MANAGEDB_COMPONENTS ){
 878     remove(TEMP_COMPONENTS);
 879     remove(fname.c_str());
 880   }
 881 }
 882
 883 void App::cb_filter_listview ()
 884 {
 885   vector<string> pos = ui_->listview_filters();
 886   log_d("App::cb_filter_listview(): pos: " + utils::to_string(pos));
 887   bool filter_expr  = ( utils::is_in( pos, string("expr") ) );
 888   bool filter_noun  = ( utils::is_in( pos, string("noun") ) );
 889   bool filter_verb  = ( utils::is_in( pos, string("verb") ) );
 890   bool filter_adj   = ( utils::is_in( pos, string("adj")  ) );
 891   vector<string> data;
 892   vector<int> ids;
 893   size_t n = 0;
 894   for ( auto &elt: listview_items_ ){
 895     auto start = elt.pos.begin();
 896     auto end = elt.pos.end();
 897     if ( filter_expr && std::find( start, end, "exp") == end )
 898       continue;
 899     if ( filter_noun && std::find( start, end, "n" ) == end )
 900       continue;
 901     if ( filter_adj && std::find_if( start, end,
 902       [](const string &s){ return strncmp(s.c_str(),"adj",3)==0;} ) == end )
 903       continue;
 904     if ( filter_verb && std::find_if( start, end,
 905       [](const string &s){ return strncmp(s.c_str(),"v",1)==0;} ) == end )
 906       continue;
 907     ids.push_back( elt.did );
 908     ids.push_back( elt.did );
 909     ids.push_back( elt.did );
 910     data.push_back ( elt.reading );
 911     data.push_back ( elt.kanji );
 912     data.push_back ( elt.sense );
 913     n++;
 914   }
 915   ui_->set_listview( data, ids );
 916   std::stringstream ss;
 917   ss << n << " results";
 918   if (  listview_items_.size() != n )
 919     ss << " (" <<  listview_items_.size()-n << " hidden)";
 920   log(ss.str());
 921   ui_->set_dic_results( ss.str() );
 922 }
 923
 924
 925 void App::cb_dicview_rightclick ( int did )
 926 {
 927   string q = "select group_concat(kanji,'') from d_kanji where did="+std::to_string(did);
 928   vector<string> res = query( q.c_str() );
 929   set<string> kanji;
 930   for ( string &c: utils::str_to_chars(res[0].c_str()) )
 931     if ( App::get()->rmn()->is_kanji(c.c_str()) )
 932       kanji.insert(c);
 933   ui_->menu_copy( vector<string>(kanji.begin(),kanji.end()) );
 934 }
 935
 936
 937 void App::apply_config ()
 938 {
 939   ui_->font_base_size( get_config<int>("font/base_size"));
 940   logger_.loglevel( get_config("log/level") );
 941   ui_->init_colors();
 942   Fl::check();
 943 }
 944
 945
 946 void App::cb_manage_db ()
 947 {
 948   auto q = query("select val from aoi where key='jmdict_version'");
 949   string jmdict_version = (q.empty()) ? "NONE":q[0];
 950   q = query("select val from aoi where key='kanjidic_version'");
 951   string kanjidic_version = q.empty() ? "NONE":q[0];
 952   q = query("select val from aoi where key='components_version'");
 953   string components_version = q.empty() ? "NONE":q[0];
 954   auto *d = ui_->dlg_manage_db();
 955   Fl_Color col;
 956   if ( !d->item(MANAGEDB_ITEM_JMDICT) ){
 957     col = ( jmdict_version == "NONE" ) ? FL_RED:FL_BLUE;
 958     d->add_item( MANAGEDB_ITEM_JMDICT, jmdict_version,
 959       "Japanese-English dictionary. Needed component.",
 960       col, scb_local_file_jmdict, scb_download_file_jmdict,
 961       (void*)this );
 962   }
 963   if ( !d->item(MANAGEDB_ITEM_KANJIDIC) ){
 964     col = ( kanjidic_version == "NONE" ) ? FL_RED:FL_BLUE;
 965     d->add_item( MANAGEDB_ITEM_KANJIDIC, kanjidic_version,
 966       "Kanji dictionary. Needed component.",
 967       col, scb_local_file_kanjidic, scb_download_file_kanjidic,
 968       (void*)this  );
 969   }
 970   if ( !d->item(MANAGEDB_ITEM_COMPONENTS) ){
 971     col = ( components_version == "NONE" ) ? FL_RED:FL_BLUE;
 972     d->add_item( MANAGEDB_ITEM_COMPONENTS, components_version,
 973       "Graphical components of kanji.",
 974       col, scb_local_file_components, scb_download_file_components,
 975       (void*)this );
 976   }
 977   d->show();
 978 }
 979
 980
 981 void App::load_config ()
 982 {
 983   log("Loading config...");
 984   vector<string> res = query("select key,val from config;");
 985   for ( size_t i=0; i < res.size(); i+=2 ){
 986     set_config( res[i], res[i+1] );
 987   }
 988   apply_config();
 989 }
 990
 991
 992 void App::save_config ( const std::map<string,aoi_config::Config::Value> &newmap)
 993 {
 994   log("Saving config...");
 995
 996   typedef std::pair<string,aoi_config::Config::Value> cfg_pair;
 997
 998   // merge new and old config
 999   for ( const cfg_pair &p: newmap )
1000     set_config( p.first, p.second.val );
1001   apply_config();
1002
1003   // prepare SQL script
1004   std::stringstream ss;
1005   ss << "BEGIN TRANSACTION;\n";
1006   ss << "DROP TABLE IF EXISTS config;\n";
1007   ss << "CREATE TABLE config (key TEXT, val TEXT);\n";
1008   for ( const cfg_pair &p: get_config_map() )
1009     ss << "INSERT INTO config (key,val) VALUES('"
1010       << p.first << "','" << p.second.val << "');\n";
1011   ss << "END TRANSACTION;\n";
1012
1013   query(ss.str().c_str());
1014
1015   // apply new fonts and colors
1016   init_dicview();
1017
1018   // TODO:
1019   //  check new keys
1020   //  check keys in oldconfig not present in newmap
1021   //  set oldkeys
1022   //  set newkeys
1023   //  mnoziny...
1024 }
1025
1026 } // namespace aoi