pyfpdb/GuiBulkImport.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 #Copyright 2008-2011 Steffen Schaumburg
   5 #This program is free software: you can redistribute it and/or modify
   6 #it under the terms of the GNU Affero General Public License as published by
   7 #the Free Software Foundation, version 3 of the License.
   8 #
   9 #This program is distributed in the hope that it will be useful,
  10 #but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 #MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  12 #GNU General Public License for more details.
  13 #
  14 #You should have received a copy of the GNU Affero General Public License
  15 #along with this program. If not, see <http://www.gnu.org/licenses/>.
  16 #In the "official" distribution you can find the license in agpl-3.0.txt.
  17
  18 import L10n
  19 _ = L10n.get_translation()
  20
  21 #    Standard Library modules
  22 import os
  23 import sys
  24 from time import time
  25 from optparse import OptionParser
  26 import traceback
  27
  28 #    pyGTK modules
  29 import pygtk
  30 pygtk.require('2.0')
  31 import gtk
  32 import gobject
  33
  34 #    fpdb/FreePokerTools modules
  35 import fpdb_import
  36 import Configuration
  37 import Exceptions
  38
  39 import logging
  40 if __name__ == "__main__":
  41     Configuration.set_logfile("fpdb-log.txt")
  42 # logging has been set up in fpdb.py or HUD_main.py, use their settings:
  43 log = logging.getLogger("importer")
  44
  45 class GuiBulkImport():
  46
  47     # CONFIGURATION  -  update these as preferred:
  48     allowThreads = False  # set to True to try out the threads field
  49
  50     def dopulse(self):
  51         self.progressbar.pulse()
  52         return True
  53
  54     def load_clicked(self, widget, data=None):
  55         if self.cbfilter.get_model()[self.cbfilter.get_active()][0] == (_("Please select site")):
  56             self.progressbar.set_text(_("Please select site"))
  57             return
  58         stored = None
  59         dups = None
  60         partial = None
  61         errs = None
  62         ttime = None
  63         # Does the lock acquisition need to be more sophisticated for multiple dirs?
  64         # (see comment above about what to do if pipe already open)
  65         if self.settings['global_lock'].acquire(wait=False, source="GuiBulkImport"):   # returns false immediately if lock not acquired
  66             #try:
  67                 self.progressbar.set_text(_("Importing"))
  68                 self.progressbar.pulse()
  69                 while gtk.events_pending(): # see http://faq.pygtk.org/index.py?req=index for more hints (3.7)
  70                     gtk.main_iteration(False)
  71                 self.timer = gobject.timeout_add(100, self.dopulse)
  72
  73                 #    get the dir to import from the chooser
  74                 selected = self.chooser.get_filenames()
  75
  76                 #    get the import settings from the gui and save in the importer
  77                 self.importer.setHandCount(int(self.spin_hands.get_text()))
  78                 self.importer.setQuiet(self.chk_st_st.get_active())
  79                 self.importer.setThreads(int(self.spin_threads.get_text()))
  80                 self.importer.setHandsInDB(self.n_hands_in_db)
  81                 cb_model = self.cb_dropindexes.get_model()
  82                 cb_index = self.cb_dropindexes.get_active()
  83                 cb_hmodel = self.cb_drophudcache.get_model()
  84                 cb_hindex = self.cb_drophudcache.get_active()
  85
  86                 #self.lab_info.set_markup('<span foreground="blue">Importing ...</span>') # uses pango markup!
  87
  88                 if cb_index:
  89                     self.importer.setDropIndexes(cb_model[cb_index][0])
  90                 else:
  91                     self.importer.setDropIndexes("auto")
  92                 if cb_hindex:
  93                     self.importer.setDropHudCache(cb_hmodel[cb_hindex][0])
  94                 else:
  95                     self.importer.setDropHudCache("auto")
  96                 sitename = self.cbfilter.get_model()[self.cbfilter.get_active()][0]
  97                 #self.importer.setFailOnError(self.chk_fail.get_active())
  98                 if self.is_archive.get_active():
  99                     if sitename == "PokerStars":
 100                         self.importer.setStarsArchive(True)
 101                     if sitename == "Full Tilt Poker":
 102                         self.importer.setFTPArchive(True)
 103
 104                 for selection in selected:
 105                     self.importer.addBulkImportImportFileOrDir(selection, site = sitename)
 106                 self.importer.setCallHud(self.cb_testmode.get_active())
 107                 self.importer.bHudTest = self.cb_testmode.get_active()
 108                 starttime = time()
 109 #                try:
 110                 (stored, dups, partial, errs, ttime) = self.importer.runImport()
 111 #                except:
 112 #                    print "*** EXCEPTION DURING BULKIMPORT!!!"
 113 #                    raise Exceptions.FpdbError
 114 #                finally:
 115                 gobject.source_remove(self.timer)
 116
 117                 ttime = time() - starttime
 118                 if ttime == 0:
 119                     ttime = 1
 120
 121                 completionMessage = _('Bulk import done: Stored: %d, Duplicates: %d, Partial: %d, Errors: %d, Time: %s seconds, Stored/second: %.0f')\
 122                     % (stored, dups, partial, errs, ttime, (stored+0.0) / ttime)
 123                 print completionMessage
 124                 log.info(completionMessage)
 125
 126                 self.importer.clearFileList()
 127
 128                 if self.n_hands_in_db == 0 and stored > 0:
 129                     self.cb_dropindexes.set_sensitive(True)
 130                     self.cb_dropindexes.set_active(0)
 131                     self.lab_drop.set_sensitive(True)
 132                     self.cb_drophudcache.set_sensitive(True)
 133                     self.cb_drophudcache.set_active(0)
 134                     self.lab_hdrop.set_sensitive(True)
 135
 136                 self.progressbar.set_text(_("Import Complete"))
 137                 self.progressbar.set_fraction(0)
 138             #except:
 139                 #err = traceback.extract_tb(sys.exc_info()[2])[-1]
 140                 #print "*** BulkImport Error: "+err[2]+"("+str(err[1])+"): "+str(sys.exc_info()[1])
 141             #self.settings['global_lock'].release()
 142                 self.settings['global_lock'].release()
 143         else:
 144             print _("bulk import aborted - global lock not available")
 145
 146     def get_vbox(self):
 147         """returns the vbox of this thread"""
 148         return self.vbox
 149
 150     def __init__(self, settings, config, sql = None, parent = None):
 151         self.settings = settings
 152         self.config = config
 153         self.parent = parent
 154
 155         self.importer = fpdb_import.Importer(self, self.settings, config, sql, parent)
 156
 157         self.vbox = gtk.VBox(False, 0)
 158         self.vbox.show()
 159
 160         self.chooser = gtk.FileChooserWidget()
 161         self.chooser.set_filename(self.settings['bulkImport-defaultPath'])
 162         self.chooser.set_select_multiple(True)
 163         self.vbox.add(self.chooser)
 164         self.chooser.show()
 165
 166 #    Table widget to hold the settings
 167         self.table = gtk.Table(rows=5, columns=5, homogeneous=False)
 168         self.vbox.add(self.table)
 169         self.table.show()
 170
 171 #    checkbox - print start/stop?
 172         self.chk_st_st = gtk.CheckButton(_('Print Start/Stop Info'))
 173         self.table.attach(self.chk_st_st, 0, 1, 0, 1, xpadding=10, ypadding=0,
 174                           yoptions=gtk.SHRINK)
 175         self.chk_st_st.show()
 176         self.chk_st_st.set_active(True)
 177
 178 #    label - status
 179         self.lab_status = gtk.Label(_("Hands/status print:"))
 180         self.table.attach(self.lab_status, 1, 2, 0, 1, xpadding=0, ypadding=0,
 181                           yoptions=gtk.SHRINK)
 182         self.lab_status.show()
 183         self.lab_status.set_justify(gtk.JUSTIFY_RIGHT)
 184         self.lab_status.set_alignment(1.0, 0.5)
 185
 186 #    spin button - status
 187         status_adj = gtk.Adjustment(value=100, lower=0, upper=300, step_incr=10,
 188                                     page_incr=1, page_size=0) #not sure what upper value should be!
 189         self.spin_status = gtk.SpinButton(adjustment=status_adj, climb_rate=0.0,
 190                                           digits=0)
 191         self.table.attach(self.spin_status, 2, 3, 0, 1, xpadding=10, ypadding=0,
 192                           yoptions=gtk.SHRINK)
 193         self.spin_status.show()
 194
 195 #    label - threads
 196         self.lab_threads = gtk.Label(_("Number of threads:"))
 197         self.table.attach(self.lab_threads, 3, 4, 0, 1, xpadding=0, ypadding=0,
 198                           yoptions=gtk.SHRINK)
 199         self.lab_threads.show()
 200         if not self.allowThreads:
 201             self.lab_threads.set_sensitive(False)
 202         self.lab_threads.set_justify(gtk.JUSTIFY_RIGHT)
 203         self.lab_threads.set_alignment(1.0, 0.5)
 204
 205 #    spin button - threads
 206         threads_adj = gtk.Adjustment(value=0, lower=0, upper=32, step_incr=1,
 207                                      page_incr=1, page_size=0) #not sure what upper value should be!
 208         self.spin_threads = gtk.SpinButton(adjustment=threads_adj, climb_rate=0.0, digits=0)
 209         self.table.attach(self.spin_threads, 4, 5, 0, 1, xpadding=10, ypadding=0,
 210                           yoptions=gtk.SHRINK)
 211         self.spin_threads.show()
 212         if not self.allowThreads:
 213             self.spin_threads.set_sensitive(False)
 214
 215 #    checkbox - archive file?
 216         self.is_archive = gtk.CheckButton(_('Archive File'))
 217         self.table.attach(self.is_archive, 0, 1, 1, 2, xpadding=10, ypadding=0, yoptions=gtk.SHRINK)
 218         self.is_archive.show()
 219
 220 #    label - hands
 221         self.lab_hands = gtk.Label(_("Hands/file:"))
 222         self.table.attach(self.lab_hands, 1, 2, 1, 2, xpadding=0, ypadding=0, yoptions=gtk.SHRINK)
 223         self.lab_hands.show()
 224         self.lab_hands.set_justify(gtk.JUSTIFY_RIGHT)
 225         self.lab_hands.set_alignment(1.0, 0.5)
 226
 227 #    spin button - hands to import
 228         hands_adj = gtk.Adjustment(value=0, lower=0, upper=10, step_incr=1,
 229                                    page_incr=1, page_size=0) #not sure what upper value should be!
 230         self.spin_hands = gtk.SpinButton(adjustment=hands_adj, climb_rate=0.0, digits=0)
 231         self.table.attach(self.spin_hands, 2, 3, 1, 2, xpadding=10, ypadding=0,
 232                           yoptions=gtk.SHRINK)
 233         self.spin_hands.show()
 234
 235 #    label - drop indexes
 236         self.lab_drop = gtk.Label(_("Drop indexes:"))
 237         self.table.attach(self.lab_drop, 3, 4, 1, 2, xpadding=0, ypadding=0,
 238                           yoptions=gtk.SHRINK)
 239         self.lab_drop.show()
 240         self.lab_drop.set_justify(gtk.JUSTIFY_RIGHT)
 241         self.lab_drop.set_alignment(1.0, 0.5)
 242
 243 #    ComboBox - drop indexes
 244         self.cb_dropindexes = gtk.combo_box_new_text()
 245         self.cb_dropindexes.append_text(_('auto'))
 246         self.cb_dropindexes.append_text(_("don't drop"))
 247         self.cb_dropindexes.append_text(_('drop'))
 248         self.cb_dropindexes.set_active(0)
 249         self.table.attach(self.cb_dropindexes, 4, 5, 1, 2, xpadding=10,
 250                           ypadding=0, yoptions=gtk.SHRINK)
 251         self.cb_dropindexes.show()
 252
 253         self.cb_testmode = gtk.CheckButton(_('HUD Test mode'))
 254         self.table.attach(self.cb_testmode, 0, 1, 2, 3, xpadding=10, ypadding=0, yoptions=gtk.SHRINK)
 255         self.cb_testmode.show()
 256
 257 #    label - filter
 258         self.lab_filter = gtk.Label(_("Site filter:"))
 259         self.table.attach(self.lab_filter, 1, 2, 2, 3, xpadding=0, ypadding=0,
 260                           yoptions=gtk.SHRINK)
 261         self.lab_filter.show()
 262         self.lab_filter.set_justify(gtk.JUSTIFY_RIGHT)
 263         self.lab_filter.set_alignment(1.0, 0.5)
 264
 265 #    ComboBox - filter
 266         self.cbfilter = gtk.combo_box_new_text()
 267         disabled_sites = []                                # move disabled sites to bottom of list
 268         self.cbfilter.append_text(_("Please select site"))
 269         for w in self.config.hhcs:
 270             try:
 271                 if self.config.supported_sites[w].enabled: # include enabled ones first
 272                     print w
 273                     self.cbfilter.append_text(w)
 274                 else:
 275                     disabled_sites.append(w)
 276             except: # self.supported_sites[w] may not exist if hud_config is bad
 277                 disabled_sites.append(w)
 278         for w in disabled_sites:                           # then disabled ones
 279             print w
 280             self.cbfilter.append_text(w)
 281         self.cbfilter.set_active(0)
 282         self.table.attach(self.cbfilter, 2, 3, 2, 3, xpadding=10, ypadding=1,
 283                           yoptions=gtk.SHRINK)
 284         self.cbfilter.show()
 285
 286 #    label - drop hudcache
 287         self.lab_hdrop = gtk.Label(_("Drop HudCache:"))
 288         self.table.attach(self.lab_hdrop, 3, 4, 2, 3, xpadding=0, ypadding=0,
 289                           yoptions=gtk.SHRINK)
 290         self.lab_hdrop.show()
 291         self.lab_hdrop.set_justify(gtk.JUSTIFY_RIGHT)
 292         self.lab_hdrop.set_alignment(1.0, 0.5)
 293
 294 #    ComboBox - drop hudcache
 295         self.cb_drophudcache = gtk.combo_box_new_text()
 296         self.cb_drophudcache.append_text(_('auto'))
 297         self.cb_drophudcache.append_text(_("don't drop"))
 298         self.cb_drophudcache.append_text(_('drop'))
 299         self.cb_drophudcache.set_active(0)
 300         self.table.attach(self.cb_drophudcache, 4, 5, 2, 3, xpadding=10,
 301                           ypadding=0, yoptions=gtk.SHRINK)
 302         self.cb_drophudcache.show()
 303
 304 #    button - Import
 305         self.load_button = gtk.Button(_('_Bulk Import'))  # todo: rename variables to import too
 306         self.load_button.connect('clicked', self.load_clicked,
 307                                  _('Import clicked'))
 308         self.table.attach(self.load_button, 2, 3, 4, 5, xpadding=0, ypadding=0,
 309                           yoptions=gtk.SHRINK)
 310         self.load_button.show()
 311
 312 #    label - spacer (keeps rows 3 & 5 apart)
 313         self.lab_spacer = gtk.Label()
 314         self.table.attach(self.lab_spacer, 3, 5, 3, 4, xpadding=0, ypadding=0,
 315                           yoptions=gtk.SHRINK)
 316         self.lab_spacer.show()
 317
 318 #    label - info
 319 #        self.lab_info = gtk.Label()
 320 #        self.table.attach(self.lab_info, 3, 5, 4, 5, xpadding = 0, ypadding = 0, yoptions=gtk.SHRINK)
 321 #        self.lab_info.show()
 322         self.progressbar = gtk.ProgressBar()
 323         self.table.attach(self.progressbar, 3, 5, 4, 5, xpadding=0, ypadding=0,
 324                           yoptions=gtk.SHRINK)
 325         self.progressbar.set_text(_("Waiting..."))
 326         self.progressbar.set_fraction(0)
 327         self.progressbar.show()
 328
 329 #    see how many hands are in the db and adjust accordingly
 330         tcursor = self.importer.database.cursor
 331         tcursor.execute("Select count(1) from Hands")
 332         row = tcursor.fetchone()
 333         tcursor.close()
 334         self.importer.database.rollback()
 335         self.n_hands_in_db = row[0]
 336         if self.n_hands_in_db == 0:
 337             self.cb_dropindexes.set_active(2)
 338             self.cb_dropindexes.set_sensitive(False)
 339             self.lab_drop.set_sensitive(False)
 340             self.cb_drophudcache.set_active(2)
 341             self.cb_drophudcache.set_sensitive(False)
 342             self.lab_hdrop.set_sensitive(False)
 343
 344 def main(argv=None):
 345     """main can also be called in the python interpreter, by supplying the command line as the argument."""
 346     if argv is None:
 347         argv = sys.argv[1:]
 348
 349     def destroy(*args):  # call back for terminating the main eventloop
 350         gtk.main_quit()
 351
 352     parser = OptionParser()
 353     parser.add_option("-f", "--file", dest="filename", metavar="FILE", default=None,
 354                     help=_("Input file"))
 355     parser.add_option("-c", "--convert", dest="filtername", default=None, metavar="FILTER",
 356                     help=_("Site")+ " (Absolute, Merge, Everleaf, Full Tilt Poker, PokerStars, ...)") #TODO: dynamically generate list
 357     parser.add_option("-x", "--failOnError", action="store_true", default=False,
 358                     help=_("If this option is used it quits with an extended error message if it encounters any error"))
 359     parser.add_option("-u", "--usage", action="store_true", dest="usage", default=False,
 360                     help=_("Print some useful one liners"))
 361     parser.add_option("-s", "--starsarchive", action="store_true", dest="starsArchive", default=False,
 362                     help=_("Do the required conversion for %s archive format (ie. as provided by support)") % "PokerStars")
 363     parser.add_option("-F", "--ftparchive", action="store_true", dest="ftpArchive", default=False,
 364                     help=_("Do the required conversion for %s archive format (ie. as provided by support)") % "Full Tilt Poker")
 365     parser.add_option("-t", "--testdata", action="store_true", dest="testData", default=False,
 366                     help=_("Generate and print test data for regression testing"))
 367     parser.add_option("-C", "--configFile", dest="config", default=None, help=_("Specifies a configuration file."))
 368     (options, argv) = parser.parse_args(args = argv)
 369
 370     if options.usage == True:
 371         #Print usage examples and exit
 372         print _("USAGE:")
 373         print ('PokerStars ' + _('converter') + ': ./GuiBulkImport.py -c PokerStars -f filename')
 374         print ('Full Tilt  ' + _('converter') + ': ./GuiBulkImport.py -c "Full Tilt Poker" -f filename')
 375         print ('Everleaf   ' + _('converter') + ': ./GuiBulkImport.py -c Everleaf -f filename')
 376         print ('Absolute   ' + _('converter') + ': ./GuiBulkImport.py -c Absolute -f filename')
 377         print ('PartyPoker ' + _('converter') + ': ./GuiBulkImport.py -c PartyPoker -f filename')
 378         sys.exit(0)
 379
 380     Configuration.set_logfile("GuiBulkImport-log.txt")
 381     if options.config:
 382         config = Configuration.Config(options.config)
 383     else:
 384         config = Configuration.Config()
 385
 386     settings = {}
 387     if os.name == 'nt': settings['os'] = 'windows'
 388     else:               settings['os'] = 'linuxmac'
 389
 390     settings.update(config.get_db_parameters())
 391     settings.update(config.get_import_parameters())
 392     settings.update(config.get_default_paths())
 393
 394     if not options.filename:
 395         i = GuiBulkImport(settings, config, None)
 396         main_window = gtk.Window()
 397         main_window.connect('destroy', destroy)
 398         main_window.add(i.vbox)
 399         main_window.show()
 400         gtk.main()
 401     else:
 402         if not options.filtername:
 403             print _("You have to select a site with the -c parameter. E.g.:"), "Everleaf   converter: ./GuiBulkImport.py -c Everleaf -f filename"
 404         #Do something useful
 405         importer = fpdb_import.Importer(False,settings, config, None)
 406         # importer.setDropIndexes("auto")
 407         importer.setDropIndexes(_("don't drop"))
 408         importer.setFailOnError(options.failOnError)
 409         importer.setThreads(-1)
 410         importer.addBulkImportImportFileOrDir(os.path.expanduser(options.filename), site=options.filtername)
 411         importer.setCallHud(False)
 412         if options.starsArchive:
 413             importer.setStarsArchive(True)
 414         if options.ftpArchive:
 415             importer.setFTPArchive(True)
 416         if options.testData:
 417             importer.setPrintTestData(True)
 418         (stored, dups, partial, errs, ttime) = importer.runImport()
 419         importer.clearFileList()
 420         print(_('Bulk import done: Stored: %d, Duplicates: %d, Partial: %d, Errors: %d, Time: %s seconds, Stored/second: %.0f')\
 421                      % (stored, dups, partial, errs, ttime, (stored+0.0) / ttime))
 422
 423
 424 if __name__ == '__main__':
 425     sys.exit(main())