pyfpdb/GuiTourneyImport.py

   1 #!/usr/bin/python
   2 # -*- coding: utf-8 -*-
   3
   4 #Copyright 2008-2010 Carl Gherardi
   5 #This program is free software: you can redistribute it and/or modify
   6 #it under the terms of the GNU Affero General Public License as published by
   7 #the Free Software Foundation, version 3 of the License.
   8 #
   9 #This program is distributed in the hope that it will be useful,
  10 #but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 #MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  12 #GNU General Public License for more details.
  13 #
  14 #You should have received a copy of the GNU Affero General Public License
  15 #along with this program. If not, see <http://www.gnu.org/licenses/>.
  16 #In the "official" distribution you can find the license in agpl-3.0.txt.
  17
  18 import L10n
  19 _ = L10n.get_translation()
  20
  21 #    Standard Library modules
  22 import os
  23 import sys
  24 from time import time
  25 import traceback
  26 import datetime
  27 import codecs
  28 import re
  29
  30 #    pyGTK modules
  31 import pygtk
  32 pygtk.require('2.0')
  33 import gtk
  34 import gobject
  35
  36 #    fpdb/FreePokerTools modules
  37 import Configuration
  38 import Options
  39 from Exceptions import FpdbParseError
  40
  41 import logging
  42 # logging has been set up in fpdb.py or HUD_main.py, use their settings:
  43 log = logging.getLogger("importer")
  44
  45
  46 class GuiTourneyImport():
  47
  48     def load_clicked(self, widget, data=None):
  49         print "DEBUG: load_clicked"
  50         stored = None
  51         dups = None
  52         partial = None
  53         errs = None
  54         ttime = None
  55
  56         if self.settings['global_lock'].acquire(wait=False, source="GuiTourneyImport"):
  57             print "DEBUG: got global lock"
  58             #    get the dir to import from the chooser
  59             selected = self.chooser.get_filenames()
  60             print "DEBUG: Files selected: %s" % selected
  61
  62             sitename = self.cbfilter.get_model()[self.cbfilter.get_active()][0]
  63
  64             for selection in selected:
  65                 self.importer.addImportFileOrDir(selection, site = sitename)
  66             starttime = time()
  67             (stored, errs) = self.importer.runImport()
  68
  69             ttime = time() - starttime
  70             if ttime == 0:
  71                 ttime = 1
  72             print _('GuiTourneyImport.load done: Stored: %d\tErrors: %d in %s seconds - %.0f/sec')\
  73                      % (stored, errs, ttime, (stored+0.0) / ttime)
  74             self.importer.clearFileList()
  75
  76             self.settings['global_lock'].release()
  77         else:
  78             print _("bulk import aborted - global lock not available")
  79
  80     def get_vbox(self):
  81         """returns the vbox of this thread"""
  82         return self.vbox
  83
  84     def __init__(self, settings, config, sql = None, parent = None):
  85         self.settings = settings
  86         self.config = config
  87         self.parent = parent
  88
  89         self.importer = SummaryImporter(config, sql, parent)
  90
  91         self.vbox = gtk.VBox(False, 0)
  92         self.vbox.show()
  93
  94         self.chooser = gtk.FileChooserWidget()
  95         self.chooser.set_filename(self.settings['bulkImport-defaultPath'])
  96         self.chooser.set_select_multiple(True)
  97         self.vbox.add(self.chooser)
  98         self.chooser.show()
  99
 100 #    Table widget to hold the settings
 101         self.table = gtk.Table(rows=5, columns=5, homogeneous=False)
 102         self.vbox.add(self.table)
 103         self.table.show()
 104
 105 #    label - tsc
 106         self.lab_filter = gtk.Label(_("Site filter:"))
 107         self.table.attach(self.lab_filter, 1, 2, 2, 3, xpadding=0, ypadding=0,
 108                           yoptions=gtk.SHRINK)
 109         self.lab_filter.show()
 110         self.lab_filter.set_justify(gtk.JUSTIFY_RIGHT)
 111         self.lab_filter.set_alignment(1.0, 0.5)
 112
 113 #    ComboBox - filter
 114         self.cbfilter = gtk.combo_box_new_text()
 115         disabled_sites = []                                # move disabled sites to bottom of list
 116         for w in self.config.hhcs:
 117             print "%s = '%s'" %(w, self.config.hhcs[w].summaryImporter)
 118             try:
 119                 # Include sites with a tourney summary importer, and enabled
 120                 if self.config.supported_sites[w].enabled and self.config.hhcs[w].summaryImporter != '':
 121                     self.cbfilter.append_text(w)
 122                 else:
 123                     disabled_sites.append(w)
 124             except: # self.supported_sites[w] may not exist if hud_config is bad
 125                 disabled_sites.append(w)
 126         for w in disabled_sites:
 127             if self.config.hhcs[w].summaryImporter != '':
 128                 self.cbfilter.append_text(w)
 129         self.cbfilter.set_active(0)
 130         self.table.attach(self.cbfilter, 2, 3, 2, 3, xpadding=10, ypadding=1,
 131                           yoptions=gtk.SHRINK)
 132         self.cbfilter.show()
 133
 134 #    button - Import
 135         self.load_button = gtk.Button(_('_Bulk Import'))  # todo: rename variables to import too
 136         self.load_button.connect('clicked', self.load_clicked,
 137                                  _('Import clicked'))
 138         self.table.attach(self.load_button, 2, 3, 4, 5, xpadding=0, ypadding=0,
 139                           yoptions=gtk.SHRINK)
 140         self.load_button.show()
 141
 142 #    label - spacer (keeps rows 3 & 5 apart)
 143         self.lab_spacer = gtk.Label()
 144         self.table.attach(self.lab_spacer, 3, 5, 3, 4, xpadding=0, ypadding=0,
 145                           yoptions=gtk.SHRINK)
 146         self.lab_spacer.show()
 147
 148 class SummaryImporter:
 149     def __init__(self, config, sql = None, parent = None):
 150         self.config     = config
 151         self.sql        = sql
 152         self.parent     = parent
 153
 154         self.filelist   = {}
 155         self.dirlist    = {}
 156
 157         self.updatedsize = {}
 158         self.updatedtime = {}
 159
 160     def addImportFile(self, filename, site = "default", tsc = "passthrough"):
 161         if filename in self.filelist or not os.path.exists(filename):
 162             print "DEBUG: addImportFile: File exists, or path non-existent"
 163             return
 164         self.filelist[filename] = [site] + [tsc]
 165
 166     def addImportDirectory(self,dir,monitor=False, site="default", tsc="passthrough"):
 167         if os.path.isdir(dir):
 168             if monitor == True:
 169                 self.monitor = True
 170                 self.dirlist[site] = [dir] + [tsc]
 171
 172             for file in os.listdir(dir):
 173                 self.addImportFile(os.path.join(dir, file), site, tsc)
 174         else:
 175             log.warning(_("Attempted to add non-directory '%s' as an import directory") % str(dir))
 176
 177     def addImportFileOrDir(self, inputPath, site = "PokerStars"):
 178         tsc = self.config.hhcs[site].summaryImporter
 179         if os.path.isdir(inputPath):
 180             for subdir in os.walk(inputPath):
 181                 for file in subdir[2]:
 182                     self.addImportFile(unicode(os.path.join(subdir[0], file),'utf-8'),
 183                                        site=site, tsc=tsc)
 184         else:
 185             self.addImportFile(unicode(inputPath,'utf-8'), site=site, tsc=tsc)
 186         pass
 187
 188     def runImport(self):
 189         start = datetime.datetime.now()
 190         starttime = time()
 191         log.info(_("Tourney Summary Import started at %s - %d files to import.") % (start, len(self.filelist)))
 192
 193         total_errors = 0
 194         total_imported = 0
 195         for f in self.filelist:
 196             (site, tsc) = self.filelist[f]
 197             imported, errs = self.importFile(f, tsc, site)
 198             total_errors += errs
 199             total_imported += imported
 200         return (total_imported, total_errors)
 201
 202
 203     def runUpdated(self):
 204         pass
 205
 206     def importFile(self, filename, tsc = "PokerStarsSummary", site = "PokerStars"):
 207         mod = __import__(tsc)
 208         obj = getattr(mod, tsc, None)
 209         if callable(obj):
 210             foabs = self.readFile(obj, filename)
 211             summaryTexts = re.split(obj.re_SplitTourneys, foabs)
 212
 213             # The summary files tend to have a header or footer
 214             # Remove the first and/or last entry if it has < 100 characters
 215             if len(summaryTexts[-1]) <= 100:
 216                 summaryTexts.pop()
 217                 log.warn(_("TourneyImport: Removing text < 100 characters from end of file"))
 218
 219             if len(summaryTexts[0]) <= 130:
 220                 del summaryTexts[0]
 221                 log.warn(_("TourneyImport: Removing text < 100 characters from start of file"))
 222
 223             print "Found %s summaries" %(len(summaryTexts))
 224             errors = 0
 225             imported = 0
 226             for j, summaryText in enumerate(summaryTexts, start=1):
 227                 try:
 228                     conv = obj(db=None, config=self.config, siteName=site, summaryText=summaryText, builtFrom = "IMAP")
 229                 except FpdbParseError, e:
 230                     errors += 1
 231                 print _("Finished importing %s/%s tournament summaries") %(j, len(summaryTexts))
 232                 imported = j
 233         return (imported - errors, errors)
 234
 235     def clearFileList(self):
 236         self.updatedsize = {}
 237         self.updatetime = {}
 238         self.filelist = {}
 239
 240     def readFile(self, tsc, filename):
 241         codepage = ["utf8", "utf16"]
 242         whole_file = None
 243         tsc.codepage
 244
 245         for kodec in codepage:
 246             try:
 247                 in_fh = codecs.open(filename, 'r', kodec)
 248                 whole_file = in_fh.read()
 249                 in_fh.close()
 250                 break
 251             except UnicodeDecodeError, e:
 252                 log.warn(_("GTI.readFile: '%s'") % e)
 253                 pass
 254
 255         return whole_file
 256
 257 def main(argv=None):
 258     """main can also be called in the python interpreter, by supplying the command line as the argument."""
 259     import SQL
 260     if argv is None:
 261         argv = sys.argv[1:]
 262
 263     (options, argv) = Options.fpdb_options()
 264
 265     if options.usage == True:
 266         #Print usage examples and exit
 267         print _("USAGE:")
 268         sys.exit(0)
 269
 270     if options.hhc == "PokerStarsToFpdb":
 271         print _("Need to define a converter")
 272         exit(0)
 273
 274     config = Configuration.Config("HUD_config.test.xml")
 275     sql = SQL.Sql(db_server = 'sqlite')
 276
 277     if options.filename == None:
 278         print _("Need a filename to import")
 279         exit(0)
 280
 281     importer = SummaryImporter(config, sql, None)
 282
 283     importer.addImportFileOrDir(options.filename, site = options.hhc)
 284     starttime = time()
 285     (stored, errs) = importer.runImport()
 286
 287     ttime = time() - starttime
 288     if ttime == 0:
 289         ttime = 1
 290     print _('GuiTourneyImport.load done: Stored: %d\tErrors: %d in %s seconds - %.0f/sec')\
 291                      % (stored, errs, ttime, (stored+0.0) / ttime)
 292     importer.clearFileList()
 293
 294
 295
 296
 297 if __name__  == '__main__':
 298     sys.exit(main())