scripts/stats.py

   1 #!/usr/bin/python2.5
   2 #
   3 # Copyright 2009 the Melange authors.
   4 #
   5 # Licensed under the Apache License, Version 2.0 (the "License");
   6 # you may not use this file except in compliance with the License.
   7 # You may obtain a copy of the License at
   8 #
   9 #   http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 # See the License for the specific language governing permissions and
  15 # limitations under the License.
  16
  17 """Starts an interactive shell with statistic helpers.
  18 """
  19
  20 __authors__ = [
  21   '"Sverre Rabbelier" <sverre@rabbelier.nl>',
  22 ]
  23
  24
  25 import cPickle
  26 import datetime
  27 import operator
  28 import sys
  29 import time
  30
  31 import interactive
  32
  33
  34 def dateFetch(queryGen, last=None, batchSize=100):
  35   """Iterator that yields an entity in batches.
  36
  37   Args:
  38     queryGen: should return a Query object
  39     last: used to .filter() for last_modified_on
  40     batchSize: how many entities to retrieve in one datastore call
  41
  42   Retrieved from http://tinyurl.com/d887ll (AppEngine cookbook).
  43   """
  44
  45   from google.appengine.ext import db
  46
  47    # AppEngine will not fetch more than 1000 results
  48   batchSize = min(batchSize,1000)
  49
  50   query = None
  51   done = False
  52   count = 0
  53
  54   while not done:
  55     print count
  56     query = queryGen()
  57     query.order('last_modified_on')
  58     if last:
  59       query.filter("last_modified_on > ", last)
  60     results = query.fetch(batchSize)
  61     for result in results:
  62       count += 1
  63       yield result
  64     if batchSize > len(results):
  65       done = True
  66     else:
  67       last = results[-1].last_modified_on
  68
  69
  70 def addKey(target, fieldname):
  71   """Adds the key of the specified field.
  72   """
  73
  74   result = target.copy()
  75   result['%s_key' % fieldname] = target[fieldname].key().name()
  76   return result
  77
  78
  79 def getEntities(model):
  80   """Returns all users as dictionary.
  81   """
  82
  83   def wrapped():
  84     gen = lambda: model.all()
  85     it = interactive.deepFetch(gen)
  86
  87     entities = [(i.key().name(), i) for i in it]
  88     return dict(entities)
  89
  90   return wrapped
  91
  92
  93 def getProps(last=None):
  94   """Returns all proposals as a list of dictionaries.
  95   """
  96
  97   key_order = [
  98       'link_id', 'scope_path', 'title', 'abstract', 'content',
  99       'additional_info', '_mentor', 'possible_mentors', 'score',
 100       'status', '_org', 'created_on', 'last_modified_on']
 101
 102   from soc.models.student_proposal import StudentProposal
 103
 104   gen = lambda: StudentProposal.all()
 105
 106   it = dateFetch(gen, last)
 107
 108   proposals = [(i.key().name(), i.toDict(key_order)) for i in it]
 109   if proposals:
 110     last = i.last_modified_on # last modified entity
 111   else:
 112     last = datetime.datetime.now()
 113
 114   return dict(proposals), last
 115
 116
 117 def orgStats(target, orgs):
 118   """Retrieves org stats.
 119   """
 120
 121   from soc.logic import dicts
 122
 123   orgs = [(v.key(), v) for k, v in orgs.iteritems()]
 124   orgs = dict(orgs)
 125
 126   grouped = dicts.groupby(target.values(), '_org')
 127
 128   grouped = [(orgs[k], v) for k, v in grouped.iteritems()]
 129   popularity = [(k.link_id, len(v)) for k, v in grouped]
 130
 131   return dict(grouped), dict(popularity)
 132
 133
 134 def countStudentsWithProposals():
 135   """Retrieves number of Students who have submitted at least one Student Proposal.
 136   """
 137
 138   proposals = getStudentProposals()
 139   students = {}
 140
 141   for proposal_key in proposals.keys():
 142     students[proposals[proposal_key].scope_path] = True
 143
 144   return len(students)
 145
 146
 147 def printPopularity(popularity):
 148   """Prints the popularity for the specified proposals.
 149   """
 150
 151   g = operator.itemgetter(1)
 152
 153   for item in sorted(popularity.iteritems(), key=g, reverse=True):
 154     print "%s: %d" % item
 155
 156
 157 def saveValues(values, saver):
 158   """Saves the specified popularities.
 159   """
 160
 161   import logging
 162   from google.appengine.ext import db
 163
 164   from soc.models.organization import Organization
 165
 166   def txn(key, value):
 167     org = Organization.get_by_key_name(key)
 168     saver(org, value)
 169     org.put()
 170
 171   for key, value in sorted(values.iteritems()):
 172     print key
 173     db.run_in_transaction_custom_retries(10, txn, key, value)
 174
 175   print "done"
 176
 177
 178 def addFollower(follower, proposals, add_public=True, add_private=True):
 179   """Adds a user as follower to the specified proposals.
 180
 181   Args:
 182     follower: the User to add as follower
 183     proposals: a list with the StudnetProposals that should be subscribed to
 184     add_public: whether the user is subscribed to public updates
 185     add_private: whether the user should be subscribed to private updates
 186   """
 187
 188   from soc.models.review_follower import ReviewFollower
 189
 190   result = []
 191
 192   for i in proposals:
 193      properties = {
 194        'user': follower,
 195        'link_id': follower.link_id,
 196        'scope': i,
 197        'scope_path': i.key().name(),
 198        'key_name': '%s/%s' % (i.key().name(), follower.link_id),
 199        'subscribed_public': add_public,
 200        'subscribed_private': add_private,
 201      }
 202
 203      entity = ReviewFollower(**properties)
 204      result.append(entity)
 205
 206   return result
 207
 208
 209 def convertProposals(org):
 210   """Convert all proposals for the specified organization.
 211
 212   Args:
 213     org: the organization for which all proposals will be converted
 214   """
 215
 216   from soc.logic.models.student_proposal import logic as proposal_logic
 217   from soc.logic.models.student_project import logic as project_logic
 218
 219   proposals = proposal_logic.getProposalsToBeAcceptedForOrg(org)
 220
 221   for proposal in proposals:
 222     fields = {
 223         'link_id': 't%i' % (int(time.time()*100)),
 224         'scope_path': proposal.org.key().id_or_name(),
 225         'scope': proposal.org,
 226         'program': proposal.program,
 227         'student': proposal.scope,
 228         'title': proposal.title,
 229         'abstract': proposal.abstract,
 230         'mentor': proposal.mentor,
 231         }
 232
 233     project = project_logic.updateOrCreateFromFields(fields, silent=True)
 234
 235     fields = {
 236         'status':'accepted',
 237         }
 238
 239     proposal_logic.updateEntityProperties(proposal, fields, silent=True)
 240
 241   fields = {
 242       'status': ['new', 'pending'],
 243       'org': org,
 244       }
 245
 246   querygen = lambda: proposal_logic.getQueryForFields(fields)
 247   proposals = [i for i in interactive.deepFetch(querygen)]
 248
 249   print "rejecting %d proposals" % len(proposals)
 250
 251   fields = {
 252       'status': 'rejected',
 253       }
 254
 255   for proposal in proposals:
 256     proposal_logic.updateEntityProperties(proposal, fields, silent=True)
 257
 258
 259 def startSpam():
 260   from soc.logic.models.job import logic as job_logic
 261   from soc.logic.models.priority_group import logic as priority_logic
 262   from soc.logic.models.program import logic as program_logic
 263
 264   program_entity = program_logic.getFromKeyName('google/gsoc2009')
 265
 266   priority_group = priority_logic.getGroup(priority_logic.EMAIL)
 267   job_fields = {
 268       'priority_group': priority_group,
 269       'task_name': 'setupStudentProposalMailing',
 270       'key_data': [program_entity.key()]}
 271
 272   job_logic.updateOrCreateFromFields(job_fields)
 273
 274
 275 def deleteEntities(model, step_size=25):
 276   """Deletes all entities of the specified type
 277   """
 278
 279   print "Deleting..."
 280   count = 0
 281
 282   while True:
 283     entities = model.all().fetch(step_size)
 284
 285     if not entities:
 286       break
 287
 288     for entity in entities:
 289       entity.delete()
 290
 291     count += step_size
 292
 293     print "deleted %d entities" % count
 294
 295   print "Done"
 296
 297 def loadPickle(name):
 298   """Loads a pickle.
 299   """
 300
 301   f = open(name + '.dat')
 302   return cPickle.load(f)
 303
 304
 305 def dumpPickle(target, name):
 306   """Dumps a pickle.
 307   """
 308
 309   f = open("%s.dat" % name, 'w')
 310   cPickle.dump(target, f)
 311
 312
 313 def main(args):
 314   """Main routine.
 315   """
 316
 317   interactive.setup()
 318
 319   from soc.models.organization import Organization
 320   from soc.models.user import User
 321   from soc.models.student import Student
 322   from soc.models.mentor import Mentor
 323   from soc.models.org_admin import OrgAdmin
 324   from soc.models.job import Job
 325   from soc.models.student_proposal import StudentProposal
 326   from soc.models.student_project import StudentProject
 327
 328   def slotSaver(org, value):
 329     org.slots = value
 330   def popSaver(org, value):
 331     org.nr_applications = value
 332   def rawSaver(org, value):
 333     org.slots_calculated = value
 334
 335   context = {
 336       'load': loadPickle,
 337       'dump': dumpPickle,
 338       'orgStats': orgStats,
 339       'printPopularity': printPopularity,
 340       'saveValues': saveValues,
 341       'getEntities': getEntities,
 342       'deleteEntities': deleteEntities,
 343       'getOrgs': getEntities(Organization),
 344       'getUsers': getEntities(User),
 345       'getStudents': getEntities(Student),
 346       'getMentors': getEntities(Mentor),
 347       'getOrgAdmins': getEntities(OrgAdmin),
 348       'getProps': getProps,
 349       'countStudentsWithProposals': countStudentsWithProposals,
 350       'convertProposals': convertProposals,
 351       'addFollower': addFollower,
 352       'Organization': Organization,
 353       'Job': Job,
 354       'User': User,
 355       'Student': Student,
 356       'Mentor': Mentor,
 357       'OrgAdmin': OrgAdmin,
 358       'StudentProject': StudentProject,
 359       'StudentProposal': StudentProposal,
 360       'slotSaver': slotSaver,
 361       'popSaver': popSaver,
 362       'rawSaver': rawSaver,
 363       'startSpam': startSpam,
 364   }
 365
 366   interactive.remote(args, context)
 367
 368 if __name__ == '__main__':
 369   if len(sys.argv) < 2:
 370     print "Usage: %s app_id [host]" % (sys.argv[0],)
 371     sys.exit(1)
 372
 373   main(sys.argv[1:])