buildbot/changes/bonsaipoller.py

   1 import time
   2 from xml.dom import minidom
   3
   4 from twisted.python import log, failure
   5 from twisted.internet import reactor
   6 from twisted.internet.task import LoopingCall
   7 from twisted.web.client import getPage
   8
   9 from buildbot.changes import base, changes
  10
  11 class InvalidResultError(Exception):
  12     def __init__(self, value="InvalidResultError"):
  13         self.value = value
  14     def __str__(self):
  15         return repr(self.value)
  16
  17 class EmptyResult(Exception):
  18     pass
  19
  20 class NoMoreCiNodes(Exception):
  21     pass
  22
  23 class NoMoreFileNodes(Exception):
  24     pass
  25
  26 class BonsaiResult:
  27     """I hold a list of CiNodes"""
  28     def __init__(self, nodes=[]):
  29         self.nodes = nodes
  30
  31     def __cmp__(self, other):
  32         if len(self.nodes) != len(other.nodes):
  33             return False
  34         for i in range(len(self.nodes)):
  35             if self.nodes[i].log != other.nodes[i].log \
  36               or self.nodes[i].who != other.nodes[i].who \
  37               or self.nodes[i].date != other.nodes[i].date \
  38               or len(self.nodes[i].files) != len(other.nodes[i].files):
  39                 return -1
  40
  41                 for j in range(len(self.nodes[i].files)):
  42                     if self.nodes[i].files[j].revision \
  43                       != other.nodes[i].files[j].revision \
  44                       or self.nodes[i].files[j].filename \
  45                       != other.nodes[i].files[j].filename:
  46                         return -1
  47
  48         return 0
  49
  50 class CiNode:
  51     """I hold information baout one <ci> node, including a list of files"""
  52     def __init__(self, log="", who="", date=0, files=[]):
  53         self.log = log
  54         self.who = who
  55         self.date = date
  56         self.files = files
  57
  58 class FileNode:
  59     """I hold information about one <f> node"""
  60     def __init__(self, revision="", filename=""):
  61         self.revision = revision
  62         self.filename = filename
  63
  64 class BonsaiParser:
  65     """I parse the XML result from a bonsai cvsquery."""
  66
  67     def __init__(self, data):
  68         try:
  69         # this is a fix for non-ascii characters
  70         # because bonsai does not give us an encoding to work with
  71         # it impossible to be 100% sure what to decode it as but latin1 covers
  72         # the broadest base
  73             data = data.decode("latin1")
  74             data = data.encode("ascii", "replace")
  75             self.dom = minidom.parseString(data)
  76             log.msg(data)
  77         except:
  78             raise InvalidResultError("Malformed XML in result")
  79
  80         self.ciNodes = self.dom.getElementsByTagName("ci")
  81         self.currentCiNode = None # filled in by _nextCiNode()
  82         self.fileNodes = None # filled in by _nextCiNode()
  83         self.currentFileNode = None # filled in by _nextFileNode()
  84         self.bonsaiResult = self._parseData()
  85
  86     def getData(self):
  87         return self.bonsaiResult
  88
  89     def _parseData(self):
  90         """Returns data from a Bonsai cvsquery in a BonsaiResult object"""
  91         nodes = []
  92         try:
  93             while self._nextCiNode():
  94                 files = []
  95                 try:
  96                     while self._nextFileNode():
  97                         files.append(FileNode(self._getRevision(),
  98                                               self._getFilename()))
  99                 except NoMoreFileNodes:
 100                     pass
 101                 except InvalidResultError:
 102                     raise
 103                 cinode = CiNode(self._getLog(), self._getWho(),
 104                                 self._getDate(), files)
 105                 # hack around bonsai xml output bug for empty check-in comments
 106                 if not cinode.log and nodes and \
 107                         not nodes[-1].log and \
 108                         cinode.who == nodes[-1].who and \
 109                         cinode.date == nodes[-1].date:
 110                     nodes[-1].files += cinode.files
 111                 else:
 112                     nodes.append(cinode)
 113
 114         except NoMoreCiNodes:
 115             pass
 116         except InvalidResultError, EmptyResult:
 117             raise
 118
 119         return BonsaiResult(nodes)
 120
 121
 122     def _nextCiNode(self):
 123         """Iterates to the next <ci> node and fills self.fileNodes with
 124            child <f> nodes"""
 125         try:
 126             self.currentCiNode = self.ciNodes.pop(0)
 127             if len(self.currentCiNode.getElementsByTagName("files")) > 1:
 128                 raise InvalidResultError("Multiple <files> for one <ci>")
 129
 130             self.fileNodes = self.currentCiNode.getElementsByTagName("f")
 131         except IndexError:
 132             # if there was zero <ci> nodes in the result
 133             if not self.currentCiNode:
 134                 raise EmptyResult
 135             else:
 136                 raise NoMoreCiNodes
 137
 138         return True
 139
 140     def _nextFileNode(self):
 141         """Iterates to the next <f> node"""
 142         try:
 143             self.currentFileNode = self.fileNodes.pop(0)
 144         except IndexError:
 145             raise NoMoreFileNodes
 146
 147         return True
 148
 149     def _getLog(self):
 150         """Returns the log of the current <ci> node"""
 151         logs = self.currentCiNode.getElementsByTagName("log")
 152         if len(logs) < 1:
 153             raise InvalidResultError("No log present")
 154         elif len(logs) > 1:
 155             raise InvalidResultError("Multiple logs present")
 156
 157         # catch empty check-in comments
 158         if logs[0].firstChild:
 159             return logs[0].firstChild.data
 160         return ''
 161
 162     def _getWho(self):
 163         """Returns the e-mail address of the commiter"""
 164         # convert unicode string to regular string
 165         return str(self.currentCiNode.getAttribute("who"))
 166
 167     def _getDate(self):
 168         """Returns the date (unix time) of the commit"""
 169         # convert unicode number to regular one
 170         try:
 171             commitDate = int(self.currentCiNode.getAttribute("date"))
 172         except ValueError:
 173             raise InvalidResultError
 174
 175         return commitDate
 176
 177     def _getFilename(self):
 178         """Returns the filename of the current <f> node"""
 179         try:
 180             filename = self.currentFileNode.firstChild.data
 181         except AttributeError:
 182             raise InvalidResultError("Missing filename")
 183
 184         return filename
 185
 186     def _getRevision(self):
 187         return self.currentFileNode.getAttribute("rev")
 188
 189
 190 class BonsaiPoller(base.ChangeSource):
 191     """This source will poll a bonsai server for changes and submit
 192     them to the change master."""
 193
 194     compare_attrs = ["bonsaiURL", "pollInterval", "tree",
 195                      "module", "branch", "cvsroot"]
 196
 197     parent = None # filled in when we're added
 198     loop = None
 199     volatile = ['loop']
 200     working = False
 201
 202     def __init__(self, bonsaiURL, module, branch, tree="default",
 203                  cvsroot="/cvsroot", pollInterval=30):
 204         """
 205         @type   bonsaiURL:      string
 206         @param  bonsaiURL:      The base URL of the Bonsai server
 207                                 (ie. http://bonsai.mozilla.org)
 208         @type   module:         string
 209         @param  module:         The module to look for changes in. Commonly
 210                                 this is 'all'
 211         @type   branch:         string
 212         @param  branch:         The branch to look for changes in. This must
 213                                 match the
 214                                 'branch' option for the Scheduler.
 215         @type   tree:           string
 216         @param  tree:           The tree to look for changes in. Commonly this
 217                                 is 'all'
 218         @type   cvsroot:        string
 219         @param  cvsroot:        The cvsroot of the repository. Usually this is
 220                                 '/cvsroot'
 221         @type   pollInterval:   int
 222         @param  pollInterval:   The time (in seconds) between queries for
 223                                 changes
 224         """
 225
 226         self.bonsaiURL = bonsaiURL
 227         self.module = module
 228         self.branch = branch
 229         self.tree = tree
 230         self.cvsroot = cvsroot
 231         self.pollInterval = pollInterval
 232         self.lastChange = time.time()
 233         self.lastPoll = time.time()
 234
 235     def startService(self):
 236         self.loop = LoopingCall(self.poll)
 237         base.ChangeSource.startService(self)
 238
 239         reactor.callLater(0, self.loop.start, self.pollInterval)
 240
 241     def stopService(self):
 242         self.loop.stop()
 243         return base.ChangeSource.stopService(self)
 244
 245     def describe(self):
 246         str = ""
 247         str += "Getting changes from the Bonsai service running at %s " \
 248                 % self.bonsaiURL
 249         str += "<br>Using tree: %s, branch: %s, and module: %s" % (self.tree, \
 250                 self.branch, self.module)
 251         return str
 252
 253     def poll(self):
 254         if self.working:
 255             log.msg("Not polling Bonsai because last poll is still working")
 256         else:
 257             self.working = True
 258             d = self._get_changes()
 259             d.addCallback(self._process_changes)
 260             d.addCallbacks(self._finished_ok, self._finished_failure)
 261         return
 262
 263     def _finished_ok(self, res):
 264         assert self.working
 265         self.working = False
 266
 267         # check for failure -- this is probably never hit but the twisted docs
 268         # are not clear enough to be sure. it is being kept "just in case"
 269         if isinstance(res, failure.Failure):
 270             log.msg("Bonsai poll failed: %s" % res)
 271         return res
 272
 273     def _finished_failure(self, res):
 274         log.msg("Bonsai poll failed: %s" % res)
 275         assert self.working
 276         self.working = False
 277         return None # eat the failure
 278
 279     def _make_url(self):
 280         args = ["treeid=%s" % self.tree, "module=%s" % self.module,
 281                 "branch=%s" % self.branch, "branchtype=match",
 282                 "sortby=Date", "date=explicit",
 283                 "mindate=%d" % self.lastChange,
 284                 "maxdate=%d" % int(time.time()),
 285                 "cvsroot=%s" % self.cvsroot, "xml=1"]
 286         # build the bonsai URL
 287         url = self.bonsaiURL
 288         url += "/cvsquery.cgi?"
 289         url += "&".join(args)
 290
 291         return url
 292
 293     def _get_changes(self):
 294         url = self._make_url()
 295         log.msg("Polling Bonsai tree at %s" % url)
 296
 297         self.lastPoll = time.time()
 298         # get the page, in XML format
 299         return getPage(url, timeout=self.pollInterval)
 300
 301     def _process_changes(self, query):
 302         try:
 303             bp = BonsaiParser(query)
 304             result = bp.getData()
 305         except InvalidResultError, e:
 306             log.msg("Could not process Bonsai query: " + e.value)
 307             return
 308         except EmptyResult:
 309             return
 310
 311         for cinode in result.nodes:
 312             files = [file.filename + ' (revision '+file.revision+')'
 313                      for file in cinode.files]
 314             c = changes.Change(who = cinode.who,
 315                                files = files,
 316                                comments = cinode.log,
 317                                when = cinode.date,
 318                                branch = self.branch)
 319             self.parent.addChange(c)
 320             self.lastChange = self.lastPoll