master/buildbot/schedulers/trysched.py

   1 # This file is part of Buildbot.  Buildbot is free software: you can
   2 # redistribute it and/or modify it under the terms of the GNU General Public
   3 # License as published by the Free Software Foundation, version 2.
   4 #
   5 # This program is distributed in the hope that it will be useful, but WITHOUT
   6 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
   7 # FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
   8 # details.
   9 #
  10 # You should have received a copy of the GNU General Public License along with
  11 # this program; if not, write to the Free Software Foundation, Inc., 51
  12 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  13 #
  14 # Copyright Buildbot Team Members
  15
  16 import os.path
  17
  18 from twisted.internet import defer
  19 from twisted.python import log, runtime
  20 from twisted.protocols import basic
  21
  22 from buildbot import pbutil
  23 from buildbot.sourcestamp import SourceStamp
  24 from buildbot.changes.maildir import MaildirService
  25 from buildbot.process.properties import Properties
  26 from buildbot.schedulers import base
  27 from buildbot.status.builder import BuildSetStatus
  28
  29
  30 class TryBase(base.BaseScheduler):
  31
  32     def run(self):
  33         # triggered by external events, not DB changes or timers
  34         return None
  35
  36     def filterBuilderList(self, builderNames):
  37         # self.builderNames is the configured list of builders
  38         # available for try.  If the user supplies a list of builders,
  39         # it must be restricted to the configured list.  If not, build
  40         # on all of the configured builders.
  41         if builderNames:
  42             for b in builderNames:
  43                 if not b in self.builderNames:
  44                     log.msg("%s got with builder %s" % (self, b))
  45                     log.msg(" but that wasn't in our list: %s"
  46                             % (self.builderNames,))
  47                     return []
  48         else:
  49             builderNames = self.builderNames
  50         return builderNames
  51
  52 class BadJobfile(Exception):
  53     pass
  54
  55 class JobFileScanner(basic.NetstringReceiver):
  56     def __init__(self):
  57         self.strings = []
  58         self.transport = self # so transport.loseConnection works
  59         self.error = False
  60
  61     def stringReceived(self, s):
  62         self.strings.append(s)
  63
  64     def loseConnection(self):
  65         self.error = True
  66
  67 class Try_Jobdir(TryBase):
  68     compare_attrs = ( 'name', 'builderNames', 'jobdir', 'properties' )
  69
  70     def __init__(self, name, builderNames, jobdir,
  71                  properties={}):
  72         base.BaseScheduler.__init__(self, name, builderNames, properties)
  73         self.jobdir = jobdir
  74         self.watcher = MaildirService()
  75         self.watcher.setServiceParent(self)
  76
  77     def setServiceParent(self, parent):
  78         sm = parent
  79         m = sm.parent
  80         self.watcher.setBasedir(os.path.join(m.basedir, self.jobdir))
  81         TryBase.setServiceParent(self, parent)
  82
  83     def parseJob(self, f):
  84         # jobfiles are serialized build requests. Each is a list of
  85         # serialized netstrings, in the following order:
  86         #  "1", the version number of this format
  87         #  buildsetID, arbitrary string, used to find the buildSet later
  88         #  branch name, "" for default-branch
  89         #  base revision, "" for HEAD
  90         #  patchlevel, usually "1"
  91         #  patch
  92         #  builderNames...
  93         p = JobFileScanner()
  94         p.dataReceived(f.read())
  95         if p.error:
  96             raise BadJobfile("unable to parse netstrings")
  97         s = p.strings
  98         ver = s.pop(0)
  99         if ver == "1":
 100             buildsetID, branch, baserev, patchlevel, diff = s[:5]
 101             builderNames = s[5:]
 102             if branch == "":
 103                 branch = None
 104             if baserev == "":
 105                 baserev = None
 106             patchlevel = int(patchlevel)
 107             patch = (patchlevel, diff)
 108             ss = SourceStamp("Old client", branch, baserev, patch)
 109         elif ver == "2": # introduced the repository and project property
 110             buildsetID, branch, baserev, patchlevel, diff, repository, project = s[:7]
 111             builderNames = s[7:]
 112             if branch == "":
 113                 branch = None
 114             if baserev == "":
 115                 baserev = None
 116             patchlevel = int(patchlevel)
 117             patch = (patchlevel, diff)
 118             ss = SourceStamp(branch, baserev, patch, repository=repository,
 119                              project=project)
 120         else:
 121             raise BadJobfile("unknown version '%s'" % ver)
 122         return builderNames, ss, buildsetID
 123
 124     def messageReceived(self, filename):
 125         md = os.path.join(self.parent.parent.basedir, self.jobdir)
 126         if runtime.platformType == "posix":
 127             # open the file before moving it, because I'm afraid that once
 128             # it's in cur/, someone might delete it at any moment
 129             path = os.path.join(md, "new", filename)
 130             f = open(path, "r")
 131             os.rename(os.path.join(md, "new", filename),
 132                       os.path.join(md, "cur", filename))
 133         else:
 134             # do this backwards under windows, because you can't move a file
 135             # that somebody is holding open. This was causing a Permission
 136             # Denied error on bear's win32-twisted1.3 buildslave.
 137             os.rename(os.path.join(md, "new", filename),
 138                       os.path.join(md, "cur", filename))
 139             path = os.path.join(md, "cur", filename)
 140             f = open(path, "r")
 141
 142         try:
 143             builderNames, ss, jobid = self.parseJob(f)
 144         except BadJobfile:
 145             log.msg("%s reports a bad jobfile in %s" % (self, filename))
 146             log.err()
 147             return
 148         # Validate/fixup the builder names.
 149         builderNames = self.filterBuilderList(builderNames)
 150         if not builderNames:
 151             return
 152         reason = "'try' job"
 153         d = self.parent.db.runInteraction(self._try, ss, builderNames, reason)
 154         def _done(ign):
 155             self.parent.loop_done() # so it will notify builder loop
 156         d.addCallback(_done)
 157         return d
 158
 159     def _try(self, t, ss, builderNames, reason):
 160         db = self.parent.db
 161         ssid = db.get_sourcestampid(ss, t)
 162         bsid = self.create_buildset(ssid, reason, t, builderNames=builderNames)
 163         return bsid
 164
 165
 166 class Try_Userpass(TryBase):
 167     compare_attrs = ( 'name', 'builderNames', 'port', 'userpass', 'properties' )
 168
 169     def __init__(self, name, builderNames, port, userpass,
 170                  properties={}):
 171         base.BaseScheduler.__init__(self, name, builderNames, properties)
 172         self.port = port
 173         self.userpass = userpass
 174         self.properties = properties
 175
 176     def startService(self):
 177         TryBase.startService(self)
 178         master = self.parent.parent
 179
 180         # register each user/passwd with the pbmanager
 181         def factory(mind, username):
 182             return Try_Userpass_Perspective(self, username)
 183         self.registrations = []
 184         for user, passwd in self.userpass:
 185             self.registrations.append(
 186                     master.pbmanager.register(self.port, user, passwd, factory))
 187
 188     def stopService(self):
 189         d = defer.maybeDeferred(TryBase.stopService, self)
 190         def unreg(_):
 191             return defer.gatherResults(
 192                 [ reg.unregister() for reg in self.registrations ])
 193         d.addCallback(unreg)
 194
 195
 196 class Try_Userpass_Perspective(pbutil.NewCredPerspective):
 197     def __init__(self, parent, username):
 198         self.parent = parent
 199         self.username = username
 200
 201     def perspective_try(self, branch, revision, patch, repository, project,
 202                         builderNames, properties={}, ):
 203         log.msg("user %s requesting build on builders %s" % (self.username,
 204                                                              builderNames))
 205         # build the intersection of the request and our configured list
 206         builderNames = self.parent.filterBuilderList(builderNames)
 207         if not builderNames:
 208             return
 209         ss = SourceStamp(branch, revision, patch, repository=repository,
 210                          project=project)
 211         reason = "'try' job from user %s" % self.username
 212
 213         # roll the specified props in with our inherited props
 214         combined_props = Properties()
 215         combined_props.updateFromProperties(self.parent.properties)
 216         combined_props.update(properties, "try build")
 217
 218         status = self.parent.parent.parent.status
 219         db = self.parent.parent.db
 220         d = db.runInteraction(self._try, ss, builderNames, reason,
 221                               combined_props, db)
 222         def _done(bsid):
 223             # return a remotely-usable BuildSetStatus object
 224             bss = BuildSetStatus(bsid, status, db)
 225             from buildbot.status.client import makeRemote
 226             r = makeRemote(bss)
 227             #self.parent.parent.loop_done() # so it will notify builder loop
 228             return r
 229         d.addCallback(_done)
 230         return d
 231
 232     def _try(self, t, ss, builderNames, reason, combined_props, db):
 233         ssid = db.get_sourcestampid(ss, t)
 234         bsid = self.parent.create_buildset(ssid, reason, t,
 235                                            props=combined_props,
 236                                            builderNames=builderNames)
 237         return bsid
 238
 239     def perspective_getAvailableBuilderNames(self):
 240         # Return a list of builder names that are configured
 241         # for the try service
 242         # This is mostly intended for integrating try services
 243         # into other applications
 244         return self.parent.listBuilderNames()