diff -N -r -u buildbot-0.8.14/buildbot/buildslave.py buildbot/buildslave.py --- buildbot-0.8.14/buildbot/buildslave.py 1970-01-01 01:00:00.000000000 +0100 +++ buildbot/buildslave.py 2012-06-24 22:30:04.000000000 +0200 @@ -0,0 +1,1012 @@ +# This file is part of Buildbot. Buildbot is free software: you can +# redistribute it and/or modify it under the terms of the GNU General Public +# License as published by the Free Software Foundation, version 2. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +# details. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, write to the Free Software Foundation, Inc., 51 +# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Portions Copyright Buildbot Team Members +# Portions Copyright Canonical Ltd. 2009 + +import time +from email.Message import Message +from email.Utils import formatdate +from zope.interface import implements +from twisted.python import log, failure +from twisted.internet import defer, reactor +from twisted.application import service +from twisted.spread import pb +from twisted.python.reflect import namedModule + +from buildbot.status.slave import SlaveStatus +from buildbot.status.mail import MailNotifier +from buildbot.process import metrics, botmaster +from buildbot.interfaces import IBuildSlave, ILatentBuildSlave +from buildbot.process.properties import Properties +from buildbot.locks import LockAccess +from buildbot.util import subscription +from buildbot import config + +class AbstractBuildSlave(config.ReconfigurableServiceMixin, pb.Avatar, + service.MultiService): + """This is the master-side representative for a remote buildbot slave. + There is exactly one for each slave described in the config file (the + c['slaves'] list). When buildbots connect in (.attach), they get a + reference to this instance. The BotMaster object is stashed as the + .botmaster attribute. The BotMaster is also our '.parent' Service. + + I represent a build slave -- a remote machine capable of + running builds. I am instantiated by the configuration file, and can be + subclassed to add extra functionality.""" + + implements(IBuildSlave) + keepalive_timer = None + keepalive_interval = None + + # reconfig slaves after builders + reconfig_priority = 64 + + def __init__(self, name, password, max_builds=None, + notify_on_missing=[], missing_timeout=3600, + properties={}, locks=None, keepalive_interval=3600): + """ + @param name: botname this machine will supply when it connects + @param password: password this machine will supply when + it connects + @param max_builds: maximum number of simultaneous builds that will + be run concurrently on this buildslave (the + default is None for no limit) + @param properties: properties that will be applied to builds run on + this slave + @type properties: dictionary + @param locks: A list of locks that must be acquired before this slave + can be used + @type locks: dictionary + """ + service.MultiService.__init__(self) + self.slavename = name + self.password = password + + # PB registration + self.registration = None + self.registered_port = None + + # these are set when the service is started, and unset when it is + # stopped + self.botmaster = None + self.master = None + + self.slave_status = SlaveStatus(name) + self.slave = None # a RemoteReference to the Bot, when connected + self.slave_commands = None + self.slavebuilders = {} + self.max_builds = max_builds + self.access = [] + if locks: + self.access = locks + self.lock_subscriptions = [] + + self.properties = Properties() + self.properties.update(properties, "BuildSlave") + self.properties.setProperty("slavename", name, "BuildSlave") + + self.lastMessageReceived = 0 + if isinstance(notify_on_missing, str): + notify_on_missing = [notify_on_missing] + self.notify_on_missing = notify_on_missing + for i in notify_on_missing: + if not isinstance(i, str): + config.error( + 'notify_on_missing arg %r is not a string' % (i,)) + self.missing_timeout = missing_timeout + self.missing_timer = None + self.keepalive_interval = keepalive_interval + + self.detached_subs = None + + self._old_builder_list = None + + def __repr__(self): + return "<%s %r>" % (self.__class__.__name__, self.slavename) + + def updateLocks(self): + """Convert the L{LockAccess} objects in C{self.locks} into real lock + objects, while also maintaining the subscriptions to lock releases.""" + # unsubscribe from any old locks + for s in self.lock_subscriptions: + s.unsubscribe() + + # convert locks into their real form + locks = [] + for access in self.access: + if not isinstance(access, LockAccess): + access = access.defaultAccess() + lock = self.botmaster.getLockByID(access.lockid) + locks.append((lock, access)) + self.locks = [(l.getLock(self), la) for l, la in locks] + self.lock_subscriptions = [ l.subscribeToReleases(self._lockReleased) + for l, la in self.locks ] + + def locksAvailable(self): + """ + I am called to see if all the locks I depend on are available, + in which I return True, otherwise I return False + """ + if not self.locks: + return True + for lock, access in self.locks: + if not lock.isAvailable(access): + return False + return True + + def acquireLocks(self): + """ + I am called when a build is preparing to run. I try to claim all + the locks that are needed for a build to happen. If I can't, then + my caller should give up the build and try to get another slave + to look at it. + """ + log.msg("acquireLocks(slave %s, locks %s)" % (self, self.locks)) + if not self.locksAvailable(): + log.msg("slave %s can't lock, giving up" % (self, )) + return False + # all locks are available, claim them all + for lock, access in self.locks: + lock.claim(self, access) + return True + + def releaseLocks(self): + """ + I am called to release any locks after a build has finished + """ + log.msg("releaseLocks(%s): %s" % (self, self.locks)) + for lock, access in self.locks: + lock.release(self, access) + + def _lockReleased(self): + """One of the locks for this slave was released; try scheduling + builds.""" + if not self.botmaster: + return # oh well.. + self.botmaster.maybeStartBuildsForSlave(self.slavename) + + def setServiceParent(self, parent): + # botmaster needs to set before setServiceParent which calls startService + self.botmaster = parent + self.master = parent.master + service.MultiService.setServiceParent(self, parent) + + def startService(self): + self.updateLocks() + self.startMissingTimer() + return service.MultiService.startService(self) + + def reconfigService(self, new_config): + # Given a new BuildSlave, configure this one identically. Because + # BuildSlave objects are remotely referenced, we can't replace them + # without disconnecting the slave, yet there's no reason to do that. + new = self.findNewSlaveInstance(new_config) + + assert self.slavename == new.slavename + + # do we need to re-register? + if (not self.registration or + self.password != new.password or + new_config.slavePortnum != self.registered_port): + if self.registration: + self.registration.unregister() + self.password = new.password + self.registered_port = new_config.slavePortnum + self.registration = self.master.pbmanager.register( + self.registered_port, self.slavename, + self.password, self.getPerspective) + + # adopt new instance's configuration parameters + self.max_builds = new.max_builds + self.access = new.access + self.notify_on_missing = new.notify_on_missing + self.keepalive_interval = new.keepalive_interval + + if self.missing_timeout != new.missing_timeout: + running_missing_timer = self.missing_timer + self.stopMissingTimer() + self.missing_timeout = new.missing_timeout + if running_missing_timer: + self.startMissingTimer() + + properties = Properties() + properties.updateFromProperties(new.properties) + self.properties = properties + + self.updateLocks() + + # update the attached slave's notion of which builders are attached. + # This assumes that the relevant builders have already been configured, + # which is why the reconfig_priority is set low in this class. + d = self.updateSlave() + + # and chain up + d.addCallback(lambda _ : + config.ReconfigurableServiceMixin.reconfigService(self, + new_config)) + + return d + + def stopService(self): + if self.registration: + self.registration.unregister() + self.stopMissingTimer() + return service.MultiService.stopService(self) + + def findNewSlaveInstance(self, new_config): + # TODO: called multiple times per reconfig; use 1-element cache? + for sl in new_config.slaves: + if sl.slavename == self.slavename: + return sl + assert 0, "no new slave named '%s'" % self.slavename + + def startMissingTimer(self): + if self.notify_on_missing and self.missing_timeout and self.parent: + self.stopMissingTimer() # in case it's already running + self.missing_timer = reactor.callLater(self.missing_timeout, + self._missing_timer_fired) + + def stopMissingTimer(self): + if self.missing_timer: + self.missing_timer.cancel() + self.missing_timer = None + + def getPerspective(self, mind, slavename): + assert slavename == self.slavename + metrics.MetricCountEvent.log("attached_slaves", 1) + + # record when this connection attempt occurred + if self.slave_status: + self.slave_status.recordConnectTime() + + + if self.isConnected(): + # duplicate slave - send it to arbitration + arb = botmaster.DuplicateSlaveArbitrator(self) + return arb.getPerspective(mind, slavename) + else: + log.msg("slave '%s' attaching from %s" % (slavename, mind.broker.transport.getPeer())) + return self + + def doKeepalive(self): + self.keepalive_timer = reactor.callLater(self.keepalive_interval, + self.doKeepalive) + if not self.slave: + return + d = self.slave.callRemote("print", "Received keepalive from master") + d.addErrback(log.msg, "Keepalive failed for '%s'" % (self.slavename, )) + + def stopKeepaliveTimer(self): + if self.keepalive_timer: + self.keepalive_timer.cancel() + + def startKeepaliveTimer(self): + assert self.keepalive_interval + log.msg("Starting buildslave keepalive timer for '%s'" % \ + (self.slavename, )) + self.doKeepalive() + + def isConnected(self): + return self.slave + + def _missing_timer_fired(self): + self.missing_timer = None + # notify people, but only if we're still in the config + if not self.parent: + return + + buildmaster = self.botmaster.master + status = buildmaster.getStatus() + text = "The Buildbot working for '%s'\n" % status.getTitle() + text += ("has noticed that the buildslave named %s went away\n" % + self.slavename) + text += "\n" + text += ("It last disconnected at %s (buildmaster-local time)\n" % + time.ctime(time.time() - self.missing_timeout)) # approx + text += "\n" + text += "The admin on record (as reported by BUILDSLAVE:info/admin)\n" + text += "was '%s'.\n" % self.slave_status.getAdmin() + text += "\n" + text += "Sincerely,\n" + text += " The Buildbot\n" + text += " %s\n" % status.getTitleURL() + subject = "Buildbot: buildslave %s was lost" % self.slavename + return self._mail_missing_message(subject, text) + + + def updateSlave(self): + """Called to add or remove builders after the slave has connected. + + @return: a Deferred that indicates when an attached slave has + accepted the new builders and/or released the old ones.""" + if self.slave: + return self.sendBuilderList() + else: + return defer.succeed(None) + + def updateSlaveStatus(self, buildStarted=None, buildFinished=None): + if buildStarted: + self.slave_status.buildStarted(buildStarted) + if buildFinished: + self.slave_status.buildFinished(buildFinished) + + @metrics.countMethod('AbstractBuildSlave.attached()') + def attached(self, bot): + """This is called when the slave connects. + + @return: a Deferred that fires when the attachment is complete + """ + + # the botmaster should ensure this. + assert not self.isConnected() + + metrics.MetricCountEvent.log("AbstractBuildSlave.attached_slaves", 1) + + # set up the subscription point for eventual detachment + self.detached_subs = subscription.SubscriptionPoint("detached") + + # now we go through a sequence of calls, gathering information, then + # tell the Botmaster that it can finally give this slave to all the + # Builders that care about it. + + # we accumulate slave information in this 'state' dictionary, then + # set it atomically if we make it far enough through the process + state = {} + + # Reset graceful shutdown status + self.slave_status.setGraceful(False) + # We want to know when the graceful shutdown flag changes + self.slave_status.addGracefulWatcher(self._gracefulChanged) + + d = defer.succeed(None) + def _log_attachment_on_slave(res): + d1 = bot.callRemote("print", "attached") + d1.addErrback(lambda why: None) + return d1 + d.addCallback(_log_attachment_on_slave) + + def _get_info(res): + d1 = bot.callRemote("getSlaveInfo") + def _got_info(info): + log.msg("Got slaveinfo from '%s'" % self.slavename) + # TODO: info{} might have other keys + state["admin"] = info.get("admin") + state["host"] = info.get("host") + state["access_uri"] = info.get("access_uri", None) + state["slave_environ"] = info.get("environ", {}) + state["slave_basedir"] = info.get("basedir", None) + state["slave_system"] = info.get("system", None) + def _info_unavailable(why): + why.trap(pb.NoSuchMethod) + # maybe an old slave, doesn't implement remote_getSlaveInfo + log.msg("BuildSlave.info_unavailable") + log.err(why) + d1.addCallbacks(_got_info, _info_unavailable) + return d1 + d.addCallback(_get_info) + self.startKeepaliveTimer() + + def _get_version(res): + d = bot.callRemote("getVersion") + def _got_version(version): + state["version"] = version + def _version_unavailable(why): + why.trap(pb.NoSuchMethod) + # probably an old slave + state["version"] = '(unknown)' + d.addCallbacks(_got_version, _version_unavailable) + return d + d.addCallback(_get_version) + + def _get_commands(res): + d1 = bot.callRemote("getCommands") + def _got_commands(commands): + state["slave_commands"] = commands + def _commands_unavailable(why): + # probably an old slave + log.msg("BuildSlave._commands_unavailable") + if why.check(AttributeError): + return + log.err(why) + d1.addCallbacks(_got_commands, _commands_unavailable) + return d1 + d.addCallback(_get_commands) + + def _accept_slave(res): + self.slave_status.setAdmin(state.get("admin")) + self.slave_status.setHost(state.get("host")) + self.slave_status.setAccessURI(state.get("access_uri")) + self.slave_status.setVersion(state.get("version")) + self.slave_status.setConnected(True) + self.slave_commands = state.get("slave_commands") + self.slave_environ = state.get("slave_environ") + self.slave_basedir = state.get("slave_basedir") + self.slave_system = state.get("slave_system") + self.slave = bot + if self.slave_system == "win32": + self.path_module = namedModule("win32path") + else: + # most eveything accepts / as separator, so posix should be a + # reasonable fallback + self.path_module = namedModule("posixpath") + log.msg("bot attached") + self.messageReceivedFromSlave() + self.stopMissingTimer() + self.botmaster.master.status.slaveConnected(self.slavename) + + return self.updateSlave() + d.addCallback(_accept_slave) + d.addCallback(lambda _: + self.botmaster.maybeStartBuildsForSlave(self.slavename)) + + # Finally, the slave gets a reference to this BuildSlave. They + # receive this later, after we've started using them. + d.addCallback(lambda _: self) + return d + + def messageReceivedFromSlave(self): + now = time.time() + self.lastMessageReceived = now + self.slave_status.setLastMessageReceived(now) + + def detached(self, mind): + metrics.MetricCountEvent.log("AbstractBuildSlave.attached_slaves", -1) + self.slave = None + self._old_builder_list = [] + self.slave_status.removeGracefulWatcher(self._gracefulChanged) + self.slave_status.setConnected(False) + log.msg("BuildSlave.detached(%s)" % self.slavename) + self.botmaster.master.status.slaveDisconnected(self.slavename) + self.stopKeepaliveTimer() + self.releaseLocks() + + # notify watchers, but do so in the next reactor iteration so that + # any further detached() action by subclasses happens first + def notif(): + subs = self.detached_subs + self.detached_subs = None + subs.deliver() + reactor.callLater(0, notif) + + def subscribeToDetach(self, callback): + """ + Request that C{callable} be invoked with no arguments when the + L{detached} method is invoked. + + @returns: L{Subscription} + """ + assert self.detached_subs, "detached_subs is only set if attached" + return self.detached_subs.subscribe(callback) + + def disconnect(self): + """Forcibly disconnect the slave. + + This severs the TCP connection and returns a Deferred that will fire + (with None) when the connection is probably gone. + + If the slave is still alive, they will probably try to reconnect + again in a moment. + + This is called in two circumstances. The first is when a slave is + removed from the config file. In this case, when they try to + reconnect, they will be rejected as an unknown slave. The second is + when we wind up with two connections for the same slave, in which + case we disconnect the older connection. + """ + + if not self.slave: + return defer.succeed(None) + log.msg("disconnecting old slave %s now" % self.slavename) + # When this Deferred fires, we'll be ready to accept the new slave + return self._disconnect(self.slave) + + def _disconnect(self, slave): + # all kinds of teardown will happen as a result of + # loseConnection(), but it happens after a reactor iteration or + # two. Hook the actual disconnect so we can know when it is safe + # to connect the new slave. We have to wait one additional + # iteration (with callLater(0)) to make sure the *other* + # notifyOnDisconnect handlers have had a chance to run. + d = defer.Deferred() + + # notifyOnDisconnect runs the callback with one argument, the + # RemoteReference being disconnected. + def _disconnected(rref): + reactor.callLater(0, d.callback, None) + slave.notifyOnDisconnect(_disconnected) + tport = slave.broker.transport + # this is the polite way to request that a socket be closed + tport.loseConnection() + try: + # but really we don't want to wait for the transmit queue to + # drain. The remote end is unlikely to ACK the data, so we'd + # probably have to wait for a (20-minute) TCP timeout. + #tport._closeSocket() + # however, doing _closeSocket (whether before or after + # loseConnection) somehow prevents the notifyOnDisconnect + # handlers from being run. Bummer. + tport.offset = 0 + tport.dataBuffer = "" + except: + # however, these hacks are pretty internal, so don't blow up if + # they fail or are unavailable + log.msg("failed to accelerate the shutdown process") + log.msg("waiting for slave to finish disconnecting") + + return d + + def sendBuilderList(self): + our_builders = self.botmaster.getBuildersForSlave(self.slavename) + blist = [(b.name, b.config.slavebuilddir) for b in our_builders] + if blist == self._old_builder_list: + return defer.succeed(None) + + d = self.slave.callRemote("setBuilderList", blist) + def sentBuilderList(ign): + self._old_builder_list = blist + return ign + d.addCallback(sentBuilderList) + return d + + def perspective_keepalive(self): + self.messageReceivedFromSlave() + + def perspective_shutdown(self): + log.msg("slave %s wants to shut down" % self.slavename) + self.slave_status.setGraceful(True) + + def addSlaveBuilder(self, sb): + self.slavebuilders[sb.builder_name] = sb + + def removeSlaveBuilder(self, sb): + try: + del self.slavebuilders[sb.builder_name] + except KeyError: + pass + + def buildFinished(self, sb): + """This is called when a build on this slave is finished.""" + self.botmaster.maybeStartBuildsForSlave(self.slavename) + + def canStartBuild(self): + """ + I am called when a build is requested to see if this buildslave + can start a build. This function can be used to limit overall + concurrency on the buildslave. + + Note for subclassers: if a slave can become willing to start a build + without any action on that slave (for example, by a resource in use on + another slave becoming available), then you must arrange for + L{maybeStartBuildsForSlave} to be called at that time, or builds on + this slave will not start. + """ + # If we're waiting to shutdown gracefully, then we shouldn't + # accept any new jobs. + if self.slave_status.getGraceful(): + return False + + if self.max_builds: + active_builders = [sb for sb in self.slavebuilders.values() + if sb.isBusy()] + if len(active_builders) >= self.max_builds: + return False + + if not self.locksAvailable(): + return False + + return True + + def _mail_missing_message(self, subject, text): + # first, see if we have a MailNotifier we can use. This gives us a + # fromaddr and a relayhost. + buildmaster = self.botmaster.master + for st in buildmaster.status: + if isinstance(st, MailNotifier): + break + else: + # if not, they get a default MailNotifier, which always uses SMTP + # to localhost and uses a dummy fromaddr of "buildbot". + log.msg("buildslave-missing msg using default MailNotifier") + st = MailNotifier("buildbot") + # now construct the mail + + m = Message() + m.set_payload(text) + m['Date'] = formatdate(localtime=True) + m['Subject'] = subject + m['From'] = st.fromaddr + recipients = self.notify_on_missing + m['To'] = ", ".join(recipients) + d = st.sendMessage(m, recipients) + # return the Deferred for testing purposes + return d + + def _gracefulChanged(self, graceful): + """This is called when our graceful shutdown setting changes""" + self.maybeShutdown() + + @defer.deferredGenerator + def shutdown(self): + """Shutdown the slave""" + if not self.slave: + log.msg("no remote; slave is already shut down") + return + + # First, try the "new" way - calling our own remote's shutdown + # method. The method was only added in 0.8.3, so ignore NoSuchMethod + # failures. + def new_way(): + d = self.slave.callRemote('shutdown') + d.addCallback(lambda _ : True) # successful shutdown request + def check_nsm(f): + f.trap(pb.NoSuchMethod) + return False # fall through to the old way + d.addErrback(check_nsm) + def check_connlost(f): + f.trap(pb.PBConnectionLost) + return True # the slave is gone, so call it finished + d.addErrback(check_connlost) + return d + + wfd = defer.waitForDeferred(new_way()) + yield wfd + if wfd.getResult(): + return # done! + + # Now, the old way. Look for a builder with a remote reference to the + # client side slave. If we can find one, then call "shutdown" on the + # remote builder, which will cause the slave buildbot process to exit. + def old_way(): + d = None + for b in self.slavebuilders.values(): + if b.remote: + d = b.remote.callRemote("shutdown") + break + + if d: + log.msg("Shutting down (old) slave: %s" % self.slavename) + # The remote shutdown call will not complete successfully since the + # buildbot process exits almost immediately after getting the + # shutdown request. + # Here we look at the reason why the remote call failed, and if + # it's because the connection was lost, that means the slave + # shutdown as expected. + def _errback(why): + if why.check(pb.PBConnectionLost): + log.msg("Lost connection to %s" % self.slavename) + else: + log.err("Unexpected error when trying to shutdown %s" % self.slavename) + d.addErrback(_errback) + return d + log.err("Couldn't find remote builder to shut down slave") + return defer.succeed(None) + wfd = defer.waitForDeferred(old_way()) + yield wfd + wfd.getResult() + + def maybeShutdown(self): + """Shut down this slave if it has been asked to shut down gracefully, + and has no active builders.""" + if not self.slave_status.getGraceful(): + return + active_builders = [sb for sb in self.slavebuilders.values() + if sb.isBusy()] + if active_builders: + return + d = self.shutdown() + d.addErrback(log.err, 'error while shutting down slave') + +class BuildSlave(AbstractBuildSlave): + + def sendBuilderList(self): + d = AbstractBuildSlave.sendBuilderList(self) + def _sent(slist): + # Nothing has changed, so don't need to re-attach to everything + if not slist: + return + dl = [] + for name, remote in slist.items(): + # use get() since we might have changed our mind since then + b = self.botmaster.builders.get(name) + if b: + d1 = b.attached(self, remote, self.slave_commands) + dl.append(d1) + return defer.DeferredList(dl) + def _set_failed(why): + log.msg("BuildSlave.sendBuilderList (%s) failed" % self) + log.err(why) + # TODO: hang up on them?, without setBuilderList we can't use + # them + d.addCallbacks(_sent, _set_failed) + return d + + def detached(self, mind): + AbstractBuildSlave.detached(self, mind) + self.botmaster.slaveLost(self) + self.startMissingTimer() + + def buildFinished(self, sb): + """This is called when a build on this slave is finished.""" + AbstractBuildSlave.buildFinished(self, sb) + + # If we're gracefully shutting down, and we have no more active + # builders, then it's safe to disconnect + self.maybeShutdown() + +class AbstractLatentBuildSlave(AbstractBuildSlave): + """A build slave that will start up a slave instance when needed. + + To use, subclass and implement start_instance and stop_instance. + + See ec2buildslave.py for a concrete example. Also see the stub example in + test/test_slaves.py. + """ + + implements(ILatentBuildSlave) + + substantiated = False + substantiation_deferred = None + substantiation_build = None + build_wait_timer = None + _shutdown_callback_handle = None + + def __init__(self, name, password, max_builds=None, + notify_on_missing=[], missing_timeout=60*20, + build_wait_timeout=60*10, + properties={}, locks=None): + AbstractBuildSlave.__init__( + self, name, password, max_builds, notify_on_missing, + missing_timeout, properties, locks) + self.building = set() + self.build_wait_timeout = build_wait_timeout + + def start_instance(self, build): + # responsible for starting instance that will try to connect with this + # master. Should return deferred with either True (instance started) + # or False (instance not started, so don't run a build here). Problems + # should use an errback. + raise NotImplementedError + + def stop_instance(self, fast=False): + # responsible for shutting down instance. + raise NotImplementedError + + def substantiate(self, sb, build): + if self.substantiated: + self._clearBuildWaitTimer() + self._setBuildWaitTimer() + return defer.succeed(True) + if self.substantiation_deferred is None: + if self.parent and not self.missing_timer: + # start timer. if timer times out, fail deferred + self.missing_timer = reactor.callLater( + self.missing_timeout, + self._substantiation_failed, defer.TimeoutError()) + self.substantiation_deferred = defer.Deferred() + self.substantiation_build = build + if self.slave is None: + d = self._substantiate(build) # start up instance + d.addErrback(log.err, "while substantiating") + # else: we're waiting for an old one to detach. the _substantiate + # will be done in ``detached`` below. + return self.substantiation_deferred + + def _substantiate(self, build): + # register event trigger + d = self.start_instance(build) + self._shutdown_callback_handle = reactor.addSystemEventTrigger( + 'before', 'shutdown', self._soft_disconnect, fast=True) + def start_instance_result(result): + # If we don't report success, then preparation failed. + if not result: + log.msg("Slave '%s' doesn not want to substantiate at this time" % (self.slavename,)) + d = self.substantiation_deferred + self.substantiation_deferred = None + d.callback(False) + return result + def clean_up(failure): + if self.missing_timer is not None: + self.missing_timer.cancel() + self._substantiation_failed(failure) + if self._shutdown_callback_handle is not None: + handle = self._shutdown_callback_handle + del self._shutdown_callback_handle + reactor.removeSystemEventTrigger(handle) + return failure + d.addCallbacks(start_instance_result, clean_up) + return d + + def attached(self, bot): + if self.substantiation_deferred is None: + msg = 'Slave %s received connection while not trying to ' \ + 'substantiate. Disconnecting.' % (self.slavename,) + log.msg(msg) + self._disconnect(bot) + return defer.fail(RuntimeError(msg)) + return AbstractBuildSlave.attached(self, bot) + + def detached(self, mind): + AbstractBuildSlave.detached(self, mind) + if self.substantiation_deferred is not None: + d = self._substantiate(self.substantiation_build) + d.addErrback(log.err, 'while re-substantiating') + + def _substantiation_failed(self, failure): + self.missing_timer = None + if self.substantiation_deferred: + d = self.substantiation_deferred + self.substantiation_deferred = None + self.substantiation_build = None + d.errback(failure) + self.insubstantiate() + # notify people, but only if we're still in the config + if not self.parent or not self.notify_on_missing: + return + + buildmaster = self.botmaster.master + status = buildmaster.getStatus() + text = "The Buildbot working for '%s'\n" % status.getTitle() + text += ("has noticed that the latent buildslave named %s \n" % + self.slavename) + text += "never substantiated after a request\n" + text += "\n" + text += ("The request was made at %s (buildmaster-local time)\n" % + time.ctime(time.time() - self.missing_timeout)) # approx + text += "\n" + text += "Sincerely,\n" + text += " The Buildbot\n" + text += " %s\n" % status.getTitleURL() + subject = "Buildbot: buildslave %s never substantiated" % self.slavename + return self._mail_missing_message(subject, text) + + def buildStarted(self, sb): + assert self.substantiated + self._clearBuildWaitTimer() + self.building.add(sb.builder_name) + + def buildFinished(self, sb): + AbstractBuildSlave.buildFinished(self, sb) + + self.building.remove(sb.builder_name) + if not self.building: + self._setBuildWaitTimer() + + def _clearBuildWaitTimer(self): + if self.build_wait_timer is not None: + if self.build_wait_timer.active(): + self.build_wait_timer.cancel() + self.build_wait_timer = None + + def _setBuildWaitTimer(self): + self._clearBuildWaitTimer() + self.build_wait_timer = reactor.callLater( + self.build_wait_timeout, self._soft_disconnect) + + def insubstantiate(self, fast=False): + self._clearBuildWaitTimer() + d = self.stop_instance(fast) + if self._shutdown_callback_handle is not None: + handle = self._shutdown_callback_handle + del self._shutdown_callback_handle + reactor.removeSystemEventTrigger(handle) + self.substantiated = False + self.building.clear() # just to be sure + return d + + def _soft_disconnect(self, fast=False): + d = AbstractBuildSlave.disconnect(self) + if self.slave is not None: + # this could be called when the slave needs to shut down, such as + # in BotMaster.removeSlave, *or* when a new slave requests a + # connection when we already have a slave. It's not clear what to + # do in the second case: this shouldn't happen, and if it + # does...if it's a latent slave, shutting down will probably kill + # something we want...but we can't know what the status is. So, + # here, we just do what should be appropriate for the first case, + # and put our heads in the sand for the second, at least for now. + # The best solution to the odd situation is removing it as a + # possibilty: make the master in charge of connecting to the + # slave, rather than vice versa. TODO. + d = defer.DeferredList([d, self.insubstantiate(fast)]) + else: + if self.substantiation_deferred is not None: + # unlike the previous block, we don't expect this situation when + # ``attached`` calls ``disconnect``, only when we get a simple + # request to "go away". + d = self.substantiation_deferred + self.substantiation_deferred = None + self.substantiation_build = None + d.errback(failure.Failure( + RuntimeError("soft disconnect aborted substantiation"))) + if self.missing_timer: + self.missing_timer.cancel() + self.missing_timer = None + self.stop_instance() + return d + + def disconnect(self): + # This returns a Deferred but we don't use it + self._soft_disconnect() + # this removes the slave from all builders. It won't come back + # without a restart (or maybe a sighup) + self.botmaster.slaveLost(self) + + def stopService(self): + res = defer.maybeDeferred(AbstractBuildSlave.stopService, self) + if self.slave is not None: + d = self._soft_disconnect() + res = defer.DeferredList([res, d]) + return res + + def updateSlave(self): + """Called to add or remove builders after the slave has connected. + + Also called after botmaster's builders are initially set. + + @return: a Deferred that indicates when an attached slave has + accepted the new builders and/or released the old ones.""" + for b in self.botmaster.getBuildersForSlave(self.slavename): + if b.name not in self.slavebuilders: + b.addLatentSlave(self) + return AbstractBuildSlave.updateSlave(self) + + def sendBuilderList(self): + d = AbstractBuildSlave.sendBuilderList(self) + def _sent(slist): + if not slist: + return + dl = [] + for name, remote in slist.items(): + # use get() since we might have changed our mind since then. + # we're checking on the builder in addition to the + # slavebuilders out of a bit of paranoia. + b = self.botmaster.builders.get(name) + sb = self.slavebuilders.get(name) + if b and sb: + d1 = sb.attached(self, remote, self.slave_commands) + dl.append(d1) + return defer.DeferredList(dl) + def _set_failed(why): + log.msg("BuildSlave.sendBuilderList (%s) failed" % self) + log.err(why) + # TODO: hang up on them?, without setBuilderList we can't use + # them + if self.substantiation_deferred: + d = self.substantiation_deferred + self.substantiation_deferred = None + self.substantiation_build = None + d.errback(why) + if self.missing_timer: + self.missing_timer.cancel() + self.missing_timer = None + # TODO: maybe log? send an email? + return why + d.addCallbacks(_sent, _set_failed) + def _substantiated(res): + log.msg("Slave %s substantiated \o/" % self.slavename) + self.substantiated = True + if not self.substantiation_deferred: + log.msg("No substantiation deferred for %s" % self.slavename) + if self.substantiation_deferred: + log.msg("Firing %s substantiation deferred with success" % self.slavename) + d = self.substantiation_deferred + self.substantiation_deferred = None + self.substantiation_build = None + d.callback(True) + # note that the missing_timer is already handled within + # ``attached`` + if not self.building: + self._setBuildWaitTimer() + d.addCallback(_substantiated) + return d diff -N -r -u buildbot-0.8.14/buildbot/changes/hgbuildbot.py buildbot/changes/hgbuildbot.py --- buildbot-0.8.14/buildbot/changes/hgbuildbot.py 1970-01-01 01:00:00.000000000 +0100 +++ buildbot/changes/hgbuildbot.py 2012-06-24 22:30:04.000000000 +0200 @@ -0,0 +1,166 @@ +# This file is part of Buildbot. Buildbot is free software: you can +# redistribute it and/or modify it under the terms of the GNU General Public +# License as published by the Free Software Foundation, version 2. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +# details. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, write to the Free Software Foundation, Inc., 51 +# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Portions Copyright Buildbot Team Members +# Portions Copyright 2007 Frederic Leroy + +# hook extension to send change notifications to buildbot when a changeset is +# brought into the repository from elsewhere. +# +# See the Buildbot manual for configuration instructions. + +import os + +from mercurial.node import bin, hex, nullid #@UnresolvedImport + +# mercurial's on-demand-importing hacks interfere with the: +#from zope.interface import Interface +# that Twisted needs to do, so disable it. +try: + from mercurial import demandimport + demandimport.disable() +except ImportError: + pass + +# In Mercurial post-1.7, some strings might be stored as a +# encoding.localstr class. encoding.fromlocal will translate +# those back to UTF-8 strings. +try: + from mercurial.encoding import fromlocal + _hush_pyflakes = [fromlocal] + del _hush_pyflakes +except ImportError: + def fromlocal(s): + return s + +def hook(ui, repo, hooktype, node=None, source=None, **kwargs): + # read config parameters + baseurl = ui.config('hgbuildbot', 'baseurl', + ui.config('web', 'baseurl', '')) + master = ui.config('hgbuildbot', 'master') + if master: + branchtype = ui.config('hgbuildbot', 'branchtype', 'inrepo') + branch = ui.config('hgbuildbot', 'branch') + fork = ui.configbool('hgbuildbot', 'fork', False) + # notify also has this setting + stripcount = int(ui.config('notify','strip') or ui.config('hgbuildbot','strip',3)) + category = ui.config('hgbuildbot', 'category', None) + project = ui.config('hgbuildbot', 'project', '') + auth = ui.config('hgbuildbot', 'auth', None) + else: + ui.write("* You must add a [hgbuildbot] section to .hg/hgrc in " + "order to use buildbot hook\n") + return + + if hooktype != "changegroup": + ui.status("hgbuildbot: hooktype %s not supported.\n" % hooktype) + return + + if fork: + child_pid = os.fork() + if child_pid == 0: + #child + pass + else: + #parent + ui.status("Notifying buildbot...\n") + return + + # only import inside the fork if forked + from buildbot.clients import sendchange + from twisted.internet import defer, reactor + + if branch is None: + if branchtype == 'dirname': + branch = os.path.basename(repo.root) + + if not auth: + auth = 'change:changepw' + auth = auth.split(':', 1) + + s = sendchange.Sender(master, auth=auth) + d = defer.Deferred() + reactor.callLater(0, d.callback, None) + # process changesets + def _send(res, c): + if not fork: + ui.status("rev %s sent\n" % c['revision']) + return s.send(c['branch'], c['revision'], c['comments'], + c['files'], c['username'], category=category, + repository=repository, project=project, vc='hg', + properties=c['properties']) + + try: # first try Mercurial 1.1+ api + start = repo[node].rev() + end = len(repo) + except TypeError: # else fall back to old api + start = repo.changelog.rev(bin(node)) + end = repo.changelog.count() + + repository = strip(repo.root, stripcount) + repository = baseurl + repository + + for rev in xrange(start, end): + # send changeset + node = repo.changelog.node(rev) + manifest, user, (time, timezone), files, desc, extra = repo.changelog.read(node) + parents = filter(lambda p: not p == nullid, repo.changelog.parents(node)) + if branchtype == 'inrepo': + branch = extra['branch'] + is_merge = len(parents) > 1 + # merges don't always contain files, but at least one file is required by buildbot + if is_merge and not files: + files = ["merge"] + properties = {'is_merge': is_merge} + if branch: + branch = fromlocal(branch) + change = { + 'master': master, + 'username': fromlocal(user), + 'revision': hex(node), + 'comments': fromlocal(desc), + 'files': files, + 'branch': branch, + 'properties':properties + } + d.addCallback(_send, change) + + def _printSuccess(res): + ui.status(s.getSuccessString(res) + '\n') + + def _printFailure(why): + ui.warn(s.getFailureString(why) + '\n') + + d.addCallbacks(_printSuccess, _printFailure) + d.addBoth(lambda _ : reactor.stop()) + reactor.run() + + if fork: + os._exit(os.EX_OK) + else: + return + +# taken from the mercurial notify extension +def strip(path, count): + '''Strip the count first slash of the path''' + + # First normalize it + path = '/'.join(path.split(os.sep)) + # and strip it part after part + while count > 0: + c = path.find('/') + if c == -1: + break + path = path[c + 1:] + count -= 1 + return path diff -N -r -u buildbot-0.8.14/buildbot/changes/pb.py buildbot/changes/pb.py --- buildbot-0.8.14/buildbot/changes/pb.py 2016-07-15 13:42:03.000000000 +0200 +++ buildbot/changes/pb.py 2016-11-27 06:30:46.000000000 +0100 @@ -14,6 +14,7 @@ # Copyright Buildbot Team Members +import urllib from twisted.internet import defer from twisted.python import log @@ -25,9 +26,10 @@ class ChangePerspective(NewCredPerspective): - def __init__(self, master, prefix): + def __init__(self, master, prefix, revlinktmpl=''): self.master = master self.prefix = prefix + self.revlinktmpl = revlinktmpl def attached(self, mind): return self @@ -40,6 +42,10 @@ if 'revlink' in changedict and not changedict['revlink']: changedict['revlink'] = '' + if self.revlinktmpl: + revision = changedict.get('revision') + if revision: + changedict['revlink'] = self.revlinktmpl % urllib.quote_plus(str(revision)) if 'repository' in changedict and not changedict['repository']: changedict['repository'] = '' if 'project' in changedict and not changedict['project']: @@ -105,7 +111,7 @@ compare_attrs = ["user", "passwd", "port", "prefix", "port"] def __init__(self, user="change", passwd="changepw", port=None, - prefix=None): + prefix=None, revlinktmpl=''): self.user = user self.passwd = passwd @@ -113,6 +119,7 @@ self.prefix = prefix self.registration = None self.registered_port = None + self.revlinktmpl = revlinktmpl def describe(self): portname = self.registered_port @@ -161,4 +168,4 @@ def getPerspective(self, mind, username): assert username == self.user - return ChangePerspective(self.master, self.prefix) + return ChangePerspective(self.master, self.prefix, self.revlinktmpl) diff -N -r -u buildbot-0.8.14/buildbot/db/migrate/README buildbot/db/migrate/README --- buildbot-0.8.14/buildbot/db/migrate/README 2016-07-15 13:42:03.000000000 +0200 +++ buildbot/db/migrate/README 1970-01-01 01:00:00.000000000 +0100 @@ -1,4 +0,0 @@ -This is a database migration repository. - -More information at -http://code.google.com/p/sqlalchemy-migrate/ diff -N -r -u buildbot-0.8.14/buildbot/process/mtrlogobserver.py buildbot/process/mtrlogobserver.py --- buildbot-0.8.14/buildbot/process/mtrlogobserver.py 1970-01-01 01:00:00.000000000 +0100 +++ buildbot/process/mtrlogobserver.py 2012-06-24 22:30:04.000000000 +0200 @@ -0,0 +1,483 @@ +# This file is part of Buildbot. Buildbot is free software: you can +# redistribute it and/or modify it under the terms of the GNU General Public +# License as published by the Free Software Foundation, version 2. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +# details. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, write to the Free Software Foundation, Inc., 51 +# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Copyright Buildbot Team Members + +import sys +import re +from twisted.python import log +from twisted.internet import defer +from twisted.enterprise import adbapi +from buildbot.process.buildstep import LogLineObserver +from buildbot.steps.shell import Test + +class EqConnectionPool(adbapi.ConnectionPool): + """This class works the same way as +twisted.enterprise.adbapi.ConnectionPool. But it adds the ability to +compare connection pools for equality (by comparing the arguments +passed to the constructor). + +This is useful when passing the ConnectionPool to a BuildStep, as +otherwise Buildbot will consider the buildstep (and hence the +containing buildfactory) to have changed every time the configuration +is reloaded. + +It also sets some defaults differently from adbapi.ConnectionPool that +are more suitable for use in MTR. +""" + def __init__(self, *args, **kwargs): + self._eqKey = (args, kwargs) + return adbapi.ConnectionPool.__init__(self, + cp_reconnect=True, cp_min=1, cp_max=3, + *args, **kwargs) + + def __eq__(self, other): + if isinstance(other, EqConnectionPool): + return self._eqKey == other._eqKey + else: + return False + + def __ne__(self, other): + return not self.__eq__(other) + + +class MtrTestFailData: + def __init__(self, testname, variant, result, info, text, callback): + self.testname = testname + self.variant = variant + self.result = result + self.info = info + self.text = text + self.callback = callback + + def add(self, line): + self.text+= line + + def fireCallback(self): + return self.callback(self.testname, self.variant, self.result, self.info, self.text) + + +class MtrLogObserver(LogLineObserver): + """ + Class implementing a log observer (can be passed to + BuildStep.addLogObserver(). + + It parses the output of mysql-test-run.pl as used in MySQL, + MariaDB, Drizzle, etc. + + It counts number of tests run and uses it to provide more accurate + completion estimates. + + It parses out test failures from the output and summarises the results on + the Waterfall page. It also passes the information to methods that can be + overridden in a subclass to do further processing on the information.""" + + _line_re = re.compile(r"^([-._0-9a-zA-z]+)( '[-_ a-zA-Z]+')?\s+(w[0-9]+\s+)?\[ (fail|pass) \]\s*(.*)$") + _line_re2 = re.compile(r"^[-._0-9a-zA-z]+( '[-_ a-zA-Z]+')?\s+(w[0-9]+\s+)?\[ [-a-z]+ \]") + _line_re3 = re.compile(r"^\*\*\*Warnings generated in error logs during shutdown after running tests: (.*)") + _line_re4 = re.compile(r"^The servers were restarted [0-9]+ times$") + _line_re5 = re.compile(r"^Only\s+[0-9]+\s+of\s+[0-9]+\s+completed.$") + + def __init__(self, textLimit=5, testNameLimit=16, testType=None): + self.textLimit = textLimit + self.testNameLimit = testNameLimit + self.testType = testType + self.numTests = 0 + self.testFail = None + self.failList = [] + self.warnList = [] + LogLineObserver.__init__(self) + + def setLog(self, loog): + LogLineObserver.setLog(self, loog) + d= loog.waitUntilFinished() + d.addCallback(lambda l: self.closeTestFail()) + + def outLineReceived(self, line): + stripLine = line.strip("\r\n") + m = self._line_re.search(stripLine) + if m: + testname, variant, worker, result, info = m.groups() + self.closeTestFail() + self.numTests += 1 + self.step.setProgress('tests', self.numTests) + + if result == "fail": + if variant == None: + variant = "" + else: + variant = variant[2:-1] + self.openTestFail(testname, variant, result, info, stripLine + "\n") + + else: + m = self._line_re3.search(stripLine) + if m: + stuff = m.group(1) + self.closeTestFail() + testList = stuff.split(" ") + self.doCollectWarningTests(testList) + + elif (self._line_re2.search(stripLine) or + self._line_re4.search(stripLine) or + self._line_re5.search(stripLine) or + stripLine == "Test suite timeout! Terminating..." or + stripLine.startswith("mysql-test-run: *** ERROR: Not all tests completed") or + (stripLine.startswith("------------------------------------------------------------") + and self.testFail != None)): + self.closeTestFail() + + else: + self.addTestFailOutput(stripLine + "\n") + + def openTestFail(self, testname, variant, result, info, line): + self.testFail = MtrTestFailData(testname, variant, result, info, line, self.doCollectTestFail) + + def addTestFailOutput(self, line): + if self.testFail != None: + self.testFail.add(line) + + def closeTestFail(self): + if self.testFail != None: + self.testFail.fireCallback() + self.testFail = None + + def addToText(self, src, dst): + lastOne = None + count = 0 + for t in src: + if t != lastOne: + dst.append(t) + count += 1 + if count >= self.textLimit: + break + + def makeText(self, done): + if done: + text = ["test"] + else: + text = ["testing"] + if self.testType: + text.append(self.testType) + fails = self.failList[:] + fails.sort() + self.addToText(fails, text) + warns = self.warnList[:] + warns.sort() + self.addToText(warns, text) + return text + + # Update waterfall status. + def updateText(self): + self.step.step_status.setText(self.makeText(False)) + + strip_re = re.compile(r"^[a-z]+\.") + + def displayTestName(self, testname): + + displayTestName = self.strip_re.sub("", testname) + + if len(displayTestName) > self.testNameLimit: + displayTestName = displayTestName[:(self.testNameLimit-2)] + "..." + return displayTestName + + def doCollectTestFail(self, testname, variant, result, info, text): + self.failList.append("F:" + self.displayTestName(testname)) + self.updateText() + self.collectTestFail(testname, variant, result, info, text) + + def doCollectWarningTests(self, testList): + for t in testList: + self.warnList.append("W:" + self.displayTestName(t)) + self.updateText() + self.collectWarningTests(testList) + + # These two methods are overridden to actually do something with the data. + def collectTestFail(self, testname, variant, result, info, text): + pass + def collectWarningTests(self, testList): + pass + +class MTR(Test): + """ + Build step that runs mysql-test-run.pl, as used in MySQL, Drizzle, + MariaDB, etc. + + It uses class MtrLogObserver to parse test results out from the + output of mysql-test-run.pl, providing better completion time + estimates and summarising test failures on the waterfall page. + + It also provides access to mysqld server error logs from the test + run to help debugging any problems. + + Optionally, it can insert into a database data about the test run, + including details of any test failures. + + Parameters: + + textLimit + Maximum number of test failures to show on the waterfall page + (to not flood the page in case of a large number of test + failures. Defaults to 5. + + testNameLimit + Maximum length of test names to show unabbreviated in the + waterfall page, to avoid excessive column width. Defaults to 16. + + parallel + Value of --parallel option used for mysql-test-run.pl (number + of processes used to run the test suite in parallel). Defaults + to 4. This is used to determine the number of server error log + files to download from the slave. Specifying a too high value + does not hurt (as nonexisting error logs will be ignored), + however if using --parallel value greater than the default it + needs to be specified, or some server error logs will be + missing. + + dbpool + An instance of twisted.enterprise.adbapi.ConnectionPool, or None. + Defaults to None. If specified, results are inserted into the database + using the ConnectionPool. + + The class process.mtrlogobserver.EqConnectionPool subclass of + ConnectionPool can be useful to pass as value for dbpool, to + avoid having config reloads think the Buildstep is changed + just because it gets a new ConnectionPool instance (even + though connection parameters are unchanged). + + autoCreateTables + Boolean, defaults to False. If True (and dbpool is specified), the + necessary database tables will be created automatically if they do + not exist already. Alternatively, the tables can be created manually + from the SQL statements found in the mtrlogobserver.py source file. + + test_type + test_info + Two descriptive strings that will be inserted in the database tables if + dbpool is specified. The test_type string, if specified, will also + appear on the waterfall page.""" + + renderables = [ 'mtr_subdir' ] + + def __init__(self, dbpool=None, test_type=None, test_info="", + description=None, descriptionDone=None, + autoCreateTables=False, textLimit=5, testNameLimit=16, + parallel=4, logfiles = {}, lazylogfiles = True, + warningPattern="MTR's internal check of the test case '.*' failed", + mtr_subdir="mysql-test", **kwargs): + + if description is None: + description = ["testing"] + if test_type: + description.append(test_type) + if descriptionDone is None: + descriptionDone = ["test"] + if test_type: + descriptionDone.append(test_type) + Test.__init__(self, logfiles=logfiles, lazylogfiles=lazylogfiles, + description=description, descriptionDone=descriptionDone, + warningPattern=warningPattern, **kwargs) + self.dbpool = dbpool + self.test_type = test_type + self.test_info = test_info + self.autoCreateTables = autoCreateTables + self.textLimit = textLimit + self.testNameLimit = testNameLimit + self.parallel = parallel + self.mtr_subdir = mtr_subdir + self.progressMetrics += ('tests',) + + self.addFactoryArguments(dbpool=self.dbpool, + test_type=self.test_type, + test_info=self.test_info, + autoCreateTables=self.autoCreateTables, + textLimit=self.textLimit, + testNameLimit=self.testNameLimit, + parallel=self.parallel, + mtr_subdir=self.mtr_subdir) + + def start(self): + # Add mysql server logfiles. + for mtr in range(0, self.parallel+1): + for mysqld in range(1, 4+1): + if mtr == 0: + logname = "mysqld.%d.err" % mysqld + filename = "var/log/mysqld.%d.err" % mysqld + else: + logname = "mysqld.%d.err.%d" % (mysqld, mtr) + filename = "var/%d/log/mysqld.%d.err" % (mtr, mysqld) + self.addLogFile(logname, self.mtr_subdir + "/" + filename) + + self.myMtr = self.MyMtrLogObserver(textLimit=self.textLimit, + testNameLimit=self.testNameLimit, + testType=self.test_type) + self.addLogObserver("stdio", self.myMtr) + # Insert a row for this test run into the database and set up + # build properties, then start the command proper. + d = self.registerInDB() + d.addCallback(self.afterRegisterInDB) + d.addErrback(self.failed) + + def getText(self, command, results): + return self.myMtr.makeText(True) + + def runInteractionWithRetry(self, actionFn, *args, **kw): + """ + Run a database transaction with dbpool.runInteraction, but retry the + transaction in case of a temporary error (like connection lost). + + This is needed to be robust against things like database connection + idle timeouts. + + The passed callable that implements the transaction must be retryable, + ie. it must not have any destructive side effects in the case where + an exception is thrown and/or rollback occurs that would prevent it + from functioning correctly when called again.""" + + def runWithRetry(txn, *args, **kw): + retryCount = 0 + while(True): + try: + return actionFn(txn, *args, **kw) + except txn.OperationalError: + retryCount += 1 + if retryCount >= 5: + raise + excType, excValue, excTraceback = sys.exc_info() + log.msg("Database transaction failed (caught exception %s(%s)), retrying ..." % (excType, excValue)) + txn.close() + txn.reconnect() + txn.reopen() + + return self.dbpool.runInteraction(runWithRetry, *args, **kw) + + def runQueryWithRetry(self, *args, **kw): + """ + Run a database query, like with dbpool.runQuery, but retry the query in + case of a temporary error (like connection lost). + + This is needed to be robust against things like database connection + idle timeouts.""" + + def runQuery(txn, *args, **kw): + txn.execute(*args, **kw) + return txn.fetchall() + + return self.runInteractionWithRetry(runQuery, *args, **kw) + + def registerInDB(self): + if self.dbpool: + return self.runInteractionWithRetry(self.doRegisterInDB) + else: + return defer.succeed(0) + + # The real database work is done in a thread in a synchronous way. + def doRegisterInDB(self, txn): + # Auto create tables. + # This is off by default, as it gives warnings in log file + # about tables already existing (and I did not find the issue + # important enough to find a better fix). + if self.autoCreateTables: + txn.execute(""" +CREATE TABLE IF NOT EXISTS test_run( + id INT PRIMARY KEY AUTO_INCREMENT, + branch VARCHAR(100), + revision VARCHAR(32) NOT NULL, + platform VARCHAR(100) NOT NULL, + dt TIMESTAMP NOT NULL, + bbnum INT NOT NULL, + typ VARCHAR(32) NOT NULL, + info VARCHAR(255), + KEY (branch, revision), + KEY (dt), + KEY (platform, bbnum) +) ENGINE=innodb +""") + txn.execute(""" +CREATE TABLE IF NOT EXISTS test_failure( + test_run_id INT NOT NULL, + test_name VARCHAR(100) NOT NULL, + test_variant VARCHAR(16) NOT NULL, + info_text VARCHAR(255), + failure_text TEXT, + PRIMARY KEY (test_run_id, test_name, test_variant) +) ENGINE=innodb +""") + txn.execute(""" +CREATE TABLE IF NOT EXISTS test_warnings( + test_run_id INT NOT NULL, + list_id INT NOT NULL, + list_idx INT NOT NULL, + test_name VARCHAR(100) NOT NULL, + PRIMARY KEY (test_run_id, list_id, list_idx) +) ENGINE=innodb +""") + + revision = self.getProperty("got_revision") + if revision is None: + revision = self.getProperty("revision") + typ = "mtr" + if self.test_type: + typ = self.test_type + txn.execute(""" +INSERT INTO test_run(branch, revision, platform, dt, bbnum, typ, info) +VALUES (%s, %s, %s, CURRENT_TIMESTAMP(), %s, %s, %s) +""", (self.getProperty("branch"), revision, + self.getProperty("buildername"), self.getProperty("buildnumber"), + typ, self.test_info)) + + return txn.lastrowid + + def afterRegisterInDB(self, insert_id): + self.setProperty("mtr_id", insert_id) + self.setProperty("mtr_warn_id", 0) + + Test.start(self) + + def reportError(self, err): + log.msg("Error in async insert into database: %s" % err) + + class MyMtrLogObserver(MtrLogObserver): + def collectTestFail(self, testname, variant, result, info, text): + # Insert asynchronously into database. + dbpool = self.step.dbpool + run_id = self.step.getProperty("mtr_id") + if dbpool == None: + return defer.succeed(None) + if variant == None: + variant = "" + d = self.step.runQueryWithRetry(""" +INSERT INTO test_failure(test_run_id, test_name, test_variant, info_text, failure_text) +VALUES (%s, %s, %s, %s, %s) +""", (run_id, testname, variant, info, text)) + + d.addErrback(self.step.reportError) + return d + + def collectWarningTests(self, testList): + # Insert asynchronously into database. + dbpool = self.step.dbpool + if dbpool == None: + return defer.succeed(None) + run_id = self.step.getProperty("mtr_id") + warn_id = self.step.getProperty("mtr_warn_id") + self.step.setProperty("mtr_warn_id", warn_id + 1) + q = ("INSERT INTO test_warnings(test_run_id, list_id, list_idx, test_name) " + + "VALUES " + ", ".join(map(lambda x: "(%s, %s, %s, %s)", testList))) + v = [] + idx = 0 + for t in testList: + v.extend([run_id, warn_id, idx, t]) + idx = idx + 1 + d = self.step.runQueryWithRetry(q, tuple(v)) + d.addErrback(self.step.reportError) + return d diff -N -r -u buildbot-0.8.14/buildbot/scripts/startup.py buildbot/scripts/startup.py --- buildbot-0.8.14/buildbot/scripts/startup.py 1970-01-01 01:00:00.000000000 +0100 +++ buildbot/scripts/startup.py 2012-06-24 22:30:04.000000000 +0200 @@ -0,0 +1,106 @@ +# This file is part of Buildbot. Buildbot is free software: you can +# redistribute it and/or modify it under the terms of the GNU General Public +# License as published by the Free Software Foundation, version 2. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +# details. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, write to the Free Software Foundation, Inc., 51 +# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Copyright Buildbot Team Members + + +import os, sys, time + +class Follower: + def follow(self): + from twisted.internet import reactor + from buildbot.scripts.logwatcher import LogWatcher + self.rc = 0 + print "Following twistd.log until startup finished.." + lw = LogWatcher("twistd.log") + d = lw.start() + d.addCallbacks(self._success, self._failure) + reactor.run() + return self.rc + + def _success(self, _): + from twisted.internet import reactor + print "The buildmaster appears to have (re)started correctly." + self.rc = 0 + reactor.stop() + + def _failure(self, why): + from twisted.internet import reactor + from buildbot.scripts.logwatcher import BuildmasterTimeoutError + from buildbot.scripts.logwatcher import ReconfigError + if why.check(BuildmasterTimeoutError): + print """ +The buildmaster took more than 10 seconds to start, so we were unable to +confirm that it started correctly. Please 'tail twistd.log' and look for a +line that says 'configuration update complete' to verify correct startup. +""" + elif why.check(ReconfigError): + print """ +The buildmaster appears to have encountered an error in the master.cfg config +file during startup. Please inspect and fix master.cfg, then restart the +buildmaster. +""" + else: + print """ +Unable to confirm that the buildmaster started correctly. You may need to +stop it, fix the config file, and restart. +""" + print why + self.rc = 1 + reactor.stop() + + +def start(config): + os.chdir(config['basedir']) + if not os.path.exists("buildbot.tac"): + print "This doesn't look like a buildbot base directory:" + print "No buildbot.tac file." + print "Giving up!" + sys.exit(1) + if config['quiet']: + return launch(config) + + # we probably can't do this os.fork under windows + from twisted.python.runtime import platformType + if platformType == "win32": + return launch(config) + + # fork a child to launch the daemon, while the parent process tails the + # logfile + if os.fork(): + # this is the parent + rc = Follower().follow() + sys.exit(rc) + # this is the child: give the logfile-watching parent a chance to start + # watching it before we start the daemon + time.sleep(0.2) + launch(config) + +def launch(config): + sys.path.insert(0, os.path.abspath(os.getcwd())) + + # see if we can launch the application without actually having to + # spawn twistd, since spawning processes correctly is a real hassle + # on windows. + argv = ["twistd", + "--no_save", + "--logfile=twistd.log", # windows doesn't use the same default + "--python=buildbot.tac"] + sys.argv = argv + + # this is copied from bin/twistd. twisted-2.0.0 through 2.4.0 use + # _twistw.run . Twisted-2.5.0 and later use twistd.run, even for + # windows. + from twisted.scripts import twistd + twistd.run() + diff -N -r -u buildbot-0.8.14/buildbot/status/builder.py buildbot/status/builder.py --- buildbot-0.8.14/buildbot/status/builder.py 2016-07-15 13:42:03.000000000 +0200 +++ buildbot/status/builder.py 2016-11-27 06:30:46.000000000 +0100 @@ -190,8 +190,9 @@ def loadBuildFromFile(self, number): filename = self.makeBuildFilename(number) try: - log.msg("Loading builder %s's build %d from on-disk pickle" - % (self.name, number)) + # disabled MvL + #log.msg("Loading builder %s's build %d from on-disk pickle" + # % (self.name, number)) with open(filename, "rb") as f: build = load(f) build.setProcessObjects(self, self.master) diff -N -r -u buildbot-0.8.14/buildbot/status/mail.py buildbot/status/mail.py --- buildbot-0.8.14/buildbot/status/mail.py 2016-07-15 13:42:03.000000000 +0200 +++ buildbot/status/mail.py 2016-11-27 06:30:46.000000000 +0100 @@ -71,6 +71,52 @@ LOG_ENCODING = 'utf-8' +# GB: NEW: log interpreting function + +failed_re = re.compile('\d+ tests? failed') + +def interpret_test_logfile(lines): + state = '' + out = [] + for line in lines: + line = line.rstrip('\n') + if state == '': + if line.startswith('Traceback (most recent call last):'): + state = 'tb' + out.append('') + out.append(line) + elif line.startswith('======'): + state = 'unittest' + out.append('') + out.append(line) + elif failed_re.match(line): + state = 'failed' + out.append('') + out.append(line) + elif line.startswith('process killed by') or \ + line.startswith('program finished with') or \ + line.startswith('command timed out') or \ + line.startswith('make: ***'): + out.append('') + out.append(line) + elif state == 'tb': + out.append(line) + if not line.startswith(' '): + state = '' + elif state == 'unittest': + out.append(line) + if line.startswith('------'): + state = '' + elif state == 'failed': + if not line.startswith(' '): + state = '' + else: + out.append(line) + return '\n'.join(out) + +# end log interpreting function + + class Domain(util.ComparableMixin): implements(interfaces.IEmailLookup) compare_attrs = ["domain"] @@ -742,6 +788,16 @@ d = self.createEmail(msgdict, name, self.master_status.getTitle(), results, builds, patches, logs) + # GB: NEW: add test logfile excerpts + if "test" in t: # only if the test went wrong + for log in build.getLogs(): + if "test" in log.getStep().getName(): + text += "\n" + text += "Excerpt from the test logfile:" + text += interpret_test_logfile(log.getText().splitlines()) + text += "\n" + break + @d.addCallback def getRecipients(m): # now, who is this message going to? diff -N -r -u buildbot-0.8.14/buildbot/status/web/baseweb.py buildbot/status/web/baseweb.py --- buildbot-0.8.14/buildbot/status/web/baseweb.py 2016-07-15 13:42:03.000000000 +0200 +++ buildbot/status/web/baseweb.py 2016-11-27 06:30:46.000000000 +0100 @@ -420,7 +420,7 @@ return wrapper def setupUsualPages(self, numbuilds, num_events, num_events_max): - # self.putChild("", IndexOrWaterfallRedirection()) + self.putChild("", Redirect("waterfall")) self.putChild("waterfall", WaterfallStatusResource(num_events=num_events, num_events_max=num_events_max)) self.putChild("grid", GridStatusResource()) diff -N -r -u buildbot-0.8.14/buildbot/status/web/console.py buildbot/status/web/console.py --- buildbot-0.8.14/buildbot/status/web/console.py 2016-07-15 13:42:03.000000000 +0200 +++ buildbot/status/web/console.py 2016-11-27 06:30:46.000000000 +0100 @@ -316,7 +316,7 @@ for tag in builderList: count += len(builderList[tag]) - tags = sorted(builderList.keys()) + tags = util.naturalSort(builderList.keys()) cs = [] diff -N -r -u buildbot-0.8.14/buildbot/status/web/files/readme_svg.txt buildbot/status/web/files/readme_svg.txt --- buildbot-0.8.14/buildbot/status/web/files/readme_svg.txt 2016-07-15 13:42:03.000000000 +0200 +++ buildbot/status/web/files/readme_svg.txt 1970-01-01 01:00:00.000000000 +0100 @@ -1,136 +0,0 @@ -This directory contains SVG templates to produce the Buildbot PngStatusResource -files that tell us the build status for a given build of a given builder. - -The SVG files has been created with Inkscape-0.48.4 under Gentoo GNU/Linux for -x86_64 architecture. - - -Howto generate the PNG files? -============================= -There are two different ways for generate the PNG files using inkscape. - - 1) For non GUI users - Just edit the SVG files (as those are just XML files) with your - favourite text editor (vi, emacs, nano, joe...). The SVG image is - composed by layers so for any buildbot status we have a layer with - two components, the status text and the background color, there are - some other layers containing glows and static text that never changes - (like thw "build" word) so you dont have to care about. - - Each layer has a descriptive name, for exmaple, the layer for failed - builds is called "rectFailed", "rectSuccess" for success, "rectExcept" - for exceptions and so on. - - Each status layer has two components, the statsu text and the status - background, text is just a "tspan" markup and looks like this: - - - success - - - The entire should be inside a markup that we don't care - about if we are not going to increade the size or the family of the - used font. - - The background component is not more complex that the previous one, it - is built using a markup that looks like: - - - - The properties are pretty self descriptives, with the "style" property - we control the look of the rect, in the previous example we are - generating a rect with filled purple color without opacity no stroke - and displayed inline, the rest of properties are just size and position - in the canvas. - - To change teh color we just need to modify the #hexcolor in the style - property. - - As we said before, some layers are just hidden while other are visible, - to export one build status we need to make visible the related layers - to the status we want to generate. - - The layers have also a "style" property in their definitions, the - hidden layers have it setted as "display:none" while the visible ones - have it as "display:inline" show or hidden layers is as easy as change - the values for layers that we want to hide or show. - - To create a new group we only have to copy a full layer block and paste - it modifying the layer name and the new property values we require - (that usually are just the backgound color and the text of tspan). - - When we are ok with our changes to the SVG file we can just execute - inkscape in the command line to export it to PNG format: - - inkscape -z -e status_size.png status_size.svg - - The -z option tells inkscape to don't use a graphical user interface, - the -e is just "export command" followed by the name of the new - exported file (status_size.png in our example) and the last argument - is just the name of the SVG file that we modified previously. - - 2) For GUI users - Well, if you never used Inkscape before, I suggest to check the basic - tutorial in their website, and then come back here: - - http://inkscape.org/doc/basic/tutorial-basic.html - - Our SVG files are compound by layers, some layers are pretty static and - them are "locked", that means that you can't click on the components - inside the layers so you can't select them to modify it. - - The first thing you have to do is show the layers window using the - shortcut Ctrl+Shift+L - - In this window you can see all the layers thart build the SVG file, you - can hide or make them visible clicking in the eye at the most left side - of the layer name. Next to this eye button is a "chain" button that is - used to lock or unlock any layer. - - Each status layer has a descriptive name like "rectSuccess" for the - success buildbot status (rectFailed for failed etc). Them are compound - of two shapes: - - The status text - The status background - - To change the text we have to select it and then click on the Text - tool in the left side of the main window, then the cursor over the - text shape in the draw should change and we can just write on it. On - the top of the main window we can see a toolset for adjust properties - of the text (I recommend the reading of Inkscape tutorials to really - understand how all this works). - - To change the background color for the status we need to show the - "Fill and Stroke" menu using the shortcut "Ctrl+Shift+F" - - Then if we click on the rect shape we can check and modify all the - related properties and values about the object look. - - To change the color we should modify the RGBA (in hex format) value - in the color form. Note that the last two characters of the hex format - controls the object alpha layer so FF is no opacity at all while 00 - is total opacity, be careful and rememeber that when you modify the - color by hand. - - To create a new layer you can just do right click on any of the already - existing status layers and select "Duplicate layer" in the context menu - that should be shown. Then you can modify the layer as your needs and - save it - - To understand how to export from the SVG file to PNG format I reommend - to read this article about Inkscape export system: - - http://inkscape.org/doc/basic/tutorial-basic.html - - Anyway my preffer method is the one described in "Non GUI Users" just - using the command line. diff -N -r -u buildbot-0.8.14/buildbot/status/web/files/status_large.svg buildbot/status/web/files/status_large.svg --- buildbot-0.8.14/buildbot/status/web/files/status_large.svg 2016-07-15 13:42:03.000000000 +0200 +++ buildbot/status/web/files/status_large.svg 1970-01-01 01:00:00.000000000 +0100 @@ -1,352 +0,0 @@ - - - - - - - - - - - - - - - - image/svg+xml - - - - - - - - - - - - - - - - - - - build status - - - - - - - - retrybuild - - - - - - diff -N -r -u buildbot-0.8.14/buildbot/status/web/files/status_small.svg buildbot/status/web/files/status_small.svg --- buildbot-0.8.14/buildbot/status/web/files/status_small.svg 2016-07-15 13:42:03.000000000 +0200 +++ buildbot/status/web/files/status_small.svg 1970-01-01 01:00:00.000000000 +0100 @@ -1,290 +0,0 @@ - - - - - - - - - - - - - - - - image/svg+xml - - - - - - - - - - W - - - - - - - - - diff -N -r -u buildbot-0.8.14/buildbot/status/web/files/status.svg buildbot/status/web/files/status.svg --- buildbot-0.8.14/buildbot/status/web/files/status.svg 2016-07-15 13:42:03.000000000 +0200 +++ buildbot/status/web/files/status.svg 1970-01-01 01:00:00.000000000 +0100 @@ -1,361 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - image/svg+xml - - - - - - - - - - - - - - unknown - - - - - build - - - - - diff -N -r -u buildbot-0.8.14/buildbot/status/web/grid.py buildbot/status/web/grid.py --- buildbot-0.8.14/buildbot/status/web/grid.py 2016-07-15 13:42:03.000000000 +0200 +++ buildbot/status/web/grid.py 2016-11-27 06:30:46.000000000 +0100 @@ -13,6 +13,7 @@ # # Copyright Buildbot Team Members +from buildbot import util from buildbot.sourcestamp import SourceStamp from buildbot.status.web.base import HtmlResource from buildbot.status.web.base import build_get_class @@ -205,7 +206,7 @@ 'stamps': [map(SourceStamp.asDict, sstamp) for sstamp in stamps], }) - sortedBuilderNames = sorted(status.getBuilderNames()) + sortedBuilderNames = util.naturalSort(status.getBuilderNames()) cxt['builders'] = [] @@ -274,7 +275,7 @@ 'stamps': [map(SourceStamp.asDict, sstamp) for sstamp in stamps], }) - sortedBuilderNames = sorted(status.getBuilderNames()) + sortedBuilderNames = util.naturalSort(status.getBuilderNames()) cxt['sorted_builder_names'] = sortedBuilderNames cxt['builder_builds'] = builder_builds = [] diff -N -r -u buildbot-0.8.14/buildbot/status/web/templates/forms.html buildbot/status/web/templates/forms.html --- buildbot-0.8.14/buildbot/status/web/templates/forms.html 2016-07-15 13:42:03.000000000 +0200 +++ buildbot/status/web/templates/forms.html 2016-11-27 06:30:46.000000000 +0100 @@ -172,6 +172,13 @@

To force a build, fill out the following fields and push the 'Force Build' button

{% endif %} +

For custom builders, "Repo path" is + your repository's relative path to http://hg.python.org/ (for example, + "features/pep-1234"). The build will be done from named branch + "default" unless you select a specific revision, tag or branch name in + "Revision to build". +

+ {% for f in sch.all_fields %} {{ force_build_scheduler_parameter(f, authz, request, sch, default_props) }} diff -N -r -u buildbot-0.8.14/buildbot/status/web/waterfall.py buildbot/status/web/waterfall.py --- buildbot-0.8.14/buildbot/status/web/waterfall.py 2016-07-15 13:42:03.000000000 +0200 +++ buildbot/status/web/waterfall.py 2016-11-27 06:30:46.000000000 +0100 @@ -250,6 +250,7 @@ debug = False e = g.next() + have_connect = e.getText() and e.getText()[0] in ('connect', 'disconnect') starts, finishes = e.getTimes() if debug: log.msg("E0", starts, finishes) @@ -270,6 +271,10 @@ while True: e = g.next() + # MvL: skip every connect/disconnect except for the first one + if e.getText() and e.getText()[0] in ('connect', 'disconnect'): + if have_connect: continue + have_connect = True if not showEvents and isinstance(e, builder.Event): continue starts, finishes = e.getTimes() @@ -325,6 +330,7 @@ for bldr in builders: tags = bldr.getTags() allTags.update(tags or []) + allTags = util.naturalSort(list(allTags)) cxt['show_tags'] = show_tags cxt['all_tags'] = allTags @@ -585,7 +591,7 @@ for builderName in builderNames: builder = status.getBuilder(builderName) tags.update(builder.getTags() or []) - tags = sorted(tags) + tags = util.naturalSort(list(tags)) ctx['tags'] = tags template = request.site.buildbot_service.templates.get_template("waterfall.html") @@ -788,7 +794,7 @@ stuff.append(today) lastDate = today stuff.append( - time.strftime("%H:%M:%S", + time.strftime("%H:%M", # MvL took off seconds time.localtime(timestamps[r]))) grid[0].append(Box(text=stuff, class_="Time", valign="bottom", align="center")) diff -N -r -u buildbot-0.8.14/buildbot/steps/blocker.py buildbot/steps/blocker.py --- buildbot-0.8.14/buildbot/steps/blocker.py 1970-01-01 01:00:00.000000000 +0100 +++ buildbot/steps/blocker.py 2012-06-24 22:30:04.000000000 +0200 @@ -0,0 +1,367 @@ +# This file is part of Buildbot. Buildbot is free software: you can +# redistribute it and/or modify it under the terms of the GNU General Public +# License as published by the Free Software Foundation, version 2. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +# details. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, write to the Free Software Foundation, Inc., 51 +# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Copyright Buildbot Team Members + +from twisted.python import log, failure +from twisted.internet import reactor +from buildbot.process.buildstep import BuildStep +from buildbot.status import builder, buildstep +from buildbot import config + +class BadStepError(Exception): + """Raised by Blocker when it is passed an upstream step that cannot + be found or is in a bad state.""" + pass + +class Blocker(BuildStep): + """ + Build step that blocks until at least one other step finishes. + + @ivar upstreamSteps: a non-empty list of (builderName, stepName) tuples + identifying the other build steps that must + complete in order to unblock this Blocker. + @ivar idlePolicy: string: what to do if one of the upstream builders is + idle when this Blocker starts; one of: + \"error\": just blow up (the Blocker will fail with + status EXCEPTION) + \"ignore\": carry on as if the referenced build step + was not mentioned (or is already complete) + \"block\": block until the referenced builder starts + a build, and then block until the referenced build + step in that build finishes + @ivar timeout: int: how long to block, in seconds, before giving up and + failing (default: None, meaning block forever) + """ + parms = (BuildStep.parms + + ['upstreamSteps', + 'idlePolicy', + 'timeout', + ]) + + flunkOnFailure = True # override BuildStep's default + upstreamSteps = None + idlePolicy = "block" + timeout = None + + VALID_IDLE_POLICIES = ("error", "ignore", "block") + + def __init__(self, **kwargs): + BuildStep.__init__(self, **kwargs) + if self.upstreamSteps is None: + config.error( + "you must supply upstreamSteps") + if len(self.upstreamSteps) < 1: + config.error( + "upstreamSteps must be a non-empty list") + if self.idlePolicy not in self.VALID_IDLE_POLICIES: + config.error( + "invalid value for idlePolicy: %r (must be one of %s)" + % (self.idlePolicy, + ", ".join(map(repr, self.VALID_IDLE_POLICIES)))) + + # list of build steps (as BuildStepStatus objects) that we're + # currently waiting on + self._blocking_steps = [] + + # set of builders (as BuilderStatus objects) that have to start + # a Build before we can block on one of their BuildSteps + self._blocking_builders = set() + + self._overall_code = builder.SUCCESS # assume the best + self._overall_text = [] + + self._timer = None # object returned by reactor.callLater() + self._timed_out = False + + def __str__(self): + return self.name + + def __repr__(self): + return "<%s %x: %s>" % (self.__class__.__name__, id(self), self.name) + + def _log(self, message, *args): + log.msg(repr(self) + ": " + (message % args)) + + def buildsMatch(self, buildStatus1, buildStatus2): + """ + Return true if buildStatus1 and buildStatus2 are from related + builds, i.e. a Blocker step running in buildStatus2 should be + blocked by an upstream step in buildStatus1. Return false if + they are unrelated. + + Default implementation simply raises NotImplementedError: you + *must* subclass Blocker and implement this method, because + BuildBot currently provides no way to relate different builders. + This might change if ticket #875 (\"build flocks\") is + implemented. + """ + raise NotImplementedError( + "abstract method: you must subclass Blocker " + "and implement buildsMatch()") + + def _getBuildStatus(self, botmaster, builderName): + try: + # Get the buildbot.process.builder.Builder object for the + # requested upstream builder: this is a long-lived object + # that exists and has useful info in it whether or not a + # build is currently running under it. + builder = botmaster.builders[builderName] + except KeyError: + raise BadStepError( + "no builder named %r" % builderName) + + # The Builder's BuilderStatus object is where we can find out + # what's going on right now ... like, say, the list of + # BuildStatus objects representing any builds running now. + myBuildStatus = self.build.getStatus() + builderStatus = builder.builder_status + matchingBuild = None + + # Get a list of all builds in this builder, past and present. + # This is subtly broken because BuilderStatus does not expose + # the information we need; in fact, it doesn't even necessarily + # *have* that information. The contents of the build cache can + # change unpredictably if someone happens to view the waterfall + # at an inopportune moment: yikes! The right fix is to keep old + # builds in the database and for BuilderStatus to expose the + # needed information. When that is implemented, then Blocker + # needs to be adapted to use it, and *then* Blocker should be + # safe to use. + all_builds = (builderStatus.buildCache.values() + + builderStatus.getCurrentBuilds()) + + for buildStatus in all_builds: + if self.buildsMatch(myBuildStatus, buildStatus): + matchingBuild = buildStatus + break + + if matchingBuild is None: + msg = "no matching builds found in builder %r" % builderName + if self.idlePolicy == "error": + raise BadStepError(msg + " (is it idle?)") + elif self.idlePolicy == "ignore": + # don't hang around waiting (assume the build has finished) + self._log(msg + ": skipping it") + return None + elif self.idlePolicy == "block": + self._log(msg + ": will block until it starts a build") + self._blocking_builders.add(builderStatus) + return None + + self._log("found builder %r: %r", builderName, builder) + return matchingBuild + + def _getStepStatus(self, buildStatus, stepName): + for step_status in buildStatus.getSteps(): + if step_status.name == stepName: + self._log("found build step %r in builder %r: %r", + stepName, buildStatus.getBuilder().getName(), step_status) + return step_status + raise BadStepError( + "builder %r has no step named %r" + % (buildStatus.builder.name, stepName)) + + def _getFullnames(self): + if len(self.upstreamSteps) == 1: + fullnames = ["(%s:%s)" % self.upstreamSteps[0]] + else: + fullnames = [] + fullnames.append("(%s:%s," % self.upstreamSteps[0]) + fullnames.extend(["%s:%s," % pair for pair in self.upstreamSteps[1:-1]]) + fullnames.append("%s:%s)" % self.upstreamSteps[-1]) + return fullnames + + def _getBlockingStatusText(self): + return [self.name+":", "blocking on"] + self._getFullnames() + + def _getFinishStatusText(self, code, elapsed): + meaning = builder.Results[code] + text = [self.name+":", + "upstream %s" % meaning, + "after %.1f sec" % elapsed] + if code != builder.SUCCESS: + text += self._getFullnames() + return text + + def _getTimeoutStatusText(self): + return [self.name+":", "timed out", "(%.1f sec)" % self.timeout] + + def start(self): + self.step_status.setText(self._getBlockingStatusText()) + + if self.timeout is not None: + self._timer = reactor.callLater(self.timeout, self._timeoutExpired) + + self._log("searching for upstream build steps") + botmaster = self.build.slavebuilder.slave.parent + errors = [] # list of strings + for (builderName, stepName) in self.upstreamSteps: + buildStatus = stepStatus = None + try: + buildStatus = self._getBuildStatus(botmaster, builderName) + if buildStatus is not None: + stepStatus = self._getStepStatus(buildStatus, stepName) + except BadStepError, err: + errors.append(err.message) + if stepStatus is not None: + # Make sure newly-discovered blocking steps are all + # added to _blocking_steps before we subscribe to their + # "finish" events! + self._blocking_steps.append(stepStatus) + + if len(errors) == 1: + raise BadStepError(errors[0]) + elif len(errors) > 1: + raise BadStepError("multiple errors:\n" + "\n".join(errors)) + + self._log("will block on %d upstream build steps: %r", + len(self._blocking_steps), self._blocking_steps) + if self._blocking_builders: + self._log("will also block on %d builders starting a build: %r", + len(self._blocking_builders), self._blocking_builders) + + # Now we can register with each blocking step (BuildStepStatus + # objects, actually) that we want a callback when the step + # finishes. Need to iterate over a copy of _blocking_steps + # because it can be modified while we iterate: if any upstream + # step is already finished, the _upstreamStepFinished() callback + # will be called immediately. + for stepStatus in self._blocking_steps[:]: + self._awaitStepFinished(stepStatus) + self._log("after registering for each upstream build step, " + "_blocking_steps = %r", + self._blocking_steps) + + # Subscribe to each builder that we're waiting on to start. + for bs in self._blocking_builders: + bs.subscribe(BuilderStatusReceiver(self, bs)) + + def _awaitStepFinished(self, stepStatus): + # N.B. this will callback *immediately* (i.e. even before we + # relinquish control to the reactor) if the upstream step in + # question has already finished. + d = stepStatus.waitUntilFinished() + d.addCallback(self._upstreamStepFinished) + + def _timeoutExpired(self): + # Hmmm: this step has failed. But it is still subscribed to + # various upstream events, so if they happen despite this + # timeout, various callbacks in this object will still be + # called. This could be confusing and is definitely a bit + # untidy: probably we should unsubscribe from all those various + # events. Even better if we had a test case to ensure that we + # really do. + self._log("timeout (%.1f sec) expired", self.timeout) + self.step_status.setColor("red") + self.step_status.setText(self._getTimeoutStatusText()) + self.finished(builder.FAILURE) + self._timed_out = True + + def _upstreamStepFinished(self, stepStatus): + assert isinstance(stepStatus, buildstep.BuildStepStatus) + self._log("upstream build step %s:%s finished; results=%r", + stepStatus.getBuild().builder.getName(), + stepStatus.getName(), + stepStatus.getResults()) + + if self._timed_out: + # don't care about upstream steps: just clean up and get out + self._blocking_steps.remove(stepStatus) + return + + (code, text) = stepStatus.getResults() + if code != builder.SUCCESS and self._overall_code == builder.SUCCESS: + # first non-SUCCESS result wins + self._overall_code = code + self._overall_text.extend(text) + self._log("now _overall_code=%r, _overall_text=%r", + self._overall_code, self._overall_text) + + self._blocking_steps.remove(stepStatus) + self._checkFinished() + + def _upstreamBuildStarted(self, builderStatus, buildStatus, receiver): + assert isinstance(builderStatus, builder.BuilderStatus) + self._log("builder %r (%r) started a build; buildStatus=%r", + builderStatus, builderStatus.getName(), buildStatus) + + myBuildStatus = self.build.getStatus() + if not self.buildsMatch(myBuildStatus, buildStatus): + self._log("but the just-started build does not match: " + "ignoring it") + return + + builderStatus.unsubscribe(receiver) + + # Need to accumulate newly-discovered steps separately, so we + # can add them to _blocking_steps en masse before subscribing to + # their "finish" events. + new_blocking_steps = [] + for (builderName, stepName) in self.upstreamSteps: + if builderName == builderStatus.getName(): + try: + stepStatus = self._getStepStatus(buildStatus, stepName) + except BadStepError: + self.failed(failure.Failure()) + #log.err() + #self._overall_code = builder.EXCEPTION + #self._overall_text.append(str(err)) + else: + new_blocking_steps.append(stepStatus) + + self._blocking_steps.extend(new_blocking_steps) + for stepStatus in new_blocking_steps: + self._awaitStepFinished(stepStatus) + + self._blocking_builders.remove(builderStatus) + self._checkFinished() + + def _checkFinished(self): + if self.step_status.isFinished(): + # this can happen if _upstreamBuildStarted() catches BadStepError + # and fails the step + self._log("_checkFinished: already finished, so nothing to do here") + return + + self._log("_checkFinished: _blocking_steps=%r, _blocking_builders=%r", + self._blocking_steps, self._blocking_builders) + + if not self._blocking_steps and not self._blocking_builders: + if self.timeout: + self._timer.cancel() + + self.finished(self._overall_code) + self.step_status.setText2(self._overall_text) + (start, finish) = self.step_status.getTimes() + self.step_status.setText( + self._getFinishStatusText(self._overall_code, finish - start)) + +class BuilderStatusReceiver: + def __init__(self, blocker, builderStatus): + # the Blocker step that wants to find out when a Builder starts + # a Build + self.blocker = blocker + self.builderStatus = builderStatus + + def builderChangedState(self, *args): + pass + + def buildStarted(self, name, buildStatus): + log.msg("BuilderStatusReceiver: " + "apparently, builder %r has started build %r" + % (name, buildStatus)) + self.blocker._upstreamBuildStarted(self.builderStatus, buildStatus, self) + + def buildFinished(self, *args): + pass diff -N -r -u buildbot-0.8.14/buildbot/steps/vstudio.py buildbot/steps/vstudio.py --- buildbot-0.8.14/buildbot/steps/vstudio.py 2016-07-15 13:42:03.000000000 +0200 +++ buildbot/steps/vstudio.py 2016-11-27 06:30:46.000000000 +0100 @@ -198,27 +198,27 @@ class VC6(VisualStudio): - default_installdir = 'C:\\Program Files\\Microsoft Visual Studio' + default_installdir = r'C:\Program Files\Microsoft Visual Studio' def setupEnvironment(self, cmd): VisualStudio.setupEnvironment(self, cmd) # Root of Visual Developer Studio Common files. - VSCommonDir = self.installdir + '\\Common' - MSVCDir = self.installdir + '\\VC98' - MSDevDir = VSCommonDir + '\\msdev98' - - addEnvPath(cmd.args['env'], "PATH", MSDevDir + '\\BIN') - addEnvPath(cmd.args['env'], "PATH", MSVCDir + '\\BIN') - addEnvPath(cmd.args['env'], "PATH", VSCommonDir + '\\TOOLS\\WINNT') - addEnvPath(cmd.args['env'], "PATH", VSCommonDir + '\\TOOLS') - - addEnvPath(cmd.args['env'], "INCLUDE", MSVCDir + '\\INCLUDE') - addEnvPath(cmd.args['env'], "INCLUDE", MSVCDir + '\\ATL\\INCLUDE') - addEnvPath(cmd.args['env'], "INCLUDE", MSVCDir + '\\MFC\\INCLUDE') + VSCommonDir = self.installdir + r'\Common' + MSVCDir = self.installdir + r'\VC98' + MSDevDir = VSCommonDir + r'\msdev98' + + addEnvPath(cmd.args['env'], "PATH", MSDevDir + r'\BIN') + addEnvPath(cmd.args['env'], "PATH", MSVCDir + r'\BIN') + addEnvPath(cmd.args['env'], "PATH", VSCommonDir + r'\TOOLS\WINNT') + addEnvPath(cmd.args['env'], "PATH", VSCommonDir + r'\TOOLS') + + addEnvPath(cmd.args['env'], "INCLUDE", MSVCDir + r'\INCLUDE') + addEnvPath(cmd.args['env'], "INCLUDE", MSVCDir + r'\ATL\INCLUDE') + addEnvPath(cmd.args['env'], "INCLUDE", MSVCDir + r'\MFC\INCLUDE') - addEnvPath(cmd.args['env'], "LIB", MSVCDir + '\\LIB') - addEnvPath(cmd.args['env'], "LIB", MSVCDir + '\\MFC\\LIB') + addEnvPath(cmd.args['env'], "LIB", MSVCDir + r'\LIB') + addEnvPath(cmd.args['env'], "LIB", MSVCDir + r'\MFC\LIB') def start(self): command = ["msdev"] @@ -241,29 +241,29 @@ class VC7(VisualStudio): - default_installdir = 'C:\\Program Files\\Microsoft Visual Studio .NET 2003' + default_installdir = r'C:\Program Files\Microsoft Visual Studio .NET 2003' def setupEnvironment(self, cmd): VisualStudio.setupEnvironment(self, cmd) - VSInstallDir = self.installdir + '\\Common7\\IDE' + VSInstallDir = self.installdir + r'\Common7\IDE' VCInstallDir = self.installdir - MSVCDir = self.installdir + '\\VC7' + MSVCDir = self.installdir + r'\VC7' addEnvPath(cmd.args['env'], "PATH", VSInstallDir) - addEnvPath(cmd.args['env'], "PATH", MSVCDir + '\\BIN') - addEnvPath(cmd.args['env'], "PATH", VCInstallDir + '\\Common7\\Tools') - addEnvPath(cmd.args['env'], "PATH", VCInstallDir + '\\Common7\\Tools\\bin') - - addEnvPath(cmd.args['env'], "INCLUDE", MSVCDir + '\\INCLUDE') - addEnvPath(cmd.args['env'], "INCLUDE", MSVCDir + '\\ATLMFC\\INCLUDE') - addEnvPath(cmd.args['env'], "INCLUDE", MSVCDir + '\\PlatformSDK\\include') - addEnvPath(cmd.args['env'], "INCLUDE", VCInstallDir + '\\SDK\\v1.1\\include') - - addEnvPath(cmd.args['env'], "LIB", MSVCDir + '\\LIB') - addEnvPath(cmd.args['env'], "LIB", MSVCDir + '\\ATLMFC\\LIB') - addEnvPath(cmd.args['env'], "LIB", MSVCDir + '\\PlatformSDK\\lib') - addEnvPath(cmd.args['env'], "LIB", VCInstallDir + '\\SDK\\v1.1\\lib') + addEnvPath(cmd.args['env'], "PATH", MSVCDir + r'\BIN') + addEnvPath(cmd.args['env'], "PATH", VCInstallDir + r'\Common7\Tools') + addEnvPath(cmd.args['env'], "PATH", VCInstallDir + r'\Common7\Tools\bin') + + addEnvPath(cmd.args['env'], "INCLUDE", MSVCDir + r'\INCLUDE') + addEnvPath(cmd.args['env'], "INCLUDE", MSVCDir + r'\ATLMFC\INCLUDE') + addEnvPath(cmd.args['env'], "INCLUDE", MSVCDir + r'\PlatformSDK\include') + addEnvPath(cmd.args['env'], "INCLUDE", VCInstallDir + r'\SDK\v1.1\include') + + addEnvPath(cmd.args['env'], "LIB", MSVCDir + r'\LIB') + addEnvPath(cmd.args['env'], "LIB", MSVCDir + r'\ATLMFC\LIB') + addEnvPath(cmd.args['env'], "LIB", MSVCDir + r'\PlatformSDK\lib') + addEnvPath(cmd.args['env'], "LIB", VCInstallDir + r'\SDK\v1.1\lib') def start(self): command = ["devenv.com"] @@ -291,7 +291,7 @@ # Our ones arch = None - default_installdir = 'C:\\Program Files\\Microsoft Visual Studio 8' + default_installdir = r'C:\Program Files\Microsoft Visual Studio 8' renderables = ['arch'] @@ -305,30 +305,30 @@ VisualStudio.setupEnvironment(self, cmd) VSInstallDir = self.installdir - VCInstallDir = self.installdir + '\\VC' + VCInstallDir = self.installdir + r'\VC' - addEnvPath(cmd.args['env'], "PATH", VSInstallDir + '\\Common7\\IDE') + addEnvPath(cmd.args['env'], "PATH", VSInstallDir + r'\Common7\IDE') if self.arch == "x64": - addEnvPath(cmd.args['env'], "PATH", VCInstallDir + '\\BIN\\x86_amd64') - addEnvPath(cmd.args['env'], "PATH", VCInstallDir + '\\BIN') - addEnvPath(cmd.args['env'], "PATH", VSInstallDir + '\\Common7\\Tools') - addEnvPath(cmd.args['env'], "PATH", VSInstallDir + '\\Common7\\Tools\\bin') - addEnvPath(cmd.args['env'], "PATH", VCInstallDir + '\\PlatformSDK\\bin') - addEnvPath(cmd.args['env'], "PATH", VSInstallDir + '\\SDK\\v2.0\\bin') - addEnvPath(cmd.args['env'], "PATH", VCInstallDir + '\\VCPackages') + addEnvPath(cmd.args['env'], "PATH", VCInstallDir + r'\BIN\x86_amd64') + addEnvPath(cmd.args['env'], "PATH", VCInstallDir + r'\BIN') + addEnvPath(cmd.args['env'], "PATH", VSInstallDir + r'\Common7\Tools') + addEnvPath(cmd.args['env'], "PATH", VSInstallDir + r'\Common7\Tools\bin') + addEnvPath(cmd.args['env'], "PATH", VCInstallDir + r'\PlatformSDK\bin') + addEnvPath(cmd.args['env'], "PATH", VSInstallDir + r'\SDK\v2.0\bin') + addEnvPath(cmd.args['env'], "PATH", VCInstallDir + r'\VCPackages') addEnvPath(cmd.args['env'], "PATH", r'${PATH}') - addEnvPath(cmd.args['env'], "INCLUDE", VCInstallDir + '\\INCLUDE') - addEnvPath(cmd.args['env'], "INCLUDE", VCInstallDir + '\\ATLMFC\\include') - addEnvPath(cmd.args['env'], "INCLUDE", VCInstallDir + '\\PlatformSDK\\include') + addEnvPath(cmd.args['env'], "INCLUDE", VCInstallDir + r'\INCLUDE') + addEnvPath(cmd.args['env'], "INCLUDE", VCInstallDir + r'\ATLMFC\include') + addEnvPath(cmd.args['env'], "INCLUDE", VCInstallDir + r'\PlatformSDK\include') archsuffix = '' if self.arch == "x64": - archsuffix = '\\amd64' - addEnvPath(cmd.args['env'], "LIB", VCInstallDir + '\\LIB' + archsuffix) - addEnvPath(cmd.args['env'], "LIB", VCInstallDir + '\\ATLMFC\\LIB' + archsuffix) - addEnvPath(cmd.args['env'], "LIB", VCInstallDir + '\\PlatformSDK\\lib' + archsuffix) - addEnvPath(cmd.args['env'], "LIB", VSInstallDir + '\\SDK\\v2.0\\lib' + archsuffix) + archsuffix = r'\amd64' + addEnvPath(cmd.args['env'], "LIB", VCInstallDir + r'\LIB' + archsuffix) + addEnvPath(cmd.args['env'], "LIB", VCInstallDir + r'\ATLMFC\LIB' + archsuffix) + addEnvPath(cmd.args['env'], "LIB", VCInstallDir + r'\PlatformSDK\lib' + archsuffix) + addEnvPath(cmd.args['env'], "LIB", VSInstallDir + r'\SDK\v2.0\lib' + archsuffix) # alias VC8 as VS2005 VS2005 = VC8 @@ -354,38 +354,42 @@ self.setCommand(command) return VisualStudio.start(self) -# Add first support for VC9 (Same as VC8, with a different installdir) - +# Add first support for VC9 (Same as VC8, with a different installdir) class VC9(VC8): - default_installdir = 'C:\\Program Files\\Microsoft Visual Studio 9.0' + default_installdir = r'C:\Program Files\Microsoft Visual Studio 9.0' VS2008 = VC9 -# VC10 doesn't look like it needs extra stuff. - +# VC10 doesn't look like it needs extra stuff. class VC10(VC9): - default_installdir = 'C:\\Program Files\\Microsoft Visual Studio 10.0' + default_installdir = r'C:\Program Files\Microsoft Visual Studio 10.0' VS2010 = VC10 -# VC11 doesn't look like it needs extra stuff. - +# VC11 doesn't look like it needs extra stuff. class VC11(VC10): - default_installdir = 'C:\\Program Files\\Microsoft Visual Studio 11.0' + default_installdir = r'C:\Program Files\Microsoft Visual Studio 11.0' VS2012 = VC11 # VC12 doesn't look like it needs extra stuff. class VC12(VC11): - default_installdir = 'C:\\Program Files\\Microsoft Visual Studio 12.0' + default_installdir = r'C:\Program Files\Microsoft Visual Studio 12.0' VS2013 = VC12 +# VC14 doesn't look like it needs extra stuff. +class VC14(VC12): + default_installdir = r'C:\Program Files (x86)\Microsoft Visual Studio 14.0' + +VS2015 = VC14 + + class MsBuild4(VisualStudio): platform = None vcenv_bat = r"${VS110COMNTOOLS}..\..\VC\vcvarsall.bat" @@ -430,3 +434,7 @@ class MsBuild12(MsBuild4): vcenv_bat = r"${VS120COMNTOOLS}..\..\VC\vcvarsall.bat" + + +class MsBuild14(MsBuild4): + vcenv_bat = r"${VS140COMNTOOLS}..\..\VC\vcvarsall.bat" diff -N -r -u buildbot-0.8.14/buildbot/test/unit/test_buildslave.py buildbot/test/unit/test_buildslave.py --- buildbot-0.8.14/buildbot/test/unit/test_buildslave.py 1970-01-01 01:00:00.000000000 +0100 +++ buildbot/test/unit/test_buildslave.py 2012-06-24 22:30:04.000000000 +0200 @@ -0,0 +1,251 @@ +# This file is part of Buildbot. Buildbot is free software: you can +# redistribute it and/or modify it under the terms of the GNU General Public +# License as published by the Free Software Foundation, version 2. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +# details. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, write to the Free Software Foundation, Inc., 51 +# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Copyright Buildbot Team Members + +import mock +from twisted.trial import unittest +from twisted.internet import defer +from buildbot import buildslave, config, locks +from buildbot.test.fake import fakemaster, pbmanager +from buildbot.test.fake.botmaster import FakeBotMaster + +class TestAbstractBuildSlave(unittest.TestCase): + + class ConcreteBuildSlave(buildslave.AbstractBuildSlave): + pass + + def test_constructor_minimal(self): + bs = self.ConcreteBuildSlave('bot', 'pass') + self.assertEqual(bs.slavename, 'bot') + self.assertEqual(bs.password, 'pass') + self.assertEqual(bs.max_builds, None) + self.assertEqual(bs.notify_on_missing, []) + self.assertEqual(bs.missing_timeout, 3600) + self.assertEqual(bs.properties.getProperty('slavename'), 'bot') + self.assertEqual(bs.access, []) + self.assertEqual(bs.keepalive_interval, 3600) + + def test_constructor_full(self): + lock1, lock2 = mock.Mock(name='lock1'), mock.Mock(name='lock2') + bs = self.ConcreteBuildSlave('bot', 'pass', + max_builds=2, + notify_on_missing=['me@me.com'], + missing_timeout=120, + properties={'a':'b'}, + locks=[lock1, lock2], + keepalive_interval=60) + self.assertEqual(bs.max_builds, 2) + self.assertEqual(bs.notify_on_missing, ['me@me.com']) + self.assertEqual(bs.missing_timeout, 120) + self.assertEqual(bs.properties.getProperty('a'), 'b') + self.assertEqual(bs.access, [lock1, lock2]) + self.assertEqual(bs.keepalive_interval, 60) + + def test_constructor_notify_on_missing_not_list(self): + bs = self.ConcreteBuildSlave('bot', 'pass', + notify_on_missing='foo@foo.com') + # turned into a list: + self.assertEqual(bs.notify_on_missing, ['foo@foo.com']) + + def test_constructor_notify_on_missing_not_string(self): + self.assertRaises(config.ConfigErrors, lambda : + self.ConcreteBuildSlave('bot', 'pass', + notify_on_missing=['a@b.com', 13])) + + @defer.deferredGenerator + def do_test_reconfigService(self, old, old_port, new, new_port): + master = self.master = fakemaster.make_master() + old.master = master + if old_port: + self.old_registration = old.registration = \ + pbmanager.FakeRegistration(master.pbmanager, old_port, old.slavename) + old.registered_port = old_port + old.missing_timer = mock.Mock(name='missing_timer') + old.startService() + + new_config = mock.Mock() + new_config.slavePortnum = new_port + new_config.slaves = [ new ] + + wfd = defer.waitForDeferred( + old.reconfigService(new_config)) + yield wfd + wfd.getResult() + + @defer.deferredGenerator + def test_reconfigService_attrs(self): + old = self.ConcreteBuildSlave('bot', 'pass', + max_builds=2, + notify_on_missing=['me@me.com'], + missing_timeout=120, + properties={'a':'b'}, + keepalive_interval=60) + new = self.ConcreteBuildSlave('bot', 'pass', + max_builds=3, + notify_on_missing=['her@me.com'], + missing_timeout=121, + properties={'a':'c'}, + keepalive_interval=61) + + old.updateSlave = mock.Mock(side_effect=lambda : defer.succeed(None)) + + wfd = defer.waitForDeferred( + self.do_test_reconfigService(old, 'tcp:1234', new, 'tcp:1234')) + yield wfd + wfd.getResult() + + self.assertEqual(old.max_builds, 3) + self.assertEqual(old.notify_on_missing, ['her@me.com']) + self.assertEqual(old.missing_timeout, 121) + self.assertEqual(old.properties.getProperty('a'), 'c') + self.assertEqual(old.keepalive_interval, 61) + self.assertEqual(self.master.pbmanager._registrations, []) + self.assertTrue(old.updateSlave.called) + + @defer.deferredGenerator + def test_reconfigService_has_properties(self): + old = self.ConcreteBuildSlave('bot', 'pass') + + wfd = defer.waitForDeferred( + self.do_test_reconfigService(old, 'tcp:1234', old, 'tcp:1234')) + yield wfd + wfd.getResult() + + self.assertTrue(old.properties.getProperty('slavename'), 'bot') + + @defer.deferredGenerator + def test_reconfigService_initial_registration(self): + old = self.ConcreteBuildSlave('bot', 'pass') + + wfd = defer.waitForDeferred( + self.do_test_reconfigService(old, None, old, 'tcp:1234')) + yield wfd + wfd.getResult() + + self.assertEqual(self.master.pbmanager._registrations, [('tcp:1234', 'bot', 'pass')]) + + @defer.deferredGenerator + def test_reconfigService_reregister_password(self): + old = self.ConcreteBuildSlave('bot', 'pass') + new = self.ConcreteBuildSlave('bot', 'newpass') + + wfd = defer.waitForDeferred( + self.do_test_reconfigService(old, 'tcp:1234', new, 'tcp:1234')) + yield wfd + wfd.getResult() + + self.assertEqual(old.password, 'newpass') + self.assertEqual(self.master.pbmanager._unregistrations, [('tcp:1234', 'bot')]) + self.assertEqual(self.master.pbmanager._registrations, [('tcp:1234', 'bot', 'newpass')]) + + @defer.deferredGenerator + def test_reconfigService_reregister_port(self): + old = self.ConcreteBuildSlave('bot', 'pass') + new = self.ConcreteBuildSlave('bot', 'pass') + + wfd = defer.waitForDeferred( + self.do_test_reconfigService(old, 'tcp:1234', new, 'tcp:5678')) + yield wfd + wfd.getResult() + + self.assertEqual(self.master.pbmanager._unregistrations, [('tcp:1234', 'bot')]) + self.assertEqual(self.master.pbmanager._registrations, [('tcp:5678', 'bot', 'pass')]) + + @defer.inlineCallbacks + def test_stopService(self): + master = self.master = fakemaster.make_master() + slave = self.ConcreteBuildSlave('bot', 'pass') + slave.master = master + slave.startService() + + config = mock.Mock() + config.slavePortnum = "tcp:1234" + config.slaves = [ slave ] + + yield slave.reconfigService(config) + yield slave.stopService() + + self.assertEqual(self.master.pbmanager._unregistrations, [('tcp:1234', 'bot')]) + self.assertEqual(self.master.pbmanager._registrations, [('tcp:1234', 'bot', 'pass')]) + + # FIXME: Test that reconfig properly deals with + # 1) locks + # 2) telling slave about builder + # 3) missing timer + # in both the initial config and a reconfiguration. + + def test_startMissingTimer_no_parent(self): + bs = self.ConcreteBuildSlave('bot', 'pass', + notify_on_missing=['abc'], + missing_timeout=10) + bs.startMissingTimer() + self.assertEqual(bs.missing_timer, None) + + def test_startMissingTimer_no_timeout(self): + bs = self.ConcreteBuildSlave('bot', 'pass', + notify_on_missing=['abc'], + missing_timeout=0) + bs.parent = mock.Mock() + bs.startMissingTimer() + self.assertEqual(bs.missing_timer, None) + + def test_startMissingTimer_no_notify(self): + bs = self.ConcreteBuildSlave('bot', 'pass', + missing_timeout=3600) + bs.parent = mock.Mock() + bs.startMissingTimer() + self.assertEqual(bs.missing_timer, None) + + def test_missing_timer(self): + bs = self.ConcreteBuildSlave('bot', 'pass', + notify_on_missing=['abc'], + missing_timeout=100) + bs.parent = mock.Mock() + bs.startMissingTimer() + self.assertNotEqual(bs.missing_timer, None) + bs.stopMissingTimer() + self.assertEqual(bs.missing_timer, None) + + def test_setServiceParent_started(self): + master = self.master = fakemaster.make_master() + botmaster = FakeBotMaster(master) + botmaster.startService() + bs = self.ConcreteBuildSlave('bot', 'pass') + bs.setServiceParent(botmaster) + self.assertEqual(bs.botmaster, botmaster) + self.assertEqual(bs.master, master) + + def test_setServiceParent_masterLocks(self): + """ + http://trac.buildbot.net/ticket/2278 + """ + master = self.master = fakemaster.make_master() + botmaster = FakeBotMaster(master) + botmaster.startService() + lock = locks.MasterLock('masterlock') + bs = self.ConcreteBuildSlave('bot', 'pass', locks = [lock]) + bs.setServiceParent(botmaster) + + def test_setServiceParent_slaveLocks(self): + """ + http://trac.buildbot.net/ticket/2278 + """ + master = self.master = fakemaster.make_master() + botmaster = FakeBotMaster(master) + botmaster.startService() + lock = locks.SlaveLock('lock') + bs = self.ConcreteBuildSlave('bot', 'pass', locks = [lock]) + bs.setServiceParent(botmaster) + + test_setServiceParent_slaveLocks.todo = "SlaveLock not support for slave lock" diff -N -r -u buildbot-0.8.14/buildbot/test/unit/test_process_botmaster_BuildRequestDistributor.py buildbot/test/unit/test_process_botmaster_BuildRequestDistributor.py --- buildbot-0.8.14/buildbot/test/unit/test_process_botmaster_BuildRequestDistributor.py 1970-01-01 01:00:00.000000000 +0100 +++ buildbot/test/unit/test_process_botmaster_BuildRequestDistributor.py 2012-06-24 22:30:04.000000000 +0200 @@ -0,0 +1,255 @@ +# This file is part of Buildbot. Buildbot is free software: you can +# redistribute it and/or modify it under the terms of the GNU General Public +# License as published by the Free Software Foundation, version 2. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +# details. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, write to the Free Software Foundation, Inc., 51 +# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Copyright Buildbot Team Members + +import mock +from twisted.trial import unittest +from twisted.internet import defer, reactor +from twisted.python import failure +from buildbot.test.util import compat +from buildbot.process import botmaster +from buildbot.util import epoch2datetime + +class Test(unittest.TestCase): + + def setUp(self): + self.botmaster = mock.Mock(name='botmaster') + self.botmaster.builders = {} + def prioritizeBuilders(master, builders): + # simple sort-by-name by default + return sorted(builders, lambda b1,b2 : cmp(b1.name, b2.name)) + self.master = self.botmaster.master = mock.Mock(name='master') + self.master.config.prioritizeBuilders = prioritizeBuilders + self.brd = botmaster.BuildRequestDistributor(self.botmaster) + self.brd.startService() + + # TODO: this is a terrible way to detect the "end" of the test - + # it regularly completes too early after a simple modification of + # a test. Is there a better way? + self.quiet_deferred = defer.Deferred() + def _quiet(): + if self.quiet_deferred: + d, self.quiet_deferred = self.quiet_deferred, None + d.callback(None) + else: + self.fail("loop has already gone quiet once") + self.brd._quiet = _quiet + + self.maybeStartBuild_calls = [] + self.builders = {} + + def tearDown(self): + if self.brd.running: + return self.brd.stopService() + + def addBuilders(self, names): + for name in names: + bldr = mock.Mock(name=name) + self.botmaster.builders[name] = bldr + self.builders[name] = bldr + def maybeStartBuild(n=name): + self.maybeStartBuild_calls.append(n) + d = defer.Deferred() + reactor.callLater(0, d.callback, None) + return d + bldr.maybeStartBuild = maybeStartBuild + bldr.name = name + + def removeBuilder(self, name): + del self.builders[name] + del self.botmaster.builders[name] + + # tests + + def test_maybeStartBuildsOn_simple(self): + self.addBuilders(['bldr1']) + self.brd.maybeStartBuildsOn(['bldr1']) + def check(_): + self.assertEqual(self.maybeStartBuild_calls, ['bldr1']) + self.quiet_deferred.addCallback(check) + return self.quiet_deferred + + def test_maybeStartBuildsOn_parallel(self): + # test 15 "parallel" invocations of maybeStartBuildsOn, with a + # _sortBuilders that takes a while. This is a regression test for bug + # #1979. + builders = ['bldr%02d' % i for i in xrange(15) ] + + def slow_sorter(master, bldrs): + bldrs.sort(lambda b1, b2 : cmp(b1.name, b2.name)) + d = defer.Deferred() + reactor.callLater(0, d.callback, bldrs) + def done(_): + return _ + d.addCallback(done) + return d + self.master.config.prioritizeBuilders = slow_sorter + + self.addBuilders(builders) + for bldr in builders: + self.brd.maybeStartBuildsOn([bldr]) + def check(_): + self.assertEqual(self.maybeStartBuild_calls, builders) + self.quiet_deferred.addCallback(check) + return self.quiet_deferred + + @compat.usesFlushLoggedErrors + def test_maybeStartBuildsOn_exception(self): + self.addBuilders(['bldr1']) + + def _callABuilder(n): + # fail slowly, so that the activity loop doesn't go quiet too soon + d = defer.Deferred() + reactor.callLater(0, + d.errback, failure.Failure(RuntimeError("oh noes"))) + return d + self.brd._callABuilder = _callABuilder + + self.brd.maybeStartBuildsOn(['bldr1']) + def check(_): + self.assertEqual(len(self.flushLoggedErrors(RuntimeError)), 1) + self.quiet_deferred.addCallback(check) + return self.quiet_deferred + + def test_maybeStartBuildsOn_collapsing(self): + self.addBuilders(['bldr1', 'bldr2', 'bldr3']) + self.brd.maybeStartBuildsOn(['bldr3']) + self.brd.maybeStartBuildsOn(['bldr2', 'bldr1']) + self.brd.maybeStartBuildsOn(['bldr4']) # should be ignored + self.brd.maybeStartBuildsOn(['bldr2']) # already queued - ignored + self.brd.maybeStartBuildsOn(['bldr3', 'bldr2']) + def check(_): + # bldr3 gets invoked twice, since it's considered to have started + # already when the first call to maybeStartBuildsOn returns + self.assertEqual(self.maybeStartBuild_calls, + ['bldr3', 'bldr1', 'bldr2', 'bldr3']) + self.quiet_deferred.addCallback(check) + return self.quiet_deferred + + def test_maybeStartBuildsOn_builders_missing(self): + self.addBuilders(['bldr1', 'bldr2', 'bldr3']) + self.brd.maybeStartBuildsOn(['bldr1', 'bldr2', 'bldr3']) + # bldr1 is already run, so surreptitiously remove the other + # two - nothing should crash, but the builders should not run + self.removeBuilder('bldr2') + self.removeBuilder('bldr3') + def check(_): + self.assertEqual(self.maybeStartBuild_calls, ['bldr1']) + self.quiet_deferred.addCallback(check) + return self.quiet_deferred + + def do_test_sortBuilders(self, prioritizeBuilders, oldestRequestTimes, + expected, returnDeferred=False): + self.addBuilders(oldestRequestTimes.keys()) + self.master.config.prioritizeBuilders = prioritizeBuilders + + def mklambda(t): # work around variable-binding issues + if returnDeferred: + return lambda : defer.succeed(t) + else: + return lambda : t + + for n, t in oldestRequestTimes.iteritems(): + if t is not None: + t = epoch2datetime(t) + self.builders[n].getOldestRequestTime = mklambda(t) + + d = self.brd._sortBuilders(oldestRequestTimes.keys()) + def check(result): + self.assertEqual(result, expected) + d.addCallback(check) + return d + + def test_sortBuilders_default_sync(self): + return self.do_test_sortBuilders(None, # use the default sort + dict(bldr1=777, bldr2=999, bldr3=888), + ['bldr1', 'bldr3', 'bldr2']) + + def test_sortBuilders_default_asyn(self): + return self.do_test_sortBuilders(None, # use the default sort + dict(bldr1=777, bldr2=999, bldr3=888), + ['bldr1', 'bldr3', 'bldr2'], + returnDeferred=True) + + def test_sortBuilders_default_None(self): + return self.do_test_sortBuilders(None, # use the default sort + dict(bldr1=777, bldr2=None, bldr3=888), + ['bldr1', 'bldr3', 'bldr2']) + + def test_sortBuilders_custom(self): + def prioritizeBuilders(master, builders): + self.assertIdentical(master, self.master) + return sorted(builders, key=lambda b : b.name) + + return self.do_test_sortBuilders(prioritizeBuilders, + dict(bldr1=1, bldr2=1, bldr3=1), + ['bldr1', 'bldr2', 'bldr3']) + + def test_sortBuilders_custom_async(self): + def prioritizeBuilders(master, builders): + self.assertIdentical(master, self.master) + return defer.succeed(sorted(builders, key=lambda b : b.name)) + + return self.do_test_sortBuilders(prioritizeBuilders, + dict(bldr1=1, bldr2=1, bldr3=1), + ['bldr1', 'bldr2', 'bldr3']) + + @compat.usesFlushLoggedErrors + def test_sortBuilders_custom_exception(self): + self.addBuilders(['x', 'y']) + def fail(m, b): + raise RuntimeError("oh noes") + self.master.config.prioritizeBuilders = fail + + # expect to get the builders back in the same order in the event of an + # exception + d = self.brd._sortBuilders(['y', 'x']) + def check(result): + self.assertEqual(result, ['y', 'x']) + + # and expect the exception to be logged + self.assertEqual(len(self.flushLoggedErrors(RuntimeError)), 1) + d.addCallback(check) + return d + + def test_stopService(self): + # check that stopService waits for a builder run to complete, but does not + # allow a subsequent run to start + self.addBuilders(['A', 'B']) + + # patch the maybeStartBuild method for A to stop the service and wait a + # beat, with some extra logging + def msb_stopNow(): + self.maybeStartBuild_calls.append('A') + stop_d = self.brd.stopService() + stop_d.addCallback(lambda _ : + self.maybeStartBuild_calls.append('(stopped)')) + + d = defer.Deferred() + def a_finished(): + self.maybeStartBuild_calls.append('A-finished') + d.callback(None) + reactor.callLater(0, a_finished) + return d + self.builders['A'].maybeStartBuild = msb_stopNow + + # start both builds; A should start and complete *before* the service stops, + # and B should not run. + self.brd.maybeStartBuildsOn(['A', 'B']) + + def check(_): + self.assertEqual(self.maybeStartBuild_calls, + ['A', 'A-finished', '(stopped)']) + self.quiet_deferred.addCallback(check) + return self.quiet_deferred diff -N -r -u buildbot-0.8.14/buildbot/test/unit/test_status_web_change_hook.py buildbot/test/unit/test_status_web_change_hook.py --- buildbot-0.8.14/buildbot/test/unit/test_status_web_change_hook.py 1970-01-01 01:00:00.000000000 +0100 +++ buildbot/test/unit/test_status_web_change_hook.py 2012-06-24 22:30:04.000000000 +0200 @@ -0,0 +1,168 @@ +# This file is part of Buildbot. Buildbot is free software: you can +# redistribute it and/or modify it under the terms of the GNU General Public +# License as published by the Free Software Foundation, version 2. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +# details. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, write to the Free Software Foundation, Inc., 51 +# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Copyright Buildbot Team Members + +from buildbot.status.web import change_hook +from buildbot.util import json +from buildbot.test.util import compat +from buildbot.test.fake.web import FakeRequest + +from twisted.trial import unittest + +class TestChangeHookUnconfigured(unittest.TestCase): + def setUp(self): + self.request = FakeRequest() + self.changeHook = change_hook.ChangeHookResource() + + # A bad URI should cause an exception inside check_hook. + # After writing the test, it became apparent this can't happen. + # I'll leave the test anyway + def testDialectReMatchFail(self): + self.request.uri = "/garbage/garbage" + self.request.method = "GET" + d = self.request.test_render(self.changeHook) + def check(ret): + expected = "URI doesn't match change_hook regex: /garbage/garbage" + self.assertEqual(self.request.written, expected) + self.request.setResponseCode.assert_called_with(400, expected) + d.addCallback(check) + return d + + def testUnkownDialect(self): + self.request.uri = "/change_hook/garbage" + self.request.method = "GET" + d = self.request.test_render(self.changeHook) + def check(ret): + expected = "The dialect specified, 'garbage', wasn't whitelisted in change_hook" + self.assertEqual(self.request.written, expected) + self.request.setResponseCode.assert_called_with(400, expected) + d.addCallback(check) + return d + + def testDefaultDialect(self): + self.request.uri = "/change_hook/" + self.request.method = "GET" + d = self.request.test_render(self.changeHook) + def check(ret): + expected = "The dialect specified, 'base', wasn't whitelisted in change_hook" + self.assertEqual(self.request.written, expected) + self.request.setResponseCode.assert_called_with(400, expected) + d.addCallback(check) + return d + +class TestChangeHookConfigured(unittest.TestCase): + def setUp(self): + self.request = FakeRequest() + self.changeHook = change_hook.ChangeHookResource(dialects={'base' : True}) + + def testDefaultDialectGetNullChange(self): + self.request.uri = "/change_hook/" + self.request.method = "GET" + d = self.request.test_render(self.changeHook) + def check_changes(r): + self.assertEquals(len(self.request.addedChanges), 1) + change = self.request.addedChanges[0] + self.assertEquals(change["category"], None) + self.assertEquals(len(change["files"]), 0) + self.assertEquals(change["repository"], None) + self.assertEquals(change["when"], None) + self.assertEquals(change["author"], None) + self.assertEquals(change["revision"], None) + self.assertEquals(change["comments"], None) + self.assertEquals(change["project"], None) + self.assertEquals(change["branch"], None) + self.assertEquals(change["revlink"], None) + self.assertEquals(len(change["properties"]), 0) + self.assertEquals(change["revision"], None) + d.addCallback(check_changes) + return d + + # Test 'base' hook with attributes. We should get a json string representing + # a Change object as a dictionary. All values show be set. + def testDefaultDialectWithChange(self): + self.request.uri = "/change_hook/" + self.request.method = "GET" + self.request.args = { "category" : ["mycat"], + "files" : [json.dumps(['file1', 'file2'])], + "repository" : ["myrepo"], + "when" : [1234], + "author" : ["Santa Claus"], + "number" : [2], + "comments" : ["a comment"], + "project" : ["a project"], + "at" : ["sometime"], + "branch" : ["a branch"], + "revlink" : ["a revlink"], + "properties" : [json.dumps( { "prop1" : "val1", "prop2" : "val2" })], + "revision" : [99] } + d = self.request.test_render(self.changeHook) + def check_changes(r): + self.assertEquals(len(self.request.addedChanges), 1) + change = self.request.addedChanges[0] + self.assertEquals(change["category"], "mycat") + self.assertEquals(change["repository"], "myrepo") + self.assertEquals(change["when"], 1234) + self.assertEquals(change["author"], "Santa Claus") + self.assertEquals(change["src"], None) + self.assertEquals(change["revision"], 99) + self.assertEquals(change["comments"], "a comment") + self.assertEquals(change["project"], "a project") + self.assertEquals(change["branch"], "a branch") + self.assertEquals(change["revlink"], "a revlink") + self.assertEquals(change['properties'], dict(prop1='val1', prop2='val2')) + self.assertEquals(change['files'], ['file1', 'file2']) + d.addCallback(check_changes) + return d + + @compat.usesFlushLoggedErrors + def testDefaultWithNoChange(self): + self.request = FakeRequest() + self.request.uri = "/change_hook/" + self.request.method = "GET" + def namedModuleMock(name): + if name == 'buildbot.status.web.hooks.base': + class mock_hook_module(object): + def getChanges(self, request, options): + raise AssertionError + return mock_hook_module() + self.patch(change_hook, "namedModule", namedModuleMock) + + d = self.request.test_render(self.changeHook) + def check_changes(r): + expected = "Error processing changes." + self.assertEquals(len(self.request.addedChanges), 0) + self.assertEqual(self.request.written, expected) + self.request.setResponseCode.assert_called_with(500, expected) + self.assertEqual(len(self.flushLoggedErrors(AssertionError)), 1) + + d.addCallback(check_changes) + return d + +class TestChangeHookConfiguredBogus(unittest.TestCase): + def setUp(self): + self.request = FakeRequest() + self.changeHook = change_hook.ChangeHookResource(dialects={'garbage' : True}) + + @compat.usesFlushLoggedErrors + def testBogusDialect(self): + self.request.uri = "/change_hook/garbage" + self.request.method = "GET" + d = self.request.test_render(self.changeHook) + def check(ret): + expected = "Error processing changes." + self.assertEqual(self.request.written, expected) + self.request.setResponseCode.assert_called_with(500, expected) + self.assertEqual(len(self.flushLoggedErrors()), 1) + d.addCallback(check) + return d diff -N -r -u buildbot-0.8.14/buildbot/test/unit/test_status_web_change_hooks_github.py buildbot/test/unit/test_status_web_change_hooks_github.py --- buildbot-0.8.14/buildbot/test/unit/test_status_web_change_hooks_github.py 1970-01-01 01:00:00.000000000 +0100 +++ buildbot/test/unit/test_status_web_change_hooks_github.py 2012-06-24 22:30:04.000000000 +0200 @@ -0,0 +1,205 @@ +# This file is part of Buildbot. Buildbot is free software: you can +# redistribute it and/or modify it under the terms of the GNU General Public +# License as published by the Free Software Foundation, version 2. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +# details. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, write to the Free Software Foundation, Inc., 51 +# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Copyright Buildbot Team Members + +import buildbot.status.web.change_hook as change_hook +from buildbot.test.fake.web import FakeRequest +from buildbot.test.util import compat + +from twisted.trial import unittest + +# Sample GITHUB commit payload from http://help.github.com/post-receive-hooks/ +# Added "modfied" and "removed", and change email + +gitJsonPayload = """ +{ + "before": "5aef35982fb2d34e9d9d4502f6ede1072793222d", + "repository": { + "url": "http://github.com/defunkt/github", + "name": "github", + "description": "You're lookin' at it.", + "watchers": 5, + "forks": 2, + "private": 1, + "owner": { + "email": "fred@flinstone.org", + "name": "defunkt" + } + }, + "commits": [ + { + "id": "41a212ee83ca127e3c8cf465891ab7216a705f59", + "url": "http://github.com/defunkt/github/commit/41a212ee83ca127e3c8cf465891ab7216a705f59", + "author": { + "email": "fred@flinstone.org", + "name": "Fred Flinstone" + }, + "message": "okay i give in", + "timestamp": "2008-02-15T14:57:17-08:00", + "added": ["filepath.rb"] + }, + { + "id": "de8251ff97ee194a289832576287d6f8ad74e3d0", + "url": "http://github.com/defunkt/github/commit/de8251ff97ee194a289832576287d6f8ad74e3d0", + "author": { + "email": "fred@flinstone.org", + "name": "Fred Flinstone" + }, + "message": "update pricing a tad", + "timestamp": "2008-02-15T14:36:34-08:00", + "modified": ["modfile"], + "removed": ["removedFile"] + } + ], + "after": "de8251ff97ee194a289832576287d6f8ad74e3d0", + "ref": "refs/heads/master" +} +""" + +gitJsonPayloadNonBranch = """ +{ + "before": "5aef35982fb2d34e9d9d4502f6ede1072793222d", + "repository": { + "url": "http://github.com/defunkt/github", + "name": "github", + "description": "You're lookin' at it.", + "watchers": 5, + "forks": 2, + "private": 1, + "owner": { + "email": "fred@flinstone.org", + "name": "defunkt" + } + }, + "commits": [ + { + "id": "41a212ee83ca127e3c8cf465891ab7216a705f59", + "url": "http://github.com/defunkt/github/commit/41a212ee83ca127e3c8cf465891ab7216a705f59", + "author": { + "email": "fred@flinstone.org", + "name": "Fred Flinstone" + }, + "message": "okay i give in", + "timestamp": "2008-02-15T14:57:17-08:00", + "added": ["filepath.rb"] + } + ], + "after": "de8251ff97ee194a289832576287d6f8ad74e3d0", + "ref": "refs/garbage/master" +} +""" + +gitJsonPayloadEmpty = """ +{ + "before": "5aef35982fb2d34e9d9d4502f6ede1072793222d", + "repository": { + "url": "http://github.com/defunkt/github", + "name": "github", + "description": "You're lookin' at it.", + "watchers": 5, + "forks": 2, + "private": 1, + "owner": { + "email": "fred@flinstone.org", + "name": "defunkt" + } + }, + "commits": [ + ], + "after": "de8251ff97ee194a289832576287d6f8ad74e3d0", + "ref": "refs/heads/master" +} +""" +class TestChangeHookConfiguredWithGitChange(unittest.TestCase): + def setUp(self): + self.changeHook = change_hook.ChangeHookResource(dialects={'github' : True}) + + # Test 'base' hook with attributes. We should get a json string representing + # a Change object as a dictionary. All values show be set. + def testGitWithChange(self): + changeDict={"payload" : [gitJsonPayload]} + self.request = FakeRequest(changeDict) + self.request.uri = "/change_hook/github" + self.request.method = "GET" + d = self.request.test_render(self.changeHook) + def check_changes(r): + self.assertEquals(len(self.request.addedChanges), 2) + change = self.request.addedChanges[0] + + self.assertEquals(change['files'], ['filepath.rb']) + self.assertEquals(change["repository"], "http://github.com/defunkt/github") + self.assertEquals(change["when"], 1203116237) + self.assertEquals(change["who"], "Fred Flinstone ") + self.assertEquals(change["revision"], '41a212ee83ca127e3c8cf465891ab7216a705f59') + self.assertEquals(change["comments"], "okay i give in") + self.assertEquals(change["branch"], "master") + self.assertEquals(change["revlink"], "http://github.com/defunkt/github/commit/41a212ee83ca127e3c8cf465891ab7216a705f59") + + change = self.request.addedChanges[1] + self.assertEquals(change['files'], [ 'modfile', 'removedFile' ]) + self.assertEquals(change["repository"], "http://github.com/defunkt/github") + self.assertEquals(change["when"], 1203114994) + self.assertEquals(change["who"], "Fred Flinstone ") + self.assertEquals(change["src"], "git") + self.assertEquals(change["revision"], 'de8251ff97ee194a289832576287d6f8ad74e3d0') + self.assertEquals(change["comments"], "update pricing a tad") + self.assertEquals(change["branch"], "master") + self.assertEquals(change["revlink"], "http://github.com/defunkt/github/commit/de8251ff97ee194a289832576287d6f8ad74e3d0") + + d.addCallback(check_changes) + return d + + @compat.usesFlushLoggedErrors + def testGitWithNoJson(self): + self.request = FakeRequest() + self.request.uri = "/change_hook/github" + self.request.method = "GET" + d = self.request.test_render(self.changeHook) + def check_changes(r): + expected = "Error processing changes." + self.assertEquals(len(self.request.addedChanges), 0) + self.assertEqual(self.request.written, expected) + self.request.setResponseCode.assert_called_with(500, expected) + self.assertEqual(len(self.flushLoggedErrors()), 1) + + d.addCallback(check_changes) + return d + + def testGitWithNoChanges(self): + changeDict={"payload" : [gitJsonPayloadEmpty]} + self.request = FakeRequest(changeDict) + self.request.uri = "/change_hook/github" + self.request.method = "GET" + d = self.request.test_render(self.changeHook) + def check_changes(r): + expected = "no changes found" + self.assertEquals(len(self.request.addedChanges), 0) + self.assertEqual(self.request.written, expected) + + d.addCallback(check_changes) + return d + + def testGitWithNonBranchChanges(self): + changeDict={"payload" : [gitJsonPayloadNonBranch]} + self.request = FakeRequest(changeDict) + self.request.uri = "/change_hook/github" + self.request.method = "GET" + d = self.request.test_render(self.changeHook) + def check_changes(r): + expected = "no changes found" + self.assertEquals(len(self.request.addedChanges), 0) + self.assertEqual(self.request.written, expected) + + d.addCallback(check_changes) + return d diff -N -r -u buildbot-0.8.14/buildbot/test/unit/test_status_web_change_hooks_googlecode.py buildbot/test/unit/test_status_web_change_hooks_googlecode.py --- buildbot-0.8.14/buildbot/test/unit/test_status_web_change_hooks_googlecode.py 1970-01-01 01:00:00.000000000 +0100 +++ buildbot/test/unit/test_status_web_change_hooks_googlecode.py 2012-06-24 22:30:04.000000000 +0200 @@ -0,0 +1,90 @@ +# This file is part of Buildbot. Buildbot is free software: you can +# redistribute it and/or modify it under the terms of the GNU General Public +# License as published by the Free Software Foundation, version 2. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +# details. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, write to the Free Software Foundation, Inc., 51 +# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Copyright 2011 Louis Opter +# +# Written from the github change hook unit test + +import StringIO + +import buildbot.status.web.change_hook as change_hook +from buildbot.test.fake.web import FakeRequest + +from twisted.trial import unittest + +# Sample Google Code commit payload extracted from a Google Code test project +# { +# "repository_path": "https://code.google.com/p/webhook-test/", +# "project_name": "webhook-test", +# "revision_count": 1, +# "revisions": [ +# { +# "added": [], +# "parents": ["6574485e26a09a0e743e0745374056891d6a836a"], +# "author": "Louis Opter \\u003Clouis@lse.epitech.net\\u003E", +# "url": "http://webhook-test.googlecode.com/hg-history/68e5df283a8e751cdbf95516b20357b2c46f93d4/", +# "timestamp": 1324082130, +# "message": "Print a message", +# "path_count": 1, +# "removed": [], +# "modified": ["/CMakeLists.txt"], +# "revision": "68e5df283a8e751cdbf95516b20357b2c46f93d4" +# } +# ] +# } +googleCodeJsonBody = '{"repository_path":"https://code.google.com/p/webhook-test/","project_name":"webhook-test","revisions":[{"added":[],"parents":["6574485e26a09a0e743e0745374056891d6a836a"],"author":"Louis Opter \u003Clouis@lse.epitech.net\u003E","url":"http://webhook-test.googlecode.com/hg-history/68e5df283a8e751cdbf95516b20357b2c46f93d4/","timestamp":1324082130,"message":"Print a message","path_count":1,"removed":[],"modified":["/CMakeLists.txt"],"revision":"68e5df283a8e751cdbf95516b20357b2c46f93d4"}],"revision_count":1}' + +class TestChangeHookConfiguredWithGoogleCodeChange(unittest.TestCase): + def setUp(self): + self.request = FakeRequest() + # Google Code simply transmit the payload as an UTF-8 JSON body + self.request.content = StringIO.StringIO(googleCodeJsonBody) + self.request.received_headers = { + 'Google-Code-Project-Hosting-Hook-Hmac': '85910bf93ba5c266402d9328b0c7a856', + 'Content-Length': '509', + 'Accept-Encoding': 'gzip', + 'User-Agent': 'Google Code Project Hosting (+http://code.google.com/p/support/wiki/PostCommitWebHooks)', + 'Host': 'buildbot6-lopter.dotcloud.com:19457', + 'Content-Type': 'application/json; charset=UTF-8' + } + + self.changeHook = change_hook.ChangeHookResource(dialects={ + 'googlecode': { + 'secret_key': 'FSP3p-Ghdn4T0oqX', + 'branch': 'test' + } + }) + + # Test 'base' hook with attributes. We should get a json string representing + # a Change object as a dictionary. All values show be set. + def testGoogleCodeWithHgChange(self): + self.request.uri = "/change_hook/googlecode" + self.request.method = "GET" + d = self.request.test_render(self.changeHook) + def check_changes(r): + # Only one changeset has been submitted. + self.assertEquals(len(self.request.addedChanges), 1) + + # First changeset. + change = self.request.addedChanges[0] + self.assertEquals(change['files'], ['/CMakeLists.txt']) + self.assertEquals(change["repository"], "https://code.google.com/p/webhook-test/") + self.assertEquals(change["when"], 1324082130) + self.assertEquals(change["author"], "Louis Opter ") + self.assertEquals(change["revision"], '68e5df283a8e751cdbf95516b20357b2c46f93d4') + self.assertEquals(change["comments"], "Print a message") + self.assertEquals(change["branch"], "test") + self.assertEquals(change["revlink"], "http://webhook-test.googlecode.com/hg-history/68e5df283a8e751cdbf95516b20357b2c46f93d4/") + + d.addCallback(check_changes) + return d diff -N -r -u buildbot-0.8.14/buildbot/test/unit/test_steps_blocker.py buildbot/test/unit/test_steps_blocker.py --- buildbot-0.8.14/buildbot/test/unit/test_steps_blocker.py 1970-01-01 01:00:00.000000000 +0100 +++ buildbot/test/unit/test_steps_blocker.py 2012-06-24 22:30:04.000000000 +0200 @@ -0,0 +1,113 @@ +# This file is part of Buildbot. Buildbot is free software: you can +# redistribute it and/or modify it under the terms of the GNU General Public +# License as published by the Free Software Foundation, version 2. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +# details. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, write to the Free Software Foundation, Inc., 51 +# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Copyright Buildbot Team Members + +from twisted.trial import unittest + +from buildbot.status import builder +from buildbot.steps import blocker +from buildbot import config + +class TestBlockerTrivial(unittest.TestCase): + """ + Trivial test cases that don't require a whole BuildMaster/BotMaster/ + Builder/Build object graph. + """ + + def assertRaises(self, exception, callable, *args, **kwargs): + """ + Variation on default assertRaises() that takes either an exception class + or an exception object. For an exception object, it compares + str() values of the expected exception and the actual raised exception. + """ + if isinstance(exception, type(Exception)): # it's an exception class + unittest.TestCase.assertRaises( + self, exception, callable, *args, **kwargs) + elif isinstance(exception, Exception): # it's an exception object + exc_name = exception.__class__.__name__ + try: + callable(*args, **kwargs) + except exception.__class__, actual: + self.assertEquals( + str(exception), str(actual), + "expected %s: %r, but got %r" + % (exc_name, str(exception), str(actual))) + else: + self.fail("%s not raised" % exc_name) + else: + raise TypeError("'exception' must be an exception class " + "or exception object (not %r)" + % exception) + + def testConstructor(self): + # upstreamSteps must be supplied... + self.assertRaises(config.ConfigErrors, lambda : + blocker.Blocker()) + # ...and must be a non-empty list + self.assertRaises(config.ConfigErrors, lambda : + blocker.Blocker(upstreamSteps=[])) + + # builder name and step name do not matter to constructor + blocker.Blocker(upstreamSteps=[("b1", "s1"), ("b1", "s3")]) + + # test validation of idlePolicy arg + self.assertRaises(config.ConfigErrors, lambda : + blocker.Blocker(upstreamSteps=[('b', 's')], idlePolicy="foo")) + + def testFullnames(self): + bstep = blocker.Blocker(upstreamSteps=[("b1", "s1")]) + self.assertEqual(["(b1:s1)"], bstep._getFullnames()) + + bstep = blocker.Blocker(upstreamSteps=[("b1", "s1"), ("b1", "s3")]) + self.assertEqual(["(b1:s1,", "b1:s3)"], bstep._getFullnames()) + + bstep = blocker.Blocker(upstreamSteps=[("b1", "s1"), + ("b1", "s3"), + ("b2", "s1")]) + self.assertEqual(["(b1:s1,", "b1:s3,", "b2:s1)"], bstep._getFullnames()) + + def testStatusText(self): + bstep = blocker.Blocker( + name="block-something", + upstreamSteps=([("builder1", "step1"), ("builder3", "step4")]), + ) + + # A Blocker can be in various states, each of which has a + # distinct status text: + # 1) blocking, ie. waiting for upstream builders/steps + # 2) successfully done (upstream steps all succeeded) + # 3) failed (at least upstream step failed) + # 4) timed out (waited too long) + + self.assertEqual(["block-something:", + "blocking on", + "(builder1:step1,", + "builder3:step4)"], + bstep._getBlockingStatusText()) + self.assertEqual(["block-something:", + "upstream success", + "after 4.3 sec"], + bstep._getFinishStatusText(builder.SUCCESS, 4.3)) + self.assertEqual(["block-something:", + "upstream failure", + "after 11.0 sec", + "(builder1:step1,", + "builder3:step4)"], + bstep._getFinishStatusText(builder.FAILURE, 11.0)) + + bstep.timeout = 1.5 + self.assertEqual(["block-something:", + "timed out", + "(1.5 sec)"], + bstep._getTimeoutStatusText()) diff -N -r -u buildbot-0.8.14/buildbot/test/unit/test_steps_source_oldsource_Source.py buildbot/test/unit/test_steps_source_oldsource_Source.py --- buildbot-0.8.14/buildbot/test/unit/test_steps_source_oldsource_Source.py 1970-01-01 01:00:00.000000000 +0100 +++ buildbot/test/unit/test_steps_source_oldsource_Source.py 2012-06-24 22:30:04.000000000 +0200 @@ -0,0 +1,144 @@ +# This file is part of Buildbot. Buildbot is free software: you can +# redistribute it and/or modify it under the terms of the GNU General Public +# License as published by the Free Software Foundation, version 2. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +# details. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, write to the Free Software Foundation, Inc., 51 +# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Copyright Buildbot Team Members + +import mock +from twisted.trial import unittest + +from buildbot.interfaces import IRenderable +from buildbot.process.properties import Properties, WithProperties +from buildbot.steps.source import _ComputeRepositoryURL, Source +from buildbot.test.util import steps, sourcesteps + + +class SourceStamp(object): + repository = "test" + +class Build(object): + s = SourceStamp() + props = Properties(foo = "bar") + def getSourceStamp(self): + return self.s + def getProperties(self): + return self.props + def render(self, value): + self.props.build = self + return IRenderable(value).getRenderingFor(self.props) + +class RepoURL(unittest.TestCase): + def setUp(self): + self.build = Build() + + def test_backward_compatibility(self): + url = _ComputeRepositoryURL("repourl") + self.assertEqual(self.build.render(url), "repourl") + + def test_format_string(self): + url = _ComputeRepositoryURL("http://server/%s") + self.assertEquals(self.build.render(url), "http://server/test") + + def test_dict(self): + dict = {} + dict['test'] = "ssh://server/testrepository" + url = _ComputeRepositoryURL(dict) + self.assertEquals(self.build.render(url), "ssh://server/testrepository") + + def test_callable(self): + func = lambda x: x[::-1] + url = _ComputeRepositoryURL(func) + self.assertEquals(self.build.render(url), "tset") + + def test_backward_compatibility_render(self): + url = _ComputeRepositoryURL(WithProperties("repourl%(foo)s")) + self.assertEquals(self.build.render(url), "repourlbar") + + def test_dict_render(self): + d = dict(test=WithProperties("repourl%(foo)s")) + url = _ComputeRepositoryURL(d) + self.assertEquals(self.build.render(url), "repourlbar") + + def test_callable_render(self): + func = lambda x: WithProperties(x+"%(foo)s") + url = _ComputeRepositoryURL(func) + self.assertEquals(self.build.render(url), "testbar") + + +class TestSourceDescription(steps.BuildStepMixin, unittest.TestCase): + + def setUp(self): + return self.setUpBuildStep() + + def tearDown(self): + return self.tearDownBuildStep() + + def test_constructor_args_strings(self): + step = Source(workdir='build', + description='svn update (running)', + descriptionDone='svn update') + self.assertEqual(step.description, ['svn update (running)']) + self.assertEqual(step.descriptionDone, ['svn update']) + + def test_constructor_args_lists(self): + step = Source(workdir='build', + description=['svn', 'update', '(running)'], + descriptionDone=['svn', 'update']) + self.assertEqual(step.description, ['svn', 'update', '(running)']) + self.assertEqual(step.descriptionDone, ['svn', 'update']) + +class TestSource(sourcesteps.SourceStepMixin, unittest.TestCase): + + def setUp(self): + return self.setUpBuildStep() + + def tearDown(self): + return self.tearDownBuildStep() + + def test_start_alwaysUseLatest_True(self): + step = self.setupStep(Source(alwaysUseLatest=True), + { + 'branch': 'other-branch', + 'revision': 'revision', + }, + patch = 'patch' + ) + step.branch = 'branch' + step.startVC = mock.Mock() + + step.startStep(mock.Mock()) + + self.assertEqual(step.startVC.call_args, (('branch', None, None), {})) + + def test_start_alwaysUseLatest_False(self): + step = self.setupStep(Source(), + { + 'branch': 'other-branch', + 'revision': 'revision', + }, + patch = 'patch' + ) + step.branch = 'branch' + step.startVC = mock.Mock() + + step.startStep(mock.Mock()) + + self.assertEqual(step.startVC.call_args, (('other-branch', 'revision', 'patch'), {})) + + def test_start_alwaysUseLatest_False_no_branch(self): + step = self.setupStep(Source()) + step.branch = 'branch' + step.startVC = mock.Mock() + + step.startStep(mock.Mock()) + + self.assertEqual(step.startVC.call_args, (('branch', None, None), {})) diff -N -r -u buildbot-0.8.14/buildbot/test/unit/test_util_loop.py buildbot/test/unit/test_util_loop.py --- buildbot-0.8.14/buildbot/test/unit/test_util_loop.py 1970-01-01 01:00:00.000000000 +0100 +++ buildbot/test/unit/test_util_loop.py 2012-06-24 22:30:04.000000000 +0200 @@ -0,0 +1,220 @@ +# This file is part of Buildbot. Buildbot is free software: you can +# redistribute it and/or modify it under the terms of the GNU General Public +# License as published by the Free Software Foundation, version 2. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +# details. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, write to the Free Software Foundation, Inc., 51 +# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Copyright Buildbot Team Members + +from twisted.trial import unittest +from twisted.internet import reactor, defer, task +from twisted.application import service + +from buildbot.test.fake.state import State + +from buildbot.util import loop, eventual + +class TestLoopMixin(object): + def setUpTestLoop(self, skipConstructor=False): + if not skipConstructor: + self.loop = loop.Loop() + self.loop.startService() + self.results = [] + + def tearDownTestLoop(self): + eventual._setReactor(None) + assert self.loop.is_quiet() + return self.loop.stopService() + + # create a simple common callback with a tag that will + # appear in self.results + def make_cb(self, tag, selfTrigger=None): + state = State(triggerCount = selfTrigger) + def cb(): + if selfTrigger: + self.results.append("%s-%d" % (tag, state.triggerCount)) + state.triggerCount -= 1 + if state.triggerCount > 0: + self.loop.trigger() + else: + self.results.append(tag) + return cb + + # wait for quiet, and call check with self.results. + # Returns a deferred. + def whenQuiet(self, check=lambda res : None): + d = self.loop.when_quiet() + d.addCallback(lambda _ : check(self.results)) + return d + +class Loop(unittest.TestCase, TestLoopMixin): + + def setUp(self): + self.setUpTestLoop() + + def tearDown(self): + self.tearDownTestLoop() + + def test_single_processor(self): + self.loop.add(self.make_cb('x')) + self.loop.trigger() + def check(res): + self.assertEqual(res, ['x']) + return self.whenQuiet(check) + + def test_multi_processor(self): + self.loop.add(self.make_cb('x')) + self.loop.add(self.make_cb('y')) + self.loop.add(self.make_cb('z')) + self.loop.trigger() + def check(res): + self.assertEqual(sorted(res), ['x', 'y', 'z']) + return self.whenQuiet(check) + + def test_big_multi_processor(self): + for i in range(300): + self.loop.add(self.make_cb(i)) + def check(res): + self.assertEqual(sorted(res), range(300)) + self.loop.trigger() + d = self.whenQuiet(check) + return d + + def test_simple_selfTrigger(self): + self.loop.add(self.make_cb('c', selfTrigger=3)) + self.loop.trigger() + def check(res): + self.assertEqual(sorted(res), ['c-1', 'c-2', 'c-3']) + return self.whenQuiet(check) + + def test_multi_selfTrigger(self): + self.loop.add(self.make_cb('c', selfTrigger=3)) + self.loop.add(self.make_cb('d')) # will get taken along + self.loop.trigger() + def check(res): + self.assertEqual(sorted(res), ['c-1', 'c-2', 'c-3', 'd', 'd', 'd']) + return self.whenQuiet(check) + + def test_multi_sequential(self): + state = State(in_proc=False, entries=0) + for i in range(10): + def cb(): + state.entries += 1 + assert not state.in_proc + state.in_proc = True + d = defer.Deferred() + def finish(): + assert state.in_proc + state.in_proc = False + d.callback(None) + reactor.callLater(0, finish) + return d + self.loop.add(cb) + self.loop.trigger() + def check(res): + self.assertEqual(state.entries, 10) + return self.whenQuiet(check) + + def test_sleep(self): + clock_reactor = self.loop._reactor = task.Clock() + eventual._setReactor(clock_reactor) + state = State(count=5) + def proc(): + self.results.append(clock_reactor.seconds()) + state.count -= 1 + if state.count: + return defer.succeed(clock_reactor.seconds() + 10.0) + self.loop.add(proc) + self.loop.trigger() + def check(ign): + clock_reactor.pump((0,) + (1,)*50) # run for 50 fake seconds + self.assertEqual(self.results, [ 0.0, 10.0, 20.0, 30.0, 40.0 ]) + d = eventual.flushEventualQueue() + d.addCallback(check) + return d + + def test_loop_done(self): + # monkey-patch the instance to have a 'loop_done' method + # which just modifies 'done' to indicate that it was called + def loop_done(): + self.results.append("done") + self.patch(self.loop, "loop_done", loop_done) + + self.loop.add(self.make_cb('t', selfTrigger=2)) + self.loop.trigger() + def check(res): + self.assertEqual(res, [ 't-2', 't-1', 'done' ]) + return self.whenQuiet(check) + + def test_mergeTriggers(self): + state = State(count=4) + def make_proc(): # without this, proc will only be added to the loop once + def proc(): + self.results.append("p") + if state.count > 0: + self.results.append("t") + self.loop.trigger() + state.count -= 1 + return proc + self.loop.add(make_proc()) + self.loop.add(make_proc()) + self.loop.trigger() + # there should be four triggers, and three runs of the loop + def check(res): + self.assertEqual(res, [ 'p', 't', 'p', 't', 'p', 't', 'p', 't', 'p', 'p' ]) + return self.whenQuiet(check) + +class DelegateLoop(unittest.TestCase, TestLoopMixin): + + def setUp(self): + self.setUpTestLoop(skipConstructor=True) + + def tearDown(self): + self.tearDownTestLoop() + + # DelegateLoop doesn't contain much logic, so we don't re-test all of the + # functionality we tested with Loop + + def test_get_processors(self): + def get_processors(): + def proc(): self.results.append("here") + return [ proc ] + self.loop = loop.DelegateLoop(get_processors) + self.loop.startService() + self.loop.trigger() + def check(res): + self.assertEqual(res, [ "here" ]) + return self.whenQuiet(check) + +class MultiServiceLoop(unittest.TestCase, TestLoopMixin): + + def setUp(self): + self.setUpTestLoop(skipConstructor=True) + self.loop = loop.MultiServiceLoop() + + def tearDown(self): + self.tearDownTestLoop() + + # MultiServiceLoop doesn't contain much logic, so we don't re-test all of + # the functionality we tested with Loop + + class ServiceWithProcess(service.Service): + did_run = False + def run(self): + self.did_run = True + + def test_serviceChild(self): + childsvc = self.ServiceWithProcess() + childsvc.setServiceParent(self.loop) + self.loop.startService() + self.loop.trigger() + def check(res): + self.assertTrue(childsvc.did_run) + return self.whenQuiet(check) diff -N -r -u buildbot-0.8.14/buildbot/util/loop.py buildbot/util/loop.py --- buildbot-0.8.14/buildbot/util/loop.py 1970-01-01 01:00:00.000000000 +0100 +++ buildbot/util/loop.py 2012-06-24 22:30:04.000000000 +0200 @@ -0,0 +1,232 @@ +# This file is part of Buildbot. Buildbot is free software: you can +# redistribute it and/or modify it under the terms of the GNU General Public +# License as published by the Free Software Foundation, version 2. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +# details. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, write to the Free Software Foundation, Inc., 51 +# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Copyright Buildbot Team Members + +""" +One-at-a-time notification-triggered Deferred event loop. Each such loop has a +'doorbell' named trigger() and a set of processing functions. The processing +functions are expected to be callables like Scheduler methods, which examine a +database for work to do. The doorbell will be rung by other code that writes +into the database (possibly in a separate process). + +At some point after the doorbell is rung, each function will be run in turn, +one at a time. Each function can return a Deferred, and the next function will +not be run until the previous one's Deferred has fired. That is, at all times, +at most one processing function will be active. + +If the doorbell is rung during a run, the loop will be run again later. +Multiple rings may be handled by a single run, but the class guarantees that +there will be at least one full run that begins after the last ring. The +relative order of processing functions within a run is not preserved. If a +processing function is added to the loop more than once, it will still only be +called once per run. + +If the Deferred returned by the processing function fires with a number, the +event loop will call that function again at or after the given time +(expressed as seconds since epoch). This can be used by processing functions +when they want to 'sleep' until some amount of time has passed, such as for a +Scheduler that is waiting for a tree-stable-timer to expire, or a Periodic +scheduler that wants to fire once every six hours. This delayed call will +obey the same one-at-a-time behavior as the run-everything trigger. + +Each function's return-value-timer value will replace the previous timer. Any +outstanding timer will be cancelled just before invoking a processing +function. As a result, these functions should basically be idempotent: if the +database says that the Scheduler needs to wake up at 5pm, it should keep +returning '5pm' until it gets called after 5pm, at which point it should +start returning None. + +The functions should also add an epsilon (perhaps one second) to their +desired wakeup time, so that rounding errors or low-resolution system timers +don't cause 'OCD Alarm Clock Syndrome' (in which they get woken up a moment +too early and then try to sleep repeatedly for zero seconds). The event loop +will silently impose a 5-second minimum delay time to avoid this. + +Any errors in the processing functions are written to log.err and then +ignored. +""" + + +from twisted.internet import reactor, defer +from twisted.application import service +from twisted.python import log + +from buildbot.util.eventual import eventually +from buildbot import util +from buildbot.process.metrics import Timer, countMethod + +class LoopBase(service.MultiService): + OCD_MINIMUM_DELAY = 5.0 + + def __init__(self): + service.MultiService.__init__(self) + self._loop_running = False + self._everything_needs_to_run = False + self._wakeup_timer = None + self._timers = {} + self._when_quiet_waiters = set() + self._start_timer = None + self._reactor = reactor # seam for tests to use t.i.t.Clock + self._remaining = [] + + def stopService(self): + if self._start_timer and self._start_timer.active(): + self._start_timer.cancel() + if self._wakeup_timer and self._wakeup_timer.active(): + self._wakeup_timer.cancel() + return service.MultiService.stopService(self) + + def is_quiet(self): + return not self._loop_running + + def when_quiet(self): + d = defer.Deferred() + self._when_quiet_waiters.add(d) + return d + + def trigger(self): + # if we're triggered while not running, ignore it. We'll automatically + # trigger when the service starts + if not self.running: + log.msg("loop triggered while service disabled; ignoring trigger") + return + self._mark_runnable(run_everything=True) + + def _mark_runnable(self, run_everything): + if run_everything: + self._everything_needs_to_run = True + # timers are now redundant, so cancel any existing ones + self._timers.clear() + self._set_wakeup_timer() + if self._loop_running: + return + self._loop_running = True + self._start_timer = self._reactor.callLater(0, self._loop_start) + + def get_processors(self): + raise Exception('subclasses must implement get_processors()') + + _loop_timer = Timer('Loop.run') + @_loop_timer.startTimer + @countMethod('Loop._loop_start()') + def _loop_start(self): + if self._everything_needs_to_run: + self._everything_needs_to_run = False + self._timers.clear() + self._set_wakeup_timer() + self._remaining = list(self.get_processors()) + else: + self._remaining = [] + now = util.now(self._reactor) + all_processors = self.get_processors() + for p in list(self._timers.keys()): + if self._timers[p] <= now: + del self._timers[p] + # don't run a processor that was removed while it still + # had a timer running + if p in all_processors: + self._remaining.append(p) + # consider sorting by 'when' + self._loop_next() + + @countMethod('Loop._loop_next()') + def _loop_next(self): + if not self._remaining: + return self._loop_done() + p = self._remaining.pop(0) + self._timers.pop(p, None) + now = util.now(self._reactor) + d = defer.maybeDeferred(p) + d.addCallback(self._set_timer, now, p) + d.addErrback(log.err) + d.addBoth(self._one_done) + return None # no long Deferred chains + + def _one_done(self, ignored): + eventually(self._loop_next) + + @_loop_timer.stopTimer + def _loop_done(self): + if self._everything_needs_to_run: + self._loop_start() + return + self._loop_running = False + self._set_wakeup_timer() + if not self._timers: + # we're really idle, so notify waiters (used by unit tests) + while self._when_quiet_waiters: + d = self._when_quiet_waiters.pop() + self._reactor.callLater(0, d.callback, None) + self.loop_done() + + def loop_done(self): + # this can be overridden by subclasses to do more work when we've + # finished a pass through the loop and don't need to immediately + # start a new one + pass + + def _set_timer(self, res, now, p): + if isinstance(res, (int, float)): + assert res > now # give me absolute time, not an interval + # don't wake up right away. By doing this here instead of in + # _set_wakeup_timer, we avoid penalizing unrelated jobs which + # want to wake up a few seconds apart + when = max(res, now+self.OCD_MINIMUM_DELAY) + self._timers[p] = when + + def _set_wakeup_timer(self): + if not self._timers: + if self._wakeup_timer: + self._wakeup_timer.cancel() + self._wakeup_timer = None + return + when = min(self._timers.values()) + # to avoid waking too frequently, this could be: + # delay=max(when-now,OCD_MINIMUM_DELAY) + # but that delays unrelated jobs that want to wake few seconds apart + delay = max(0, when - util.now(self._reactor)) + if self._wakeup_timer: + self._wakeup_timer.reset(delay) + else: + self._wakeup_timer = self._reactor.callLater(delay, self._wakeup) + + def _wakeup(self): + self._wakeup_timer = None + self._mark_runnable(run_everything=False) + +class Loop(LoopBase): + def __init__(self): + LoopBase.__init__(self) + self.processors = set() + + def add(self, processor): + self.processors.add(processor) + + def remove(self, processor): + self.processors.remove(processor) + + def get_processors(self): + return self.processors.copy() + +class DelegateLoop(LoopBase): + def __init__(self, get_processors_function): + LoopBase.__init__(self) + self.get_processors = get_processors_function + +class MultiServiceLoop(LoopBase): + """I am a Loop which gets my processors from my service children. When I + run, I iterate over each of them, invoking their 'run' method.""" + + def get_processors(self): + return [child.run for child in self] diff -N -r -u buildbot-0.8.14/buildbot/util/monkeypatches.py buildbot/util/monkeypatches.py --- buildbot-0.8.14/buildbot/util/monkeypatches.py 1970-01-01 01:00:00.000000000 +0100 +++ buildbot/util/monkeypatches.py 2012-06-24 22:30:04.000000000 +0200 @@ -0,0 +1,44 @@ +# This file is part of Buildbot. Buildbot is free software: you can +# redistribute it and/or modify it under the terms of the GNU General Public +# License as published by the Free Software Foundation, version 2. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +# details. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, write to the Free Software Foundation, Inc., 51 +# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Copyright Buildbot Team Members + +import sys +import twisted +from twisted.trial import unittest + +def add_debugging_monkeypatches(): + """ + DO NOT CALL THIS DIRECTLY + + This adds a few "harmless" monkeypatches which make it easier to debug + failing tests. It is called automatically by buildbot.test.__init__. + """ + from twisted.application.service import Service + old_startService = Service.startService + old_stopService = Service.stopService + def startService(self): + assert not self.running + return old_startService(self) + def stopService(self): + assert self.running + return old_stopService(self) + Service.startService = startService + Service.stopService = stopService + + # versions of Twisted before 9.0.0 did not have a UnitTest.patch that worked + # on Python-2.7 + if twisted.version.major <= 9 and sys.version_info[:2] == (2,7): + def nopatch(self, *args): + raise unittest.SkipTest('unittest.TestCase.patch is not available') + unittest.TestCase.patch = nopatch diff -N -r -u buildbot-0.8.14/buildbot/VERSION buildbot/VERSION --- buildbot-0.8.14/buildbot/VERSION 2016-08-11 20:20:16.000000000 +0200 +++ buildbot/VERSION 2016-11-28 05:02:16.000000000 +0100 @@ -1 +1 @@ -0.8.14 \ Pas de fin de ligne à la fin du fichier +0.8.14py1 \ Pas de fin de ligne à la fin du fichier