buildbot.buildslave

1 # Portions copyright Canonical Ltd. 2009 2 3 import time 4 from email.Message import Message 5 from email.Utils import formatdate 6 from zope.interface import implements 7 from twisted.python import log 8 from twisted.internet import defer, reactor 9 from twisted.application import service 10 import twisted.spread.pb 11 12 from buildbot.pbutil import NewCredPerspective 13 from buildbot.status.builder import SlaveStatus 14 from buildbot.status.mail import MailNotifier 15 from buildbot.interfaces import IBuildSlave, ILatentBuildSlave 16 from buildbot.process.properties import Properties 17 18 import sys 19 if sys.version_info[:3] < (2,4,0): 20 from sets import Set as set 21

22 -class AbstractBuildSlave(NewCredPerspective, service.MultiService):

23 """This is the master-side representative for a remote buildbot slave. 24 There is exactly one for each slave described in the config file (the 25 c['slaves'] list). When buildbots connect in (.attach), they get a 26 reference to this instance. The BotMaster object is stashed as the 27 .botmaster attribute. The BotMaster is also our '.parent' Service. 28 29 I represent a build slave -- a remote machine capable of 30 running builds. I am instantiated by the configuration file, and can be 31 subclassed to add extra functionality.""" 32 33 implements(IBuildSlave) 34

35 - def __init__(self, name, password, max_builds=None, 36 notify_on_missing=[], missing_timeout=3600, 37 properties={}):

38 """ 39 @param name: botname this machine will supply when it connects 40 @param password: password this machine will supply when 41 it connects 42 @param max_builds: maximum number of simultaneous builds that will 43 be run concurrently on this buildslave (the 44 default is None for no limit) 45 @param properties: properties that will be applied to builds run on 46 this slave 47 @type properties: dictionary 48 """ 49 service.MultiService.__init__(self) 50 self.slavename = name 51 self.password = password 52 self.botmaster = None # no buildmaster yet 53 self.slave_status = SlaveStatus(name) 54 self.slave = None # a RemoteReference to the Bot, when connected 55 self.slave_commands = None 56 self.slavebuilders = {} 57 self.max_builds = max_builds 58 59 self.properties = Properties() 60 self.properties.update(properties, "BuildSlave") 61 self.properties.setProperty("slavename", name, "BuildSlave") 62 63 self.lastMessageReceived = 0 64 if isinstance(notify_on_missing, str): 65 notify_on_missing = [notify_on_missing] 66 self.notify_on_missing = notify_on_missing 67 for i in notify_on_missing: 68 assert isinstance(i, str) 69 self.missing_timeout = missing_timeout 70 self.missing_timer = None

71

72 - def update(self, new):

73 """ 74 Given a new BuildSlave, configure this one identically. Because 75 BuildSlave objects are remotely referenced, we can't replace them 76 without disconnecting the slave, yet there's no reason to do that. 77 """ 78 # the reconfiguration logic should guarantee this: 79 assert self.slavename == new.slavename 80 assert self.password == new.password 81 assert self.__class__ == new.__class__ 82 self.max_builds = new.max_builds

83

84 - def __repr__(self):

85 if self.botmaster: 86 builders = self.botmaster.getBuildersForSlave(self.slavename) 87 return "<%s '%s', current builders: %s>" % \ 88 (self.__class__.__name__, self.slavename, 89 ','.join(map(lambda b: b.name, builders))) 90 else: 91 return "<%s '%s', (no builders yet)>" % \ 92 (self.__class__.__name__, self.slavename)

93

94 - def setBotmaster(self, botmaster):

95 assert not self.botmaster, "BuildSlave already has a botmaster" 96 self.botmaster = botmaster 97 self.startMissingTimer()

98

99 - def stopMissingTimer(self):

100 if self.missing_timer: 101 self.missing_timer.cancel() 102 self.missing_timer = None

103

104 - def startMissingTimer(self):

105 if self.notify_on_missing and self.missing_timeout and self.parent: 106 self.stopMissingTimer() # in case it's already running 107 self.missing_timer = reactor.callLater(self.missing_timeout, 108 self._missing_timer_fired)

109

110 - def _missing_timer_fired(self):

111 self.missing_timer = None 112 # notify people, but only if we're still in the config 113 if not self.parent: 114 return 115 116 buildmaster = self.botmaster.parent 117 status = buildmaster.getStatus() 118 text = "The Buildbot working for '%s'\n" % status.getProjectName() 119 text += ("has noticed that the buildslave named %s went away\n" % 120 self.slavename) 121 text += "\n" 122 text += ("It last disconnected at %s (buildmaster-local time)\n" % 123 time.ctime(time.time() - self.missing_timeout)) # approx 124 text += "\n" 125 text += "The admin on record (as reported by BUILDSLAVE:info/admin)\n" 126 text += "was '%s'.\n" % self.slave_status.getAdmin() 127 text += "\n" 128 text += "Sincerely,\n" 129 text += " The Buildbot\n" 130 text += " %s\n" % status.getProjectURL() 131 subject = "Buildbot: buildslave %s was lost" % self.slavename 132 return self._mail_missing_message(subject, text)

133 134

135 - def updateSlave(self):

136 """Called to add or remove builders after the slave has connected. 137 138 @return: a Deferred that indicates when an attached slave has 139 accepted the new builders and/or released the old ones.""" 140 if self.slave: 141 return self.sendBuilderList() 142 else: 143 return defer.succeed(None)

144

145 - def updateSlaveStatus(self, buildStarted=None, buildFinished=None):

146 if buildStarted: 147 self.slave_status.buildStarted(buildStarted) 148 if buildFinished: 149 self.slave_status.buildFinished(buildFinished)

150

151 - def attached(self, bot):

152 """This is called when the slave connects. 153 154 @return: a Deferred that fires with a suitable pb.IPerspective to 155 give to the slave (i.e. 'self')""" 156 157 if self.slave: 158 # uh-oh, we've got a duplicate slave. The most likely 159 # explanation is that the slave is behind a slow link, thinks we 160 # went away, and has attempted to reconnect, so we've got two 161 # "connections" from the same slave, but the previous one is 162 # stale. Give the new one precedence. 163 log.msg("duplicate slave %s replacing old one" % self.slavename) 164 165 # just in case we've got two identically-configured slaves, 166 # report the IP addresses of both so someone can resolve the 167 # squabble 168 tport = self.slave.broker.transport 169 log.msg("old slave was connected from", tport.getPeer()) 170 log.msg("new slave is from", bot.broker.transport.getPeer()) 171 d = self.disconnect() 172 else: 173 d = defer.succeed(None) 174 # now we go through a sequence of calls, gathering information, then 175 # tell the Botmaster that it can finally give this slave to all the 176 # Builders that care about it. 177 178 # we accumulate slave information in this 'state' dictionary, then 179 # set it atomically if we make it far enough through the process 180 state = {} 181 182 # Reset graceful shutdown status 183 self.slave_status.setGraceful(False) 184 # We want to know when the graceful shutdown flag changes 185 self.slave_status.addGracefulWatcher(self._gracefulChanged) 186 187 def _log_attachment_on_slave(res): 188 d1 = bot.callRemote("print", "attached") 189 d1.addErrback(lambda why: None) 190 return d1

191 d.addCallback(_log_attachment_on_slave) 192 193 def _get_info(res): 194 d1 = bot.callRemote("getSlaveInfo") 195 def _got_info(info): 196 log.msg("Got slaveinfo from '%s'" % self.slavename) 197 # TODO: info{} might have other keys 198 state["admin"] = info.get("admin") 199 state["host"] = info.get("host")

200 def _info_unavailable(why): 201 # maybe an old slave, doesn't implement remote_getSlaveInfo 202 log.msg("BuildSlave.info_unavailable") 203 log.err(why) 204 d1.addCallbacks(_got_info, _info_unavailable) 205 return d1 206 d.addCallback(_get_info) 207 208 def _get_commands(res): 209 d1 = bot.callRemote("getCommands") 210 def _got_commands(commands): 211 state["slave_commands"] = commands 212 def _commands_unavailable(why): 213 # probably an old slave 214 log.msg("BuildSlave._commands_unavailable") 215 if why.check(AttributeError): 216 return 217 log.err(why) 218 d1.addCallbacks(_got_commands, _commands_unavailable) 219 return d1 220 d.addCallback(_get_commands) 221 222 def _accept_slave(res): 223 self.slave_status.setAdmin(state.get("admin")) 224 self.slave_status.setHost(state.get("host")) 225 self.slave_status.setConnected(True) 226 self.slave_commands = state.get("slave_commands") 227 self.slave = bot 228 log.msg("bot attached") 229 self.messageReceivedFromSlave() 230 self.stopMissingTimer() 231 232 return self.updateSlave() 233 d.addCallback(_accept_slave) 234 d.addCallback(lambda res: self.botmaster.maybeStartAllBuilds()) 235 236 # Finally, the slave gets a reference to this BuildSlave. They 237 # receive this later, after we've started using them. 238 d.addCallback(lambda res: self) 239 return d 240

241 - def messageReceivedFromSlave(self):

242 now = time.time() 243 self.lastMessageReceived = now 244 self.slave_status.setLastMessageReceived(now)

245

246 - def detached(self, mind):

247 self.slave = None 248 self.slave_status.removeGracefulWatcher(self._gracefulChanged) 249 self.slave_status.setConnected(False) 250 log.msg("BuildSlave.detached(%s)" % self.slavename)

251

252 - def disconnect(self):

253 """Forcibly disconnect the slave. 254 255 This severs the TCP connection and returns a Deferred that will fire 256 (with None) when the connection is probably gone. 257 258 If the slave is still alive, they will probably try to reconnect 259 again in a moment. 260 261 This is called in two circumstances. The first is when a slave is 262 removed from the config file. In this case, when they try to 263 reconnect, they will be rejected as an unknown slave. The second is 264 when we wind up with two connections for the same slave, in which 265 case we disconnect the older connection. 266 """ 267 268 if not self.slave: 269 return defer.succeed(None) 270 log.msg("disconnecting old slave %s now" % self.slavename) 271 # When this Deferred fires, we'll be ready to accept the new slave 272 return self._disconnect(self.slave)

273

274 - def _disconnect(self, slave):

275 # all kinds of teardown will happen as a result of 276 # loseConnection(), but it happens after a reactor iteration or 277 # two. Hook the actual disconnect so we can know when it is safe 278 # to connect the new slave. We have to wait one additional 279 # iteration (with callLater(0)) to make sure the *other* 280 # notifyOnDisconnect handlers have had a chance to run. 281 d = defer.Deferred() 282 283 # notifyOnDisconnect runs the callback with one argument, the 284 # RemoteReference being disconnected. 285 def _disconnected(rref): 286 reactor.callLater(0, d.callback, None)

287 slave.notifyOnDisconnect(_disconnected) 288 tport = slave.broker.transport 289 # this is the polite way to request that a socket be closed 290 tport.loseConnection() 291 try: 292 # but really we don't want to wait for the transmit queue to 293 # drain. The remote end is unlikely to ACK the data, so we'd 294 # probably have to wait for a (20-minute) TCP timeout. 295 #tport._closeSocket() 296 # however, doing _closeSocket (whether before or after 297 # loseConnection) somehow prevents the notifyOnDisconnect 298 # handlers from being run. Bummer. 299 tport.offset = 0 300 tport.dataBuffer = "" 301 except: 302 # however, these hacks are pretty internal, so don't blow up if 303 # they fail or are unavailable 304 log.msg("failed to accelerate the shutdown process") 305 pass 306 log.msg("waiting for slave to finish disconnecting") 307 308 return d 309

310 - def sendBuilderList(self):

311 our_builders = self.botmaster.getBuildersForSlave(self.slavename) 312 blist = [(b.name, b.builddir) for b in our_builders] 313 d = self.slave.callRemote("setBuilderList", blist) 314 return d

315

316 - def perspective_keepalive(self):

317 pass

318

319 - def addSlaveBuilder(self, sb):

320 if sb.builder_name not in self.slavebuilders: 321 log.msg("%s adding %s" % (self, sb)) 322 elif sb is not self.slavebuilders[sb.builder_name]: 323 log.msg("%s replacing %s" % (self, sb)) 324 else: 325 return 326 self.slavebuilders[sb.builder_name] = sb

327

328 - def removeSlaveBuilder(self, sb):

329 try: 330 del self.slavebuilders[sb.builder_name] 331 except KeyError: 332 pass 333 else: 334 log.msg("%s removed %s" % (self, sb))

335

336 - def canStartBuild(self):

337 """ 338 I am called when a build is requested to see if this buildslave 339 can start a build. This function can be used to limit overall 340 concurrency on the buildslave. 341 """ 342 # If we're waiting to shutdown gracefully, then we shouldn't 343 # accept any new jobs. 344 if self.slave_status.getGraceful(): 345 return False 346 347 if self.max_builds: 348 active_builders = [sb for sb in self.slavebuilders.values() 349 if sb.isBusy()] 350 if len(active_builders) >= self.max_builds: 351 return False 352 return True

353

354 - def _mail_missing_message(self, subject, text):

355 # first, see if we have a MailNotifier we can use. This gives us a 356 # fromaddr and a relayhost. 357 buildmaster = self.botmaster.parent 358 for st in buildmaster.statusTargets: 359 if isinstance(st, MailNotifier): 360 break 361 else: 362 # if not, they get a default MailNotifier, which always uses SMTP 363 # to localhost and uses a dummy fromaddr of "buildbot". 364 log.msg("buildslave-missing msg using default MailNotifier") 365 st = MailNotifier("buildbot") 366 # now construct the mail 367 368 m = Message() 369 m.set_payload(text) 370 m['Date'] = formatdate(localtime=True) 371 m['Subject'] = subject 372 m['From'] = st.fromaddr 373 recipients = self.notify_on_missing 374 m['To'] = ", ".join(recipients) 375 d = st.sendMessage(m, recipients) 376 # return the Deferred for testing purposes 377 return d

378

379 - def _gracefulChanged(self, graceful):

380 """This is called when our graceful shutdown setting changes""" 381 if graceful: 382 active_builders = [sb for sb in self.slavebuilders.values() 383 if sb.isBusy()] 384 if len(active_builders) == 0: 385 # Shut down! 386 self.shutdown()

387

388 - def shutdown(self):

389 """Shutdown the slave""" 390 # Look for a builder with a remote reference to the client side 391 # slave. If we can find one, then call "shutdown" on the remote 392 # builder, which will cause the slave buildbot process to exit. 393 d = None 394 for b in self.slavebuilders.values(): 395 if b.remote: 396 d = b.remote.callRemote("shutdown") 397 break 398 399 if d: 400 log.msg("Shutting down slave: %s" % self.slavename) 401 # The remote shutdown call will not complete successfully since the 402 # buildbot process exits almost immediately after getting the 403 # shutdown request. 404 # Here we look at the reason why the remote call failed, and if 405 # it's because the connection was lost, that means the slave 406 # shutdown as expected. 407 def _errback(why): 408 if why.check(twisted.spread.pb.PBConnectionLost): 409 log.msg("Lost connection to %s" % self.slavename) 410 else: 411 log.err("Unexpected error when trying to shutdown %s" % self.slavename)

412 d.addErrback(_errback) 413 return d 414 log.err("Couldn't find remote builder to shut down slave") 415 return defer.succeed(None) 416

417 -class BuildSlave(AbstractBuildSlave):

418

419 - def sendBuilderList(self):

420 d = AbstractBuildSlave.sendBuilderList(self) 421 def _sent(slist): 422 dl = [] 423 for name, remote in slist.items(): 424 # use get() since we might have changed our mind since then 425 b = self.botmaster.builders.get(name) 426 if b: 427 d1 = b.attached(self, remote, self.slave_commands) 428 dl.append(d1) 429 return defer.DeferredList(dl)

430 def _set_failed(why): 431 log.msg("BuildSlave.sendBuilderList (%s) failed" % self) 432 log.err(why)

433 # TODO: hang up on them?, without setBuilderList we can't use 434 # them 435 d.addCallbacks(_sent, _set_failed) 436 return d 437

438 - def detached(self, mind):

439 AbstractBuildSlave.detached(self, mind) 440 self.botmaster.slaveLost(self) 441 self.startMissingTimer()

442

443 - def buildFinished(self, sb):

444 """This is called when a build on this slave is finished.""" 445 # If we're gracefully shutting down, and we have no more active 446 # builders, then it's safe to disconnect 447 if self.slave_status.getGraceful(): 448 active_builders = [sb for sb in self.slavebuilders.values() 449 if sb.isBusy()] 450 if len(active_builders) == 0: 451 # Shut down! 452 return self.shutdown() 453 return defer.succeed(None)

454

455 -class AbstractLatentBuildSlave(AbstractBuildSlave):

456 """A build slave that will start up a slave instance when needed. 457 458 To use, subclass and implement start_instance and stop_instance. 459 460 See ec2buildslave.py for a concrete example. Also see the stub example in 461 test/test_slaves.py. 462 """ 463 464 implements(ILatentBuildSlave) 465 466 substantiated = False 467 substantiation_deferred = None 468 build_wait_timer = None 469 _start_result = _shutdown_callback_handle = None 470

471 - def __init__(self, name, password, max_builds=None, 472 notify_on_missing=[], missing_timeout=60*20, 473 build_wait_timeout=60*10, 474 properties={}):

475 AbstractBuildSlave.__init__( 476 self, name, password, max_builds, notify_on_missing, 477 missing_timeout, properties) 478 self.building = set() 479 self.build_wait_timeout = build_wait_timeout

480

481 - def start_instance(self):

482 # responsible for starting instance that will try to connect with 483 # this master. Should return deferred. Problems should use an 484 # errback. 485 raise NotImplementedError

486

487 - def stop_instance(self, fast=False):

488 # responsible for shutting down instance. 489 raise NotImplementedError

490

491 - def substantiate(self, sb):

492 if self.substantiated: 493 self._clearBuildWaitTimer() 494 self._setBuildWaitTimer() 495 return defer.succeed(self) 496 if self.substantiation_deferred is None: 497 if self.parent and not self.missing_timer: 498 # start timer. if timer times out, fail deferred 499 self.missing_timer = reactor.callLater( 500 self.missing_timeout, 501 self._substantiation_failed, defer.TimeoutError()) 502 self.substantiation_deferred = defer.Deferred() 503 if self.slave is None: 504 self._substantiate() # start up instance 505 # else: we're waiting for an old one to detach. the _substantiate 506 # will be done in ``detached`` below. 507 return self.substantiation_deferred

508

509 - def _substantiate(self):

510 # register event trigger 511 d = self.start_instance() 512 self._shutdown_callback_handle = reactor.addSystemEventTrigger( 513 'before', 'shutdown', self._soft_disconnect, fast=True) 514 def stash_reply(result): 515 self._start_result = result

516 def clean_up(failure): 517 if self.missing_timer is not None: 518 self.missing_timer.cancel() 519 self._substantiation_failed(failure) 520 if self._shutdown_callback_handle is not None: 521 handle = self._shutdown_callback_handle 522 del self._shutdown_callback_handle 523 reactor.removeSystemEventTrigger(handle) 524 return failure

525 d.addCallbacks(stash_reply, clean_up) 526 return d 527

528 - def attached(self, bot):

529 if self.substantiation_deferred is None: 530 msg = 'Slave %s received connection while not trying to ' \ 531 'substantiate. Disconnecting.' % (self.slavename,) 532 log.msg(msg) 533 self._disconnect(bot) 534 return defer.fail(RuntimeError(msg)) 535 return AbstractBuildSlave.attached(self, bot)

536

537 - def detached(self, mind):

538 AbstractBuildSlave.detached(self, mind) 539 if self.substantiation_deferred is not None: 540 self._substantiate()

541

542 - def _substantiation_failed(self, failure):

543 d = self.substantiation_deferred 544 self.substantiation_deferred = None 545 self.missing_timer = None 546 d.errback(failure) 547 self.insubstantiate() 548 # notify people, but only if we're still in the config 549 if not self.parent or not self.notify_on_missing: 550 return 551 552 buildmaster = self.botmaster.parent 553 status = buildmaster.getStatus() 554 text = "The Buildbot working for '%s'\n" % status.getProjectName() 555 text += ("has noticed that the latent buildslave named %s \n" % 556 self.slavename) 557 text += "never substantiated after a request\n" 558 text += "\n" 559 text += ("The request was made at %s (buildmaster-local time)\n" % 560 time.ctime(time.time() - self.missing_timeout)) # approx 561 text += "\n" 562 text += "Sincerely,\n" 563 text += " The Buildbot\n" 564 text += " %s\n" % status.getProjectURL() 565 subject = "Buildbot: buildslave %s never substantiated" % self.slavename 566 return self._mail_missing_message(subject, text)

567

568 - def buildStarted(self, sb):

569 assert self.substantiated 570 self._clearBuildWaitTimer() 571 self.building.add(sb.builder_name)

572

573 - def buildFinished(self, sb):

574 self.building.remove(sb.builder_name) 575 if not self.building: 576 self._setBuildWaitTimer()

577

578 - def _clearBuildWaitTimer(self):

579 if self.build_wait_timer is not None: 580 if self.build_wait_timer.active(): 581 self.build_wait_timer.cancel() 582 self.build_wait_timer = None

583

584 - def _setBuildWaitTimer(self):

585 self._clearBuildWaitTimer() 586 self.build_wait_timer = reactor.callLater( 587 self.build_wait_timeout, self._soft_disconnect)

588

589 - def insubstantiate(self, fast=False):

590 self._clearBuildWaitTimer() 591 d = self.stop_instance(fast) 592 if self._shutdown_callback_handle is not None: 593 handle = self._shutdown_callback_handle 594 del self._shutdown_callback_handle 595 reactor.removeSystemEventTrigger(handle) 596 self.substantiated = False 597 self.building.clear() # just to be sure 598 return d

599

600 - def _soft_disconnect(self, fast=False):

601 d = AbstractBuildSlave.disconnect(self) 602 if self.slave is not None: 603 # this could be called when the slave needs to shut down, such as 604 # in BotMaster.removeSlave, *or* when a new slave requests a 605 # connection when we already have a slave. It's not clear what to 606 # do in the second case: this shouldn't happen, and if it 607 # does...if it's a latent slave, shutting down will probably kill 608 # something we want...but we can't know what the status is. So, 609 # here, we just do what should be appropriate for the first case, 610 # and put our heads in the sand for the second, at least for now. 611 # The best solution to the odd situation is removing it as a 612 # possibilty: make the master in charge of connecting to the 613 # slave, rather than vice versa. TODO. 614 d = defer.DeferredList([d, self.insubstantiate(fast)]) 615 else: 616 if self.substantiation_deferred is not None: 617 # unlike the previous block, we don't expect this situation when 618 # ``attached`` calls ``disconnect``, only when we get a simple 619 # request to "go away". 620 self.substantiation_deferred.errback() 621 self.substantiation_deferred = None 622 if self.missing_timer: 623 self.missing_timer.cancel() 624 self.missing_timer = None 625 self.stop_instance() 626 return d

627

628 - def disconnect(self):

629 d = self._soft_disconnect() 630 # this removes the slave from all builders. It won't come back 631 # without a restart (or maybe a sighup) 632 self.botmaster.slaveLost(self)

633

634 - def stopService(self):

635 res = defer.maybeDeferred(AbstractBuildSlave.stopService, self) 636 if self.slave is not None: 637 d = self._soft_disconnect() 638 res = defer.DeferredList([res, d]) 639 return res

640

641 - def updateSlave(self):

642 """Called to add or remove builders after the slave has connected. 643 644 Also called after botmaster's builders are initially set. 645 646 @return: a Deferred that indicates when an attached slave has 647 accepted the new builders and/or released the old ones.""" 648 for b in self.botmaster.getBuildersForSlave(self.slavename): 649 if b.name not in self.slavebuilders: 650 b.addLatentSlave(self) 651 return AbstractBuildSlave.updateSlave(self)

652

653 - def sendBuilderList(self):

654 d = AbstractBuildSlave.sendBuilderList(self) 655 def _sent(slist): 656 dl = [] 657 for name, remote in slist.items(): 658 # use get() since we might have changed our mind since then. 659 # we're checking on the builder in addition to the 660 # slavebuilders out of a bit of paranoia. 661 b = self.botmaster.builders.get(name) 662 sb = self.slavebuilders.get(name) 663 if b and sb: 664 d1 = sb.attached(self, remote, self.slave_commands) 665 dl.append(d1) 666 return defer.DeferredList(dl)

667 def _set_failed(why): 668 log.msg("BuildSlave.sendBuilderList (%s) failed" % self) 669 log.err(why) 670 # TODO: hang up on them?, without setBuilderList we can't use 671 # them 672 if self.substantiation_deferred: 673 self.substantiation_deferred.errback() 674 self.substantiation_deferred = None 675 if self.missing_timer: 676 self.missing_timer.cancel() 677 self.missing_timer = None 678 # TODO: maybe log? send an email? 679 return why 680 d.addCallbacks(_sent, _set_failed) 681 def _substantiated(res): 682 self.substantiated = True 683 if self.substantiation_deferred: 684 d = self.substantiation_deferred 685 del self.substantiation_deferred 686 res = self._start_result 687 del self._start_result 688 d.callback(res) 689 # note that the missing_timer is already handled within 690 # ``attached`` 691 if not self.building: 692 self._setBuildWaitTimer() 693 d.addCallback(_substantiated) 694 return d 695

Source Code for Module buildbot.buildslave