buildbot.master

1 # -*- test-case-name: buildbot.test.test_run -*- 2 3 import os 4 signal = None 5 try: 6 import signal 7 except ImportError: 8 pass 9 from cPickle import load 10 import warnings 11 12 from zope.interface import implements 13 from twisted.python import log, components 14 from twisted.python.failure import Failure 15 from twisted.internet import defer, reactor 16 from twisted.spread import pb 17 from twisted.cred import portal, checkers 18 from twisted.application import service, strports 19 from twisted.persisted import styles 20 21 import buildbot 22 # sibling imports 23 from buildbot.util import now 24 from buildbot.pbutil import NewCredPerspective 25 from buildbot.process.builder import Builder, IDLE 26 from buildbot.process.base import BuildRequest 27 from buildbot.status.builder import Status 28 from buildbot.changes.changes import Change, ChangeMaster, TestChangeMaster 29 from buildbot.sourcestamp import SourceStamp 30 from buildbot.buildslave import BuildSlave 31 from buildbot import interfaces, locks 32 from buildbot.process.properties import Properties 33 34 ######################################## 35

36 -class BotMaster(service.MultiService):

37 38 """This is the master-side service which manages remote buildbot slaves. 39 It provides them with BuildSlaves, and distributes file change 40 notification messages to them. 41 """ 42 43 debug = 0 44

45 - def __init__(self):

46 service.MultiService.__init__(self) 47 self.builders = {} 48 self.builderNames = [] 49 # builders maps Builder names to instances of bb.p.builder.Builder, 50 # which is the master-side object that defines and controls a build. 51 # They are added by calling botmaster.addBuilder() from the startup 52 # code. 53 54 # self.slaves contains a ready BuildSlave instance for each 55 # potential buildslave, i.e. all the ones listed in the config file. 56 # If the slave is connected, self.slaves[slavename].slave will 57 # contain a RemoteReference to their Bot instance. If it is not 58 # connected, that attribute will hold None. 59 self.slaves = {} # maps slavename to BuildSlave 60 self.statusClientService = None 61 self.watchers = {} 62 63 # self.locks holds the real Lock instances 64 self.locks = {} 65 66 # self.mergeRequests is the callable override for merging build 67 # requests 68 self.mergeRequests = None 69 70 # self.prioritizeBuilders is the callable override for builder order 71 # traversal 72 self.prioritizeBuilders = None

73 74 # these four are convenience functions for testing 75

76 - def waitUntilBuilderAttached(self, name):

77 b = self.builders[name] 78 #if b.slaves: 79 # return defer.succeed(None) 80 d = defer.Deferred() 81 b.watchers['attach'].append(d) 82 return d

83

84 - def waitUntilBuilderDetached(self, name):

85 b = self.builders.get(name) 86 if not b or not b.slaves: 87 return defer.succeed(None) 88 d = defer.Deferred() 89 b.watchers['detach'].append(d) 90 return d

91

92 - def waitUntilBuilderFullyDetached(self, name):

93 b = self.builders.get(name) 94 # TODO: this looks too deeply inside the Builder object 95 if not b or not b.slaves: 96 return defer.succeed(None) 97 d = defer.Deferred() 98 b.watchers['detach_all'].append(d) 99 return d

100

101 - def waitUntilBuilderIdle(self, name):

102 b = self.builders[name] 103 # TODO: this looks way too deeply inside the Builder object 104 for sb in b.slaves: 105 if sb.state != IDLE: 106 d = defer.Deferred() 107 b.watchers['idle'].append(d) 108 return d 109 return defer.succeed(None)

110

111 - def loadConfig_Slaves(self, new_slaves):

112 old_slaves = [c for c in list(self) 113 if interfaces.IBuildSlave.providedBy(c)] 114 115 # identify added/removed slaves. For each slave we construct a tuple 116 # of (name, password, class), and we consider the slave to be already 117 # present if the tuples match. (we include the class to make sure 118 # that BuildSlave(name,pw) is different than 119 # SubclassOfBuildSlave(name,pw) ). If the password or class has 120 # changed, we will remove the old version of the slave and replace it 121 # with a new one. If anything else has changed, we just update the 122 # old BuildSlave instance in place. If the name has changed, of 123 # course, it looks exactly the same as deleting one slave and adding 124 # an unrelated one. 125 old_t = {} 126 for s in old_slaves: 127 old_t[(s.slavename, s.password, s.__class__)] = s 128 new_t = {} 129 for s in new_slaves: 130 new_t[(s.slavename, s.password, s.__class__)] = s 131 removed = [old_t[t] 132 for t in old_t 133 if t not in new_t] 134 added = [new_t[t] 135 for t in new_t 136 if t not in old_t] 137 remaining_t = [t 138 for t in new_t 139 if t in old_t] 140 # removeSlave will hang up on the old bot 141 dl = [] 142 for s in removed: 143 dl.append(self.removeSlave(s)) 144 d = defer.DeferredList(dl, fireOnOneErrback=True) 145 def _add(res): 146 for s in added: 147 self.addSlave(s) 148 for t in remaining_t: 149 old_t[t].update(new_t[t])

150 d.addCallback(_add) 151 return d

152

153 - def addSlave(self, s):

154 s.setServiceParent(self) 155 s.setBotmaster(self) 156 self.slaves[s.slavename] = s

157

158 - def removeSlave(self, s):

159 # TODO: technically, disownServiceParent could return a Deferred 160 s.disownServiceParent() 161 d = self.slaves[s.slavename].disconnect() 162 del self.slaves[s.slavename] 163 return d

164

165 - def slaveLost(self, bot):

166 for name, b in self.builders.items(): 167 if bot.slavename in b.slavenames: 168 b.detached(bot)

169

170 - def getBuildersForSlave(self, slavename):

171 return [b 172 for b in self.builders.values() 173 if slavename in b.slavenames]

174

175 - def getBuildernames(self):

176 return self.builderNames

177

178 - def getBuilders(self):

179 allBuilders = [self.builders[name] for name in self.builderNames] 180 return allBuilders

181

182 - def setBuilders(self, builders):

183 self.builders = {} 184 self.builderNames = [] 185 for b in builders: 186 for slavename in b.slavenames: 187 # this is actually validated earlier 188 assert slavename in self.slaves 189 self.builders[b.name] = b 190 self.builderNames.append(b.name) 191 b.setBotmaster(self) 192 d = self._updateAllSlaves() 193 return d

194

195 - def _updateAllSlaves(self):

196 """Notify all buildslaves about changes in their Builders.""" 197 dl = [s.updateSlave() for s in self.slaves.values()] 198 return defer.DeferredList(dl)

199

200 - def maybeStartAllBuilds(self):

201 builders = self.builders.values() 202 if self.prioritizeBuilders is not None: 203 try: 204 builders = self.prioritizeBuilders(self.parent, builders) 205 except: 206 log.msg("Exception prioritizing builders") 207 log.err(Failure()) 208 return 209 else: 210 def _sortfunc(b1, b2): 211 t1 = b1.getOldestRequestTime() 212 t2 = b2.getOldestRequestTime() 213 # If t1 or t2 is None, then there are no build requests, 214 # so sort it at the end 215 if t1 is None: 216 return 1 217 if t2 is None: 218 return -1 219 return cmp(t1, t2)

220 builders.sort(_sortfunc) 221 try: 222 for b in builders: 223 b.maybeStartBuild() 224 except: 225 log.msg("Exception starting builds") 226 log.err(Failure()) 227

228 - def shouldMergeRequests(self, builder, req1, req2):

229 """Determine whether two BuildRequests should be merged for 230 the given builder. 231 232 """ 233 if self.mergeRequests is not None: 234 return self.mergeRequests(builder, req1, req2) 235 return req1.canBeMergedWith(req2)

236

237 - def getPerspective(self, slavename):

238 return self.slaves[slavename]

239

240 - def shutdownSlaves(self):

241 # TODO: make this into a bot method rather than a builder method 242 for b in self.slaves.values(): 243 b.shutdownSlave()

244

245 - def stopService(self):

246 for b in self.builders.values(): 247 b.builder_status.addPointEvent(["master", "shutdown"]) 248 b.builder_status.saveYourself() 249 return service.Service.stopService(self)

250

251 - def getLockByID(self, lockid):

252 """Convert a Lock identifier into an actual Lock instance. 253 @param lockid: a locks.MasterLock or locks.SlaveLock instance 254 @return: a locks.RealMasterLock or locks.RealSlaveLock instance 255 """ 256 assert isinstance(lockid, (locks.MasterLock, locks.SlaveLock)) 257 if not lockid in self.locks: 258 self.locks[lockid] = lockid.lockClass(lockid) 259 # if the master.cfg file has changed maxCount= on the lock, the next 260 # time a build is started, they'll get a new RealLock instance. Note 261 # that this requires that MasterLock and SlaveLock (marker) instances 262 # be hashable and that they should compare properly. 263 return self.locks[lockid]

264 265 ######################################## 266 267 268

269 -class DebugPerspective(NewCredPerspective):

270 - def attached(self, mind):

271 return self

272 - def detached(self, mind):

273 pass

274

275 - def perspective_requestBuild(self, buildername, reason, branch, revision, properties={}):

276 c = interfaces.IControl(self.master) 277 bc = c.getBuilder(buildername) 278 ss = SourceStamp(branch, revision) 279 bpr = Properties() 280 bpr.update(properties, "remote requestBuild") 281 br = BuildRequest(reason, ss, builderName=buildername, properties=bpr) 282 bc.requestBuild(br)

283

284 - def perspective_pingBuilder(self, buildername):

285 c = interfaces.IControl(self.master) 286 bc = c.getBuilder(buildername) 287 bc.ping()

288

289 - def perspective_fakeChange(self, file, revision=None, who="fakeUser", 290 branch=None):

291 change = Change(who, [file], "some fake comments\n", 292 branch=branch, revision=revision) 293 c = interfaces.IControl(self.master) 294 c.addChange(change)

295

296 - def perspective_setCurrentState(self, buildername, state):

297 builder = self.botmaster.builders.get(buildername) 298 if not builder: return 299 if state == "offline": 300 builder.statusbag.currentlyOffline() 301 if state == "idle": 302 builder.statusbag.currentlyIdle() 303 if state == "waiting": 304 builder.statusbag.currentlyWaiting(now()+10) 305 if state == "building": 306 builder.statusbag.currentlyBuilding(None)

307 - def perspective_reload(self):

308 print "doing reload of the config file" 309 self.master.loadTheConfigFile()

310 - def perspective_pokeIRC(self):

311 print "saying something on IRC" 312 from buildbot.status import words 313 for s in self.master: 314 if isinstance(s, words.IRC): 315 bot = s.f 316 for channel in bot.channels: 317 print " channel", channel 318 bot.p.msg(channel, "Ow, quit it")

319

320 - def perspective_print(self, msg):

321 print "debug", msg

322

323 -class Dispatcher(styles.Versioned):

324 implements(portal.IRealm) 325 persistenceVersion = 2 326

327 - def __init__(self):

328 self.names = {}

329

330 - def upgradeToVersion1(self):

331 self.master = self.botmaster.parent

332 - def upgradeToVersion2(self):

333 self.names = {}

334

335 - def register(self, name, afactory):

336 self.names[name] = afactory

337 - def unregister(self, name):

338 del self.names[name]

339

340 - def requestAvatar(self, avatarID, mind, interface):

341 assert interface == pb.IPerspective 342 afactory = self.names.get(avatarID) 343 if afactory: 344 p = afactory.getPerspective() 345 elif avatarID == "debug": 346 p = DebugPerspective() 347 p.master = self.master 348 p.botmaster = self.botmaster 349 elif avatarID == "statusClient": 350 p = self.statusClientService.getPerspective() 351 else: 352 # it must be one of the buildslaves: no other names will make it 353 # past the checker 354 p = self.botmaster.getPerspective(avatarID) 355 356 if not p: 357 raise ValueError("no perspective for '%s'" % avatarID) 358 359 d = defer.maybeDeferred(p.attached, mind) 360 d.addCallback(self._avatarAttached, mind) 361 return d

362

363 - def _avatarAttached(self, p, mind):

364 return (pb.IPerspective, p, lambda p=p,mind=mind: p.detached(mind))

365 366 ######################################## 367 368 # service hierarchy: 369 # BuildMaster 370 # BotMaster 371 # ChangeMaster 372 # all IChangeSource objects 373 # StatusClientService 374 # TCPClient(self.ircFactory) 375 # TCPServer(self.slaveFactory) -> dispatcher.requestAvatar 376 # TCPServer(self.site) 377 # UNIXServer(ResourcePublisher(self.site)) 378 379

380 -class BuildMaster(service.MultiService, styles.Versioned):

381 debug = 0 382 persistenceVersion = 3 383 manhole = None 384 debugPassword = None 385 projectName = "(unspecified)" 386 projectURL = None 387 buildbotURL = None 388 change_svc = None 389 properties = Properties() 390

391 - def __init__(self, basedir, configFileName="master.cfg"):

392 service.MultiService.__init__(self) 393 self.setName("buildmaster") 394 self.basedir = basedir 395 self.configFileName = configFileName 396 397 # the dispatcher is the realm in which all inbound connections are 398 # looked up: slave builders, change notifications, status clients, and 399 # the debug port 400 dispatcher = Dispatcher() 401 dispatcher.master = self 402 self.dispatcher = dispatcher 403 self.checker = checkers.InMemoryUsernamePasswordDatabaseDontUse() 404 # the checker starts with no user/passwd pairs: they are added later 405 p = portal.Portal(dispatcher) 406 p.registerChecker(self.checker) 407 self.slaveFactory = pb.PBServerFactory(p) 408 self.slaveFactory.unsafeTracebacks = True # let them see exceptions 409 410 self.slavePortnum = None 411 self.slavePort = None 412 413 self.botmaster = BotMaster() 414 self.botmaster.setName("botmaster") 415 self.botmaster.setServiceParent(self) 416 dispatcher.botmaster = self.botmaster 417 418 self.status = Status(self.botmaster, self.basedir) 419 420 self.statusTargets = [] 421 422 # this ChangeMaster is a dummy, only used by tests. In the real 423 # buildmaster, where the BuildMaster instance is activated 424 # (startService is called) by twistd, this attribute is overwritten. 425 self.useChanges(TestChangeMaster()) 426 427 self.readConfig = False

428

429 - def upgradeToVersion1(self):

430 self.dispatcher = self.slaveFactory.root.portal.realm

431

432 - def upgradeToVersion2(self): # post-0.4.3

433 self.webServer = self.webTCPPort 434 del self.webTCPPort 435 self.webDistribServer = self.webUNIXPort 436 del self.webUNIXPort 437 self.configFileName = "master.cfg"

438

439 - def upgradeToVersion3(self):

440 # post 0.6.3, solely to deal with the 0.6.3 breakage. Starting with 441 # 0.6.5 I intend to do away with .tap files altogether 442 self.services = [] 443 self.namedServices = {} 444 del self.change_svc

445

446 - def startService(self):

447 service.MultiService.startService(self) 448 self.loadChanges() # must be done before loading the config file 449 if not self.readConfig: 450 # TODO: consider catching exceptions during this call to 451 # loadTheConfigFile and bailing (reactor.stop) if it fails, 452 # since without a config file we can't do anything except reload 453 # the config file, and it would be nice for the user to discover 454 # this quickly. 455 self.loadTheConfigFile() 456 if signal and hasattr(signal, "SIGHUP"): 457 signal.signal(signal.SIGHUP, self._handleSIGHUP) 458 for b in self.botmaster.builders.values(): 459 b.builder_status.addPointEvent(["master", "started"]) 460 b.builder_status.saveYourself()

461

462 - def useChanges(self, changes):

463 if self.change_svc: 464 # TODO: can return a Deferred 465 self.change_svc.disownServiceParent() 466 self.change_svc = changes 467 self.change_svc.basedir = self.basedir 468 self.change_svc.setName("changemaster") 469 self.dispatcher.changemaster = self.change_svc 470 self.change_svc.setServiceParent(self)

471

472 - def loadChanges(self):

473 filename = os.path.join(self.basedir, "changes.pck") 474 try: 475 changes = load(open(filename, "rb")) 476 styles.doUpgrade() 477 except IOError: 478 log.msg("changes.pck missing, using new one") 479 changes = ChangeMaster() 480 except EOFError: 481 log.msg("corrupted changes.pck, using new one") 482 changes = ChangeMaster() 483 self.useChanges(changes)

484

485 - def _handleSIGHUP(self, *args):

486 reactor.callLater(0, self.loadTheConfigFile)

487

488 - def getStatus(self):

489 """ 490 @rtype: L{buildbot.status.builder.Status} 491 """ 492 return self.status

493

494 - def loadTheConfigFile(self, configFile=None):

495 if not configFile: 496 configFile = os.path.join(self.basedir, self.configFileName) 497 498 log.msg("Creating BuildMaster -- buildbot.version: %s" % buildbot.version) 499 log.msg("loading configuration from %s" % configFile) 500 configFile = os.path.expanduser(configFile) 501 502 try: 503 f = open(configFile, "r") 504 except IOError, e: 505 log.msg("unable to open config file '%s'" % configFile) 506 log.msg("leaving old configuration in place") 507 log.err(e) 508 return 509 510 try: 511 self.loadConfig(f) 512 except: 513 log.msg("error during loadConfig") 514 log.err() 515 log.msg("The new config file is unusable, so I'll ignore it.") 516 log.msg("I will keep using the previous config file instead.") 517 f.close()

518

519 - def loadConfig(self, f):

520 """Internal function to load a specific configuration file. Any 521 errors in the file will be signalled by raising an exception. 522 523 @return: a Deferred that will fire (with None) when the configuration 524 changes have been completed. This may involve a round-trip to each 525 buildslave that was involved.""" 526 527 localDict = {'basedir': os.path.expanduser(self.basedir)} 528 try: 529 exec f in localDict 530 except: 531 log.msg("error while parsing config file") 532 raise 533 534 try: 535 config = localDict['BuildmasterConfig'] 536 except KeyError: 537 log.err("missing config dictionary") 538 log.err("config file must define BuildmasterConfig") 539 raise 540 541 known_keys = ("bots", "slaves", 542 "sources", "change_source", 543 "schedulers", "builders", "mergeRequests", 544 "slavePortnum", "debugPassword", "logCompressionLimit", 545 "manhole", "status", "projectName", "projectURL", 546 "buildbotURL", "properties", "prioritizeBuilders", 547 "eventHorizon", "buildCacheSize", "logHorizon", "buildHorizon", 548 "changeHorizon", 549 ) 550 for k in config.keys(): 551 if k not in known_keys: 552 log.msg("unknown key '%s' defined in config dictionary" % k) 553 554 try: 555 # required 556 schedulers = config['schedulers'] 557 builders = config['builders'] 558 for k in builders: 559 if k['name'].startswith("_"): 560 errmsg = ("builder names must not start with an " 561 "underscore: " + k['name']) 562 log.err(errmsg) 563 raise ValueError(errmsg) 564 565 slavePortnum = config['slavePortnum'] 566 #slaves = config['slaves'] 567 #change_source = config['change_source'] 568 569 # optional 570 debugPassword = config.get('debugPassword') 571 manhole = config.get('manhole') 572 status = config.get('status', []) 573 projectName = config.get('projectName') 574 projectURL = config.get('projectURL') 575 buildbotURL = config.get('buildbotURL') 576 properties = config.get('properties', {}) 577 buildCacheSize = config.get('buildCacheSize', None) 578 eventHorizon = config.get('eventHorizon', None) 579 logHorizon = config.get('logHorizon', None) 580 buildHorizon = config.get('buildHorizon', None) 581 logCompressionLimit = config.get('logCompressionLimit') 582 if logCompressionLimit is not None and not \ 583 isinstance(logCompressionLimit, int): 584 raise ValueError("logCompressionLimit needs to be bool or int") 585 mergeRequests = config.get('mergeRequests') 586 if mergeRequests is not None and not callable(mergeRequests): 587 raise ValueError("mergeRequests must be a callable") 588 prioritizeBuilders = config.get('prioritizeBuilders') 589 if prioritizeBuilders is not None and not callable(prioritizeBuilders): 590 raise ValueError("prioritizeBuilders must be callable") 591 changeHorizon = config.get("changeHorizon") 592 if changeHorizon is not None and not isinstance(changeHorizon, int): 593 raise ValueError("changeHorizon needs to be an int") 594 595 except KeyError, e: 596 log.msg("config dictionary is missing a required parameter") 597 log.msg("leaving old configuration in place") 598 raise 599 600 #if "bots" in config: 601 # raise KeyError("c['bots'] is no longer accepted") 602 603 slaves = config.get('slaves', []) 604 if "bots" in config: 605 m = ("c['bots'] is deprecated as of 0.7.6 and will be " 606 "removed by 0.8.0 . Please use c['slaves'] instead.") 607 log.msg(m) 608 warnings.warn(m, DeprecationWarning) 609 for name, passwd in config['bots']: 610 slaves.append(BuildSlave(name, passwd)) 611 612 if "bots" not in config and "slaves" not in config: 613 log.msg("config dictionary must have either 'bots' or 'slaves'") 614 log.msg("leaving old configuration in place") 615 raise KeyError("must have either 'bots' or 'slaves'") 616 617 #if "sources" in config: 618 # raise KeyError("c['sources'] is no longer accepted") 619 620 if changeHorizon is not None: 621 self.change_svc.changeHorizon = changeHorizon 622 623 change_source = config.get('change_source', []) 624 if isinstance(change_source, (list, tuple)): 625 change_sources = change_source 626 else: 627 change_sources = [change_source] 628 if "sources" in config: 629 m = ("c['sources'] is deprecated as of 0.7.6 and will be " 630 "removed by 0.8.0 . Please use c['change_source'] instead.") 631 log.msg(m) 632 warnings.warn(m, DeprecationWarning) 633 for s in config['sources']: 634 change_sources.append(s) 635 636 # do some validation first 637 for s in slaves: 638 assert interfaces.IBuildSlave.providedBy(s) 639 if s.slavename in ("debug", "change", "status"): 640 raise KeyError( 641 "reserved name '%s' used for a bot" % s.slavename) 642 if config.has_key('interlocks'): 643 raise KeyError("c['interlocks'] is no longer accepted") 644 645 assert isinstance(change_sources, (list, tuple)) 646 for s in change_sources: 647 assert interfaces.IChangeSource(s, None) 648 # this assertion catches c['schedulers'] = Scheduler(), since 649 # Schedulers are service.MultiServices and thus iterable. 650 errmsg = "c['schedulers'] must be a list of Scheduler instances" 651 assert isinstance(schedulers, (list, tuple)), errmsg 652 for s in schedulers: 653 assert interfaces.IScheduler(s, None), errmsg 654 assert isinstance(status, (list, tuple)) 655 for s in status: 656 assert interfaces.IStatusReceiver(s, None) 657 658 slavenames = [s.slavename for s in slaves] 659 buildernames = [] 660 dirnames = [] 661 for b in builders: 662 if type(b) is tuple: 663 raise ValueError("builder %s must be defined with a dict, " 664 "not a tuple" % b[0]) 665 if b.has_key('slavename') and b['slavename'] not in slavenames: 666 raise ValueError("builder %s uses undefined slave %s" \ 667 % (b['name'], b['slavename'])) 668 for n in b.get('slavenames', []): 669 if n not in slavenames: 670 raise ValueError("builder %s uses undefined slave %s" \ 671 % (b['name'], n)) 672 if b['name'] in buildernames: 673 raise ValueError("duplicate builder name %s" 674 % b['name']) 675 buildernames.append(b['name']) 676 if b['builddir'] in dirnames: 677 raise ValueError("builder %s reuses builddir %s" 678 % (b['name'], b['builddir'])) 679 dirnames.append(b['builddir']) 680 681 unscheduled_buildernames = buildernames[:] 682 schedulernames = [] 683 for s in schedulers: 684 for b in s.listBuilderNames(): 685 assert b in buildernames, \ 686 "%s uses unknown builder %s" % (s, b) 687 if b in unscheduled_buildernames: 688 unscheduled_buildernames.remove(b) 689 690 if s.name in schedulernames: 691 # TODO: schedulers share a namespace with other Service 692 # children of the BuildMaster node, like status plugins, the 693 # Manhole, the ChangeMaster, and the BotMaster (although most 694 # of these don't have names) 695 msg = ("Schedulers must have unique names, but " 696 "'%s' was a duplicate" % (s.name,)) 697 raise ValueError(msg) 698 schedulernames.append(s.name) 699 700 if unscheduled_buildernames: 701 log.msg("Warning: some Builders have no Schedulers to drive them:" 702 " %s" % (unscheduled_buildernames,)) 703 704 # assert that all locks used by the Builds and their Steps are 705 # uniquely named. 706 lock_dict = {} 707 for b in builders: 708 for l in b.get('locks', []): 709 if isinstance(l, locks.LockAccess): # User specified access to the lock 710 l = l.lockid 711 if lock_dict.has_key(l.name): 712 if lock_dict[l.name] is not l: 713 raise ValueError("Two different locks (%s and %s) " 714 "share the name %s" 715 % (l, lock_dict[l.name], l.name)) 716 else: 717 lock_dict[l.name] = l 718 # TODO: this will break with any BuildFactory that doesn't use a 719 # .steps list, but I think the verification step is more 720 # important. 721 for s in b['factory'].steps: 722 for l in s[1].get('locks', []): 723 if isinstance(l, locks.LockAccess): # User specified access to the lock 724 l = l.lockid 725 if lock_dict.has_key(l.name): 726 if lock_dict[l.name] is not l: 727 raise ValueError("Two different locks (%s and %s)" 728 " share the name %s" 729 % (l, lock_dict[l.name], l.name)) 730 else: 731 lock_dict[l.name] = l 732 733 if not isinstance(properties, dict): 734 raise ValueError("c['properties'] must be a dictionary") 735 736 # slavePortnum supposed to be a strports specification 737 if type(slavePortnum) is int: 738 slavePortnum = "tcp:%d" % slavePortnum 739 740 # now we're committed to implementing the new configuration, so do 741 # it atomically 742 # TODO: actually, this is spread across a couple of Deferreds, so it 743 # really isn't atomic. 744 745 d = defer.succeed(None) 746 747 self.projectName = projectName 748 self.projectURL = projectURL 749 self.buildbotURL = buildbotURL 750 751 self.properties = Properties() 752 self.properties.update(properties, self.configFileName) 753 if logCompressionLimit is not None: 754 self.status.logCompressionLimit = logCompressionLimit 755 if mergeRequests is not None: 756 self.botmaster.mergeRequests = mergeRequests 757 if prioritizeBuilders is not None: 758 self.botmaster.prioritizeBuilders = prioritizeBuilders 759 760 self.buildCacheSize = buildCacheSize 761 self.eventHorizon = eventHorizon 762 self.logHorizon = logHorizon 763 self.buildHorizon = buildHorizon 764 765 # self.slaves: Disconnect any that were attached and removed from the 766 # list. Update self.checker with the new list of passwords, including 767 # debug/change/status. 768 d.addCallback(lambda res: self.loadConfig_Slaves(slaves)) 769 770 # self.debugPassword 771 if debugPassword: 772 self.checker.addUser("debug", debugPassword) 773 self.debugPassword = debugPassword 774 775 # self.manhole 776 if manhole != self.manhole: 777 # changing 778 if self.manhole: 779 # disownServiceParent may return a Deferred 780 d.addCallback(lambda res: self.manhole.disownServiceParent()) 781 def _remove(res): 782 self.manhole = None 783 return res

784 d.addCallback(_remove) 785 if manhole: 786 def _add(res): 787 self.manhole = manhole 788 manhole.setServiceParent(self) 789 d.addCallback(_add) 790 791 # add/remove self.botmaster.builders to match builders. The 792 # botmaster will handle startup/shutdown issues. 793 d.addCallback(lambda res: self.loadConfig_Builders(builders)) 794 795 d.addCallback(lambda res: self.loadConfig_status(status)) 796 797 # Schedulers are added after Builders in case they start right away 798 d.addCallback(lambda res: self.loadConfig_Schedulers(schedulers)) 799 # and Sources go after Schedulers for the same reason 800 d.addCallback(lambda res: self.loadConfig_Sources(change_sources)) 801 802 # self.slavePort 803 if self.slavePortnum != slavePortnum: 804 if self.slavePort: 805 def closeSlavePort(res): 806 d1 = self.slavePort.disownServiceParent() 807 self.slavePort = None 808 return d1 809 d.addCallback(closeSlavePort) 810 if slavePortnum is not None: 811 def openSlavePort(res): 812 self.slavePort = strports.service(slavePortnum, 813 self.slaveFactory) 814 self.slavePort.setServiceParent(self) 815 d.addCallback(openSlavePort) 816 log.msg("BuildMaster listening on port %s" % slavePortnum) 817 self.slavePortnum = slavePortnum 818 819 log.msg("configuration update started") 820 def _done(res): 821 self.readConfig = True 822 log.msg("configuration update complete") 823 d.addCallback(_done) 824 d.addCallback(lambda res: self.botmaster.maybeStartAllBuilds()) 825 return d 826

827 - def loadConfig_Slaves(self, new_slaves):

828 # set up the Checker with the names and passwords of all valid bots 829 self.checker.users = {} # violates abstraction, oh well 830 for s in new_slaves: 831 self.checker.addUser(s.slavename, s.password) 832 self.checker.addUser("change", "changepw") 833 # let the BotMaster take care of the rest 834 return self.botmaster.loadConfig_Slaves(new_slaves)

835

836 - def loadConfig_Sources(self, sources):

837 if not sources: 838 log.msg("warning: no ChangeSources specified in c['change_source']") 839 # shut down any that were removed, start any that were added 840 deleted_sources = [s for s in self.change_svc if s not in sources] 841 added_sources = [s for s in sources if s not in self.change_svc] 842 log.msg("adding %d new changesources, removing %d" % 843 (len(added_sources), len(deleted_sources))) 844 dl = [self.change_svc.removeSource(s) for s in deleted_sources] 845 def addNewOnes(res): 846 [self.change_svc.addSource(s) for s in added_sources]

847 d = defer.DeferredList(dl, fireOnOneErrback=1, consumeErrors=0) 848 d.addCallback(addNewOnes) 849 return d 850

851 - def allSchedulers(self):

852 return [child for child in self 853 if interfaces.IScheduler.providedBy(child)]

854 855

856 - def loadConfig_Schedulers(self, newschedulers):

857 oldschedulers = self.allSchedulers() 858 removed = [s for s in oldschedulers if s not in newschedulers] 859 added = [s for s in newschedulers if s not in oldschedulers] 860 dl = [defer.maybeDeferred(s.disownServiceParent) for s in removed] 861 def addNewOnes(res): 862 log.msg("adding %d new schedulers, removed %d" % 863 (len(added), len(dl))) 864 for s in added: 865 s.setServiceParent(self)

866 d = defer.DeferredList(dl, fireOnOneErrback=1) 867 d.addCallback(addNewOnes) 868 if removed or added: 869 # notify Downstream schedulers to potentially pick up 870 # new schedulers now that we have removed and added some 871 def updateDownstreams(res): 872 log.msg("notifying downstream schedulers of changes") 873 for s in newschedulers: 874 if interfaces.IDownstreamScheduler.providedBy(s): 875 s.checkUpstreamScheduler() 876 d.addCallback(updateDownstreams) 877 return d 878

879 - def loadConfig_Builders(self, newBuilderData):

880 somethingChanged = False 881 newList = {} 882 newBuilderNames = [] 883 allBuilders = self.botmaster.builders.copy() 884 for data in newBuilderData: 885 name = data['name'] 886 newList[name] = data 887 newBuilderNames.append(name) 888 889 # identify all that were removed 890 for oldname in self.botmaster.getBuildernames(): 891 if oldname not in newList: 892 log.msg("removing old builder %s" % oldname) 893 del allBuilders[oldname] 894 somethingChanged = True 895 # announce the change 896 self.status.builderRemoved(oldname) 897 898 # everything in newList is either unchanged, changed, or new 899 for name, data in newList.items(): 900 old = self.botmaster.builders.get(name) 901 basedir = data['builddir'] # used on both master and slave 902 #name, slave, builddir, factory = data 903 if not old: # new 904 # category added after 0.6.2 905 category = data.get('category', None) 906 log.msg("adding new builder %s for category %s" % 907 (name, category)) 908 statusbag = self.status.builderAdded(name, basedir, category) 909 builder = Builder(data, statusbag) 910 allBuilders[name] = builder 911 somethingChanged = True 912 elif old.compareToSetup(data): 913 # changed: try to minimize the disruption and only modify the 914 # pieces that really changed 915 diffs = old.compareToSetup(data) 916 log.msg("updating builder %s: %s" % (name, "\n".join(diffs))) 917 918 statusbag = old.builder_status 919 statusbag.saveYourself() # seems like a good idea 920 # TODO: if the basedir was changed, we probably need to make 921 # a new statusbag 922 new_builder = Builder(data, statusbag) 923 new_builder.consumeTheSoulOfYourPredecessor(old) 924 # that migrates any retained slavebuilders too 925 926 # point out that the builder was updated. On the Waterfall, 927 # this will appear just after any currently-running builds. 928 statusbag.addPointEvent(["config", "updated"]) 929 930 allBuilders[name] = new_builder 931 somethingChanged = True 932 else: 933 # unchanged: leave it alone 934 log.msg("builder %s is unchanged" % name) 935 pass 936 937 # regardless of whether anything changed, get each builder status 938 # to update its config 939 for builder in allBuilders.values(): 940 builder.builder_status.reconfigFromBuildmaster(self) 941 942 # and then tell the botmaster if anything's changed 943 if somethingChanged: 944 sortedAllBuilders = [allBuilders[name] for name in newBuilderNames] 945 d = self.botmaster.setBuilders(sortedAllBuilders) 946 return d 947 return None

948

949 - def loadConfig_status(self, status):

950 dl = [] 951 952 # remove old ones 953 for s in self.statusTargets[:]: 954 if not s in status: 955 log.msg("removing IStatusReceiver", s) 956 d = defer.maybeDeferred(s.disownServiceParent) 957 dl.append(d) 958 self.statusTargets.remove(s) 959 # after those are finished going away, add new ones 960 def addNewOnes(res): 961 for s in status: 962 if not s in self.statusTargets: 963 log.msg("adding IStatusReceiver", s) 964 s.setServiceParent(self) 965 self.statusTargets.append(s)

966 d = defer.DeferredList(dl, fireOnOneErrback=1) 967 d.addCallback(addNewOnes) 968 return d 969 970

971 - def addChange(self, change):

972 for s in self.allSchedulers(): 973 s.addChange(change)

974

975 - def submitBuildSet(self, bs):

976 # determine the set of Builders to use 977 builders = [] 978 for name in bs.builderNames: 979 b = self.botmaster.builders.get(name) 980 if b: 981 if b not in builders: 982 builders.append(b) 983 continue 984 # TODO: add aliases like 'all' 985 raise KeyError("no such builder named '%s'" % name) 986 987 # now tell the BuildSet to create BuildRequests for all those 988 # Builders and submit them 989 bs.start(builders) 990 self.status.buildsetSubmitted(bs.status)

991 992

993 -class Control:

994 implements(interfaces.IControl) 995

996 - def __init__(self, master):

997 self.master = master

998

999 - def addChange(self, change):

1000 self.master.change_svc.addChange(change)

1001

1002 - def submitBuildSet(self, bs):

1003 self.master.submitBuildSet(bs)

1004

1005 - def getBuilder(self, name):

1006 b = self.master.botmaster.builders[name] 1007 return interfaces.IBuilderControl(b)

1008 1009 components.registerAdapter(Control, BuildMaster, interfaces.IControl) 1010 1011 # so anybody who can get a handle on the BuildMaster can cause a build with: 1012 # IControl(master).getBuilder("full-2.3").requestBuild(buildrequest) 1013

Source Code for Module buildbot.master