buildbot.process.base

1 # -*- test-case-name: buildbot.test.test_step -*- 2 3 import types 4 5 from zope.interface import implements 6 from twisted.python import log 7 from twisted.python.failure import Failure 8 from twisted.internet import reactor, defer, error 9 10 from buildbot import interfaces, locks 11 from buildbot.status.builder import SUCCESS, WARNINGS, FAILURE, EXCEPTION 12 from buildbot.status.builder import Results, BuildRequestStatus 13 from buildbot.status.progress import BuildProgress 14 from buildbot.process.properties import Properties 15

16 -class BuildRequest:

17 """I represent a request to a specific Builder to run a single build. 18 19 I have a SourceStamp which specifies what sources I will build. This may 20 specify a specific revision of the source tree (so source.branch, 21 source.revision, and source.patch are used). The .patch attribute is 22 either None or a tuple of (patchlevel, diff), consisting of a number to 23 use in 'patch -pN', and a unified-format context diff. 24 25 Alternatively, the SourceStamp may specify a set of Changes to be built, 26 contained in source.changes. In this case, I may be mergeable with other 27 BuildRequests on the same branch. 28 29 I may be part of a BuildSet, in which case I will report status results 30 to it. 31 32 I am paired with a BuildRequestStatus object, to which I feed status 33 information. 34 35 @type source: a L{buildbot.sourcestamp.SourceStamp} instance. 36 @ivar source: the source code that this BuildRequest use 37 38 @type reason: string 39 @ivar reason: the reason this Build is being requested. Schedulers 40 provide this, but for forced builds the user requesting the 41 build will provide a string. 42 43 @type properties: Properties object 44 @ivar properties: properties that should be applied to this build 45 'owner' property is used by Build objects to collect 46 the list returned by getInterestedUsers 47 48 @ivar status: the IBuildStatus object which tracks our status 49 50 @ivar submittedAt: a timestamp (seconds since epoch) when this request 51 was submitted to the Builder. This is used by the CVS 52 step to compute a checkout timestamp, as well as the 53 master to prioritize build requests from oldest to 54 newest. 55 """ 56 57 source = None 58 builder = None 59 startCount = 0 # how many times we have tried to start this build 60 submittedAt = None 61 62 implements(interfaces.IBuildRequestControl) 63

64 - def __init__(self, reason, source, builderName, properties=None):

65 assert interfaces.ISourceStamp(source, None) 66 self.reason = reason 67 self.source = source 68 69 self.properties = Properties() 70 if properties: 71 self.properties.updateFromProperties(properties) 72 73 self.start_watchers = [] 74 self.finish_watchers = [] 75 self.status = BuildRequestStatus(source, builderName)

76

77 - def canBeMergedWith(self, other):

78 return self.source.canBeMergedWith(other.source)

79

80 - def mergeWith(self, others):

81 return self.source.mergeWith([o.source for o in others])

82

83 - def mergeReasons(self, others):

84 """Return a reason for the merged build request.""" 85 reasons = [] 86 for req in [self] + others: 87 if req.reason and req.reason not in reasons: 88 reasons.append(req.reason) 89 return ", ".join(reasons)

90

91 - def waitUntilFinished(self):

92 """Get a Deferred that will fire (with a 93 L{buildbot.interfaces.IBuildStatus} instance when the build 94 finishes.""" 95 d = defer.Deferred() 96 self.finish_watchers.append(d) 97 return d

98 99 # these are called by the Builder 100

101 - def requestSubmitted(self, builder):

102 # the request has been placed on the queue 103 self.builder = builder

104

105 - def buildStarted(self, build, buildstatus):

106 """This is called by the Builder when a Build has been started in the 107 hopes of satifying this BuildRequest. It may be called multiple 108 times, since interrupted builds and lost buildslaves may force 109 multiple Builds to be run until the fate of the BuildRequest is known 110 for certain.""" 111 for o in self.start_watchers[:]: 112 # these observers get the IBuildControl 113 o(build) 114 # while these get the IBuildStatus 115 self.status.buildStarted(buildstatus)

116

117 - def finished(self, buildstatus):

118 """This is called by the Builder when the BuildRequest has been 119 retired. This happens when its Build has either succeeded (yay!) or 120 failed (boo!). TODO: If it is halted due to an exception (oops!), or 121 some other retryable error, C{finished} will not be called yet.""" 122 123 for w in self.finish_watchers: 124 w.callback(buildstatus) 125 self.finish_watchers = []

126 127 # IBuildRequestControl 128

131 - def unsubscribe(self, observer):

132 self.start_watchers.remove(observer)

133

134 - def cancel(self):

135 """Cancel this request. This can only be successful if the Build has 136 not yet been started. 137 138 @return: a boolean indicating if the cancel was successful.""" 139 if self.builder: 140 return self.builder.cancelBuildRequest(self) 141 return False

142

143 - def setSubmitTime(self, t):

144 self.submittedAt = t 145 self.status.setSubmitTime(t)

146

147 - def getSubmitTime(self):

148 return self.submittedAt

149 150

151 -class Build:

152 """I represent a single build by a single slave. Specialized Builders can 153 use subclasses of Build to hold status information unique to those build 154 processes. 155 156 I control B{how} the build proceeds. The actual build is broken up into a 157 series of steps, saved in the .buildSteps[] array as a list of 158 L{buildbot.process.step.BuildStep} objects. Each step is a single remote 159 command, possibly a shell command. 160 161 During the build, I put status information into my C{BuildStatus} 162 gatherer. 163 164 After the build, I go away. 165 166 I can be used by a factory by setting buildClass on 167 L{buildbot.process.factory.BuildFactory} 168 169 @ivar requests: the list of L{BuildRequest}s that triggered me 170 @ivar build_status: the L{buildbot.status.builder.BuildStatus} that 171 collects our status 172 """ 173 174 implements(interfaces.IBuildControl) 175 176 workdir = "build" 177 build_status = None 178 reason = "changes" 179 finished = False 180 results = None 181

182 - def __init__(self, requests):

183 self.requests = requests 184 for req in self.requests: 185 req.startCount += 1 186 self.locks = [] 187 # build a source stamp 188 self.source = requests[0].mergeWith(requests[1:]) 189 self.reason = requests[0].mergeReasons(requests[1:]) 190 191 self.progress = None 192 self.currentStep = None 193 self.slaveEnvironment = {} 194 195 self.terminate = False

196

197 - def setBuilder(self, builder):

198 """ 199 Set the given builder as our builder. 200 201 @type builder: L{buildbot.process.builder.Builder} 202 """ 203 self.builder = builder

204

205 - def setLocks(self, locks):

206 self.locks = locks

207

208 - def setSlaveEnvironment(self, env):

209 self.slaveEnvironment = env

210

211 - def getSourceStamp(self):

212 return self.source

213

214 - def setProperty(self, propname, value, source):

215 """Set a property on this build. This may only be called after the 216 build has started, so that it has a BuildStatus object where the 217 properties can live.""" 218 self.build_status.setProperty(propname, value, source)

219

220 - def getProperties(self):

221 return self.build_status.getProperties()

222

223 - def getProperty(self, propname):

224 return self.build_status.getProperty(propname)

225

226 - def allChanges(self):

227 return self.source.changes

228

229 - def allFiles(self):

230 # return a list of all source files that were changed 231 files = [] 232 havedirs = 0 233 for c in self.allChanges(): 234 for f in c.files: 235 files.append(f) 236 if c.isdir: 237 havedirs = 1 238 return files

239

240 - def __repr__(self):

241 return "<Build %s>" % (self.builder.name,)

242

243 - def __getstate__(self):

244 d = self.__dict__.copy() 245 if d.has_key('remote'): 246 del d['remote'] 247 return d

248

249 - def blamelist(self):

250 blamelist = [] 251 for c in self.allChanges(): 252 if c.who not in blamelist: 253 blamelist.append(c.who) 254 blamelist.sort() 255 return blamelist

256

257 - def changesText(self):

258 changetext = "" 259 for c in self.allChanges(): 260 changetext += "-" * 60 + "\n\n" + c.asText() + "\n" 261 # consider sorting these by number 262 return changetext

263

264 - def setStepFactories(self, step_factories):

265 """Set a list of 'step factories', which are tuples of (class, 266 kwargs), where 'class' is generally a subclass of step.BuildStep . 267 These are used to create the Steps themselves when the Build starts 268 (as opposed to when it is first created). By creating the steps 269 later, their __init__ method will have access to things like 270 build.allFiles() .""" 271 self.stepFactories = list(step_factories)

272 273 274 275 useProgress = True 276

277 - def getSlaveCommandVersion(self, command, oldversion=None):

278 return self.slavebuilder.getSlaveCommandVersion(command, oldversion)

279 - def getSlaveName(self):

280 return self.slavebuilder.slave.slavename

281

282 - def setupProperties(self):

283 props = self.getProperties() 284 285 # start with global properties from the configuration 286 buildmaster = self.builder.botmaster.parent 287 props.updateFromProperties(buildmaster.properties) 288 289 # get any properties from requests (this is the path through 290 # which schedulers will send us properties) 291 for rq in self.requests: 292 props.updateFromProperties(rq.properties) 293 294 # now set some properties of our own, corresponding to the 295 # build itself 296 props.setProperty("buildername", self.builder.name, "Build") 297 props.setProperty("buildnumber", self.build_status.number, "Build") 298 props.setProperty("branch", self.source.branch, "Build") 299 props.setProperty("revision", self.source.revision, "Build")

300

301 - def setupSlaveBuilder(self, slavebuilder):

302 self.slavebuilder = slavebuilder 303 304 # navigate our way back to the L{buildbot.buildslave.BuildSlave} 305 # object that came from the config, and get its properties 306 buildslave_properties = slavebuilder.slave.properties 307 self.getProperties().updateFromProperties(buildslave_properties) 308 309 self.slavename = slavebuilder.slave.slavename 310 self.build_status.setSlavename(self.slavename)

311

312 - def startBuild(self, build_status, expectations, slavebuilder):

313 """This method sets up the build, then starts it by invoking the 314 first Step. It returns a Deferred which will fire when the build 315 finishes. This Deferred is guaranteed to never errback.""" 316 317 # we are taking responsibility for watching the connection to the 318 # remote. This responsibility was held by the Builder until our 319 # startBuild was called, and will not return to them until we fire 320 # the Deferred returned by this method. 321 322 log.msg("%s.startBuild" % self) 323 self.build_status = build_status 324 # now that we have a build_status, we can set properties 325 self.setupProperties() 326 self.setupSlaveBuilder(slavebuilder) 327 slavebuilder.slave.updateSlaveStatus(buildStarted=build_status) 328 329 # convert all locks into their real forms 330 lock_list = [] 331 for access in self.locks: 332 if not isinstance(access, locks.LockAccess): 333 # Buildbot 0.7.7 compability: user did not specify access 334 access = access.defaultAccess() 335 lock = self.builder.botmaster.getLockByID(access.lockid) 336 lock_list.append((lock, access)) 337 self.locks = lock_list 338 # then narrow SlaveLocks down to the right slave 339 self.locks = [(l.getLock(self.slavebuilder), la) 340 for l, la in self.locks] 341 self.remote = slavebuilder.remote 342 self.remote.notifyOnDisconnect(self.lostRemote) 343 d = self.deferred = defer.Deferred() 344 def _release_slave(res, slave, bs): 345 self.slavebuilder.buildFinished() 346 slave.updateSlaveStatus(buildFinished=bs) 347 return res

348 d.addCallback(_release_slave, self.slavebuilder.slave, build_status) 349 350 try: 351 self.setupBuild(expectations) # create .steps 352 except: 353 # the build hasn't started yet, so log the exception as a point 354 # event instead of flunking the build. TODO: associate this 355 # failure with the build instead. this involves doing 356 # self.build_status.buildStarted() from within the exception 357 # handler 358 log.msg("Build.setupBuild failed") 359 log.err(Failure()) 360 self.builder.builder_status.addPointEvent(["setupBuild", 361 "exception"]) 362 self.finished = True 363 self.results = FAILURE 364 self.deferred = None 365 d.callback(self) 366 return d 367 368 self.acquireLocks().addCallback(self._startBuild_2) 369 return d

370

371 - def acquireLocks(self, res=None):

372 log.msg("acquireLocks(step %s, locks %s)" % (self, self.locks)) 373 if not self.locks: 374 return defer.succeed(None) 375 for lock, access in self.locks: 376 if not lock.isAvailable(access): 377 log.msg("Build %s waiting for lock %s" % (self, lock)) 378 d = lock.waitUntilMaybeAvailable(self, access) 379 d.addCallback(self.acquireLocks) 380 return d 381 # all locks are available, claim them all 382 for lock, access in self.locks: 383 lock.claim(self, access) 384 return defer.succeed(None)

385

386 - def _startBuild_2(self, res):

387 self.build_status.buildStarted(self) 388 self.startNextStep()

389

390 - def setupBuild(self, expectations):

391 # create the actual BuildSteps. If there are any name collisions, we 392 # add a count to the loser until it is unique. 393 self.steps = [] 394 self.stepStatuses = {} 395 stepnames = {} 396 sps = [] 397 398 for factory, args in self.stepFactories: 399 args = args.copy() 400 try: 401 step = factory(**args) 402 except: 403 log.msg("error while creating step, factory=%s, args=%s" 404 % (factory, args)) 405 raise 406 step.setBuild(self) 407 step.setBuildSlave(self.slavebuilder.slave) 408 step.setDefaultWorkdir(self.workdir) 409 name = step.name 410 if stepnames.has_key(name): 411 count = stepnames[name] 412 count += 1 413 stepnames[name] = count 414 name = step.name + "_%d" % count 415 else: 416 stepnames[name] = 0 417 step.name = name 418 self.steps.append(step) 419 420 # tell the BuildStatus about the step. This will create a 421 # BuildStepStatus and bind it to the Step. 422 step_status = self.build_status.addStepWithName(name) 423 step.setStepStatus(step_status) 424 425 sp = None 426 if self.useProgress: 427 # XXX: maybe bail if step.progressMetrics is empty? or skip 428 # progress for that one step (i.e. "it is fast"), or have a 429 # separate "variable" flag that makes us bail on progress 430 # tracking 431 sp = step.setupProgress() 432 if sp: 433 sps.append(sp) 434 435 # Create a buildbot.status.progress.BuildProgress object. This is 436 # called once at startup to figure out how to build the long-term 437 # Expectations object, and again at the start of each build to get a 438 # fresh BuildProgress object to track progress for that individual 439 # build. TODO: revisit at-startup call 440 441 if self.useProgress: 442 self.progress = BuildProgress(sps) 443 if self.progress and expectations: 444 self.progress.setExpectationsFrom(expectations) 445 446 # we are now ready to set up our BuildStatus. 447 self.build_status.setSourceStamp(self.source) 448 self.build_status.setRequests([req.status for req in self.requests]) 449 self.build_status.setReason(self.reason) 450 self.build_status.setBlamelist(self.blamelist()) 451 self.build_status.setProgress(self.progress) 452 453 # gather owners from build requests 454 owners = [r.properties['owner'] for r in self.requests 455 if r.properties.has_key('owner')] 456 if owners: self.setProperty('owners', owners, self.reason) 457 458 self.results = [] # list of FAILURE, SUCCESS, WARNINGS, SKIPPED 459 self.result = SUCCESS # overall result, may downgrade after each step 460 self.text = [] # list of text string lists (text2)

461

462 - def getNextStep(self):

463 """This method is called to obtain the next BuildStep for this build. 464 When it returns None (or raises a StopIteration exception), the build 465 is complete.""" 466 if not self.steps: 467 return None 468 if self.terminate: 469 while True: 470 s = self.steps.pop(0) 471 if s.alwaysRun: 472 return s 473 if not self.steps: 474 return None 475 else: 476 return self.steps.pop(0)

477

478 - def startNextStep(self):

479 try: 480 s = self.getNextStep() 481 except StopIteration: 482 s = None 483 if not s: 484 return self.allStepsDone() 485 self.currentStep = s 486 d = defer.maybeDeferred(s.startStep, self.remote) 487 d.addCallback(self._stepDone, s) 488 d.addErrback(self.buildException)

489

490 - def _stepDone(self, results, step):

491 self.currentStep = None 492 if self.finished: 493 return # build was interrupted, don't keep building 494 terminate = self.stepDone(results, step) # interpret/merge results 495 if terminate: 496 self.terminate = True 497 return self.startNextStep()

498

499 - def stepDone(self, result, step):

500 """This method is called when the BuildStep completes. It is passed a 501 status object from the BuildStep and is responsible for merging the 502 Step's results into those of the overall Build.""" 503 504 terminate = False 505 text = None 506 if type(result) == types.TupleType: 507 result, text = result 508 assert type(result) == type(SUCCESS) 509 log.msg(" step '%s' complete: %s" % (step.name, Results[result])) 510 self.results.append(result) 511 if text: 512 self.text.extend(text) 513 if not self.remote: 514 terminate = True 515 if result == FAILURE: 516 if step.warnOnFailure: 517 if self.result != FAILURE: 518 self.result = WARNINGS 519 if step.flunkOnFailure: 520 self.result = FAILURE 521 if step.haltOnFailure: 522 terminate = True 523 elif result == WARNINGS: 524 if step.warnOnWarnings: 525 if self.result != FAILURE: 526 self.result = WARNINGS 527 if step.flunkOnWarnings: 528 self.result = FAILURE 529 elif result == EXCEPTION: 530 self.result = EXCEPTION 531 terminate = True 532 return terminate

533

534 - def lostRemote(self, remote=None):

535 # the slave went away. There are several possible reasons for this, 536 # and they aren't necessarily fatal. For now, kill the build, but 537 # TODO: see if we can resume the build when it reconnects. 538 log.msg("%s.lostRemote" % self) 539 self.remote = None 540 if self.currentStep: 541 # this should cause the step to finish. 542 log.msg(" stopping currentStep", self.currentStep) 543 self.currentStep.interrupt(Failure(error.ConnectionLost()))

544

545 - def stopBuild(self, reason="<no reason given>"):

546 # the idea here is to let the user cancel a build because, e.g., 547 # they realized they committed a bug and they don't want to waste 548 # the time building something that they know will fail. Another 549 # reason might be to abandon a stuck build. We want to mark the 550 # build as failed quickly rather than waiting for the slave's 551 # timeout to kill it on its own. 552 553 log.msg(" %s: stopping build: %s" % (self, reason)) 554 if self.finished: 555 return 556 # TODO: include 'reason' in this point event 557 self.builder.builder_status.addPointEvent(['interrupt']) 558 self.currentStep.interrupt(reason) 559 if 0: 560 # TODO: maybe let its deferred do buildFinished 561 if self.currentStep and self.currentStep.progress: 562 # XXX: really .fail or something 563 self.currentStep.progress.finish() 564 text = ["stopped", reason] 565 self.buildFinished(text, FAILURE)

566

567 - def allStepsDone(self):

568 if self.result == FAILURE: 569 text = ["failed"] 570 elif self.result == WARNINGS: 571 text = ["warnings"] 572 elif self.result == EXCEPTION: 573 text = ["exception"] 574 else: 575 text = ["build", "successful"] 576 text.extend(self.text) 577 return self.buildFinished(text, self.result)

578

579 - def buildException(self, why):

580 log.msg("%s.buildException" % self) 581 log.err(why) 582 self.buildFinished(["build", "exception"], FAILURE)

583

584 - def buildFinished(self, text, results):

585 """This method must be called when the last Step has completed. It 586 marks the Build as complete and returns the Builder to the 'idle' 587 state. 588 589 It takes two arguments which describe the overall build status: 590 text, results. 'results' is one of SUCCESS, WARNINGS, or FAILURE. 591 592 If 'results' is SUCCESS or WARNINGS, we will permit any dependant 593 builds to start. If it is 'FAILURE', those builds will be 594 abandoned.""" 595 596 self.finished = True 597 if self.remote: 598 self.remote.dontNotifyOnDisconnect(self.lostRemote) 599 self.results = results 600 601 log.msg(" %s: build finished" % self) 602 self.build_status.setText(text) 603 self.build_status.setResults(results) 604 self.build_status.buildFinished() 605 if self.progress and results == SUCCESS: 606 # XXX: also test a 'timing consistent' flag? 607 log.msg(" setting expectations for next time") 608 self.builder.setExpectations(self.progress) 609 reactor.callLater(0, self.releaseLocks) 610 self.deferred.callback(self) 611 self.deferred = None

612

613 - def releaseLocks(self):

614 log.msg("releaseLocks(%s): %s" % (self, self.locks)) 615 for lock, access in self.locks: 616 lock.release(self, access)

617 618 # IBuildControl 619

620 - def getStatus(self):

621 return self.build_status

622 623 # stopBuild is defined earlier 624

Source Code for Module buildbot.process.base