buildbot.scripts.tryclient

1 # -*- test-case-name: buildbot.test.test_scheduler,buildbot.test.test_vc -*- 2 3 import sys, os, re, time, random 4 from twisted.internet import utils, protocol, defer, reactor, task 5 from twisted.spread import pb 6 from twisted.cred import credentials 7 from twisted.python import log 8 from twisted.python.procutils import which 9 10 from buildbot.sourcestamp import SourceStamp 11 from buildbot.scripts import runner 12 from buildbot.util import now 13 from buildbot.status import builder 14

15 -class SourceStampExtractor:

16

17 - def __init__(self, treetop, branch):

18 self.treetop = treetop 19 self.branch = branch 20 self.exe = which(self.vcexe)[0]

21

22 - def dovc(self, cmd):

23 """This accepts the arguments of a command, without the actual 24 command itself.""" 25 env = os.environ.copy() 26 env['LC_ALL'] = "C" 27 d = utils.getProcessOutputAndValue(self.exe, cmd, env=env, 28 path=self.treetop) 29 d.addCallback(self._didvc, cmd) 30 return d

31 - def _didvc(self, res, cmd):

32 (stdout, stderr, code) = res 33 # 'bzr diff' sets rc=1 if there were any differences. tla, baz, and 34 # cvs do something similar, so don't bother requring rc=0. 35 return stdout

36

37 - def get(self):

38 """Return a Deferred that fires with a SourceStamp instance.""" 39 d = self.getBaseRevision() 40 d.addCallback(self.getPatch) 41 d.addCallback(self.done) 42 return d

43 - def readPatch(self, res, patchlevel):

44 self.patch = (patchlevel, res)

45 - def done(self, res):

46 # TODO: figure out the branch too 47 ss = SourceStamp(self.branch, self.baserev, self.patch) 48 return ss

49

50 -class CVSExtractor(SourceStampExtractor):

51 patchlevel = 0 52 vcexe = "cvs"

53 - def getBaseRevision(self):

54 # this depends upon our local clock and the repository's clock being 55 # reasonably synchronized with each other. We express everything in 56 # UTC because the '%z' format specifier for strftime doesn't always 57 # work. 58 self.baserev = time.strftime("%Y-%m-%d %H:%M:%S +0000", 59 time.gmtime(now())) 60 return defer.succeed(None)

61

62 - def getPatch(self, res):

63 # the -q tells CVS to not announce each directory as it works 64 if self.branch is not None: 65 # 'cvs diff' won't take both -r and -D at the same time (it 66 # ignores the -r). As best I can tell, there is no way to make 67 # cvs give you a diff relative to a timestamp on the non-trunk 68 # branch. A bare 'cvs diff' will tell you about the changes 69 # relative to your checked-out versions, but I know of no way to 70 # find out what those checked-out versions are. 71 raise RuntimeError("Sorry, CVS 'try' builds don't work with " 72 "branches") 73 args = ['-q', 'diff', '-u', '-D', self.baserev] 74 d = self.dovc(args) 75 d.addCallback(self.readPatch, self.patchlevel) 76 return d

77

78 -class SVNExtractor(SourceStampExtractor):

79 patchlevel = 0 80 vcexe = "svn" 81

82 - def getBaseRevision(self):

83 d = self.dovc(["status", "-u"]) 84 d.addCallback(self.parseStatus) 85 return d

86 - def parseStatus(self, res):

87 # svn shows the base revision for each file that has been modified or 88 # which needs an update. You can update each file to a different 89 # version, so each file is displayed with its individual base 90 # revision. It also shows the repository-wide latest revision number 91 # on the last line ("Status against revision: \d+"). 92 93 # for our purposes, we use the latest revision number as the "base" 94 # revision, and get a diff against that. This means we will get 95 # reverse-diffs for local files that need updating, but the resulting 96 # tree will still be correct. The only weirdness is that the baserev 97 # that we emit may be different than the version of the tree that we 98 # first checked out. 99 100 # to do this differently would probably involve scanning the revision 101 # numbers to find the max (or perhaps the min) revision, and then 102 # using that as a base. 103 104 for line in res.split("\n"): 105 m = re.search(r'^Status against revision:\s+(\d+)', line) 106 if m: 107 self.baserev = int(m.group(1)) 108 return 109 raise IndexError("Could not find 'Status against revision' in " 110 "SVN output: %s" % res)

111 - def getPatch(self, res):

112 d = self.dovc(["diff", "-r%d" % self.baserev]) 113 d.addCallback(self.readPatch, self.patchlevel) 114 return d

115

116 -class BazExtractor(SourceStampExtractor):

117 patchlevel = 1 118 vcexe = "baz"

119 - def getBaseRevision(self):

120 d = self.dovc(["tree-id"]) 121 d.addCallback(self.parseStatus) 122 return d

123 - def parseStatus(self, res):

124 tid = res.strip() 125 slash = tid.index("/") 126 dd = tid.rindex("--") 127 self.branch = tid[slash+1:dd] 128 self.baserev = tid[dd+2:]

129 - def getPatch(self, res):

130 d = self.dovc(["diff"]) 131 d.addCallback(self.readPatch, self.patchlevel) 132 return d

133

134 -class TlaExtractor(SourceStampExtractor):

135 patchlevel = 1 136 vcexe = "tla"

137 - def getBaseRevision(self):

138 # 'tla logs --full' gives us ARCHIVE/BRANCH--REVISION 139 # 'tla logs' gives us REVISION 140 d = self.dovc(["logs", "--full", "--reverse"]) 141 d.addCallback(self.parseStatus) 142 return d

143 - def parseStatus(self, res):

144 tid = res.split("\n")[0].strip() 145 slash = tid.index("/") 146 dd = tid.rindex("--") 147 self.branch = tid[slash+1:dd] 148 self.baserev = tid[dd+2:]

149

150 - def getPatch(self, res):

151 d = self.dovc(["changes", "--diffs"]) 152 d.addCallback(self.readPatch, self.patchlevel) 153 return d

154

155 -class BzrExtractor(SourceStampExtractor):

156 patchlevel = 0 157 vcexe = "bzr"

158 - def getBaseRevision(self):

159 d = self.dovc(["version-info"]) 160 d.addCallback(self.get_revision_number) 161 return d

162 - def get_revision_number(self, out):

163 for line in out.split("\n"): 164 colon = line.find(":") 165 if colon != -1: 166 key, value = line[:colon], line[colon+2:] 167 if key == "revno": 168 self.baserev = int(value) 169 return 170 raise ValueError("unable to find revno: in bzr output: '%s'" % out)

171

172 - def getPatch(self, res):

173 d = self.dovc(["diff"]) 174 d.addCallback(self.readPatch, self.patchlevel) 175 return d

176

177 -class MercurialExtractor(SourceStampExtractor):

178 patchlevel = 1 179 vcexe = "hg"

180 - def getBaseRevision(self):

181 d = self.dovc(["identify"]) 182 d.addCallback(self.parseStatus) 183 return d

184 - def parseStatus(self, output):

185 m = re.search(r'^(\w+)', output) 186 self.baserev = m.group(0)

187 - def getPatch(self, res):

188 d = self.dovc(["diff"]) 189 d.addCallback(self.readPatch, self.patchlevel) 190 return d

191

192 -class DarcsExtractor(SourceStampExtractor):

193 patchlevel = 1 194 vcexe = "darcs"

195 - def getBaseRevision(self):

196 d = self.dovc(["changes", "--context"]) 197 d.addCallback(self.parseStatus) 198 return d

199 - def parseStatus(self, res):

200 self.baserev = res # the whole context file

201 - def getPatch(self, res):

202 d = self.dovc(["diff", "-u"]) 203 d.addCallback(self.readPatch, self.patchlevel) 204 return d

205

206 -class GitExtractor(SourceStampExtractor):

207 patchlevel = 1 208 vcexe = "git" 209

210 - def getBaseRevision(self):

211 d = self.dovc(["branch", "--no-color", "-v", "--no-abbrev"]) 212 d.addCallback(self.parseStatus) 213 return d

214

215 - def readConfig(self):

216 d = self.dovc(["config", "-l"]) 217 d.addCallback(self.parseConfig) 218 return d

219

220 - def parseConfig(self, res):

221 git_config = {} 222 for l in res.split("\n"): 223 if l.strip(): 224 parts = l.strip().split("=", 2) 225 git_config[parts[0]] = parts[1] 226 227 # If we're tracking a remote, consider that the base. 228 remote = git_config.get("branch." + self.branch + ".remote") 229 ref = git_config.get("branch." + self.branch + ".merge") 230 if remote and ref: 231 remote_branch = ref.split("/", 3)[-1] 232 d = self.dovc(["rev-parse", remote + "/" + remote_branch]) 233 d.addCallback(self.override_baserev) 234 return d

235

236 - def override_baserev(self, res):

237 self.baserev = res.strip()

238

239 - def parseStatus(self, res):

240 # The current branch is marked by '*' at the start of the 241 # line, followed by the branch name and the SHA1. 242 # 243 # Branch names may contain pretty much anything but whitespace. 244 m = re.search(r'^\* (\S+)\s+([0-9a-f]{40})', res, re.MULTILINE) 245 if m: 246 self.baserev = m.group(2) 247 # If a branch is specified, parse out the rev it points to 248 # and extract the local name (assuming it has a slash). 249 # This may break if someone specifies the name of a local 250 # branch that has a slash in it and has no corresponding 251 # remote branch (or something similarly contrived). 252 if self.branch: 253 d = self.dovc(["rev-parse", self.branch]) 254 if '/' in self.branch: 255 self.branch = self.branch.split('/', 1)[1] 256 d.addCallback(self.override_baserev) 257 return d 258 else: 259 self.branch = m.group(1) 260 return self.readConfig() 261 raise IndexError("Could not find current GIT branch: %s" % res)

262

263 - def getPatch(self, res):

264 d = self.dovc(["diff", self.baserev]) 265 d.addCallback(self.readPatch, self.patchlevel) 266 return d

267

268 -def getSourceStamp(vctype, treetop, branch=None):

269 if vctype == "cvs": 270 e = CVSExtractor(treetop, branch) 271 elif vctype == "svn": 272 e = SVNExtractor(treetop, branch) 273 elif vctype == "baz": 274 e = BazExtractor(treetop, branch) 275 elif vctype == "bzr": 276 e = BzrExtractor(treetop, branch) 277 elif vctype == "tla": 278 e = TlaExtractor(treetop, branch) 279 elif vctype == "hg": 280 e = MercurialExtractor(treetop, branch) 281 elif vctype == "darcs": 282 e = DarcsExtractor(treetop, branch) 283 elif vctype == "git": 284 e = GitExtractor(treetop, branch) 285 else: 286 raise KeyError("unknown vctype '%s'" % vctype) 287 return e.get()

288 289

290 -def ns(s):

291 return "%d:%s," % (len(s), s)

292

293 -def createJobfile(bsid, branch, baserev, patchlevel, diff, builderNames):

294 job = "" 295 job += ns("1") 296 job += ns(bsid) 297 job += ns(branch) 298 job += ns(str(baserev)) 299 job += ns("%d" % patchlevel) 300 job += ns(diff) 301 for bn in builderNames: 302 job += ns(bn) 303 return job

304

305 -def getTopdir(topfile, start=None):

306 """walk upwards from the current directory until we find this topfile""" 307 if not start: 308 start = os.getcwd() 309 here = start 310 toomany = 20 311 while toomany > 0: 312 if os.path.exists(os.path.join(here, topfile)): 313 return here 314 next = os.path.dirname(here) 315 if next == here: 316 break # we've hit the root 317 here = next 318 toomany -= 1 319 raise ValueError("Unable to find topfile '%s' anywhere from %s upwards" 320 % (topfile, start))

321

322 -class RemoteTryPP(protocol.ProcessProtocol):

323 - def __init__(self, job):

324 self.job = job 325 self.d = defer.Deferred()

326 - def connectionMade(self):

327 self.transport.write(self.job) 328 self.transport.closeStdin()

329 - def outReceived(self, data):

330 sys.stdout.write(data)

331 - def errReceived(self, data):

332 sys.stderr.write(data)

333 - def processEnded(self, status_object):

334 sig = status_object.value.signal 335 rc = status_object.value.exitCode 336 if sig != None or rc != 0: 337 self.d.errback(RuntimeError("remote 'buildbot tryserver' failed" 338 ": sig=%s, rc=%s" % (sig, rc))) 339 return 340 self.d.callback((sig, rc))

341

342 -class BuildSetStatusGrabber:

343 retryCount = 5 # how many times to we try to grab the BuildSetStatus? 344 retryDelay = 3 # seconds to wait between attempts 345

346 - def __init__(self, status, bsid):

347 self.status = status 348 self.bsid = bsid

349

350 - def grab(self):

351 # return a Deferred that either fires with the BuildSetStatus 352 # reference or errbacks because we were unable to grab it 353 self.d = defer.Deferred() 354 # wait a second before querying to give the master's maildir watcher 355 # a chance to see the job 356 reactor.callLater(1, self.go) 357 return self.d

358

359 - def go(self, dummy=None):

360 if self.retryCount == 0: 361 raise RuntimeError("couldn't find matching buildset") 362 self.retryCount -= 1 363 d = self.status.callRemote("getBuildSets") 364 d.addCallback(self._gotSets)

365

366 - def _gotSets(self, buildsets):

367 for bs,bsid in buildsets: 368 if bsid == self.bsid: 369 # got it 370 self.d.callback(bs) 371 return 372 d = defer.Deferred() 373 d.addCallback(self.go) 374 reactor.callLater(self.retryDelay, d.callback, None)

375 376

377 -class Try(pb.Referenceable):

378 buildsetStatus = None 379 quiet = False 380

381 - def __init__(self, config):

382 self.config = config 383 self.opts = runner.loadOptions() 384 self.connect = self.getopt('connect', 'try_connect') 385 assert self.connect, "you must specify a connect style: ssh or pb" 386 self.builderNames = self.getopt('builders', 'try_builders')

387

388 - def getopt(self, config_name, options_name, default=None):

389 value = self.config.get(config_name) 390 if value is None or value == []: 391 value = self.opts.get(options_name) 392 if value is None or value == []: 393 value = default 394 return value

395

396 - def createJob(self):

397 # returns a Deferred which fires when the job parameters have been 398 # created 399 opts = self.opts 400 # generate a random (unique) string. It would make sense to add a 401 # hostname and process ID here, but a) I suspect that would cause 402 # windows portability problems, and b) really this is good enough 403 self.bsid = "%d-%s" % (time.time(), random.randint(0, 1000000)) 404 405 # common options 406 branch = self.getopt("branch", "try_branch") 407 408 difffile = self.config.get("diff") 409 if difffile: 410 baserev = self.config.get("baserev") 411 if difffile == "-": 412 diff = sys.stdin.read() 413 else: 414 diff = open(difffile,"r").read() 415 patch = (self.config['patchlevel'], diff) 416 ss = SourceStamp(branch, baserev, patch) 417 d = defer.succeed(ss) 418 else: 419 vc = self.getopt("vc", "try_vc") 420 if vc in ("cvs", "svn"): 421 # we need to find the tree-top 422 topdir = self.getopt("try_topdir", "try_topdir") 423 if topdir: 424 treedir = os.path.expanduser(topdir) 425 else: 426 topfile = self.getopt("try-topfile", "try_topfile") 427 treedir = getTopdir(topfile) 428 else: 429 treedir = os.getcwd() 430 d = getSourceStamp(vc, treedir, branch) 431 d.addCallback(self._createJob_1) 432 return d

433

434 - def _createJob_1(self, ss):

435 self.sourcestamp = ss 436 if self.connect == "ssh": 437 patchlevel, diff = ss.patch 438 revspec = ss.revision 439 if revspec is None: 440 revspec = "" 441 self.jobfile = createJobfile(self.bsid, 442 ss.branch or "", revspec, 443 patchlevel, diff, 444 self.builderNames)

445

446 - def fakeDeliverJob(self):

447 # Display the job to be delivered, but don't perform delivery. 448 ss = self.sourcestamp 449 print ("Job:\n\tBranch: %s\n\tRevision: %s\n\tBuilders: %s\n%s" 450 % (ss.branch, 451 ss.revision, 452 self.builderNames, 453 ss.patch[1])) 454 d = defer.Deferred() 455 d.callback(True) 456 return d

457

458 - def deliverJob(self):

459 # returns a Deferred that fires when the job has been delivered 460 opts = self.opts 461 462 if self.connect == "ssh": 463 tryhost = self.getopt("tryhost", "try_host") 464 tryuser = self.getopt("username", "try_username") 465 trydir = self.getopt("trydir", "try_dir") 466 467 argv = ["ssh", "-l", tryuser, tryhost, 468 "buildbot", "tryserver", "--jobdir", trydir] 469 # now run this command and feed the contents of 'job' into stdin 470 471 pp = RemoteTryPP(self.jobfile) 472 p = reactor.spawnProcess(pp, argv[0], argv, os.environ) 473 d = pp.d 474 return d 475 if self.connect == "pb": 476 user = self.getopt("username", "try_username") 477 passwd = self.getopt("passwd", "try_password") 478 master = self.getopt("master", "try_master") 479 tryhost, tryport = master.split(":") 480 tryport = int(tryport) 481 f = pb.PBClientFactory() 482 d = f.login(credentials.UsernamePassword(user, passwd)) 483 reactor.connectTCP(tryhost, tryport, f) 484 d.addCallback(self._deliverJob_pb) 485 return d 486 raise RuntimeError("unknown connecttype '%s', should be 'ssh' or 'pb'" 487 % self.connect)

488

489 - def _deliverJob_pb(self, remote):

490 ss = self.sourcestamp 491 492 d = remote.callRemote("try", 493 ss.branch, 494 ss.revision, 495 ss.patch, 496 self.builderNames, 497 self.config.get('properties', {})) 498 d.addCallback(self._deliverJob_pb2) 499 return d

500 - def _deliverJob_pb2(self, status):

501 self.buildsetStatus = status 502 return status

503

504 - def getStatus(self):

505 # returns a Deferred that fires when the builds have finished, and 506 # may emit status messages while we wait 507 wait = bool(self.getopt("wait", "try_wait", False)) 508 if not wait: 509 # TODO: emit the URL where they can follow the builds. This 510 # requires contacting the Status server over PB and doing 511 # getURLForThing() on the BuildSetStatus. To get URLs for 512 # individual builds would require we wait for the builds to 513 # start. 514 print "not waiting for builds to finish" 515 return 516 d = self.running = defer.Deferred() 517 if self.buildsetStatus: 518 self._getStatus_1() 519 # contact the status port 520 # we're probably using the ssh style 521 master = self.getopt("master", "masterstatus") 522 host, port = master.split(":") 523 port = int(port) 524 self.announce("contacting the status port at %s:%d" % (host, port)) 525 f = pb.PBClientFactory() 526 creds = credentials.UsernamePassword("statusClient", "clientpw") 527 d = f.login(creds) 528 reactor.connectTCP(host, port, f) 529 d.addCallback(self._getStatus_ssh_1) 530 return self.running

531

532 - def _getStatus_ssh_1(self, remote):

533 # find a remotereference to the corresponding BuildSetStatus object 534 self.announce("waiting for job to be accepted") 535 g = BuildSetStatusGrabber(remote, self.bsid) 536 d = g.grab() 537 d.addCallback(self._getStatus_1) 538 return d

539

540 - def _getStatus_1(self, res=None):

541 if res: 542 self.buildsetStatus = res 543 # gather the set of BuildRequests 544 d = self.buildsetStatus.callRemote("getBuildRequests") 545 d.addCallback(self._getStatus_2)

546

547 - def _getStatus_2(self, brs):

548 self.builderNames = [] 549 self.buildRequests = {} 550 551 # self.builds holds the current BuildStatus object for each one 552 self.builds = {} 553 554 # self.outstanding holds the list of builderNames which haven't 555 # finished yet 556 self.outstanding = [] 557 558 # self.results holds the list of build results. It holds a tuple of 559 # (result, text) 560 self.results = {} 561 562 # self.currentStep holds the name of the Step that each build is 563 # currently running 564 self.currentStep = {} 565 566 # self.ETA holds the expected finishing time (absolute time since 567 # epoch) 568 self.ETA = {} 569 570 for n,br in brs: 571 self.builderNames.append(n) 572 self.buildRequests[n] = br 573 self.builds[n] = None 574 self.outstanding.append(n) 575 self.results[n] = [None,None] 576 self.currentStep[n] = None 577 self.ETA[n] = None 578 # get new Builds for this buildrequest. We follow each one until 579 # it finishes or is interrupted. 580 br.callRemote("subscribe", self) 581 582 # now that those queries are in transit, we can start the 583 # display-status-every-30-seconds loop 584 self.printloop = task.LoopingCall(self.printStatus) 585 self.printloop.start(3, now=False)

586 587 588 # these methods are invoked by the status objects we've subscribed to 589

590 - def remote_newbuild(self, bs, builderName):

591 if self.builds[builderName]: 592 self.builds[builderName].callRemote("unsubscribe", self) 593 self.builds[builderName] = bs 594 bs.callRemote("subscribe", self, 20) 595 d = bs.callRemote("waitUntilFinished") 596 d.addCallback(self._build_finished, builderName)

597

598 - def remote_stepStarted(self, buildername, build, stepname, step):

599 self.currentStep[buildername] = stepname

600

601 - def remote_stepFinished(self, buildername, build, stepname, step, results):

602 pass

603

604 - def remote_buildETAUpdate(self, buildername, build, eta):

605 self.ETA[buildername] = now() + eta

606

607 - def _build_finished(self, bs, builderName):

608 # we need to collect status from the newly-finished build. We don't 609 # remove the build from self.outstanding until we've collected 610 # everything we want. 611 self.builds[builderName] = None 612 self.ETA[builderName] = None 613 self.currentStep[builderName] = "finished" 614 d = bs.callRemote("getResults") 615 d.addCallback(self._build_finished_2, bs, builderName) 616 return d

617 - def _build_finished_2(self, results, bs, builderName):

618 self.results[builderName][0] = results 619 d = bs.callRemote("getText") 620 d.addCallback(self._build_finished_3, builderName) 621 return d

622 - def _build_finished_3(self, text, builderName):

623 self.results[builderName][1] = text 624 625 self.outstanding.remove(builderName) 626 if not self.outstanding: 627 # all done 628 return self.statusDone()

629

630 - def printStatus(self):

631 names = self.buildRequests.keys() 632 names.sort() 633 for n in names: 634 if n not in self.outstanding: 635 # the build is finished, and we have results 636 code,text = self.results[n] 637 t = builder.Results[code] 638 if text: 639 t += " (%s)" % " ".join(text) 640 elif self.builds[n]: 641 t = self.currentStep[n] or "building" 642 if self.ETA[n]: 643 t += " [ETA %ds]" % (self.ETA[n] - now()) 644 else: 645 t = "no build" 646 self.announce("%s: %s" % (n, t)) 647 self.announce("")

648

649 - def statusDone(self):

650 self.printloop.stop() 651 print "All Builds Complete" 652 # TODO: include a URL for all failing builds 653 names = self.buildRequests.keys() 654 names.sort() 655 happy = True 656 for n in names: 657 code,text = self.results[n] 658 t = "%s: %s" % (n, builder.Results[code]) 659 if text: 660 t += " (%s)" % " ".join(text) 661 print t 662 if self.results[n] != builder.SUCCESS: 663 happy = False 664 665 if happy: 666 self.exitcode = 0 667 else: 668 self.exitcode = 1 669 self.running.callback(self.exitcode)

670

671 - def announce(self, message):

672 if not self.quiet: 673 print message

674

675 - def run(self):

676 # we can't do spawnProcess until we're inside reactor.run(), so get 677 # funky 678 print "using '%s' connect method" % self.connect 679 self.exitcode = 0 680 d = defer.Deferred() 681 d.addCallback(lambda res: self.createJob()) 682 d.addCallback(lambda res: self.announce("job created")) 683 deliver = self.deliverJob 684 if bool(self.config.get("dryrun")): 685 deliver = self.fakeDeliverJob 686 d.addCallback(lambda res: deliver()) 687 d.addCallback(lambda res: self.announce("job has been delivered")) 688 d.addCallback(lambda res: self.getStatus()) 689 d.addErrback(log.err) 690 d.addCallback(self.cleanup) 691 d.addCallback(lambda res: reactor.stop()) 692 693 reactor.callLater(0, d.callback, None) 694 reactor.run() 695 sys.exit(self.exitcode)

696

697 - def logErr(self, why):

698 log.err(why) 699 print "error during 'try' processing" 700 print why

701

702 - def cleanup(self, res=None):

703 if self.buildsetStatus: 704 self.buildsetStatus.broker.transport.loseConnection()

705

Source Code for Module buildbot.scripts.tryclient