buildbot.slave.commands

1 # -*- test-case-name: buildbot.test.test_slavecommand -*- 2 3 import os, sys, re, signal, shutil, types, time, tarfile, tempfile 4 from stat import ST_CTIME, ST_MTIME, ST_SIZE 5 6 from zope.interface import implements 7 from twisted.internet.protocol import ProcessProtocol 8 from twisted.internet import reactor, defer, task 9 from twisted.python import log, failure, runtime 10 from twisted.python.procutils import which 11 12 from buildbot.slave.interfaces import ISlaveCommand 13 from buildbot.slave.registry import registerSlaveCommand 14 from buildbot.util import to_text 15 16 # this used to be a CVS $-style "Revision" auto-updated keyword, but since I 17 # moved to Darcs as the primary repository, this is updated manually each 18 # time this file is changed. The last cvs_ver that was here was 1.51 . 19 command_version = "2.8" 20 21 # version history: 22 # >=1.17: commands are interruptable 23 # >=1.28: Arch understands 'revision', added Bazaar 24 # >=1.33: Source classes understand 'retry' 25 # >=1.39: Source classes correctly handle changes in branch (except Git) 26 # Darcs accepts 'revision' (now all do but Git) (well, and P4Sync) 27 # Arch/Baz should accept 'build-config' 28 # >=1.51: (release 0.7.3) 29 # >= 2.1: SlaveShellCommand now accepts 'initial_stdin', 'keep_stdin_open', 30 # and 'logfiles'. It now sends 'log' messages in addition to 31 # stdout/stdin/header/rc. It acquired writeStdin/closeStdin methods, 32 # but these are not remotely callable yet. 33 # (not externally visible: ShellCommandPP has writeStdin/closeStdin. 34 # ShellCommand accepts new arguments (logfiles=, initialStdin=, 35 # keepStdinOpen=) and no longer accepts stdin=) 36 # (release 0.7.4) 37 # >= 2.2: added monotone, uploadFile, and downloadFile (release 0.7.5) 38 # >= 2.3: added bzr (release 0.7.6) 39 # >= 2.4: Git understands 'revision' and branches 40 # >= 2.5: workaround added for remote 'hg clone --rev REV' when hg<0.9.2 41 # >= 2.6: added uploadDirectory 42 # >= 2.7: added usePTY option to SlaveShellCommand 43 # >= 2.8: added username and password args to SVN class 44

45 -class CommandInterrupted(Exception):

46 pass

47 -class TimeoutError(Exception):

48 pass

49

50 -class Obfuscated:

51 """An obfuscated string in a command"""

52 - def __init__(self, real, fake):

53 self.real = real 54 self.fake = fake

55

56 - def __str__(self):

57 return self.fake

58

59 - def __repr__(self):

60 return `self.fake`

61

62 - def get_real(command):

63 rv = command 64 if type(command) == types.ListType: 65 rv = [] 66 for elt in command: 67 if isinstance(elt, Obfuscated): 68 rv.append(elt.real) 69 else: 70 rv.append(to_text(elt)) 71 return rv

72 get_real = staticmethod(get_real) 73

74 - def get_fake(command):

75 rv = command 76 if type(command) == types.ListType: 77 rv = [] 78 for elt in command: 79 if isinstance(elt, Obfuscated): 80 rv.append(elt.fake) 81 else: 82 rv.append(to_text(elt)) 83 return rv

84 get_fake = staticmethod(get_fake)

85

86 -class AbandonChain(Exception):

87 """A series of chained steps can raise this exception to indicate that 88 one of the intermediate ShellCommands has failed, such that there is no 89 point in running the remainder. 'rc' should be the non-zero exit code of 90 the failing ShellCommand.""" 91

92 - def __repr__(self):

93 return "<AbandonChain rc=%s>" % self.args[0]

94

95 -def getCommand(name):

96 possibles = which(name) 97 if not possibles: 98 raise RuntimeError("Couldn't find executable for '%s'" % name) 99 return possibles[0]

100

101 -def rmdirRecursive(dir):

102 """This is a replacement for shutil.rmtree that works better under 103 windows. Thanks to Bear at the OSAF for the code.""" 104 if not os.path.exists(dir): 105 return 106 107 if os.path.islink(dir): 108 os.remove(dir) 109 return 110 111 # Verify the directory is read/write/execute for the current user 112 os.chmod(dir, 0700) 113 114 for name in os.listdir(dir): 115 full_name = os.path.join(dir, name) 116 # on Windows, if we don't have write permission we can't remove 117 # the file/directory either, so turn that on 118 if os.name == 'nt': 119 if not os.access(full_name, os.W_OK): 120 # I think this is now redundant, but I don't have an NT 121 # machine to test on, so I'm going to leave it in place 122 # -warner 123 os.chmod(full_name, 0600) 124 125 if os.path.isdir(full_name): 126 rmdirRecursive(full_name) 127 else: 128 os.chmod(full_name, 0700) 129 os.remove(full_name) 130 os.rmdir(dir)

131

132 -class ShellCommandPP(ProcessProtocol):

133 debug = False 134

135 - def __init__(self, command):

136 self.command = command 137 self.pending_stdin = "" 138 self.stdin_finished = False

139

140 - def writeStdin(self, data):

141 assert not self.stdin_finished 142 if self.connected: 143 self.transport.write(data) 144 else: 145 self.pending_stdin += data

146

147 - def closeStdin(self):

148 if self.connected: 149 if self.debug: log.msg(" closing stdin") 150 self.transport.closeStdin() 151 self.stdin_finished = True

152

153 - def connectionMade(self):

154 if self.debug: 155 log.msg("ShellCommandPP.connectionMade") 156 if not self.command.process: 157 if self.debug: 158 log.msg(" assigning self.command.process: %s" % 159 (self.transport,)) 160 self.command.process = self.transport 161 162 # TODO: maybe we shouldn't close stdin when using a PTY. I can't test 163 # this yet, recent debian glibc has a bug which causes thread-using 164 # test cases to SIGHUP trial, and the workaround is to either run 165 # the whole test with /bin/sh -c " ".join(argv) (way gross) or to 166 # not use a PTY. Once the bug is fixed, I'll be able to test what 167 # happens when you close stdin on a pty. My concern is that it will 168 # SIGHUP the child (since we are, in a sense, hanging up on them). 169 # But it may well be that keeping stdout open prevents the SIGHUP 170 # from being sent. 171 #if not self.command.usePTY: 172 173 if self.pending_stdin: 174 if self.debug: log.msg(" writing to stdin") 175 self.transport.write(self.pending_stdin) 176 if self.stdin_finished: 177 if self.debug: log.msg(" closing stdin") 178 self.transport.closeStdin()

179

180 - def outReceived(self, data):

181 if self.debug: 182 log.msg("ShellCommandPP.outReceived") 183 self.command.addStdout(data)

184

185 - def errReceived(self, data):

186 if self.debug: 187 log.msg("ShellCommandPP.errReceived") 188 self.command.addStderr(data)

189

190 - def processEnded(self, status_object):

191 if self.debug: 192 log.msg("ShellCommandPP.processEnded", status_object) 193 # status_object is a Failure wrapped around an 194 # error.ProcessTerminated or and error.ProcessDone. 195 # requires twisted >= 1.0.4 to overcome a bug in process.py 196 sig = status_object.value.signal 197 rc = status_object.value.exitCode 198 self.command.finished(sig, rc)

199

200 -class LogFileWatcher:

201 POLL_INTERVAL = 2 202

203 - def __init__(self, command, name, logfile, follow=False):

204 self.command = command 205 self.name = name 206 self.logfile = logfile 207 208 log.msg("LogFileWatcher created to watch %s" % logfile) 209 # we are created before the ShellCommand starts. If the logfile we're 210 # supposed to be watching already exists, record its size and 211 # ctime/mtime so we can tell when it starts to change. 212 self.old_logfile_stats = self.statFile() 213 self.started = False 214 215 # follow the file, only sending back lines 216 # added since we started watching 217 self.follow = follow 218 219 # every 2 seconds we check on the file again 220 self.poller = task.LoopingCall(self.poll)

221

222 - def start(self):

223 self.poller.start(self.POLL_INTERVAL).addErrback(self._cleanupPoll)

224

225 - def _cleanupPoll(self, err):

226 log.err(err, msg="Polling error") 227 self.poller = None

228

229 - def stop(self):

230 self.poll() 231 if self.poller is not None: 232 self.poller.stop() 233 if self.started: 234 self.f.close()

235

236 - def statFile(self):

237 if os.path.exists(self.logfile): 238 s = os.stat(self.logfile) 239 return (s[ST_CTIME], s[ST_MTIME], s[ST_SIZE]) 240 return None

241

242 - def poll(self):

243 if not self.started: 244 s = self.statFile() 245 if s == self.old_logfile_stats: 246 return # not started yet 247 if not s: 248 # the file was there, but now it's deleted. Forget about the 249 # initial state, clearly the process has deleted the logfile 250 # in preparation for creating a new one. 251 self.old_logfile_stats = None 252 return # no file to work with 253 self.f = open(self.logfile, "rb") 254 # if we only want new lines, seek to 255 # where we stat'd so we only find new 256 # lines 257 if self.follow: 258 self.f.seek(s[2], 0) 259 self.started = True 260 self.f.seek(self.f.tell(), 0) 261 while True: 262 data = self.f.read(10000) 263 if not data: 264 return 265 self.command.addLogfile(self.name, data)

266 267

268 -class ShellCommand:

269 # This is a helper class, used by SlaveCommands to run programs in a 270 # child shell. 271 272 notreally = False 273 BACKUP_TIMEOUT = 5 274 KILL = "KILL" 275 CHUNK_LIMIT = 128*1024 276 277 # For sending elapsed time: 278 startTime = None 279 elapsedTime = None 280 # I wish we had easy access to CLOCK_MONOTONIC in Python: 281 # http://www.opengroup.org/onlinepubs/000095399/functions/clock_getres.html 282 # Then changes to the system clock during a run wouldn't effect the "elapsed 283 # time" results. 284

285 - def __init__(self, builder, command, 286 workdir, environ=None, 287 sendStdout=True, sendStderr=True, sendRC=True, 288 timeout=None, initialStdin=None, keepStdinOpen=False, 289 keepStdout=False, keepStderr=False, logEnviron=True, 290 logfiles={}, usePTY="slave-config"):

291 """ 292 293 @param keepStdout: if True, we keep a copy of all the stdout text 294 that we've seen. This copy is available in 295 self.stdout, which can be read after the command 296 has finished. 297 @param keepStderr: same, for stderr 298 299 @param usePTY: "slave-config" -> use the SlaveBuilder's usePTY; 300 otherwise, true to use a PTY, false to not use a PTY. 301 """ 302 303 self.builder = builder 304 self.command = Obfuscated.get_real(command) 305 self.fake_command = Obfuscated.get_fake(command) 306 self.sendStdout = sendStdout 307 self.sendStderr = sendStderr 308 self.sendRC = sendRC 309 self.logfiles = logfiles 310 self.workdir = workdir 311 if not os.path.exists(workdir): 312 os.makedirs(workdir) 313 self.environ = os.environ.copy() 314 if environ: 315 if environ.has_key('PYTHONPATH'): 316 ppath = environ['PYTHONPATH'] 317 # Need to do os.pathsep translation. We could either do that 318 # by replacing all incoming ':'s with os.pathsep, or by 319 # accepting lists. I like lists better. 320 if not isinstance(ppath, str): 321 # If it's not a string, treat it as a sequence to be 322 # turned in to a string. 323 ppath = os.pathsep.join(ppath) 324 325 if self.environ.has_key('PYTHONPATH'): 326 # special case, prepend the builder's items to the 327 # existing ones. This will break if you send over empty 328 # strings, so don't do that. 329 ppath = ppath + os.pathsep + self.environ['PYTHONPATH'] 330 331 environ['PYTHONPATH'] = ppath 332 333 self.environ.update(environ) 334 self.initialStdin = initialStdin 335 self.keepStdinOpen = keepStdinOpen 336 self.logEnviron = logEnviron 337 self.timeout = timeout 338 self.timer = None 339 self.keepStdout = keepStdout 340 self.keepStderr = keepStderr 341 342 343 if usePTY == "slave-config": 344 self.usePTY = self.builder.usePTY 345 else: 346 self.usePTY = usePTY 347 348 # usePTY=True is a convenience for cleaning up all children and 349 # grandchildren of a hung command. Fall back to usePTY=False on systems 350 # and in situations where ptys cause problems. PTYs are posix-only, 351 # and for .closeStdin to matter, we must use a pipe, not a PTY 352 if runtime.platformType != "posix" or initialStdin is not None: 353 if self.usePTY and usePTY != "slave-config": 354 self.sendStatus({'header': "WARNING: disabling usePTY for this command"}) 355 self.usePTY = False 356 357 self.logFileWatchers = [] 358 for name,filevalue in self.logfiles.items(): 359 filename = filevalue 360 follow = False 361 362 # check for a dictionary of options 363 # filename is required, others are optional 364 if type(filevalue) == dict: 365 filename = filevalue['filename'] 366 follow = filevalue.get('follow', False) 367 368 w = LogFileWatcher(self, name, 369 os.path.join(self.workdir, filename), 370 follow=follow) 371 self.logFileWatchers.append(w)

372

373 - def __repr__(self):

374 return "<slavecommand.ShellCommand '%s'>" % self.fake_command

375

376 - def sendStatus(self, status):

377 self.builder.sendUpdate(status)

378

379 - def start(self):

380 # return a Deferred which fires (with the exit code) when the command 381 # completes 382 if self.keepStdout: 383 self.stdout = "" 384 if self.keepStderr: 385 self.stderr = "" 386 self.deferred = defer.Deferred() 387 try: 388 self._startCommand() 389 except: 390 log.msg("error in ShellCommand._startCommand") 391 log.err() 392 # pretend it was a shell error 393 self.deferred.errback(AbandonChain(-1)) 394 return self.deferred

395

396 - def _startCommand(self):

397 # ensure workdir exists 398 if not os.path.isdir(self.workdir): 399 os.makedirs(self.workdir) 400 log.msg("ShellCommand._startCommand") 401 if self.notreally: 402 self.sendStatus({'header': "command '%s' in dir %s" % \ 403 (self.fake_command, self.workdir)}) 404 self.sendStatus({'header': "(not really)\n"}) 405 self.finished(None, 0) 406 return 407 408 self.pp = ShellCommandPP(self) 409 410 if type(self.command) in types.StringTypes: 411 if runtime.platformType == 'win32': 412 argv = os.environ['COMSPEC'].split() # allow %COMSPEC% to have args 413 if '/c' not in argv: argv += ['/c'] 414 argv += [self.command] 415 else: 416 # for posix, use /bin/sh. for other non-posix, well, doesn't 417 # hurt to try 418 argv = ['/bin/sh', '-c', self.command] 419 display = self.fake_command 420 else: 421 if runtime.platformType == 'win32' and not self.command[0].lower().endswith(".exe"): 422 argv = os.environ['COMSPEC'].split() # allow %COMSPEC% to have args 423 if '/c' not in argv: argv += ['/c'] 424 argv += list(self.command) 425 else: 426 argv = self.command 427 display = " ".join(self.fake_command) 428 429 # $PWD usually indicates the current directory; spawnProcess may not 430 # update this value, though, so we set it explicitly here. This causes 431 # weird problems (bug #456) on msys, though.. 432 if not self.environ.get('MACHTYPE', None) == 'i686-pc-msys': 433 self.environ['PWD'] = os.path.abspath(self.workdir) 434 435 # self.stdin is handled in ShellCommandPP.connectionMade 436 437 # first header line is the command in plain text, argv joined with 438 # spaces. You should be able to cut-and-paste this into a shell to 439 # obtain the same results. If there are spaces in the arguments, too 440 # bad. 441 log.msg(" " + display) 442 self.sendStatus({'header': display+"\n"}) 443 444 # then comes the secondary information 445 msg = " in dir %s" % (self.workdir,) 446 if self.timeout: 447 msg += " (timeout %d secs)" % (self.timeout,) 448 log.msg(" " + msg) 449 self.sendStatus({'header': msg+"\n"}) 450 451 msg = " watching logfiles %s" % (self.logfiles,) 452 log.msg(" " + msg) 453 self.sendStatus({'header': msg+"\n"}) 454 455 # then the obfuscated command array for resolving unambiguity 456 msg = " argv: %s" % (self.fake_command,) 457 log.msg(" " + msg) 458 self.sendStatus({'header': msg+"\n"}) 459 460 # then the environment, since it sometimes causes problems 461 if self.logEnviron: 462 msg = " environment:\n" 463 env_names = self.environ.keys() 464 env_names.sort() 465 for name in env_names: 466 msg += " %s=%s\n" % (name, self.environ[name]) 467 log.msg(" environment: %s" % (self.environ,)) 468 self.sendStatus({'header': msg}) 469 470 if self.initialStdin: 471 msg = " writing %d bytes to stdin" % len(self.initialStdin) 472 log.msg(" " + msg) 473 self.sendStatus({'header': msg+"\n"}) 474 475 if self.keepStdinOpen: 476 msg = " leaving stdin open" 477 else: 478 msg = " closing stdin" 479 log.msg(" " + msg) 480 self.sendStatus({'header': msg+"\n"}) 481 482 msg = " using PTY: %s" % bool(self.usePTY) 483 log.msg(" " + msg) 484 self.sendStatus({'header': msg+"\n"}) 485 486 # this will be buffered until connectionMade is called 487 if self.initialStdin: 488 self.pp.writeStdin(self.initialStdin) 489 if not self.keepStdinOpen: 490 self.pp.closeStdin() 491 492 # win32eventreactor's spawnProcess (under twisted <= 2.0.1) returns 493 # None, as opposed to all the posixbase-derived reactors (which 494 # return the new Process object). This is a nuisance. We can make up 495 # for it by having the ProcessProtocol give us their .transport 496 # attribute after they get one. I'd prefer to get it from 497 # spawnProcess because I'm concerned about returning from this method 498 # without having a valid self.process to work with. (if kill() were 499 # called right after we return, but somehow before connectionMade 500 # were called, then kill() would blow up). 501 self.process = None 502 self.startTime = time.time() 503 504 p = reactor.spawnProcess(self.pp, argv[0], argv, 505 self.environ, 506 self.workdir, 507 usePTY=self.usePTY) 508 # connectionMade might have been called during spawnProcess 509 if not self.process: 510 self.process = p 511 512 # connectionMade also closes stdin as long as we're not using a PTY. 513 # This is intended to kill off inappropriately interactive commands 514 # better than the (long) hung-command timeout. ProcessPTY should be 515 # enhanced to allow the same childFDs argument that Process takes, 516 # which would let us connect stdin to /dev/null . 517 518 if self.timeout: 519 self.timer = reactor.callLater(self.timeout, self.doTimeout) 520 521 for w in self.logFileWatchers: 522 w.start()

523 524

525 - def _chunkForSend(self, data):

526 # limit the chunks that we send over PB to 128k, since it has a 527 # hardwired string-size limit of 640k. 528 LIMIT = self.CHUNK_LIMIT 529 for i in range(0, len(data), LIMIT): 530 yield data[i:i+LIMIT]

531

532 - def addStdout(self, data):

533 if self.sendStdout: 534 for chunk in self._chunkForSend(data): 535 self.sendStatus({'stdout': chunk}) 536 if self.keepStdout: 537 self.stdout += data 538 if self.timer: 539 self.timer.reset(self.timeout)

540

541 - def addStderr(self, data):

542 if self.sendStderr: 543 for chunk in self._chunkForSend(data): 544 self.sendStatus({'stderr': chunk}) 545 if self.keepStderr: 546 self.stderr += data 547 if self.timer: 548 self.timer.reset(self.timeout)

549

550 - def addLogfile(self, name, data):

551 for chunk in self._chunkForSend(data): 552 self.sendStatus({'log': (name, chunk)}) 553 if self.timer: 554 self.timer.reset(self.timeout)

555

556 - def finished(self, sig, rc):

557 self.elapsedTime = time.time() - self.startTime 558 log.msg("command finished with signal %s, exit code %s, elapsedTime: %0.6f" % (sig,rc,self.elapsedTime)) 559 for w in self.logFileWatchers: 560 # this will send the final updates 561 w.stop() 562 if sig is not None: 563 rc = -1 564 if self.sendRC: 565 if sig is not None: 566 self.sendStatus( 567 {'header': "process killed by signal %d\n" % sig}) 568 self.sendStatus({'rc': rc}) 569 self.sendStatus({'header': "elapsedTime=%0.6f\n" % self.elapsedTime}) 570 if self.timer: 571 self.timer.cancel() 572 self.timer = None 573 d = self.deferred 574 self.deferred = None 575 if d: 576 d.callback(rc) 577 else: 578 log.msg("Hey, command %s finished twice" % self)

579

580 - def failed(self, why):

581 log.msg("ShellCommand.failed: command failed: %s" % (why,)) 582 if self.timer: 583 self.timer.cancel() 584 self.timer = None 585 d = self.deferred 586 self.deferred = None 587 if d: 588 d.errback(why) 589 else: 590 log.msg("Hey, command %s finished twice" % self)

591

592 - def doTimeout(self):

593 self.timer = None 594 msg = "command timed out: %d seconds without output" % self.timeout 595 self.kill(msg)

596

597 - def kill(self, msg):

598 # This may be called by the timeout, or when the user has decided to 599 # abort this build. 600 if self.timer: 601 self.timer.cancel() 602 self.timer = None 603 if hasattr(self.process, "pid") and self.process.pid is not None: 604 msg += ", killing pid %s" % self.process.pid 605 log.msg(msg) 606 self.sendStatus({'header': "\n" + msg + "\n"}) 607 608 hit = 0 609 if runtime.platformType == "posix": 610 try: 611 # really want to kill off all child processes too. Process 612 # Groups are ideal for this, but that requires 613 # spawnProcess(usePTY=1). Try both ways in case process was 614 # not started that way. 615 616 # the test suite sets self.KILL=None to tell us we should 617 # only pretend to kill the child. This lets us test the 618 # backup timer. 619 620 sig = None 621 if self.KILL is not None: 622 sig = getattr(signal, "SIG"+ self.KILL, None) 623 624 if self.KILL == None: 625 log.msg("self.KILL==None, only pretending to kill child") 626 elif sig is None: 627 log.msg("signal module is missing SIG%s" % self.KILL) 628 elif not hasattr(os, "kill"): 629 log.msg("os module is missing the 'kill' function") 630 elif not hasattr(self.process, "pid") or self.process.pid is None: 631 log.msg("self.process has no pid") 632 else: 633 log.msg("trying os.kill(-pid, %d)" % (sig,)) 634 # TODO: maybe use os.killpg instead of a negative pid? 635 os.kill(-self.process.pid, sig) 636 log.msg(" signal %s sent successfully" % sig) 637 hit = 1 638 except OSError: 639 # probably no-such-process, maybe because there is no process 640 # group 641 pass 642 if not hit: 643 try: 644 if self.KILL is None: 645 log.msg("self.KILL==None, only pretending to kill child") 646 else: 647 log.msg("trying process.signalProcess('KILL')") 648 self.process.signalProcess(self.KILL) 649 log.msg(" signal %s sent successfully" % (self.KILL,)) 650 hit = 1 651 except OSError: 652 # could be no-such-process, because they finished very recently 653 pass 654 if not hit: 655 log.msg("signalProcess/os.kill failed both times") 656 657 if runtime.platformType == "posix": 658 # we only do this under posix because the win32eventreactor 659 # blocks here until the process has terminated, while closing 660 # stderr. This is weird. 661 self.pp.transport.loseConnection() 662 663 # finished ought to be called momentarily. Just in case it doesn't, 664 # set a timer which will abandon the command. 665 self.timer = reactor.callLater(self.BACKUP_TIMEOUT, 666 self.doBackupTimeout)

667

668 - def doBackupTimeout(self):

669 log.msg("we tried to kill the process, and it wouldn't die.." 670 " finish anyway") 671 self.timer = None 672 self.sendStatus({'header': "SIGKILL failed to kill process\n"}) 673 if self.sendRC: 674 self.sendStatus({'header': "using fake rc=-1\n"}) 675 self.sendStatus({'rc': -1}) 676 self.failed(TimeoutError("SIGKILL failed to kill process"))

677 678

679 - def writeStdin(self, data):

680 self.pp.writeStdin(data)

681

682 - def closeStdin(self):

683 self.pp.closeStdin()

684 685

686 -class Command:

687 implements(ISlaveCommand) 688 689 """This class defines one command that can be invoked by the build master. 690 The command is executed on the slave side, and always sends back a 691 completion message when it finishes. It may also send intermediate status 692 as it runs (by calling builder.sendStatus). Some commands can be 693 interrupted (either by the build master or a local timeout), in which 694 case the step is expected to complete normally with a status message that 695 indicates an error occurred. 696 697 These commands are used by BuildSteps on the master side. Each kind of 698 BuildStep uses a single Command. The slave must implement all the 699 Commands required by the set of BuildSteps used for any given build: 700 this is checked at startup time. 701 702 All Commands are constructed with the same signature: 703 c = CommandClass(builder, args) 704 where 'builder' is the parent SlaveBuilder object, and 'args' is a 705 dict that is interpreted per-command. 706 707 The setup(args) method is available for setup, and is run from __init__. 708 709 The Command is started with start(). This method must be implemented in a 710 subclass, and it should return a Deferred. When your step is done, you 711 should fire the Deferred (the results are not used). If the command is 712 interrupted, it should fire the Deferred anyway. 713 714 While the command runs. it may send status messages back to the 715 buildmaster by calling self.sendStatus(statusdict). The statusdict is 716 interpreted by the master-side BuildStep however it likes. 717 718 A separate completion message is sent when the deferred fires, which 719 indicates that the Command has finished, but does not carry any status 720 data. If the Command needs to return an exit code of some sort, that 721 should be sent as a regular status message before the deferred is fired . 722 Once builder.commandComplete has been run, no more status messages may be 723 sent. 724 725 If interrupt() is called, the Command should attempt to shut down as 726 quickly as possible. Child processes should be killed, new ones should 727 not be started. The Command should send some kind of error status update, 728 then complete as usual by firing the Deferred. 729 730 .interrupted should be set by interrupt(), and can be tested to avoid 731 sending multiple error status messages. 732 733 If .running is False, the bot is shutting down (or has otherwise lost the 734 connection to the master), and should not send any status messages. This 735 is checked in Command.sendStatus . 736 737 """ 738 739 # builder methods: 740 # sendStatus(dict) (zero or more) 741 # commandComplete() or commandInterrupted() (one, at end) 742 743 debug = False 744 interrupted = False 745 running = False # set by Builder, cleared on shutdown or when the 746 # Deferred fires 747

748 - def __init__(self, builder, stepId, args):

749 self.builder = builder 750 self.stepId = stepId # just for logging 751 self.args = args 752 self.setup(args)

753

754 - def setup(self, args):

755 """Override this in a subclass to extract items from the args dict.""" 756 pass

757

758 - def doStart(self):

759 self.running = True 760 d = defer.maybeDeferred(self.start) 761 d.addBoth(self.commandComplete) 762 return d

763

764 - def start(self):

765 """Start the command. This method should return a Deferred that will 766 fire when the command has completed. The Deferred's argument will be 767 ignored. 768 769 This method should be overridden by subclasses.""" 770 raise NotImplementedError, "You must implement this in a subclass"

771

772 - def sendStatus(self, status):

773 """Send a status update to the master.""" 774 if self.debug: 775 log.msg("sendStatus", status) 776 if not self.running: 777 log.msg("would sendStatus but not .running") 778 return 779 self.builder.sendUpdate(status)

780

781 - def doInterrupt(self):

782 self.running = False 783 self.interrupt()

784

785 - def interrupt(self):

786 """Override this in a subclass to allow commands to be interrupted. 787 May be called multiple times, test and set self.interrupted=True if 788 this matters.""" 789 pass

790

791 - def commandComplete(self, res):

792 self.running = False 793 return res

794 795 # utility methods, mostly used by SlaveShellCommand and the like 796

797 - def _abandonOnFailure(self, rc):

798 if type(rc) is not int: 799 log.msg("weird, _abandonOnFailure was given rc=%s (%s)" % \ 800 (rc, type(rc))) 801 assert isinstance(rc, int) 802 if rc != 0: 803 raise AbandonChain(rc) 804 return rc

805

806 - def _sendRC(self, res):

807 self.sendStatus({'rc': 0})

808

809 - def _checkAbandoned(self, why):

810 log.msg("_checkAbandoned", why) 811 why.trap(AbandonChain) 812 log.msg(" abandoning chain", why.value) 813 self.sendStatus({'rc': why.value.args[0]}) 814 return None

815 816 817

818 -class SlaveFileUploadCommand(Command):

819 """ 820 Upload a file from slave to build master 821 Arguments: 822 823 - ['workdir']: base directory to use 824 - ['slavesrc']: name of the slave-side file to read from 825 - ['writer']: RemoteReference to a transfer._FileWriter object 826 - ['maxsize']: max size (in bytes) of file to write 827 - ['blocksize']: max size for each data block 828 """ 829 debug = False 830

831 - def setup(self, args):

832 self.workdir = args['workdir'] 833 self.filename = args['slavesrc'] 834 self.writer = args['writer'] 835 self.remaining = args['maxsize'] 836 self.blocksize = args['blocksize'] 837 self.stderr = None 838 self.rc = 0

839

840 - def start(self):

841 if self.debug: 842 log.msg('SlaveFileUploadCommand started') 843 844 # Open file 845 self.path = os.path.join(self.builder.basedir, 846 self.workdir, 847 os.path.expanduser(self.filename)) 848 try: 849 self.fp = open(self.path, 'rb') 850 if self.debug: 851 log.msg('Opened %r for upload' % self.path) 852 except: 853 # TODO: this needs cleanup 854 self.fp = None 855 self.stderr = 'Cannot open file %r for upload' % self.path 856 self.rc = 1 857 if self.debug: 858 log.msg('Cannot open file %r for upload' % self.path) 859 860 self.sendStatus({'header': "sending %s" % self.path}) 861 862 d = defer.Deferred() 863 reactor.callLater(0, self._loop, d) 864 def _close(res): 865 # close the file, but pass through any errors from _loop 866 d1 = self.writer.callRemote("close") 867 d1.addErrback(log.err) 868 d1.addCallback(lambda ignored: res) 869 return d1

870 d.addBoth(_close) 871 d.addBoth(self.finished) 872 return d

873

874 - def _loop(self, fire_when_done):

875 d = defer.maybeDeferred(self._writeBlock) 876 def _done(finished): 877 if finished: 878 fire_when_done.callback(None) 879 else: 880 self._loop(fire_when_done)

881 def _err(why): 882 fire_when_done.errback(why) 883 d.addCallbacks(_done, _err) 884 return None 885

886 - def _writeBlock(self):

887 """Write a block of data to the remote writer""" 888 889 if self.interrupted or self.fp is None: 890 if self.debug: 891 log.msg('SlaveFileUploadCommand._writeBlock(): end') 892 return True 893 894 length = self.blocksize 895 if self.remaining is not None and length > self.remaining: 896 length = self.remaining 897 898 if length <= 0: 899 if self.stderr is None: 900 self.stderr = 'Maximum filesize reached, truncating file %r' \ 901 % self.path 902 self.rc = 1 903 data = '' 904 else: 905 data = self.fp.read(length) 906 907 if self.debug: 908 log.msg('SlaveFileUploadCommand._writeBlock(): '+ 909 'allowed=%d readlen=%d' % (length, len(data))) 910 if len(data) == 0: 911 log.msg("EOF: callRemote(close)") 912 return True 913 914 if self.remaining is not None: 915 self.remaining = self.remaining - len(data) 916 assert self.remaining >= 0 917 d = self.writer.callRemote('write', data) 918 d.addCallback(lambda res: False) 919 return d

920

921 - def interrupt(self):

922 if self.debug: 923 log.msg('interrupted') 924 if self.interrupted: 925 return 926 if self.stderr is None: 927 self.stderr = 'Upload of %r interrupted' % self.path 928 self.rc = 1 929 self.interrupted = True

930 # the next _writeBlock call will notice the .interrupted flag 931

932 - def finished(self, res):

933 if self.debug: 934 log.msg('finished: stderr=%r, rc=%r' % (self.stderr, self.rc)) 935 if self.stderr is None: 936 self.sendStatus({'rc': self.rc}) 937 else: 938 self.sendStatus({'stderr': self.stderr, 'rc': self.rc}) 939 return res

940 941 registerSlaveCommand("uploadFile", SlaveFileUploadCommand, command_version) 942 943

944 -class SlaveDirectoryUploadCommand(SlaveFileUploadCommand):

945 """ 946 Upload a directory from slave to build master 947 Arguments: 948 949 - ['workdir']: base directory to use 950 - ['slavesrc']: name of the slave-side directory to read from 951 - ['writer']: RemoteReference to a transfer._DirectoryWriter object 952 - ['maxsize']: max size (in bytes) of file to write 953 - ['blocksize']: max size for each data block 954 - ['compress']: one of [None, 'bz2', 'gz'] 955 """ 956 debug = True 957

958 - def setup(self, args):

959 self.workdir = args['workdir'] 960 self.dirname = args['slavesrc'] 961 self.writer = args['writer'] 962 self.remaining = args['maxsize'] 963 self.blocksize = args['blocksize'] 964 self.compress = args['compress'] 965 self.stderr = None 966 self.rc = 0

967

968 - def start(self):

969 if self.debug: 970 log.msg('SlaveDirectoryUploadCommand started') 971 972 self.path = os.path.join(self.builder.basedir, 973 self.workdir, 974 os.path.expanduser(self.dirname)) 975 if self.debug: 976 log.msg("path: %r" % self.path) 977 978 # Create temporary archive 979 fd, self.tarname = tempfile.mkstemp() 980 fileobj = os.fdopen(fd, 'w') 981 if self.compress == 'bz2': 982 mode='w|bz2' 983 elif self.compress == 'gz': 984 mode='w|gz' 985 else: 986 mode = 'w' 987 archive = tarfile.open(name=self.tarname, mode=mode, fileobj=fileobj) 988 archive.add(self.path, '') 989 archive.close() 990 fileobj.close() 991 992 # Transfer it 993 self.fp = open(self.tarname, 'rb') 994 995 self.sendStatus({'header': "sending %s" % self.path}) 996 997 d = defer.Deferred() 998 reactor.callLater(0, self._loop, d) 999 def unpack(res): 1000 # unpack the archive, but pass through any errors from _loop 1001 d1 = self.writer.callRemote("unpack") 1002 d1.addErrback(log.err) 1003 d1.addCallback(lambda ignored: res) 1004 return d1

1005 d.addCallback(unpack) 1006 d.addBoth(self.finished) 1007 return d

1008

1009 - def finished(self, res):

1010 self.fp.close() 1011 os.remove(self.tarname) 1012 if self.debug: 1013 log.msg('finished: stderr=%r, rc=%r' % (self.stderr, self.rc)) 1014 if self.stderr is None: 1015 self.sendStatus({'rc': self.rc}) 1016 else: 1017 self.sendStatus({'stderr': self.stderr, 'rc': self.rc}) 1018 return res

1019 1020 registerSlaveCommand("uploadDirectory", SlaveDirectoryUploadCommand, command_version) 1021 1022

1023 -class SlaveFileDownloadCommand(Command):

1024 """ 1025 Download a file from master to slave 1026 Arguments: 1027 1028 - ['workdir']: base directory to use 1029 - ['slavedest']: name of the slave-side file to be created 1030 - ['reader']: RemoteReference to a transfer._FileReader object 1031 - ['maxsize']: max size (in bytes) of file to write 1032 - ['blocksize']: max size for each data block 1033 - ['mode']: access mode for the new file 1034 """ 1035 debug = False 1036

1037 - def setup(self, args):

1038 self.workdir = args['workdir'] 1039 self.filename = args['slavedest'] 1040 self.reader = args['reader'] 1041 self.bytes_remaining = args['maxsize'] 1042 self.blocksize = args['blocksize'] 1043 self.mode = args['mode'] 1044 self.stderr = None 1045 self.rc = 0

1046

1047 - def start(self):

1048 if self.debug: 1049 log.msg('SlaveFileDownloadCommand starting') 1050 1051 # Open file 1052 self.path = os.path.join(self.builder.basedir, 1053 self.workdir, 1054 os.path.expanduser(self.filename)) 1055 1056 dirname = os.path.dirname(self.path) 1057 if not os.path.exists(dirname): 1058 os.makedirs(dirname) 1059 1060 try: 1061 self.fp = open(self.path, 'wb') 1062 if self.debug: 1063 log.msg('Opened %r for download' % self.path) 1064 if self.mode is not None: 1065 # note: there is a brief window during which the new file 1066 # will have the buildslave's default (umask) mode before we 1067 # set the new one. Don't use this mode= feature to keep files 1068 # private: use the buildslave's umask for that instead. (it 1069 # is possible to call os.umask() before and after the open() 1070 # call, but cleaning up from exceptions properly is more of a 1071 # nuisance that way). 1072 os.chmod(self.path, self.mode) 1073 except IOError: 1074 # TODO: this still needs cleanup 1075 self.fp = None 1076 self.stderr = 'Cannot open file %r for download' % self.path 1077 self.rc = 1 1078 if self.debug: 1079 log.msg('Cannot open file %r for download' % self.path) 1080 1081 d = defer.Deferred() 1082 reactor.callLater(0, self._loop, d) 1083 def _close(res): 1084 # close the file, but pass through any errors from _loop 1085 d1 = self.reader.callRemote('close') 1086 d1.addErrback(log.err) 1087 d1.addCallback(lambda ignored: res) 1088 return d1

1089 d.addBoth(_close) 1090 d.addBoth(self.finished) 1091 return d

1092

1093 - def _loop(self, fire_when_done):

1094 d = defer.maybeDeferred(self._readBlock) 1095 def _done(finished): 1096 if finished: 1097 fire_when_done.callback(None) 1098 else: 1099 self._loop(fire_when_done)

1100 def _err(why): 1101 fire_when_done.errback(why) 1102 d.addCallbacks(_done, _err) 1103 return None 1104

1105 - def _readBlock(self):

1106 """Read a block of data from the remote reader.""" 1107 1108 if self.interrupted or self.fp is None: 1109 if self.debug: 1110 log.msg('SlaveFileDownloadCommand._readBlock(): end') 1111 return True 1112 1113 length = self.blocksize 1114 if self.bytes_remaining is not None and length > self.bytes_remaining: 1115 length = self.bytes_remaining 1116 1117 if length <= 0: 1118 if self.stderr is None: 1119 self.stderr = 'Maximum filesize reached, truncating file %r' \ 1120 % self.path 1121 self.rc = 1 1122 return True 1123 else: 1124 d = self.reader.callRemote('read', length) 1125 d.addCallback(self._writeData) 1126 return d

1127

1128 - def _writeData(self, data):

1129 if self.debug: 1130 log.msg('SlaveFileDownloadCommand._readBlock(): readlen=%d' % 1131 len(data)) 1132 if len(data) == 0: 1133 return True 1134 1135 if self.bytes_remaining is not None: 1136 self.bytes_remaining = self.bytes_remaining - len(data) 1137 assert self.bytes_remaining >= 0 1138 self.fp.write(data) 1139 return False

1140

1141 - def interrupt(self):

1142 if self.debug: 1143 log.msg('interrupted') 1144 if self.interrupted: 1145 return 1146 if self.stderr is None: 1147 self.stderr = 'Download of %r interrupted' % self.path 1148 self.rc = 1 1149 self.interrupted = True

1150 # now we wait for the next read request to return. _readBlock will 1151 # abandon the file when it sees self.interrupted set. 1152

1153 - def finished(self, res):

1154 if self.fp is not None: 1155 self.fp.close() 1156 1157 if self.debug: 1158 log.msg('finished: stderr=%r, rc=%r' % (self.stderr, self.rc)) 1159 if self.stderr is None: 1160 self.sendStatus({'rc': self.rc}) 1161 else: 1162 self.sendStatus({'stderr': self.stderr, 'rc': self.rc}) 1163 return res

1164 1165 registerSlaveCommand("downloadFile", SlaveFileDownloadCommand, command_version) 1166 1167 1168

1169 -class SlaveShellCommand(Command):

1170 """This is a Command which runs a shell command. The args dict contains 1171 the following keys: 1172 1173 - ['command'] (required): a shell command to run. If this is a string, 1174 it will be run with /bin/sh (['/bin/sh', 1175 '-c', command]). If it is a list 1176 (preferred), it will be used directly. 1177 - ['workdir'] (required): subdirectory in which the command will be 1178 run, relative to the builder dir 1179 - ['env']: a dict of environment variables to augment/replace 1180 os.environ . PYTHONPATH is treated specially, and 1181 should be a list of path components to be prepended to 1182 any existing PYTHONPATH environment variable. 1183 - ['initial_stdin']: a string which will be written to the command's 1184 stdin as soon as it starts 1185 - ['keep_stdin_open']: unless True, the command's stdin will be 1186 closed as soon as initial_stdin has been 1187 written. Set this to True if you plan to write 1188 to stdin after the command has been started. 1189 - ['want_stdout']: 0 if stdout should be thrown away 1190 - ['want_stderr']: 0 if stderr should be thrown away 1191 - ['usePTY']: True or False if the command should use a PTY (defaults to 1192 configuration of the slave) 1193 - ['not_really']: 1 to skip execution and return rc=0 1194 - ['timeout']: seconds of silence to tolerate before killing command 1195 - ['logfiles']: dict mapping LogFile name to the workdir-relative 1196 filename of a local log file. This local file will be 1197 watched just like 'tail -f', and all changes will be 1198 written to 'log' status updates. 1199 1200 ShellCommand creates the following status messages: 1201 - {'stdout': data} : when stdout data is available 1202 - {'stderr': data} : when stderr data is available 1203 - {'header': data} : when headers (command start/stop) are available 1204 - {'log': (logfile_name, data)} : when log files have new contents 1205 - {'rc': rc} : when the process has terminated 1206 """ 1207

1208 - def start(self):

1209 args = self.args 1210 # args['workdir'] is relative to Builder directory, and is required. 1211 assert args['workdir'] is not None 1212 workdir = os.path.join(self.builder.basedir, args['workdir']) 1213 1214 c = ShellCommand(self.builder, args['command'], 1215 workdir, environ=args.get('env'), 1216 timeout=args.get('timeout', None), 1217 sendStdout=args.get('want_stdout', True), 1218 sendStderr=args.get('want_stderr', True), 1219 sendRC=True, 1220 initialStdin=args.get('initial_stdin'), 1221 keepStdinOpen=args.get('keep_stdin_open'), 1222 logfiles=args.get('logfiles', {}), 1223 usePTY=args.get('usePTY', "slave-config"), 1224 ) 1225 self.command = c 1226 d = self.command.start() 1227 return d

1228

1229 - def interrupt(self):

1230 self.interrupted = True 1231 self.command.kill("command interrupted")

1232

1233 - def writeStdin(self, data):

1234 self.command.writeStdin(data)

1235

1236 - def closeStdin(self):

1237 self.command.closeStdin()

1238 1239 registerSlaveCommand("shell", SlaveShellCommand, command_version) 1240 1241

1242 -class DummyCommand(Command):

1243 """ 1244 I am a dummy no-op command that by default takes 5 seconds to complete. 1245 See L{buildbot.steps.dummy.RemoteDummy} 1246 """ 1247

1248 - def start(self):

1249 self.d = defer.Deferred() 1250 log.msg(" starting dummy command [%s]" % self.stepId) 1251 self.timer = reactor.callLater(1, self.doStatus) 1252 return self.d

1253

1254 - def interrupt(self):

1255 if self.interrupted: 1256 return 1257 self.timer.cancel() 1258 self.timer = None 1259 self.interrupted = True 1260 self.finished()

1261

1262 - def doStatus(self):

1263 log.msg(" sending intermediate status") 1264 self.sendStatus({'stdout': 'data'}) 1265 timeout = self.args.get('timeout', 5) + 1 1266 self.timer = reactor.callLater(timeout - 1, self.finished)

1267

1268 - def finished(self):

1269 log.msg(" dummy command finished [%s]" % self.stepId) 1270 if self.interrupted: 1271 self.sendStatus({'rc': 1}) 1272 else: 1273 self.sendStatus({'rc': 0}) 1274 self.d.callback(0)

1275 1276 registerSlaveCommand("dummy", DummyCommand, command_version) 1277 1278 1279 # this maps handle names to a callable. When the WaitCommand starts, this 1280 # callable is invoked with no arguments. It should return a Deferred. When 1281 # that Deferred fires, our WaitCommand will finish. 1282 waitCommandRegistry = {} 1283

1284 -class WaitCommand(Command):

1285 """ 1286 I am a dummy command used by the buildbot unit test suite. I want for the 1287 unit test to tell us to finish. See L{buildbot.steps.dummy.Wait} 1288 """ 1289

1290 - def start(self):

1291 self.d = defer.Deferred() 1292 log.msg(" starting wait command [%s]" % self.stepId) 1293 handle = self.args['handle'] 1294 cb = waitCommandRegistry[handle] 1295 del waitCommandRegistry[handle] 1296 def _called(): 1297 log.msg(" wait-%s starting" % (handle,)) 1298 d = cb() 1299 def _done(res): 1300 log.msg(" wait-%s finishing: %s" % (handle, res)) 1301 return res

1302 d.addBoth(_done) 1303 d.addCallbacks(self.finished, self.failed)

1304 reactor.callLater(0, _called) 1305 return self.d 1306

1307 - def interrupt(self):

1308 log.msg(" wait command interrupted") 1309 if self.interrupted: 1310 return 1311 self.interrupted = True 1312 self.finished("interrupted")

1313

1314 - def finished(self, res):

1315 log.msg(" wait command finished [%s]" % self.stepId) 1316 if self.interrupted: 1317 self.sendStatus({'rc': 2}) 1318 else: 1319 self.sendStatus({'rc': 0}) 1320 self.d.callback(0)

1321 - def failed(self, why):

1322 log.msg(" wait command failed [%s]" % self.stepId) 1323 self.sendStatus({'rc': 1}) 1324 self.d.callback(0)

1325 1326 registerSlaveCommand("dummy.wait", WaitCommand, command_version) 1327 1328

1329 -class SourceBase(Command):

1330 """Abstract base class for Version Control System operations (checkout 1331 and update). This class extracts the following arguments from the 1332 dictionary received from the master: 1333 1334 - ['workdir']: (required) the subdirectory where the buildable sources 1335 should be placed 1336 1337 - ['mode']: one of update/copy/clobber/export, defaults to 'update' 1338 1339 - ['revision']: If not None, this is an int or string which indicates 1340 which sources (along a time-like axis) should be used. 1341 It is the thing you provide as the CVS -r or -D 1342 argument. 1343 1344 - ['patch']: If not None, this is a tuple of (striplevel, patch) 1345 which contains a patch that should be applied after the 1346 checkout has occurred. Once applied, the tree is no 1347 longer eligible for use with mode='update', and it only 1348 makes sense to use this in conjunction with a 1349 ['revision'] argument. striplevel is an int, and patch 1350 is a string in standard unified diff format. The patch 1351 will be applied with 'patch -p%d <PATCH', with 1352 STRIPLEVEL substituted as %d. The command will fail if 1353 the patch process fails (rejected hunks). 1354 1355 - ['timeout']: seconds of silence tolerated before we kill off the 1356 command 1357 1358 - ['retry']: If not None, this is a tuple of (delay, repeats) 1359 which means that any failed VC updates should be 1360 reattempted, up to REPEATS times, after a delay of 1361 DELAY seconds. This is intended to deal with slaves 1362 that experience transient network failures. 1363 """ 1364 1365 sourcedata = "" 1366

1367 - def setup(self, args):

1368 # if we need to parse the output, use this environment. Otherwise 1369 # command output will be in whatever the buildslave's native language 1370 # has been set to. 1371 self.env = os.environ.copy() 1372 self.env['LC_MESSAGES'] = "C" 1373 1374 self.workdir = args['workdir'] 1375 self.mode = args.get('mode', "update") 1376 self.revision = args.get('revision') 1377 self.patch = args.get('patch') 1378 self.timeout = args.get('timeout', 120) 1379 self.retry = args.get('retry')

1380 # VC-specific subclasses should override this to extract more args. 1381 # Make sure to upcall! 1382

1383 - def start(self):

1384 self.sendStatus({'header': "starting " + self.header + "\n"}) 1385 self.command = None 1386 1387 # self.srcdir is where the VC system should put the sources 1388 if self.mode == "copy": 1389 self.srcdir = "source" # hardwired directory name, sorry 1390 else: 1391 self.srcdir = self.workdir 1392 self.sourcedatafile = os.path.join(self.builder.basedir, 1393 self.srcdir, 1394 ".buildbot-sourcedata") 1395 1396 d = defer.succeed(None) 1397 self.maybeClobber(d) 1398 if not (self.sourcedirIsUpdateable() and self.sourcedataMatches()): 1399 # the directory cannot be updated, so we have to clobber it. 1400 # Perhaps the master just changed modes from 'export' to 1401 # 'update'. 1402 d.addCallback(self.doClobber, self.srcdir) 1403 1404 d.addCallback(self.doVC) 1405 1406 if self.mode == "copy": 1407 d.addCallback(self.doCopy) 1408 if self.patch: 1409 d.addCallback(self.doPatch) 1410 d.addCallbacks(self._sendRC, self._checkAbandoned) 1411 return d

1412

1413 - def maybeClobber(self, d):

1414 # do we need to clobber anything? 1415 if self.mode in ("copy", "clobber", "export"): 1416 d.addCallback(self.doClobber, self.workdir)

1417

1418 - def interrupt(self):

1419 self.interrupted = True 1420 if self.command: 1421 self.command.kill("command interrupted")

1422

1423 - def doVC(self, res):

1424 if self.interrupted: 1425 raise AbandonChain(1) 1426 if self.sourcedirIsUpdateable() and self.sourcedataMatches(): 1427 d = self.doVCUpdate() 1428 d.addCallback(self.maybeDoVCFallback) 1429 else: 1430 d = self.doVCFull() 1431 d.addBoth(self.maybeDoVCRetry) 1432 d.addCallback(self._abandonOnFailure) 1433 d.addCallback(self._handleGotRevision) 1434 d.addCallback(self.writeSourcedata) 1435 return d

1436

1437 - def sourcedataMatches(self):

1438 try: 1439 olddata = open(self.sourcedatafile, "r").read() 1440 if olddata != self.sourcedata: 1441 return False 1442 except IOError: 1443 return False 1444 return True

1445

1446 - def _handleGotRevision(self, res):

1447 d = defer.maybeDeferred(self.parseGotRevision) 1448 d.addCallback(lambda got_revision: 1449 self.sendStatus({'got_revision': got_revision})) 1450 return d

1451

1452 - def parseGotRevision(self):

1453 """Override this in a subclass. It should return a string that 1454 represents which revision was actually checked out, or a Deferred 1455 that will fire with such a string. If, in a future build, you were to 1456 pass this 'got_revision' string in as the 'revision' component of a 1457 SourceStamp, you should wind up with the same source code as this 1458 checkout just obtained. 1459 1460 It is probably most useful to scan self.command.stdout for a string 1461 of some sort. Be sure to set keepStdout=True on the VC command that 1462 you run, so that you'll have something available to look at. 1463 1464 If this information is unavailable, just return None.""" 1465 1466 return None

1467

1468 - def writeSourcedata(self, res):

1469 open(self.sourcedatafile, "w").write(self.sourcedata) 1470 return res

1471

1472 - def sourcedirIsUpdateable(self):

1473 raise NotImplementedError("this must be implemented in a subclass")

1474

1475 - def doVCUpdate(self):

1476 raise NotImplementedError("this must be implemented in a subclass")

1477

1478 - def doVCFull(self):

1479 raise NotImplementedError("this must be implemented in a subclass")

1480

1481 - def maybeDoVCFallback(self, rc):

1482 if type(rc) is int and rc == 0: 1483 return rc 1484 if self.interrupted: 1485 raise AbandonChain(1) 1486 msg = "update failed, clobbering and trying again" 1487 self.sendStatus({'header': msg + "\n"}) 1488 log.msg(msg) 1489 d = self.doClobber(None, self.srcdir) 1490 d.addCallback(self.doVCFallback2) 1491 return d

1492

1493 - def doVCFallback2(self, res):

1494 msg = "now retrying VC operation" 1495 self.sendStatus({'header': msg + "\n"}) 1496 log.msg(msg) 1497 d = self.doVCFull() 1498 d.addBoth(self.maybeDoVCRetry) 1499 d.addCallback(self._abandonOnFailure) 1500 return d

1501

1502 - def maybeDoVCRetry(self, res):

1503 """We get here somewhere after a VC chain has finished. res could 1504 be:: 1505 1506 - 0: the operation was successful 1507 - nonzero: the operation failed. retry if possible 1508 - AbandonChain: the operation failed, someone else noticed. retry. 1509 - Failure: some other exception, re-raise 1510 """ 1511 1512 if isinstance(res, failure.Failure): 1513 if self.interrupted: 1514 return res # don't re-try interrupted builds 1515 res.trap(AbandonChain) 1516 else: 1517 if type(res) is int and res == 0: 1518 return res 1519 if self.interrupted: 1520 raise AbandonChain(1) 1521 # if we get here, we should retry, if possible 1522 if self.retry: 1523 delay, repeats = self.retry 1524 if repeats >= 0: 1525 self.retry = (delay, repeats-1) 1526 msg = ("update failed, trying %d more times after %d seconds" 1527 % (repeats, delay)) 1528 self.sendStatus({'header': msg + "\n"}) 1529 log.msg(msg) 1530 d = defer.Deferred() 1531 self.maybeClobber(d) 1532 d.addCallback(lambda res: self.doVCFull()) 1533 d.addBoth(self.maybeDoVCRetry) 1534 reactor.callLater(delay, d.callback, None) 1535 return d 1536 return res

1537

1538 - def doClobber(self, dummy, dirname):

1539 # TODO: remove the old tree in the background 1540 ## workdir = os.path.join(self.builder.basedir, self.workdir) 1541 ## deaddir = self.workdir + ".deleting" 1542 ## if os.path.isdir(workdir): 1543 ## try: 1544 ## os.rename(workdir, deaddir) 1545 ## # might fail if deaddir already exists: previous deletion 1546 ## # hasn't finished yet 1547 ## # start the deletion in the background 1548 ## # TODO: there was a solaris/NetApp/NFS problem where a 1549 ## # process that was still running out of the directory we're 1550 ## # trying to delete could prevent the rm-rf from working. I 1551 ## # think it stalled the rm, but maybe it just died with 1552 ## # permission issues. Try to detect this. 1553 ## os.commands("rm -rf %s &" % deaddir) 1554 ## except: 1555 ## # fall back to sequential delete-then-checkout 1556 ## pass 1557 d = os.path.join(self.builder.basedir, dirname) 1558 if runtime.platformType != "posix": 1559 # if we're running on w32, use rmtree instead. It will block, 1560 # but hopefully it won't take too long. 1561 rmdirRecursive(d) 1562 return defer.succeed(0) 1563 command = ["rm", "-rf", d] 1564 c = ShellCommand(self.builder, command, self.builder.basedir, 1565 sendRC=0, timeout=self.timeout, usePTY=False) 1566 1567 self.command = c 1568 # sendRC=0 means the rm command will send stdout/stderr to the 1569 # master, but not the rc=0 when it finishes. That job is left to 1570 # _sendRC 1571 d = c.start() 1572 d.addCallback(self._abandonOnFailure) 1573 return d

1574

1575 - def doCopy(self, res):

1576 # now copy tree to workdir 1577 fromdir = os.path.join(self.builder.basedir, self.srcdir) 1578 todir = os.path.join(self.builder.basedir, self.workdir) 1579 if runtime.platformType != "posix": 1580 self.sendStatus({'header': "Since we're on a non-POSIX platform, " 1581 "we're not going to try to execute cp in a subprocess, but instead " 1582 "use shutil.copytree(), which will block until it is complete. " 1583 "fromdir: %s, todir: %s\n" % (fromdir, todir)}) 1584 shutil.copytree(fromdir, todir) 1585 return defer.succeed(0) 1586 1587 if not os.path.exists(os.path.dirname(todir)): 1588 os.makedirs(os.path.dirname(todir)) 1589 if os.path.exists(todir): 1590 # I don't think this happens, but just in case.. 1591 log.msg("cp target '%s' already exists -- cp will not do what you think!" % todir) 1592 1593 command = ['cp', '-R', '-P', '-p', fromdir, todir] 1594 c = ShellCommand(self.builder, command, self.builder.basedir, 1595 sendRC=False, timeout=self.timeout, usePTY=False) 1596 self.command = c 1597 d = c.start() 1598 d.addCallback(self._abandonOnFailure) 1599 return d

1600

1601 - def doPatch(self, res):

1602 patchlevel, diff = self.patch 1603 command = [getCommand("patch"), '-p%d' % patchlevel] 1604 dir = os.path.join(self.builder.basedir, self.workdir) 1605 # mark the directory so we don't try to update it later 1606 open(os.path.join(dir, ".buildbot-patched"), "w").write("patched\n") 1607 # now apply the patch 1608 c = ShellCommand(self.builder, command, dir, 1609 sendRC=False, timeout=self.timeout, 1610 initialStdin=diff, usePTY=False) 1611 self.command = c 1612 d = c.start() 1613 d.addCallback(self._abandonOnFailure) 1614 return d

1615 1616

1617 -class CVS(SourceBase):

1618 """CVS-specific VC operation. In addition to the arguments handled by 1619 SourceBase, this command reads the following keys: 1620 1621 ['cvsroot'] (required): the CVSROOT repository string 1622 ['cvsmodule'] (required): the module to be retrieved 1623 ['branch']: a '-r' tag or branch name to use for the checkout/update 1624 ['login']: a string for use as a password to 'cvs login' 1625 ['global_options']: a list of strings to use before the CVS verb 1626 ['checkout_options']: a list of strings to use after checkout, 1627 but before revision and branch specifiers 1628 """ 1629 1630 header = "cvs operation" 1631

1632 - def setup(self, args):

1633 SourceBase.setup(self, args) 1634 self.vcexe = getCommand("cvs") 1635 self.cvsroot = args['cvsroot'] 1636 self.cvsmodule = args['cvsmodule'] 1637 self.global_options = args.get('global_options', []) 1638 self.checkout_options = args.get('checkout_options', []) 1639 self.branch = args.get('branch') 1640 self.login = args.get('login') 1641 self.sourcedata = "%s\n%s\n%s\n" % (self.cvsroot, self.cvsmodule, 1642 self.branch)

1643

1644 - def sourcedirIsUpdateable(self):

1645 if os.path.exists(os.path.join(self.builder.basedir, 1646 self.srcdir, ".buildbot-patched")): 1647 return False 1648 return os.path.isdir(os.path.join(self.builder.basedir, 1649 self.srcdir, "CVS"))

1650

1651 - def start(self):

1652 if self.login is not None: 1653 # need to do a 'cvs login' command first 1654 d = self.builder.basedir 1655 command = ([self.vcexe, '-d', self.cvsroot] + self.global_options 1656 + ['login']) 1657 c = ShellCommand(self.builder, command, d, 1658 sendRC=False, timeout=self.timeout, 1659 initialStdin=self.login+"\n", usePTY=False) 1660 self.command = c 1661 d = c.start() 1662 d.addCallback(self._abandonOnFailure) 1663 d.addCallback(self._didLogin) 1664 return d 1665 else: 1666 return self._didLogin(None)

1667

1668 - def _didLogin(self, res):

1669 # now we really start 1670 return SourceBase.start(self)

1671

1672 - def doVCUpdate(self):

1673 d = os.path.join(self.builder.basedir, self.srcdir) 1674 command = [self.vcexe, '-z3'] + self.global_options + ['update', '-dP'] 1675 if self.branch: 1676 command += ['-r', self.branch] 1677 if self.revision: 1678 command += ['-D', self.revision] 1679 c = ShellCommand(self.builder, command, d, 1680 sendRC=False, timeout=self.timeout, usePTY=False) 1681 self.command = c 1682 return c.start()

1683

1684 - def doVCFull(self):

1685 d = self.builder.basedir 1686 if self.mode == "export": 1687 verb = "export" 1688 else: 1689 verb = "checkout" 1690 command = ([self.vcexe, '-d', self.cvsroot, '-z3'] + 1691 self.global_options + 1692 [verb, '-d', self.srcdir]) 1693 1694 if verb == "checkout": 1695 command += self.checkout_options 1696 if self.branch: 1697 command += ['-r', self.branch] 1698 if self.revision: 1699 command += ['-D', self.revision] 1700 command += [self.cvsmodule] 1701 1702 c = ShellCommand(self.builder, command, d, 1703 sendRC=False, timeout=self.timeout, usePTY=False) 1704 self.command = c 1705 return c.start()

1706

1707 - def parseGotRevision(self):

1708 # CVS does not have any kind of revision stamp to speak of. We return 1709 # the current timestamp as a best-effort guess, but this depends upon 1710 # the local system having a clock that is 1711 # reasonably-well-synchronized with the repository. 1712 return time.strftime("%Y-%m-%d %H:%M:%S +0000", time.gmtime())

1713 1714 registerSlaveCommand("cvs", CVS, command_version) 1715

1716 -class SVN(SourceBase):

1717 """Subversion-specific VC operation. In addition to the arguments 1718 handled by SourceBase, this command reads the following keys: 1719 1720 ['svnurl'] (required): the SVN repository string 1721 ['username'] Username passed to the svn command 1722 ['password'] Password passed to the svn command 1723 """ 1724 1725 header = "svn operation" 1726

1727 - def setup(self, args):

1728 SourceBase.setup(self, args) 1729 self.vcexe = getCommand("svn") 1730 self.svnurl = args['svnurl'] 1731 self.sourcedata = "%s\n" % self.svnurl 1732 1733 self.svn_args = [] 1734 if args.has_key('username'): 1735 self.svn_args.extend(["--username", args['username']]) 1736 if args.has_key('password'): 1737 self.svn_args.extend(["--password", Obfuscated(args['password'], "XXXX")]) 1738 if args.get('extra_args', None) is not None: 1739 self.svn_args.extend(args['extra_args'])

1740

1741 - def sourcedirIsUpdateable(self):

1742 if os.path.exists(os.path.join(self.builder.basedir, 1743 self.srcdir, ".buildbot-patched")): 1744 return False 1745 return os.path.isdir(os.path.join(self.builder.basedir, 1746 self.srcdir, ".svn"))

1747

1748 - def doVCUpdate(self):

1749 revision = self.args['revision'] or 'HEAD' 1750 # update: possible for mode in ('copy', 'update') 1751 d = os.path.join(self.builder.basedir, self.srcdir) 1752 command = [self.vcexe, 'update'] + \ 1753 self.svn_args + \ 1754 ['--revision', str(revision), 1755 '--non-interactive', '--no-auth-cache'] 1756 c = ShellCommand(self.builder, command, d, 1757 sendRC=False, timeout=self.timeout, 1758 keepStdout=True, usePTY=False) 1759 self.command = c 1760 return c.start()

1761

1762 - def doVCFull(self):

1763 revision = self.args['revision'] or 'HEAD' 1764 d = self.builder.basedir 1765 if self.mode == "export": 1766 command = [self.vcexe, 'export'] + \ 1767 self.svn_args + \ 1768 ['--revision', str(revision), 1769 '--non-interactive', '--no-auth-cache', 1770 self.svnurl, self.srcdir] 1771 else: 1772 # mode=='clobber', or copy/update on a broken workspace 1773 command = [self.vcexe, 'checkout'] + \ 1774 self.svn_args + \ 1775 ['--revision', str(revision), 1776 '--non-interactive', '--no-auth-cache', 1777 self.svnurl, self.srcdir] 1778 c = ShellCommand(self.builder, command, d, 1779 sendRC=False, timeout=self.timeout, 1780 keepStdout=True, usePTY=False) 1781 self.command = c 1782 return c.start()

1783

1784 - def getSvnVersionCommand(self):

1785 """ 1786 Get the (shell) command used to determine SVN revision number 1787 of checked-out code 1788 1789 return: list of strings, passable as the command argument to ShellCommand 1790 """ 1791 # svn checkout operations finish with 'Checked out revision 16657.' 1792 # svn update operations finish the line 'At revision 16654.' 1793 # But we don't use those. Instead, run 'svnversion'. 1794 svnversion_command = getCommand("svnversion") 1795 # older versions of 'svnversion' (1.1.4) require the WC_PATH 1796 # argument, newer ones (1.3.1) do not. 1797 return [svnversion_command, "."]

1798

1799 - def parseGotRevision(self):

1800 c = ShellCommand(self.builder, 1801 self.getSvnVersionCommand(), 1802 os.path.join(self.builder.basedir, self.srcdir), 1803 environ=self.env, 1804 sendStdout=False, sendStderr=False, sendRC=False, 1805 keepStdout=True, usePTY=False) 1806 d = c.start() 1807 def _parse(res): 1808 r_raw = c.stdout.strip() 1809 # Extract revision from the version "number" string 1810 r = r_raw.rstrip('MS') 1811 r = r.split(':')[-1] 1812 got_version = None 1813 try: 1814 got_version = int(r) 1815 except ValueError: 1816 msg =("SVN.parseGotRevision unable to parse output " 1817 "of svnversion: '%s'" % r_raw) 1818 log.msg(msg) 1819 self.sendStatus({'header': msg + "\n"}) 1820 return got_version

1821 d.addCallback(_parse) 1822 return d

1823 1824 1825 registerSlaveCommand("svn", SVN, command_version) 1826

1827 -class Darcs(SourceBase):

1828 """Darcs-specific VC operation. In addition to the arguments 1829 handled by SourceBase, this command reads the following keys: 1830 1831 ['repourl'] (required): the Darcs repository string 1832 """ 1833 1834 header = "darcs operation" 1835

1836 - def setup(self, args):

1837 SourceBase.setup(self, args) 1838 self.vcexe = getCommand("darcs") 1839 self.repourl = args['repourl'] 1840 self.sourcedata = "%s\n" % self.repourl 1841 self.revision = self.args.get('revision')

1842

1843 - def sourcedirIsUpdateable(self):

1844 if os.path.exists(os.path.join(self.builder.basedir, 1845 self.srcdir, ".buildbot-patched")): 1846 return False 1847 if self.revision: 1848 # checking out a specific revision requires a full 'darcs get' 1849 return False 1850 return os.path.isdir(os.path.join(self.builder.basedir, 1851 self.srcdir, "_darcs"))

1852

1853 - def doVCUpdate(self):

1854 assert not self.revision 1855 # update: possible for mode in ('copy', 'update') 1856 d = os.path.join(self.builder.basedir, self.srcdir) 1857 command = [self.vcexe, 'pull', '--all', '--verbose'] 1858 c = ShellCommand(self.builder, command, d, 1859 sendRC=False, timeout=self.timeout, usePTY=False) 1860 self.command = c 1861 return c.start()

1862

1863 - def doVCFull(self):

1864 # checkout or export 1865 d = self.builder.basedir 1866 command = [self.vcexe, 'get', '--verbose', '--partial', 1867 '--repo-name', self.srcdir] 1868 if self.revision: 1869 # write the context to a file 1870 n = os.path.join(self.builder.basedir, ".darcs-context") 1871 f = open(n, "wb") 1872 f.write(self.revision) 1873 f.close() 1874 # tell Darcs to use that context 1875 command.append('--context') 1876 command.append(n) 1877 command.append(self.repourl) 1878 1879 c = ShellCommand(self.builder, command, d, 1880 sendRC=False, timeout=self.timeout, usePTY=False) 1881 self.command = c 1882 d = c.start() 1883 if self.revision: 1884 d.addCallback(self.removeContextFile, n) 1885 return d

1886

1887 - def removeContextFile(self, res, n):

1888 os.unlink(n) 1889 return res

1890

1891 - def parseGotRevision(self):

1892 # we use 'darcs context' to find out what we wound up with 1893 command = [self.vcexe, "changes", "--context"] 1894 c = ShellCommand(self.builder, command, 1895 os.path.join(self.builder.basedir, self.srcdir), 1896 environ=self.env, 1897 sendStdout=False, sendStderr=False, sendRC=False, 1898 keepStdout=True, usePTY=False) 1899 d = c.start() 1900 d.addCallback(lambda res: c.stdout) 1901 return d

1902 1903 registerSlaveCommand("darcs", Darcs, command_version) 1904

1905 -class Monotone(SourceBase):

1906 """Monotone-specific VC operation. In addition to the arguments handled 1907 by SourceBase, this command reads the following keys: 1908 1909 ['server_addr'] (required): the address of the server to pull from 1910 ['branch'] (required): the branch the revision is on 1911 ['db_path'] (required): the local database path to use 1912 ['revision'] (required): the revision to check out 1913 ['monotone']: (required): path to monotone executable 1914 """ 1915 1916 header = "monotone operation" 1917

1918 - def setup(self, args):

1919 SourceBase.setup(self, args) 1920 self.server_addr = args["server_addr"] 1921 self.branch = args["branch"] 1922 self.db_path = args["db_path"] 1923 self.revision = args["revision"] 1924 self.monotone = args["monotone"] 1925 self._made_fulls = False 1926 self._pull_timeout = args["timeout"]

1927

1928 - def _makefulls(self):

1929 if not self._made_fulls: 1930 basedir = self.builder.basedir 1931 self.full_db_path = os.path.join(basedir, self.db_path) 1932 self.full_srcdir = os.path.join(basedir, self.srcdir) 1933 self._made_fulls = True

1934

1935 - def sourcedirIsUpdateable(self):

1936 self._makefulls() 1937 if os.path.exists(os.path.join(self.full_srcdir, 1938 ".buildbot_patched")): 1939 return False 1940 return (os.path.isfile(self.full_db_path) 1941 and os.path.isdir(os.path.join(self.full_srcdir, "MT")))

1942

1943 - def doVCUpdate(self):

1944 return self._withFreshDb(self._doUpdate)

1945

1946 - def _doUpdate(self):

1947 # update: possible for mode in ('copy', 'update') 1948 command = [self.monotone, "update", 1949 "-r", self.revision, 1950 "-b", self.branch] 1951 c = ShellCommand(self.builder, command, self.full_srcdir, 1952 sendRC=False, timeout=self.timeout, usePTY=False) 1953 self.command = c 1954 return c.start()

1955

1956 - def doVCFull(self):

1957 return self._withFreshDb(self._doFull)

1958

1959 - def _doFull(self):

1960 command = [self.monotone, "--db=" + self.full_db_path, 1961 "checkout", 1962 "-r", self.revision, 1963 "-b", self.branch, 1964 self.full_srcdir] 1965 c = ShellCommand(self.builder, command, self.builder.basedir, 1966 sendRC=False, timeout=self.timeout, usePTY=False) 1967 self.command = c 1968 return c.start()

1969

1970 - def _withFreshDb(self, callback):

1971 self._makefulls() 1972 # first ensure the db exists and is usable 1973 if os.path.isfile(self.full_db_path): 1974 # already exists, so run 'db migrate' in case monotone has been 1975 # upgraded under us 1976 command = [self.monotone, "db", "migrate", 1977 "--db=" + self.full_db_path] 1978 else: 1979 # We'll be doing an initial pull, so up the timeout to 3 hours to 1980 # make sure it will have time to complete. 1981 self._pull_timeout = max(self._pull_timeout, 3 * 60 * 60) 1982 self.sendStatus({"header": "creating database %s\n" 1983 % (self.full_db_path,)}) 1984 command = [self.monotone, "db", "init", 1985 "--db=" + self.full_db_path] 1986 c = ShellCommand(self.builder, command, self.builder.basedir, 1987 sendRC=False, timeout=self.timeout, usePTY=False) 1988 self.command = c 1989 d = c.start() 1990 d.addCallback(self._abandonOnFailure) 1991 d.addCallback(self._didDbInit) 1992 d.addCallback(self._didPull, callback) 1993 return d

1994

1995 - def _didDbInit(self, res):

1996 command = [self.monotone, "--db=" + self.full_db_path, 1997 "pull", "--ticker=dot", self.server_addr, self.branch] 1998 c = ShellCommand(self.builder, command, self.builder.basedir, 1999 sendRC=False, timeout=self._pull_timeout, usePTY=False) 2000 self.sendStatus({"header": "pulling %s from %s\n" 2001 % (self.branch, self.server_addr)}) 2002 self.command = c 2003 return c.start()

2004

2005 - def _didPull(self, res, callback):

2006 return callback()

2007 2008 registerSlaveCommand("monotone", Monotone, command_version) 2009 2010

2011 -class Git(SourceBase):

2012 """Git specific VC operation. In addition to the arguments 2013 handled by SourceBase, this command reads the following keys: 2014 2015 ['repourl'] (required): the upstream GIT repository string 2016 ['branch'] (optional): which version (i.e. branch or tag) to 2017 retrieve. Default: "master". 2018 ['submodules'] (optional): whether to initialize and update 2019 submodules. Default: False. 2020 """ 2021 2022 header = "git operation" 2023

2024 - def setup(self, args):

2025 SourceBase.setup(self, args) 2026 self.repourl = args['repourl'] 2027 self.branch = args.get('branch') 2028 if not self.branch: 2029 self.branch = "master" 2030 self.sourcedata = "%s %s\n" % (self.repourl, self.branch) 2031 self.submodules = args.get('submodules')

2032

2033 - def _fullSrcdir(self):

2034 return os.path.join(self.builder.basedir, self.srcdir)

2035

2036 - def _commitSpec(self):

2037 if self.revision: 2038 return self.revision 2039 return self.branch

2040

2041 - def sourcedirIsUpdateable(self):

2042 return os.path.isdir(os.path.join(self._fullSrcdir(), ".git"))

2043

2044 - def readSourcedata(self):

2045 return open(self.sourcedatafile, "r").read()

2046 2047 # If the repourl matches the sourcedata file, then 2048 # we can say that the sourcedata matches. We can 2049 # ignore branch changes, since Git can work with 2050 # many branches fetched, and we deal with it properly 2051 # in doVCUpdate.

2052 - def sourcedataMatches(self):

2053 try: 2054 olddata = self.readSourcedata() 2055 if not olddata.startswith(self.repourl+' '): 2056 return False 2057 except IOError: 2058 return False 2059 return True

2060

2061 - def _didSubmodules(self, res):

2062 command = ['git', 'submodule', 'update', '--init'] 2063 c = ShellCommand(self.builder, command, self._fullSrcdir(), 2064 sendRC=False, timeout=self.timeout, usePTY=False) 2065 self.command = c 2066 return c.start()

2067

2068 - def _didFetch(self, res):

2069 if self.revision: 2070 head = self.revision 2071 else: 2072 head = 'FETCH_HEAD' 2073 2074 command = ['git', 'reset', '--hard', head] 2075 c = ShellCommand(self.builder, command, self._fullSrcdir(), 2076 sendRC=False, timeout=self.timeout, usePTY=False) 2077 self.command = c 2078 d = c.start() 2079 if self.submodules: 2080 d.addCallback(self._abandonOnFailure) 2081 d.addCallback(self._didSubmodules) 2082 return d

2083 2084 # Update first runs "git clean", removing local changes, This, 2085 # combined with the later "git reset" equates clobbering the repo, 2086 # but it's much more efficient.

2087 - def doVCUpdate(self):

2088 command = ['git', 'clean', '-f', '-d', '-x'] 2089 c = ShellCommand(self.builder, command, self._fullSrcdir(), 2090 sendRC=False, timeout=self.timeout, usePTY=False) 2091 self.command = c 2092 d = c.start() 2093 d.addCallback(self._abandonOnFailure) 2094 d.addCallback(self._didClean) 2095 return d

2096

2097 - def _didClean(self, dummy):

2098 command = ['git', 'fetch', '-t', self.repourl, self.branch] 2099 self.sendStatus({"header": "fetching branch %s from %s\n" 2100 % (self.branch, self.repourl)}) 2101 c = ShellCommand(self.builder, command, self._fullSrcdir(), 2102 sendRC=False, timeout=self.timeout, usePTY=False) 2103 self.command = c 2104 d = c.start() 2105 d.addCallback(self._abandonOnFailure) 2106 d.addCallback(self._didFetch) 2107 return d

2108

2109 - def _didInit(self, res):

2110 return self.doVCUpdate()

2111

2112 - def doVCFull(self):

2113 os.mkdir(self._fullSrcdir()) 2114 c = ShellCommand(self.builder, ['git', 'init'], self._fullSrcdir(), 2115 sendRC=False, timeout=self.timeout, usePTY=False) 2116 self.command = c 2117 d = c.start() 2118 d.addCallback(self._abandonOnFailure) 2119 d.addCallback(self._didInit) 2120 return d

2121

2122 - def parseGotRevision(self):

2123 command = ['git', 'rev-parse', 'HEAD'] 2124 c = ShellCommand(self.builder, command, self._fullSrcdir(), 2125 sendRC=False, keepStdout=True, usePTY=False) 2126 d = c.start() 2127 def _parse(res): 2128 hash = c.stdout.strip() 2129 if len(hash) != 40: 2130 return None 2131 return hash

2132 d.addCallback(_parse) 2133 return d

2134 2135 registerSlaveCommand("git", Git, command_version) 2136

2137 -class Arch(SourceBase):

2138 """Arch-specific (tla-specific) VC operation. In addition to the 2139 arguments handled by SourceBase, this command reads the following keys: 2140 2141 ['url'] (required): the repository string 2142 ['version'] (required): which version (i.e. branch) to retrieve 2143 ['revision'] (optional): the 'patch-NN' argument to check out 2144 ['archive']: the archive name to use. If None, use the archive's default 2145 ['build-config']: if present, give to 'tla build-config' after checkout 2146 """ 2147 2148 header = "arch operation" 2149 buildconfig = None 2150

2151 - def setup(self, args):

2152 SourceBase.setup(self, args) 2153 self.vcexe = getCommand("tla") 2154 self.archive = args.get('archive') 2155 self.url = args['url'] 2156 self.version = args['version'] 2157 self.revision = args.get('revision') 2158 self.buildconfig = args.get('build-config') 2159 self.sourcedata = "%s\n%s\n%s\n" % (self.url, self.version, 2160 self.buildconfig)

2161

2162 - def sourcedirIsUpdateable(self):

2163 if self.revision: 2164 # Arch cannot roll a directory backwards, so if they ask for a 2165 # specific revision, clobber the directory. Technically this 2166 # could be limited to the cases where the requested revision is 2167 # later than our current one, but it's too hard to extract the 2168 # current revision from the tree. 2169 return False 2170 if os.path.exists(os.path.join(self.builder.basedir, 2171 self.srcdir, ".buildbot-patched")): 2172 return False 2173 return os.path.isdir(os.path.join(self.builder.basedir, 2174 self.srcdir, "{arch}"))

2175

2176 - def doVCUpdate(self):

2177 # update: possible for mode in ('copy', 'update') 2178 d = os.path.join(self.builder.basedir, self.srcdir) 2179 command = [self.vcexe, 'replay'] 2180 if self.revision: 2181 command.append(self.revision) 2182 c = ShellCommand(self.builder, command, d, 2183 sendRC=False, timeout=self.timeout, usePTY=False) 2184 self.command = c 2185 return c.start()

2186

2187 - def doVCFull(self):

2188 # to do a checkout, we must first "register" the archive by giving 2189 # the URL to tla, which will go to the repository at that URL and 2190 # figure out the archive name. tla will tell you the archive name 2191 # when it is done, and all further actions must refer to this name. 2192 2193 command = [self.vcexe, 'register-archive', '--force', self.url] 2194 c = ShellCommand(self.builder, command, self.builder.basedir, 2195 sendRC=False, keepStdout=True, 2196 timeout=self.timeout, usePTY=False) 2197 self.command = c 2198 d = c.start() 2199 d.addCallback(self._abandonOnFailure) 2200 d.addCallback(self._didRegister, c) 2201 return d

2202

2203 - def _didRegister(self, res, c):

2204 # find out what tla thinks the archive name is. If the user told us 2205 # to use something specific, make sure it matches. 2206 r = re.search(r'Registering archive: (\S+)\s*$', c.stdout) 2207 if r: 2208 msg = "tla reports archive name is '%s'" % r.group(1) 2209 log.msg(msg) 2210 self.builder.sendUpdate({'header': msg+"\n"}) 2211 if self.archive and r.group(1) != self.archive: 2212 msg = (" mismatch, we wanted an archive named '%s'" 2213 % self.archive) 2214 log.msg(msg) 2215 self.builder.sendUpdate({'header': msg+"\n"}) 2216 raise AbandonChain(-1) 2217 self.archive = r.group(1) 2218 assert self.archive, "need archive name to continue" 2219 return self._doGet()

2220

2221 - def _doGet(self):

2222 ver = self.version 2223 if self.revision: 2224 ver += "--%s" % self.revision 2225 command = [self.vcexe, 'get', '--archive', self.archive, 2226 '--no-pristine', 2227 ver, self.srcdir] 2228 c = ShellCommand(self.builder, command, self.builder.basedir, 2229 sendRC=False, timeout=self.timeout, usePTY=False) 2230 self.command = c 2231 d = c.start() 2232 d.addCallback(self._abandonOnFailure) 2233 if self.buildconfig: 2234 d.addCallback(self._didGet) 2235 return d

2236

2237 - def _didGet(self, res):

2238 d = os.path.join(self.builder.basedir, self.srcdir) 2239 command = [self.vcexe, 'build-config', self.buildconfig] 2240 c = ShellCommand(self.builder, command, d, 2241 sendRC=False, timeout=self.timeout, usePTY=False) 2242 self.command = c 2243 d = c.start() 2244 d.addCallback(self._abandonOnFailure) 2245 return d

2246

2247 - def parseGotRevision(self):

2248 # using code from tryclient.TlaExtractor 2249 # 'tla logs --full' gives us ARCHIVE/BRANCH--REVISION 2250 # 'tla logs' gives us REVISION 2251 command = [self.vcexe, "logs", "--full", "--reverse"] 2252 c = ShellCommand(self.builder, command, 2253 os.path.join(self.builder.basedir, self.srcdir), 2254 environ=self.env, 2255 sendStdout=False, sendStderr=False, sendRC=False, 2256 keepStdout=True, usePTY=False) 2257 d = c.start() 2258 def _parse(res): 2259 tid = c.stdout.split("\n")[0].strip() 2260 slash = tid.index("/") 2261 dd = tid.rindex("--") 2262 #branch = tid[slash+1:dd] 2263 baserev = tid[dd+2:] 2264 return baserev

2265 d.addCallback(_parse) 2266 return d

2267 2268 registerSlaveCommand("arch", Arch, command_version) 2269

2270 -class Bazaar(Arch):

2271 """Bazaar (/usr/bin/baz) is an alternative client for Arch repositories. 2272 It is mostly option-compatible, but archive registration is different 2273 enough to warrant a separate Command. 2274 2275 ['archive'] (required): the name of the archive being used 2276 """ 2277

2278 - def setup(self, args):

2279 Arch.setup(self, args) 2280 self.vcexe = getCommand("baz") 2281 # baz doesn't emit the repository name after registration (and 2282 # grepping through the output of 'baz archives' is too hard), so we 2283 # require that the buildmaster configuration to provide both the 2284 # archive name and the URL. 2285 self.archive = args['archive'] # required for Baz 2286 self.sourcedata = "%s\n%s\n%s\n" % (self.url, self.version, 2287 self.buildconfig)

2288 2289 # in _didRegister, the regexp won't match, so we'll stick with the name 2290 # in self.archive 2291

2292 - def _doGet(self):

2293 # baz prefers ARCHIVE/VERSION. This will work even if 2294 # my-default-archive is not set. 2295 ver = self.archive + "/" + self.version 2296 if self.revision: 2297 ver += "--%s" % self.revision 2298 command = [self.vcexe, 'get', '--no-pristine', 2299 ver, self.srcdir] 2300 c = ShellCommand(self.builder, command, self.builder.basedir, 2301 sendRC=False, timeout=self.timeout, usePTY=False) 2302 self.command = c 2303 d = c.start() 2304 d.addCallback(self._abandonOnFailure) 2305 if self.buildconfig: 2306 d.addCallback(self._didGet) 2307 return d

2308

2309 - def parseGotRevision(self):

2310 # using code from tryclient.BazExtractor 2311 command = [self.vcexe, "tree-id"] 2312 c = ShellCommand(self.builder, command, 2313 os.path.join(self.builder.basedir, self.srcdir), 2314 environ=self.env, 2315 sendStdout=False, sendStderr=False, sendRC=False, 2316 keepStdout=True, usePTY=False) 2317 d = c.start() 2318 def _parse(res): 2319 tid = c.stdout.strip() 2320 slash = tid.index("/") 2321 dd = tid.rindex("--") 2322 #branch = tid[slash+1:dd] 2323 baserev = tid[dd+2:] 2324 return baserev

2325 d.addCallback(_parse) 2326 return d

2327 2328 registerSlaveCommand("bazaar", Bazaar, command_version) 2329 2330

2331 -class Bzr(SourceBase):

2332 """bzr-specific VC operation. In addition to the arguments 2333 handled by SourceBase, this command reads the following keys: 2334 2335 ['repourl'] (required): the Bzr repository string 2336 """ 2337 2338 header = "bzr operation" 2339

2340 - def setup(self, args):

2341 SourceBase.setup(self, args) 2342 self.vcexe = getCommand("bzr") 2343 self.repourl = args['repourl'] 2344 self.sourcedata = "%s\n" % self.repourl 2345 self.revision = self.args.get('revision') 2346 self.forceSharedRepo = args.get('forceSharedRepo')

2347

2348 - def sourcedirIsUpdateable(self):

2349 if os.path.exists(os.path.join(self.builder.basedir, 2350 self.srcdir, ".buildbot-patched")): 2351 return False 2352 if self.revision: 2353 # checking out a specific revision requires a full 'bzr checkout' 2354 return False 2355 return os.path.isdir(os.path.join(self.builder.basedir, 2356 self.srcdir, ".bzr"))

2357

2358 - def start(self):

2359 def cont(res): 2360 # Continue with start() method in superclass. 2361 return SourceBase.start(self)

2362 2363 if self.forceSharedRepo: 2364 d = self.doForceSharedRepo(); 2365 d.addCallback(cont) 2366 return d 2367 else: 2368 return cont(None)

2369

2370 - def doVCUpdate(self):

2371 assert not self.revision 2372 # update: possible for mode in ('copy', 'update') 2373 srcdir = os.path.join(self.builder.basedir, self.srcdir) 2374 command = [self.vcexe, 'update'] 2375 c = ShellCommand(self.builder, command, srcdir, 2376 sendRC=False, timeout=self.timeout, usePTY=False) 2377 self.command = c 2378 return c.start()

2379

2380 - def doVCFull(self):

2381 # checkout or export 2382 d = self.builder.basedir 2383 if self.mode == "export": 2384 # exporting in bzr requires a separate directory 2385 return self.doVCExport() 2386 # originally I added --lightweight here, but then 'bzr revno' is 2387 # wrong. The revno reported in 'bzr version-info' is correct, 2388 # however. Maybe this is a bzr bug? 2389 # 2390 # In addition, you cannot perform a 'bzr update' on a repo pulled 2391 # from an HTTP repository that used 'bzr checkout --lightweight'. You 2392 # get a "ERROR: Cannot lock: transport is read only" when you try. 2393 # 2394 # So I won't bother using --lightweight for now. 2395 2396 command = [self.vcexe, 'checkout'] 2397 if self.revision: 2398 command.append('--revision') 2399 command.append(str(self.revision)) 2400 command.append(self.repourl) 2401 command.append(self.srcdir) 2402 2403 c = ShellCommand(self.builder, command, d, 2404 sendRC=False, timeout=self.timeout, usePTY=False) 2405 self.command = c 2406 d = c.start() 2407 return d

2408

2409 - def doVCExport(self):

2410 tmpdir = os.path.join(self.builder.basedir, "export-temp") 2411 srcdir = os.path.join(self.builder.basedir, self.srcdir) 2412 command = [self.vcexe, 'checkout', '--lightweight'] 2413 if self.revision: 2414 command.append('--revision') 2415 command.append(str(self.revision)) 2416 command.append(self.repourl) 2417 command.append(tmpdir) 2418 c = ShellCommand(self.builder, command, self.builder.basedir, 2419 sendRC=False, timeout=self.timeout, usePTY=False) 2420 self.command = c 2421 d = c.start() 2422 def _export(res): 2423 command = [self.vcexe, 'export', srcdir] 2424 c = ShellCommand(self.builder, command, tmpdir, 2425 sendRC=False, timeout=self.timeout, usePTY=False) 2426 self.command = c 2427 return c.start()

2428 d.addCallback(_export) 2429 return d 2430

2431 - def doForceSharedRepo(self):

2432 # Don't send stderr. When there is no shared repo, this might confuse 2433 # users, as they will see a bzr error message. But having no shared 2434 # repo is not an error, just an indication that we need to make one. 2435 c = ShellCommand(self.builder, [self.vcexe, 'info', '.'], 2436 self.builder.basedir, 2437 sendStderr=False, sendRC=False, usePTY=False) 2438 d = c.start() 2439 def afterCheckSharedRepo(res): 2440 if type(res) is int and res != 0: 2441 log.msg("No shared repo found, creating it") 2442 # bzr info fails, try to create shared repo. 2443 c = ShellCommand(self.builder, [self.vcexe, 'init-repo', '.'], 2444 self.builder.basedir, 2445 sendRC=False, usePTY=False) 2446 self.command = c 2447 return c.start() 2448 else: 2449 return defer.succeed(res)

2450 d.addCallback(afterCheckSharedRepo) 2451 return d 2452

2453 - def get_revision_number(self, out):

2454 # it feels like 'bzr revno' sometimes gives different results than 2455 # the 'revno:' line from 'bzr version-info', and the one from 2456 # version-info is more likely to be correct. 2457 for line in out.split("\n"): 2458 colon = line.find(":") 2459 if colon != -1: 2460 key, value = line[:colon], line[colon+2:] 2461 if key == "revno": 2462 return int(value) 2463 raise ValueError("unable to find revno: in bzr output: '%s'" % out)

2464

2465 - def parseGotRevision(self):

2466 command = [self.vcexe, "version-info"] 2467 c = ShellCommand(self.builder, command, 2468 os.path.join(self.builder.basedir, self.srcdir), 2469 environ=self.env, 2470 sendStdout=False, sendStderr=False, sendRC=False, 2471 keepStdout=True, usePTY=False) 2472 d = c.start() 2473 def _parse(res): 2474 try: 2475 return self.get_revision_number(c.stdout) 2476 except ValueError: 2477 msg =("Bzr.parseGotRevision unable to parse output " 2478 "of bzr version-info: '%s'" % c.stdout.strip()) 2479 log.msg(msg) 2480 self.sendStatus({'header': msg + "\n"}) 2481 return None

2482 d.addCallback(_parse) 2483 return d 2484 2485 registerSlaveCommand("bzr", Bzr, command_version) 2486

2487 -class Mercurial(SourceBase):

2488 """Mercurial specific VC operation. In addition to the arguments 2489 handled by SourceBase, this command reads the following keys: 2490 2491 ['repourl'] (required): the Mercurial repository string 2492 """ 2493 2494 header = "mercurial operation" 2495

2496 - def setup(self, args):

2497 SourceBase.setup(self, args) 2498 self.vcexe = getCommand("hg") 2499 self.repourl = args['repourl'] 2500 self.clobberOnBranchChange = args.get('clobberOnBranchChange', True) 2501 self.sourcedata = "%s\n" % self.repourl 2502 self.branchType = args.get('branchType', 'dirname') 2503 self.stdout = "" 2504 self.stderr = ""

2505

2506 - def sourcedirIsUpdateable(self):

2507 return os.path.isdir(os.path.join(self.builder.basedir, 2508 self.srcdir, ".hg"))

2509

2510 - def doVCUpdate(self):

2511 d = os.path.join(self.builder.basedir, self.srcdir) 2512 command = [self.vcexe, 'pull', '--verbose', self.repourl] 2513 c = ShellCommand(self.builder, command, d, 2514 sendRC=False, timeout=self.timeout, 2515 keepStdout=True, usePTY=False) 2516 self.command = c 2517 d = c.start() 2518 d.addCallback(self._handleEmptyUpdate) 2519 d.addCallback(self._update) 2520 return d

2521

2522 - def _handleEmptyUpdate(self, res):

2523 if type(res) is int and res == 1: 2524 if self.command.stdout.find("no changes found") != -1: 2525 # 'hg pull', when it doesn't have anything to do, exits with 2526 # rc=1, and there appears to be no way to shut this off. It 2527 # emits a distinctive message to stdout, though. So catch 2528 # this and pretend that it completed successfully. 2529 return 0 2530 return res

2531

2532 - def doVCFull(self):

2533 d = os.path.join(self.builder.basedir, self.srcdir) 2534 command = [self.vcexe, 'clone', '--verbose', '--noupdate'] 2535 2536 # if got revision, clobbering and in dirname, only clone to specific revision 2537 # (otherwise, do full clone to re-use .hg dir for subsequent byuilds) 2538 if self.args.get('revision') and self.mode == 'clobber' and self.branchType == 'dirname': 2539 command.extend(['--rev', self.args.get('revision')]) 2540 command.extend([self.repourl, d]) 2541 2542 c = ShellCommand(self.builder, command, self.builder.basedir, 2543 sendRC=False, timeout=self.timeout, usePTY=False) 2544 self.command = c 2545 cmd1 = c.start() 2546 cmd1.addCallback(self._update) 2547 return cmd1

2548

2549 - def _clobber(self, dummy, dirname):

2550 def _vcfull(res): 2551 return self.doVCFull()

2552 2553 c = self.doClobber(dummy, dirname) 2554 c.addCallback(_vcfull) 2555 2556 return c

2557

2558 - def _purge(self, dummy, dirname):

2559 d = os.path.join(self.builder.basedir, self.srcdir) 2560 purge = [self.vcexe, 'purge', '--all'] 2561 purgeCmd = ShellCommand(self.builder, purge, d, 2562 sendStdout=False, sendStderr=False, 2563 keepStdout=True, keepStderr=True, usePTY=False) 2564 2565 def _clobber(res): 2566 if res != 0: 2567 # purge failed, we need to switch to a classic clobber 2568 msg = "'hg purge' failed: %s\n%s. Clobbering." % (purgeCmd.stdout, purgeCmd.stderr) 2569 self.sendStatus({'header': msg + "\n"}) 2570 log.msg(msg) 2571 2572 return self._clobber(dummy, dirname) 2573 2574 # Purge was a success, then we need to update 2575 return self._update2(res)

2576 2577 p = purgeCmd.start() 2578 p.addCallback(_clobber) 2579 return p 2580

2581 - def _update(self, res):

2582 if res != 0: 2583 return res 2584 2585 # compare current branch to update 2586 self.update_branch = self.args.get('branch', 'default') 2587 2588 d = os.path.join(self.builder.basedir, self.srcdir) 2589 parentscmd = [self.vcexe, 'identify', '--num', '--branch'] 2590 cmd = ShellCommand(self.builder, parentscmd, d, 2591 sendStdout=False, sendStderr=False, 2592 keepStdout=True, keepStderr=True, usePTY=False) 2593 2594 self.clobber = None 2595 2596 def _parseIdentify(res): 2597 if res != 0: 2598 msg = "'hg identify' failed: %s\n%s" % (cmd.stdout, cmd.stderr) 2599 self.sendStatus({'header': msg + "\n"}) 2600 log.msg(msg) 2601 return res 2602 2603 log.msg('Output: %s' % cmd.stdout) 2604 2605 match = re.search(r'^(.+) (.+)$', cmd.stdout) 2606 assert match 2607 2608 rev = match.group(1) 2609 current_branch = match.group(2) 2610 2611 if rev == '-1': 2612 msg = "Fresh hg repo, don't worry about in-repo branch name" 2613 log.msg(msg) 2614 2615 elif os.path.exists(os.path.join(self.builder.basedir, 2616 self.srcdir, ".buildbot-patched")): 2617 self.clobber = self._purge 2618 2619 elif self.update_branch != current_branch: 2620 msg = "Working dir is on in-repo branch '%s' and build needs '%s'." % (current_branch, self.update_branch) 2621 if self.clobberOnBranchChange: 2622 msg += ' Cloberring.' 2623 else: 2624 msg += ' Updating.' 2625 2626 self.sendStatus({'header': msg + "\n"}) 2627 log.msg(msg) 2628 2629 # Clobbers only if clobberOnBranchChange is set 2630 if self.clobberOnBranchChange: 2631 self.clobber = self._purge 2632 2633 else: 2634 msg = "Working dir on same in-repo branch as build (%s)." % (current_branch) 2635 log.msg(msg) 2636 2637 return 0

2638 2639 def _checkRepoURL(res): 2640 parentscmd = [self.vcexe, 'paths', 'default'] 2641 cmd2 = ShellCommand(self.builder, parentscmd, d, 2642 sendStdout=False, sendStderr=False, 2643 keepStdout=True, keepStderr=True, usePTY=False) 2644 2645 def _parseRepoURL(res): 2646 if res == 1: 2647 if "not found!" == cmd2.stderr.strip(): 2648 msg = "hg default path not set. Not checking repo url for clobber test" 2649 log.msg(msg) 2650 return 0 2651 else: 2652 msg = "'hg paths default' failed: %s\n%s" % (cmd2.stdout, cmd2.stderr) 2653 log.msg(msg) 2654 return 1 2655 2656 oldurl = cmd2.stdout.strip() 2657 2658 log.msg("Repo cloned from: '%s'" % oldurl) 2659 2660 if sys.platform == "win32": 2661 oldurl = oldurl.lower().replace('\\', '/') 2662 repourl = self.repourl.lower().replace('\\', '/') 2663 if repourl.startswith('file://'): 2664 repourl = repourl.split('file://')[1] 2665 else: 2666 repourl = self.repourl 2667 2668 if oldurl != repourl: 2669 self.clobber = self._clobber 2670 msg = "RepoURL changed from '%s' in wc to '%s' in update. Clobbering" % (oldurl, repourl) 2671 log.msg(msg) 2672 2673 return 0 2674 2675 c = cmd2.start() 2676 c.addCallback(_parseRepoURL) 2677 return c 2678 2679 def _maybeClobber(res): 2680 if self.clobber: 2681 msg = "Clobber flag set. Doing clobbering" 2682 log.msg(msg) 2683 2684 def _vcfull(res): 2685 return self.doVCFull() 2686 2687 return self.clobber(None, self.srcdir) 2688 2689 return 0 2690 2691 c = cmd.start() 2692 c.addCallback(_parseIdentify) 2693 c.addCallback(_checkRepoURL) 2694 c.addCallback(_maybeClobber) 2695 c.addCallback(self._update2) 2696 return c 2697

2698 - def _update2(self, res):

2699 d = os.path.join(self.builder.basedir, self.srcdir) 2700 2701 updatecmd=[self.vcexe, 'update', '--clean', '--repository', d] 2702 if self.args.get('revision'): 2703 updatecmd.extend(['--rev', self.args['revision']]) 2704 else: 2705 updatecmd.extend(['--rev', self.args.get('branch', 'default')]) 2706 self.command = ShellCommand(self.builder, updatecmd, 2707 self.builder.basedir, sendRC=False, 2708 timeout=self.timeout, usePTY=False) 2709 return self.command.start()

2710

2711 - def parseGotRevision(self):

2712 # we use 'hg identify' to find out what we wound up with 2713 command = [self.vcexe, "identify"] 2714 c = ShellCommand(self.builder, command, 2715 os.path.join(self.builder.basedir, self.srcdir), 2716 environ=self.env, 2717 sendStdout=False, sendStderr=False, sendRC=False, 2718 keepStdout=True, usePTY=False) 2719 d = c.start() 2720 def _parse(res): 2721 m = re.search(r'^(\w+)', c.stdout) 2722 return m.group(1)

2723 d.addCallback(_parse) 2724 return d 2725 2726 registerSlaveCommand("hg", Mercurial, command_version) 2727 2728

2729 -class P4Base(SourceBase):

2730 """Base class for P4 source-updaters 2731 2732 ['p4port'] (required): host:port for server to access 2733 ['p4user'] (optional): user to use for access 2734 ['p4passwd'] (optional): passwd to try for the user 2735 ['p4client'] (optional): client spec to use 2736 """

2737 - def setup(self, args):

2738 SourceBase.setup(self, args) 2739 self.p4port = args['p4port'] 2740 self.p4client = args['p4client'] 2741 self.p4user = args['p4user'] 2742 self.p4passwd = args['p4passwd']

2743

2744 - def parseGotRevision(self):

2745 # Executes a p4 command that will give us the latest changelist number 2746 # of any file under the current (or default) client: 2747 command = ['p4'] 2748 if self.p4port: 2749 command.extend(['-p', self.p4port]) 2750 if self.p4user: 2751 command.extend(['-u', self.p4user]) 2752 if self.p4passwd: 2753 command.extend(['-P', self.p4passwd]) 2754 if self.p4client: 2755 command.extend(['-c', self.p4client]) 2756 command.extend(['changes', '-m', '1', '#have']) 2757 c = ShellCommand(self.builder, command, self.builder.basedir, 2758 environ=self.env, timeout=self.timeout, 2759 sendStdout=True, sendStderr=False, sendRC=False, 2760 keepStdout=True, usePTY=False) 2761 self.command = c 2762 d = c.start() 2763 2764 def _parse(res): 2765 # 'p4 -c clien-name change -m 1 "#have"' will produce an output like: 2766 # "Change 28147 on 2008/04/07 by p4user@hostname..." 2767 # The number after "Change" is the one we want. 2768 m = re.match('Change\s+(\d+)\s+', c.stdout) 2769 if m: 2770 return m.group(1) 2771 return None

2772 d.addCallback(_parse) 2773 return d

2774 2775

2776 -class P4(P4Base):

2777 """A P4 source-updater. 2778 2779 ['p4port'] (required): host:port for server to access 2780 ['p4user'] (optional): user to use for access 2781 ['p4passwd'] (optional): passwd to try for the user 2782 ['p4client'] (optional): client spec to use 2783 ['p4extra_views'] (optional): additional client views to use 2784 """ 2785 2786 header = "p4" 2787

2788 - def setup(self, args):

2789 P4Base.setup(self, args) 2790 self.p4base = args['p4base'] 2791 self.p4extra_views = args['p4extra_views'] 2792 self.p4mode = args['mode'] 2793 self.p4branch = args['branch'] 2794 2795 self.sourcedata = str([ 2796 # Perforce server. 2797 self.p4port, 2798 2799 # Client spec. 2800 self.p4client, 2801 2802 # Depot side of view spec. 2803 self.p4base, 2804 self.p4branch, 2805 self.p4extra_views, 2806 2807 # Local side of view spec (srcdir is made from these). 2808 self.builder.basedir, 2809 self.mode, 2810 self.workdir 2811 ])

2812 2813

2814 - def sourcedirIsUpdateable(self):

2815 if os.path.exists(os.path.join(self.builder.basedir, 2816 self.srcdir, ".buildbot-patched")): 2817 return False 2818 # We assume our client spec is still around. 2819 # We just say we aren't updateable if the dir doesn't exist so we 2820 # don't get ENOENT checking the sourcedata. 2821 return os.path.isdir(os.path.join(self.builder.basedir, 2822 self.srcdir))

2823

2824 - def doVCUpdate(self):

2825 return self._doP4Sync(force=False)

2826

2827 - def _doP4Sync(self, force):

2828 command = ['p4'] 2829 2830 if self.p4port: 2831 command.extend(['-p', self.p4port]) 2832 if self.p4user: 2833 command.extend(['-u', self.p4user]) 2834 if self.p4passwd: 2835 command.extend(['-P', self.p4passwd]) 2836 if self.p4client: 2837 command.extend(['-c', self.p4client]) 2838 command.extend(['sync']) 2839 if force: 2840 command.extend(['-f']) 2841 if self.revision: 2842 command.extend(['@' + str(self.revision)]) 2843 env = {} 2844 c = ShellCommand(self.builder, command, self.builder.basedir, 2845 environ=env, sendRC=False, timeout=self.timeout, 2846 keepStdout=True, usePTY=False) 2847 self.command = c 2848 d = c.start() 2849 d.addCallback(self._abandonOnFailure) 2850 return d

2851 2852

2853 - def doVCFull(self):

2854 env = {} 2855 command = ['p4'] 2856 client_spec = '' 2857 client_spec += "Client: %s\n\n" % self.p4client 2858 client_spec += "Owner: %s\n\n" % self.p4user 2859 client_spec += "Description:\n\tCreated by %s\n\n" % self.p4user 2860 client_spec += "Root:\t%s\n\n" % self.builder.basedir 2861 client_spec += "Options:\tallwrite rmdir\n\n" 2862 client_spec += "LineEnd:\tlocal\n\n" 2863 2864 # Setup a view 2865 client_spec += "View:\n\t%s" % (self.p4base) 2866 if self.p4branch: 2867 client_spec += "%s/" % (self.p4branch) 2868 client_spec += "... //%s/%s/...\n" % (self.p4client, self.srcdir) 2869 if self.p4extra_views: 2870 for k, v in self.p4extra_views: 2871 client_spec += "\t%s/... //%s/%s%s/...\n" % (k, self.p4client, 2872 self.srcdir, v) 2873 if self.p4port: 2874 command.extend(['-p', self.p4port]) 2875 if self.p4user: 2876 command.extend(['-u', self.p4user]) 2877 if self.p4passwd: 2878 command.extend(['-P', self.p4passwd]) 2879 command.extend(['client', '-i']) 2880 log.msg(client_spec) 2881 c = ShellCommand(self.builder, command, self.builder.basedir, 2882 environ=env, sendRC=False, timeout=self.timeout, 2883 initialStdin=client_spec, usePTY=False) 2884 self.command = c 2885 d = c.start() 2886 d.addCallback(self._abandonOnFailure) 2887 d.addCallback(lambda _: self._doP4Sync(force=True)) 2888 return d

2889

2890 - def parseGotRevision(self):

2891 rv = None 2892 if self.revision: 2893 rv = str(self.revision) 2894 return rv

2895 2896 registerSlaveCommand("p4", P4, command_version) 2897 2898

2899 -class P4Sync(P4Base):

2900 """A partial P4 source-updater. Requires manual setup of a per-slave P4 2901 environment. The only thing which comes from the master is P4PORT. 2902 'mode' is required to be 'copy'. 2903 2904 ['p4port'] (required): host:port for server to access 2905 ['p4user'] (optional): user to use for access 2906 ['p4passwd'] (optional): passwd to try for the user 2907 ['p4client'] (optional): client spec to use 2908 """ 2909 2910 header = "p4 sync" 2911

2912 - def setup(self, args):

2913 P4Base.setup(self, args) 2914 self.vcexe = getCommand("p4")

2915

2916 - def sourcedirIsUpdateable(self):

2917 return True

2918

2919 - def _doVC(self, force):

2920 d = os.path.join(self.builder.basedir, self.srcdir) 2921 command = [self.vcexe] 2922 if self.p4port: 2923 command.extend(['-p', self.p4port]) 2924 if self.p4user: 2925 command.extend(['-u', self.p4user]) 2926 if self.p4passwd: 2927 command.extend(['-P', self.p4passwd]) 2928 if self.p4client: 2929 command.extend(['-c', self.p4client]) 2930 command.extend(['sync']) 2931 if force: 2932 command.extend(['-f']) 2933 if self.revision: 2934 command.extend(['@' + self.revision]) 2935 env = {} 2936 c = ShellCommand(self.builder, command, d, environ=env, 2937 sendRC=False, timeout=self.timeout, usePTY=False) 2938 self.command = c 2939 return c.start()

2940

2941 - def doVCUpdate(self):

2942 return self._doVC(force=False)

2943

2944 - def doVCFull(self):

2945 return self._doVC(force=True)

2946

2947 - def parseGotRevision(self):

2948 rv = None 2949 if self.revision: 2950 rv = str(self.revision) 2951 return rv

2952 2953 registerSlaveCommand("p4sync", P4Sync, command_version) 2954

Source Code for Module buildbot.slave.commands