buildbot.steps.source

1 # -*- test-case-name: buildbot.test.test_vc -*- 2 3 from warnings import warn 4 from email.Utils import formatdate 5 from twisted.python import log 6 from buildbot.process.buildstep import LoggingBuildStep, LoggedRemoteCommand 7 from buildbot.interfaces import BuildSlaveTooOldError 8 from buildbot.status.builder import SKIPPED 9 10

11 -class Source(LoggingBuildStep):

12 """This is a base class to generate a source tree in the buildslave. 13 Each version control system has a specialized subclass, and is expected 14 to override __init__ and implement computeSourceRevision() and 15 startVC(). The class as a whole builds up the self.args dictionary, then 16 starts a LoggedRemoteCommand with those arguments. 17 """ 18 19 # if the checkout fails, there's no point in doing anything else 20 haltOnFailure = True 21 flunkOnFailure = True 22 notReally = False 23 24 branch = None # the default branch, should be set in __init__ 25

26 - def __init__(self, workdir=None, mode='update', alwaysUseLatest=False, 27 timeout=20*60, retry=None, **kwargs):

28 """ 29 @type workdir: string 30 @param workdir: local directory (relative to the Builder's root) 31 where the tree should be placed 32 33 @type mode: string 34 @param mode: the kind of VC operation that is desired: 35 - 'update': specifies that the checkout/update should be 36 performed directly into the workdir. Each build is performed 37 in the same directory, allowing for incremental builds. This 38 minimizes disk space, bandwidth, and CPU time. However, it 39 may encounter problems if the build process does not handle 40 dependencies properly (if you must sometimes do a 'clean 41 build' to make sure everything gets compiled), or if source 42 files are deleted but generated files can influence test 43 behavior (e.g. python's .pyc files), or when source 44 directories are deleted but generated files prevent CVS from 45 removing them. 46 47 - 'copy': specifies that the source-controlled workspace 48 should be maintained in a separate directory (called the 49 'copydir'), using checkout or update as necessary. For each 50 build, a new workdir is created with a copy of the source 51 tree (rm -rf workdir; cp -R -P -p copydir workdir). This 52 doubles the disk space required, but keeps the bandwidth low 53 (update instead of a full checkout). A full 'clean' build 54 is performed each time. This avoids any generated-file 55 build problems, but is still occasionally vulnerable to 56 problems such as a CVS repository being manually rearranged 57 (causing CVS errors on update) which are not an issue with 58 a full checkout. 59 60 - 'clobber': specifies that the working directory should be 61 deleted each time, necessitating a full checkout for each 62 build. This insures a clean build off a complete checkout, 63 avoiding any of the problems described above, but is 64 bandwidth intensive, as the whole source tree must be 65 pulled down for each build. 66 67 - 'export': is like 'clobber', except that e.g. the 'cvs 68 export' command is used to create the working directory. 69 This command removes all VC metadata files (the 70 CVS/.svn/{arch} directories) from the tree, which is 71 sometimes useful for creating source tarballs (to avoid 72 including the metadata in the tar file). Not all VC systems 73 support export. 74 75 @type alwaysUseLatest: boolean 76 @param alwaysUseLatest: whether to always update to the most 77 recent available sources for this build. 78 79 Normally the Source step asks its Build for a list of all 80 Changes that are supposed to go into the build, then computes a 81 'source stamp' (revision number or timestamp) that will cause 82 exactly that set of changes to be present in the checked out 83 tree. This is turned into, e.g., 'cvs update -D timestamp', or 84 'svn update -r revnum'. If alwaysUseLatest=True, bypass this 85 computation and always update to the latest available sources 86 for each build. 87 88 The source stamp helps avoid a race condition in which someone 89 commits a change after the master has decided to start a build 90 but before the slave finishes checking out the sources. At best 91 this results in a build which contains more changes than the 92 buildmaster thinks it has (possibly resulting in the wrong 93 person taking the blame for any problems that result), at worst 94 is can result in an incoherent set of sources (splitting a 95 non-atomic commit) which may not build at all. 96 97 @type retry: tuple of ints (delay, repeats) (or None) 98 @param retry: if provided, VC update failures are re-attempted up 99 to REPEATS times, with DELAY seconds between each 100 attempt. Some users have slaves with poor connectivity 101 to their VC repository, and they say that up to 80% of 102 their build failures are due to transient network 103 failures that could be handled by simply retrying a 104 couple times. 105 106 """ 107 108 LoggingBuildStep.__init__(self, **kwargs) 109 self.addFactoryArguments(workdir=workdir, 110 mode=mode, 111 alwaysUseLatest=alwaysUseLatest, 112 timeout=timeout, 113 retry=retry, 114 ) 115 116 assert mode in ("update", "copy", "clobber", "export") 117 if retry: 118 delay, repeats = retry 119 assert isinstance(repeats, int) 120 assert repeats > 0 121 self.args = {'mode': mode, 122 'workdir': workdir, 123 'timeout': timeout, 124 'retry': retry, 125 'patch': None, # set during .start 126 } 127 self.alwaysUseLatest = alwaysUseLatest 128 129 # Compute defaults for descriptions: 130 description = ["updating"] 131 descriptionDone = ["update"] 132 if mode == "clobber": 133 description = ["checkout"] 134 # because checkingouting takes too much space 135 descriptionDone = ["checkout"] 136 elif mode == "export": 137 description = ["exporting"] 138 descriptionDone = ["export"] 139 self.description = description 140 self.descriptionDone = descriptionDone

141

142 - def setDefaultWorkdir(self, workdir):

143 self.args['workdir'] = self.args['workdir'] or workdir

144

145 - def describe(self, done=False):

146 if done: 147 return self.descriptionDone 148 return self.description

149

150 - def computeSourceRevision(self, changes):

151 """Each subclass must implement this method to do something more 152 precise than -rHEAD every time. For version control systems that use 153 repository-wide change numbers (SVN, P4), this can simply take the 154 maximum such number from all the changes involved in this build. For 155 systems that do not (CVS), it needs to create a timestamp based upon 156 the latest Change, the Build's treeStableTimer, and an optional 157 self.checkoutDelay value.""" 158 return None

159

160 - def start(self):

161 if self.notReally: 162 log.msg("faking %s checkout/update" % self.name) 163 self.step_status.setText(["fake", self.name, "successful"]) 164 self.addCompleteLog("log", 165 "Faked %s checkout/update 'successful'\n" \ 166 % self.name) 167 return SKIPPED 168 169 # what source stamp would this build like to use? 170 s = self.build.getSourceStamp() 171 # if branch is None, then use the Step's "default" branch 172 branch = s.branch or self.branch 173 # if revision is None, use the latest sources (-rHEAD) 174 revision = s.revision 175 if not revision and not self.alwaysUseLatest: 176 revision = self.computeSourceRevision(s.changes) 177 # if patch is None, then do not patch the tree after checkout 178 179 # 'patch' is None or a tuple of (patchlevel, diff) 180 patch = s.patch 181 if patch: 182 self.addCompleteLog("patch", patch[1]) 183 184 self.startVC(branch, revision, patch)

185

186 - def commandComplete(self, cmd):

187 if cmd.updates.has_key("got_revision"): 188 got_revision = cmd.updates["got_revision"][-1] 189 if got_revision is not None: 190 self.setProperty("got_revision", str(got_revision), "Source")

191 192 193

194 -class CVS(Source):

195 """I do CVS checkout/update operations. 196 197 Note: if you are doing anonymous/pserver CVS operations, you will need 198 to manually do a 'cvs login' on each buildslave before the slave has any 199 hope of success. XXX: fix then, take a cvs password as an argument and 200 figure out how to do a 'cvs login' on each build 201 """ 202 203 name = "cvs" 204 205 #progressMetrics = ('output',) 206 # 207 # additional things to track: update gives one stderr line per directory 208 # (starting with 'cvs server: Updating ') (and is fairly stable if files 209 # is empty), export gives one line per directory (starting with 'cvs 210 # export: Updating ') and another line per file (starting with U). Would 211 # be nice to track these, requires grepping LogFile data for lines, 212 # parsing each line. Might be handy to have a hook in LogFile that gets 213 # called with each complete line. 214

215 - def __init__(self, cvsroot, cvsmodule, 216 global_options=[], branch=None, checkoutDelay=None, 217 checkout_options=[], 218 login=None, 219 **kwargs):

220 221 """ 222 @type cvsroot: string 223 @param cvsroot: CVS Repository from which the source tree should 224 be obtained. '/home/warner/Repository' for local 225 or NFS-reachable repositories, 226 ':pserver:anon@foo.com:/cvs' for anonymous CVS, 227 'user@host.com:/cvs' for non-anonymous CVS or 228 CVS over ssh. Lots of possibilities, check the 229 CVS documentation for more. 230 231 @type cvsmodule: string 232 @param cvsmodule: subdirectory of CVS repository that should be 233 retrieved 234 235 @type login: string or None 236 @param login: if not None, a string which will be provided as a 237 password to the 'cvs login' command, used when a 238 :pserver: method is used to access the repository. 239 This login is only needed once, but must be run 240 each time (just before the CVS operation) because 241 there is no way for the buildslave to tell whether 242 it was previously performed or not. 243 244 @type branch: string 245 @param branch: the default branch name, will be used in a '-r' 246 argument to specify which branch of the source tree 247 should be used for this checkout. Defaults to None, 248 which means to use 'HEAD'. 249 250 @type checkoutDelay: int or None 251 @param checkoutDelay: if not None, the number of seconds to put 252 between the last known Change and the 253 timestamp given to the -D argument. This 254 defaults to exactly half of the parent 255 Build's .treeStableTimer, but it could be 256 set to something else if your CVS change 257 notification has particularly weird 258 latency characteristics. 259 260 @type global_options: list of strings 261 @param global_options: these arguments are inserted in the cvs 262 command line, before the 263 'checkout'/'update' command word. See 264 'cvs --help-options' for a list of what 265 may be accepted here. ['-r'] will make 266 the checked out files read only. ['-r', 267 '-R'] will also assume the repository is 268 read-only (I assume this means it won't 269 use locks to insure atomic access to the 270 ,v files). 271 272 @type checkout_options: list of strings 273 @param checkout_options: these arguments are inserted in the cvs 274 command line, after 'checkout' but before 275 branch or revision specifiers. 276 """ 277 278 self.checkoutDelay = checkoutDelay 279 self.branch = branch 280 281 Source.__init__(self, **kwargs) 282 self.addFactoryArguments(cvsroot=cvsroot, 283 cvsmodule=cvsmodule, 284 global_options=global_options, 285 checkout_options=checkout_options, 286 branch=branch, 287 checkoutDelay=checkoutDelay, 288 login=login, 289 ) 290 291 self.args.update({'cvsroot': cvsroot, 292 'cvsmodule': cvsmodule, 293 'global_options': global_options, 294 'checkout_options':checkout_options, 295 'login': login, 296 })

297

298 - def computeSourceRevision(self, changes):

299 if not changes: 300 return None 301 lastChange = max([c.when for c in changes]) 302 if self.checkoutDelay is not None: 303 when = lastChange + self.checkoutDelay 304 else: 305 lastSubmit = max([r.submittedAt for r in self.build.requests]) 306 when = (lastChange + lastSubmit) / 2 307 return formatdate(when)

308

309 - def startVC(self, branch, revision, patch):

310 if self.slaveVersionIsOlderThan("cvs", "1.39"): 311 # the slave doesn't know to avoid re-using the same sourcedir 312 # when the branch changes. We have no way of knowing which branch 313 # the last build used, so if we're using a non-default branch and 314 # either 'update' or 'copy' modes, it is safer to refuse to 315 # build, and tell the user they need to upgrade the buildslave. 316 if (branch != self.branch 317 and self.args['mode'] in ("update", "copy")): 318 m = ("This buildslave (%s) does not know about multiple " 319 "branches, and using mode=%s would probably build the " 320 "wrong tree. " 321 "Refusing to build. Please upgrade the buildslave to " 322 "buildbot-0.7.0 or newer." % (self.build.slavename, 323 self.args['mode'])) 324 log.msg(m) 325 raise BuildSlaveTooOldError(m) 326 327 if branch is None: 328 branch = "HEAD" 329 self.args['branch'] = branch 330 self.args['revision'] = revision 331 self.args['patch'] = patch 332 333 if self.args['branch'] == "HEAD" and self.args['revision']: 334 # special case. 'cvs update -r HEAD -D today' gives no files 335 # TODO: figure out why, see if it applies to -r BRANCH 336 self.args['branch'] = None 337 338 # deal with old slaves 339 warnings = [] 340 slavever = self.slaveVersion("cvs", "old") 341 342 if slavever == "old": 343 # 0.5.0 344 if self.args['mode'] == "export": 345 self.args['export'] = 1 346 elif self.args['mode'] == "clobber": 347 self.args['clobber'] = 1 348 elif self.args['mode'] == "copy": 349 self.args['copydir'] = "source" 350 self.args['tag'] = self.args['branch'] 351 assert not self.args['patch'] # 0.5.0 slave can't do patch 352 353 cmd = LoggedRemoteCommand("cvs", self.args) 354 self.startCommand(cmd, warnings)

355 356

357 -class SVN(Source):

358 """I perform Subversion checkout/update operations.""" 359 360 name = 'svn' 361

362 - def __init__(self, svnurl=None, baseURL=None, defaultBranch=None, 363 directory=None, username=None, password=None, 364 extra_args=None, **kwargs):

365 """ 366 @type svnurl: string 367 @param svnurl: the URL which points to the Subversion server, 368 combining the access method (HTTP, ssh, local file), 369 the repository host/port, the repository path, the 370 sub-tree within the repository, and the branch to 371 check out. Using C{svnurl} does not enable builds of 372 alternate branches: use C{baseURL} to enable this. 373 Use exactly one of C{svnurl} and C{baseURL}. 374 375 @param baseURL: if branches are enabled, this is the base URL to 376 which a branch name will be appended. It should 377 probably end in a slash. Use exactly one of 378 C{svnurl} and C{baseURL}. 379 380 @param defaultBranch: if branches are enabled, this is the branch 381 to use if the Build does not specify one 382 explicitly. It will simply be appended 383 to C{baseURL} and the result handed to 384 the SVN command. 385 386 @param username: username to pass to svn's --username 387 @param password: username to pass to svn's --password 388 """ 389 390 if not kwargs.has_key('workdir') and directory is not None: 391 # deal with old configs 392 warn("Please use workdir=, not directory=", DeprecationWarning) 393 kwargs['workdir'] = directory 394 395 self.svnurl = svnurl 396 self.baseURL = baseURL 397 self.branch = defaultBranch 398 self.username = username 399 self.password = password 400 self.extra_args = extra_args 401 402 Source.__init__(self, **kwargs) 403 self.addFactoryArguments(svnurl=svnurl, 404 baseURL=baseURL, 405 defaultBranch=defaultBranch, 406 directory=directory, 407 username=username, 408 password=password, 409 extra_args=extra_args, 410 ) 411 412 if not svnurl and not baseURL: 413 raise ValueError("you must use exactly one of svnurl and baseURL")

414 415

416 - def computeSourceRevision(self, changes):

417 if not changes or None in [c.revision for c in changes]: 418 return None 419 lastChange = max([int(c.revision) for c in changes]) 420 return lastChange

421

422 - def startVC(self, branch, revision, patch):

423 424 # handle old slaves 425 warnings = [] 426 slavever = self.slaveVersion("svn", "old") 427 if not slavever: 428 m = "slave does not have the 'svn' command" 429 raise BuildSlaveTooOldError(m) 430 431 if self.slaveVersionIsOlderThan("svn", "1.39"): 432 # the slave doesn't know to avoid re-using the same sourcedir 433 # when the branch changes. We have no way of knowing which branch 434 # the last build used, so if we're using a non-default branch and 435 # either 'update' or 'copy' modes, it is safer to refuse to 436 # build, and tell the user they need to upgrade the buildslave. 437 if (branch != self.branch 438 and self.args['mode'] in ("update", "copy")): 439 m = ("This buildslave (%s) does not know about multiple " 440 "branches, and using mode=%s would probably build the " 441 "wrong tree. " 442 "Refusing to build. Please upgrade the buildslave to " 443 "buildbot-0.7.0 or newer." % (self.build.slavename, 444 self.args['mode'])) 445 raise BuildSlaveTooOldError(m) 446 447 if slavever == "old": 448 # 0.5.0 compatibility 449 if self.args['mode'] in ("clobber", "copy"): 450 # TODO: use some shell commands to make up for the 451 # deficiency, by blowing away the old directory first (thus 452 # forcing a full checkout) 453 warnings.append("WARNING: this slave can only do SVN updates" 454 ", not mode=%s\n" % self.args['mode']) 455 log.msg("WARNING: this slave only does mode=update") 456 if self.args['mode'] == "export": 457 raise BuildSlaveTooOldError("old slave does not have " 458 "mode=export") 459 self.args['directory'] = self.args['workdir'] 460 if revision is not None: 461 # 0.5.0 can only do HEAD. We have no way of knowing whether 462 # the requested revision is HEAD or not, and for 463 # slowly-changing trees this will probably do the right 464 # thing, so let it pass with a warning 465 m = ("WARNING: old slave can only update to HEAD, not " 466 "revision=%s" % revision) 467 log.msg(m) 468 warnings.append(m + "\n") 469 revision = "HEAD" # interprets this key differently 470 if patch: 471 raise BuildSlaveTooOldError("old slave can't do patch") 472 473 if self.svnurl: 474 assert not branch # we need baseURL= to use branches 475 self.args['svnurl'] = self.svnurl 476 else: 477 self.args['svnurl'] = self.baseURL + branch 478 self.args['revision'] = revision 479 self.args['patch'] = patch 480 481 if self.username is not None or self.password is not None: 482 if self.slaveVersionIsOlderThan("svn", "2.8"): 483 m = ("This buildslave (%s) does not support svn usernames " 484 "and passwords. " 485 "Refusing to build. Please upgrade the buildslave to " 486 "buildbot-0.7.10 or newer." % (self.build.slavename,)) 487 raise BuildSlaveTooOldError(m) 488 if self.username is not None: self.args['username'] = self.username 489 if self.password is not None: self.args['password'] = self.password 490 491 if self.extra_args is not None: 492 self.args['extra_args'] = self.extra_args 493 494 revstuff = [] 495 if branch is not None and branch != self.branch: 496 revstuff.append("[branch]") 497 if revision is not None: 498 revstuff.append("r%s" % revision) 499 if patch is not None: 500 revstuff.append("[patch]") 501 self.description.extend(revstuff) 502 self.descriptionDone.extend(revstuff) 503 504 cmd = LoggedRemoteCommand("svn", self.args) 505 self.startCommand(cmd, warnings)

506 507

508 -class Darcs(Source):

509 """Check out a source tree from a Darcs repository at 'repourl'. 510 511 Darcs has no concept of file modes. This means the eXecute-bit will be 512 cleared on all source files. As a result, you may need to invoke 513 configuration scripts with something like: 514 515 C{s(step.Configure, command=['/bin/sh', './configure'])} 516 """ 517 518 name = "darcs" 519

520 - def __init__(self, repourl=None, baseURL=None, defaultBranch=None, 521 **kwargs):

522 """ 523 @type repourl: string 524 @param repourl: the URL which points at the Darcs repository. This 525 is used as the default branch. Using C{repourl} does 526 not enable builds of alternate branches: use 527 C{baseURL} to enable this. Use either C{repourl} or 528 C{baseURL}, not both. 529 530 @param baseURL: if branches are enabled, this is the base URL to 531 which a branch name will be appended. It should 532 probably end in a slash. Use exactly one of 533 C{repourl} and C{baseURL}. 534 535 @param defaultBranch: if branches are enabled, this is the branch 536 to use if the Build does not specify one 537 explicitly. It will simply be appended to 538 C{baseURL} and the result handed to the 539 'darcs pull' command. 540 """ 541 self.repourl = repourl 542 self.baseURL = baseURL 543 self.branch = defaultBranch 544 Source.__init__(self, **kwargs) 545 self.addFactoryArguments(repourl=repourl, 546 baseURL=baseURL, 547 defaultBranch=defaultBranch, 548 ) 549 assert self.args['mode'] != "export", \ 550 "Darcs does not have an 'export' mode" 551 if (not repourl and not baseURL) or (repourl and baseURL): 552 raise ValueError("you must provide exactly one of repourl and" 553 " baseURL")

554

555 - def startVC(self, branch, revision, patch):

556 slavever = self.slaveVersion("darcs") 557 if not slavever: 558 m = "slave is too old, does not know about darcs" 559 raise BuildSlaveTooOldError(m) 560 561 if self.slaveVersionIsOlderThan("darcs", "1.39"): 562 if revision: 563 # TODO: revisit this once we implement computeSourceRevision 564 m = "0.6.6 slaves can't handle args['revision']" 565 raise BuildSlaveTooOldError(m) 566 567 # the slave doesn't know to avoid re-using the same sourcedir 568 # when the branch changes. We have no way of knowing which branch 569 # the last build used, so if we're using a non-default branch and 570 # either 'update' or 'copy' modes, it is safer to refuse to 571 # build, and tell the user they need to upgrade the buildslave. 572 if (branch != self.branch 573 and self.args['mode'] in ("update", "copy")): 574 m = ("This buildslave (%s) does not know about multiple " 575 "branches, and using mode=%s would probably build the " 576 "wrong tree. " 577 "Refusing to build. Please upgrade the buildslave to " 578 "buildbot-0.7.0 or newer." % (self.build.slavename, 579 self.args['mode'])) 580 raise BuildSlaveTooOldError(m) 581 582 if self.repourl: 583 assert not branch # we need baseURL= to use branches 584 self.args['repourl'] = self.repourl 585 else: 586 self.args['repourl'] = self.baseURL + branch 587 self.args['revision'] = revision 588 self.args['patch'] = patch 589 590 revstuff = [] 591 if branch is not None and branch != self.branch: 592 revstuff.append("[branch]") 593 self.description.extend(revstuff) 594 self.descriptionDone.extend(revstuff) 595 596 cmd = LoggedRemoteCommand("darcs", self.args) 597 self.startCommand(cmd)

598 599

600 -class Git(Source):

601 """Check out a source tree from a git repository 'repourl'.""" 602 603 name = "git" 604

605 - def __init__(self, repourl, 606 branch="master", 607 submodules=False, 608 **kwargs):

609 """ 610 @type repourl: string 611 @param repourl: the URL which points at the git repository 612 613 @type branch: string 614 @param branch: The branch or tag to check out by default. If 615 a build specifies a different branch, it will 616 be used instead of this. 617 618 @type submodules: boolean 619 @param submodules: Whether or not to update (and initialize) 620 git submodules. 621 622 """ 623 Source.__init__(self, **kwargs) 624 self.addFactoryArguments(repourl=repourl, 625 branch=branch, 626 submodules=submodules, 627 ) 628 self.args.update({'repourl': repourl, 629 'branch': branch, 630 'submodules' : submodules, 631 })

632

633 - def computeSourceRevision(self, changes):

634 if not changes: 635 return None 636 return changes[-1].revision

637

638 - def startVC(self, branch, revision, patch):

639 if branch is not None: 640 self.args['branch'] = branch 641 642 self.args['revision'] = revision 643 self.args['patch'] = patch 644 slavever = self.slaveVersion("git") 645 if not slavever: 646 raise BuildSlaveTooOldError("slave is too old, does not know " 647 "about git") 648 cmd = LoggedRemoteCommand("git", self.args) 649 self.startCommand(cmd)

650 651

652 -class Arch(Source):

653 """Check out a source tree from an Arch repository named 'archive' 654 available at 'url'. 'version' specifies which version number (development 655 line) will be used for the checkout: this is mostly equivalent to a 656 branch name. This version uses the 'tla' tool to do the checkout, to use 657 'baz' see L{Bazaar} instead. 658 """ 659 660 name = "arch" 661 # TODO: slaves >0.6.6 will accept args['build-config'], so use it 662

663 - def __init__(self, url, version, archive=None, **kwargs):

664 """ 665 @type url: string 666 @param url: the Arch coordinates of the repository. This is 667 typically an http:// URL, but could also be the absolute 668 pathname of a local directory instead. 669 670 @type version: string 671 @param version: the category--branch--version to check out. This is 672 the default branch. If a build specifies a different 673 branch, it will be used instead of this. 674 675 @type archive: string 676 @param archive: The archive name. If provided, it must match the one 677 that comes from the repository. If not, the 678 repository's default will be used. 679 """ 680 self.branch = version 681 Source.__init__(self, **kwargs) 682 self.addFactoryArguments(url=url, 683 version=version, 684 archive=archive, 685 ) 686 self.args.update({'url': url, 687 'archive': archive, 688 })

689

690 - def computeSourceRevision(self, changes):

691 # in Arch, fully-qualified revision numbers look like: 692 # arch@buildbot.sourceforge.net--2004/buildbot--dev--0--patch-104 693 # For any given builder, all of this is fixed except the patch-104. 694 # The Change might have any part of the fully-qualified string, so we 695 # just look for the last part. We return the "patch-NN" string. 696 if not changes: 697 return None 698 lastChange = None 699 for c in changes: 700 if not c.revision: 701 continue 702 if c.revision.endswith("--base-0"): 703 rev = 0 704 else: 705 i = c.revision.rindex("patch") 706 rev = int(c.revision[i+len("patch-"):]) 707 lastChange = max(lastChange, rev) 708 if lastChange is None: 709 return None 710 if lastChange == 0: 711 return "base-0" 712 return "patch-%d" % lastChange

713

714 - def checkSlaveVersion(self, cmd, branch):

715 warnings = [] 716 slavever = self.slaveVersion(cmd) 717 if not slavever: 718 m = "slave is too old, does not know about %s" % cmd 719 raise BuildSlaveTooOldError(m) 720 721 # slave 1.28 and later understand 'revision' 722 if self.slaveVersionIsOlderThan(cmd, "1.28"): 723 if not self.alwaysUseLatest: 724 # we don't know whether our requested revision is the latest 725 # or not. If the tree does not change very quickly, this will 726 # probably build the right thing, so emit a warning rather 727 # than refuse to build at all 728 m = "WARNING, buildslave is too old to use a revision" 729 log.msg(m) 730 warnings.append(m + "\n") 731 732 if self.slaveVersionIsOlderThan(cmd, "1.39"): 733 # the slave doesn't know to avoid re-using the same sourcedir 734 # when the branch changes. We have no way of knowing which branch 735 # the last build used, so if we're using a non-default branch and 736 # either 'update' or 'copy' modes, it is safer to refuse to 737 # build, and tell the user they need to upgrade the buildslave. 738 if (branch != self.branch 739 and self.args['mode'] in ("update", "copy")): 740 m = ("This buildslave (%s) does not know about multiple " 741 "branches, and using mode=%s would probably build the " 742 "wrong tree. " 743 "Refusing to build. Please upgrade the buildslave to " 744 "buildbot-0.7.0 or newer." % (self.build.slavename, 745 self.args['mode'])) 746 log.msg(m) 747 raise BuildSlaveTooOldError(m) 748 749 return warnings

750

751 - def startVC(self, branch, revision, patch):

752 self.args['version'] = branch 753 self.args['revision'] = revision 754 self.args['patch'] = patch 755 warnings = self.checkSlaveVersion("arch", branch) 756 757 revstuff = [] 758 if branch is not None and branch != self.branch: 759 revstuff.append("[branch]") 760 if revision is not None: 761 revstuff.append("patch%s" % revision) 762 self.description.extend(revstuff) 763 self.descriptionDone.extend(revstuff) 764 765 cmd = LoggedRemoteCommand("arch", self.args) 766 self.startCommand(cmd, warnings)

767 768

769 -class Bazaar(Arch):

770 """Bazaar is an alternative client for Arch repositories. baz is mostly 771 compatible with tla, but archive registration is slightly different.""" 772 773 # TODO: slaves >0.6.6 will accept args['build-config'], so use it 774

775 - def __init__(self, url, version, archive, **kwargs):

776 """ 777 @type url: string 778 @param url: the Arch coordinates of the repository. This is 779 typically an http:// URL, but could also be the absolute 780 pathname of a local directory instead. 781 782 @type version: string 783 @param version: the category--branch--version to check out 784 785 @type archive: string 786 @param archive: The archive name (required). This must always match 787 the one that comes from the repository, otherwise the 788 buildslave will attempt to get sources from the wrong 789 archive. 790 """ 791 self.branch = version 792 Source.__init__(self, **kwargs) 793 self.addFactoryArguments(url=url, 794 version=version, 795 archive=archive, 796 ) 797 self.args.update({'url': url, 798 'archive': archive, 799 })

800

801 - def startVC(self, branch, revision, patch):

802 self.args['version'] = branch 803 self.args['revision'] = revision 804 self.args['patch'] = patch 805 warnings = self.checkSlaveVersion("bazaar", branch) 806 807 revstuff = [] 808 if branch is not None and branch != self.branch: 809 revstuff.append("[branch]") 810 if revision is not None: 811 revstuff.append("patch%s" % revision) 812 self.description.extend(revstuff) 813 self.descriptionDone.extend(revstuff) 814 815 cmd = LoggedRemoteCommand("bazaar", self.args) 816 self.startCommand(cmd, warnings)

817

818 -class Bzr(Source):

819 """Check out a source tree from a bzr (Bazaar) repository at 'repourl'. 820 821 """ 822 823 name = "bzr" 824

825 - def __init__(self, repourl=None, baseURL=None, defaultBranch=None, 826 forceSharedRepo=None, 827 **kwargs):

828 """ 829 @type repourl: string 830 @param repourl: the URL which points at the bzr repository. This 831 is used as the default branch. Using C{repourl} does 832 not enable builds of alternate branches: use 833 C{baseURL} to enable this. Use either C{repourl} or 834 C{baseURL}, not both. 835 836 @param baseURL: if branches are enabled, this is the base URL to 837 which a branch name will be appended. It should 838 probably end in a slash. Use exactly one of 839 C{repourl} and C{baseURL}. 840 841 @param defaultBranch: if branches are enabled, this is the branch 842 to use if the Build does not specify one 843 explicitly. It will simply be appended to 844 C{baseURL} and the result handed to the 845 'bzr checkout pull' command. 846 847 848 @param forceSharedRepo: Boolean, defaults to False. If set to True, 849 the working directory will be made into a 850 bzr shared repository if it is not already. 851 Shared repository greatly reduces the amount 852 of history data that needs to be downloaded 853 if not using update/copy mode, or if using 854 update/copy mode with multiple branches. 855 """ 856 self.repourl = repourl 857 self.baseURL = baseURL 858 self.branch = defaultBranch 859 Source.__init__(self, **kwargs) 860 self.addFactoryArguments(repourl=repourl, 861 baseURL=baseURL, 862 defaultBranch=defaultBranch, 863 forceSharedRepo=forceSharedRepo 864 ) 865 self.args.update({'forceSharedRepo': forceSharedRepo}) 866 if (not repourl and not baseURL) or (repourl and baseURL): 867 raise ValueError("you must provide exactly one of repourl and" 868 " baseURL")

869

870 - def computeSourceRevision(self, changes):

871 if not changes: 872 return None 873 lastChange = max([int(c.revision) for c in changes]) 874 return lastChange

875

876 - def startVC(self, branch, revision, patch):

877 slavever = self.slaveVersion("bzr") 878 if not slavever: 879 m = "slave is too old, does not know about bzr" 880 raise BuildSlaveTooOldError(m) 881 882 if self.repourl: 883 assert not branch # we need baseURL= to use branches 884 self.args['repourl'] = self.repourl 885 else: 886 self.args['repourl'] = self.baseURL + branch 887 self.args['revision'] = revision 888 self.args['patch'] = patch 889 890 revstuff = [] 891 if branch is not None and branch != self.branch: 892 revstuff.append("[" + branch + "]") 893 if revision is not None: 894 revstuff.append("r%s" % revision) 895 self.description.extend(revstuff) 896 self.descriptionDone.extend(revstuff) 897 898 cmd = LoggedRemoteCommand("bzr", self.args) 899 self.startCommand(cmd)

900 901

902 -class Mercurial(Source):

903 """Check out a source tree from a mercurial repository 'repourl'.""" 904 905 name = "hg" 906

907 - def __init__(self, repourl=None, baseURL=None, defaultBranch=None, 908 branchType='dirname', clobberOnBranchChange=True, **kwargs):

909 """ 910 @type repourl: string 911 @param repourl: the URL which points at the Mercurial repository. 912 This uses the 'default' branch unless defaultBranch is 913 specified below and the C{branchType} is set to 914 'inrepo'. It is an error to specify a branch without 915 setting the C{branchType} to 'inrepo'. 916 917 @param baseURL: if 'dirname' branches are enabled, this is the base URL 918 to which a branch name will be appended. It should 919 probably end in a slash. Use exactly one of C{repourl} 920 and C{baseURL}. 921 922 @param defaultBranch: if branches are enabled, this is the branch 923 to use if the Build does not specify one 924 explicitly. 925 For 'dirname' branches, It will simply be 926 appended to C{baseURL} and the result handed to 927 the 'hg update' command. 928 For 'inrepo' branches, this specifies the named 929 revision to which the tree will update after a 930 clone. 931 932 @param branchType: either 'dirname' or 'inrepo' depending on whether 933 the branch name should be appended to the C{baseURL} 934 or the branch is a mercurial named branch and can be 935 found within the C{repourl} 936 937 @param clobberOnBranchChange: boolean, defaults to True. If set and 938 using inrepos branches, clobber the tree 939 at each branch change. Otherwise, just 940 update to the branch. 941 """ 942 self.repourl = repourl 943 self.baseURL = baseURL 944 self.branch = defaultBranch 945 self.branchType = branchType 946 self.clobberOnBranchChange = clobberOnBranchChange 947 Source.__init__(self, **kwargs) 948 self.addFactoryArguments(repourl=repourl, 949 baseURL=baseURL, 950 defaultBranch=defaultBranch, 951 branchType=branchType, 952 clobberOnBranchChange=clobberOnBranchChange, 953 ) 954 if (not repourl and not baseURL) or (repourl and baseURL): 955 raise ValueError("you must provide exactly one of repourl and" 956 " baseURL")

957

958 - def startVC(self, branch, revision, patch):

959 slavever = self.slaveVersion("hg") 960 if not slavever: 961 raise BuildSlaveTooOldError("slave is too old, does not know " 962 "about hg") 963 964 if self.repourl: 965 # we need baseURL= to use dirname branches 966 assert self.branchType == 'inrepo' or not branch 967 self.args['repourl'] = self.repourl 968 if branch: 969 self.args['branch'] = branch 970 else: 971 self.args['repourl'] = self.baseURL + (branch or '') 972 self.args['revision'] = revision 973 self.args['patch'] = patch 974 self.args['clobberOnBranchChange'] = self.clobberOnBranchChange 975 self.args['branchType'] = self.branchType 976 977 revstuff = [] 978 if branch is not None and branch != self.branch: 979 revstuff.append("[branch]") 980 self.description.extend(revstuff) 981 self.descriptionDone.extend(revstuff) 982 983 cmd = LoggedRemoteCommand("hg", self.args) 984 self.startCommand(cmd)

985

986 - def computeSourceRevision(self, changes):

987 if not changes: 988 return None 989 # without knowing the revision ancestry graph, we can't sort the 990 # changes at all. So for now, assume they were given to us in sorted 991 # order, and just pay attention to the last one. See ticket #103 for 992 # more details. 993 if len(changes) > 1: 994 log.msg("Mercurial.computeSourceRevision: warning: " 995 "there are %d changes here, assuming the last one is " 996 "the most recent" % len(changes)) 997 return changes[-1].revision

998 999

1000 -class P4(Source):

1001 """ P4 is a class for accessing perforce revision control""" 1002 name = "p4" 1003

1004 - def __init__(self, p4base, defaultBranch=None, p4port=None, p4user=None, 1005 p4passwd=None, p4extra_views=[], 1006 p4client='buildbot_%(slave)s_%(builder)s', **kwargs):

1007 """ 1008 @type p4base: string 1009 @param p4base: A view into a perforce depot, typically 1010 "//depot/proj/" 1011 1012 @type defaultBranch: string 1013 @param defaultBranch: Identify a branch to build by default. Perforce 1014 is a view based branching system. So, the branch 1015 is normally the name after the base. For example, 1016 branch=1.0 is view=//depot/proj/1.0/... 1017 branch=1.1 is view=//depot/proj/1.1/... 1018 1019 @type p4port: string 1020 @param p4port: Specify the perforce server to connection in the format 1021 <host>:<port>. Example "perforce.example.com:1666" 1022 1023 @type p4user: string 1024 @param p4user: The perforce user to run the command as. 1025 1026 @type p4passwd: string 1027 @param p4passwd: The password for the perforce user. 1028 1029 @type p4extra_views: list of tuples 1030 @param p4extra_views: Extra views to be added to 1031 the client that is being used. 1032 1033 @type p4client: string 1034 @param p4client: The perforce client to use for this buildslave. 1035 """ 1036 1037 self.branch = defaultBranch 1038 Source.__init__(self, **kwargs) 1039 self.addFactoryArguments(p4base=p4base, 1040 defaultBranch=defaultBranch, 1041 p4port=p4port, 1042 p4user=p4user, 1043 p4passwd=p4passwd, 1044 p4extra_views=p4extra_views, 1045 p4client=p4client, 1046 ) 1047 self.args['p4port'] = p4port 1048 self.args['p4user'] = p4user 1049 self.args['p4passwd'] = p4passwd 1050 self.args['p4base'] = p4base 1051 self.args['p4extra_views'] = p4extra_views 1052 self.p4client = p4client

1053

1054 - def setBuild(self, build):

1055 Source.setBuild(self, build) 1056 self.args['p4client'] = self.p4client % { 1057 'slave': build.slavename, 1058 'builder': build.builder.name, 1059 }

1060

1061 - def computeSourceRevision(self, changes):

1062 if not changes: 1063 return None 1064 lastChange = max([int(c.revision) for c in changes]) 1065 return lastChange

1066

1067 - def startVC(self, branch, revision, patch):

1068 slavever = self.slaveVersion("p4") 1069 assert slavever, "slave is too old, does not know about p4" 1070 args = dict(self.args) 1071 args['branch'] = branch or self.branch 1072 args['revision'] = revision 1073 args['patch'] = patch 1074 cmd = LoggedRemoteCommand("p4", args) 1075 self.startCommand(cmd)

1076

1077 -class P4Sync(Source):

1078 """This is a partial solution for using a P4 source repository. You are 1079 required to manually set up each build slave with a useful P4 1080 environment, which means setting various per-slave environment variables, 1081 and creating a P4 client specification which maps the right files into 1082 the slave's working directory. Once you have done that, this step merely 1083 performs a 'p4 sync' to update that workspace with the newest files. 1084 1085 Each slave needs the following environment: 1086 1087 - PATH: the 'p4' binary must be on the slave's PATH 1088 - P4USER: each slave needs a distinct user account 1089 - P4CLIENT: each slave needs a distinct client specification 1090 1091 You should use 'p4 client' (?) to set up a client view spec which maps 1092 the desired files into $SLAVEBASE/$BUILDERBASE/source . 1093 """ 1094 1095 name = "p4sync" 1096

1097 - def __init__(self, p4port, p4user, p4passwd, p4client, **kwargs):

1098 assert kwargs['mode'] == "copy", "P4Sync can only be used in mode=copy" 1099 self.branch = None 1100 Source.__init__(self, **kwargs) 1101 self.addFactoryArguments(p4port=p4port, 1102 p4user=p4user, 1103 p4passwd=p4passwd, 1104 p4client=p4client, 1105 ) 1106 self.args['p4port'] = p4port 1107 self.args['p4user'] = p4user 1108 self.args['p4passwd'] = p4passwd 1109 self.args['p4client'] = p4client

1110

1111 - def computeSourceRevision(self, changes):

1112 if not changes: 1113 return None 1114 lastChange = max([int(c.revision) for c in changes]) 1115 return lastChange

1116

1117 - def startVC(self, branch, revision, patch):

1118 slavever = self.slaveVersion("p4sync") 1119 assert slavever, "slave is too old, does not know about p4" 1120 cmd = LoggedRemoteCommand("p4sync", self.args) 1121 self.startCommand(cmd)

1122

1123 -class Monotone(Source):

1124 """Check out a revision from a monotone server at 'server_addr', 1125 branch 'branch'. 'revision' specifies which revision id to check 1126 out. 1127 1128 This step will first create a local database, if necessary, and then pull 1129 the contents of the server into the database. Then it will do the 1130 checkout/update from this database.""" 1131 1132 name = "monotone" 1133

1134 - def __init__(self, server_addr, branch, db_path="monotone.db", 1135 monotone="monotone", 1136 **kwargs):

1137 Source.__init__(self, **kwargs) 1138 self.addFactoryArguments(server_addr=server_addr, 1139 branch=branch, 1140 db_path=db_path, 1141 monotone=monotone, 1142 ) 1143 self.args.update({"server_addr": server_addr, 1144 "branch": branch, 1145 "db_path": db_path, 1146 "monotone": monotone})

1147

1148 - def computeSourceRevision(self, changes):

1149 if not changes: 1150 return None 1151 return changes[-1].revision

1152

1153 - def startVC(self):

1154 slavever = self.slaveVersion("monotone") 1155 assert slavever, "slave is too old, does not know about monotone" 1156 cmd = LoggedRemoteCommand("monotone", self.args) 1157 self.startCommand(cmd)

1158

Source Code for Module buildbot.steps.source