buildbot.steps.shell

1 # -*- test-case-name: buildbot.test.test_steps,buildbot.test.test_properties -*- 2 3 import re 4 from twisted.python import log 5 from buildbot.process.buildstep import LoggingBuildStep, RemoteShellCommand 6 from buildbot.status.builder import SUCCESS, WARNINGS, FAILURE, STDOUT, STDERR 7 8 # for existing configurations that import WithProperties from here. We like 9 # to move this class around just to keep our readers guessing. 10 from buildbot.process.properties import WithProperties 11 _hush_pyflakes = [WithProperties] 12 del _hush_pyflakes 13

14 -class ShellCommand(LoggingBuildStep):

15 """I run a single shell command on the buildslave. I return FAILURE if 16 the exit code of that command is non-zero, SUCCESS otherwise. To change 17 this behavior, override my .evaluateCommand method. 18 19 By default, a failure of this step will mark the whole build as FAILURE. 20 To override this, give me an argument of flunkOnFailure=False . 21 22 I create a single Log named 'log' which contains the output of the 23 command. To create additional summary Logs, override my .createSummary 24 method. 25 26 The shell command I run (a list of argv strings) can be provided in 27 several ways: 28 - a class-level .command attribute 29 - a command= parameter to my constructor (overrides .command) 30 - set explicitly with my .setCommand() method (overrides both) 31 32 @ivar command: a list of renderable objects (typically strings or 33 WithProperties instances). This will be used by start() 34 to create a RemoteShellCommand instance. 35 36 @ivar logfiles: a dict mapping log NAMEs to workdir-relative FILENAMEs 37 of their corresponding logfiles. The contents of the file 38 named FILENAME will be put into a LogFile named NAME, ina 39 something approximating real-time. (note that logfiles= 40 is actually handled by our parent class LoggingBuildStep) 41 42 """ 43 44 name = "shell" 45 description = None # set this to a list of short strings to override 46 descriptionDone = None # alternate description when the step is complete 47 command = None # set this to a command, or set in kwargs 48 # logfiles={} # you can also set 'logfiles' to a dictionary, and it 49 # will be merged with any logfiles= argument passed in 50 # to __init__ 51 52 # override this on a specific ShellCommand if you want to let it fail 53 # without dooming the entire build to a status of FAILURE 54 flunkOnFailure = True 55

56 - def __init__(self, workdir=None, 57 description=None, descriptionDone=None, 58 command=None, 59 usePTY="slave-config", 60 **kwargs):

61 # most of our arguments get passed through to the RemoteShellCommand 62 # that we create, but first strip out the ones that we pass to 63 # BuildStep (like haltOnFailure and friends), and a couple that we 64 # consume ourselves. 65 66 if description: 67 self.description = description 68 if isinstance(self.description, str): 69 self.description = [self.description] 70 if descriptionDone: 71 self.descriptionDone = descriptionDone 72 if isinstance(self.descriptionDone, str): 73 self.descriptionDone = [self.descriptionDone] 74 if command: 75 self.setCommand(command) 76 77 # pull out the ones that LoggingBuildStep wants, then upcall 78 buildstep_kwargs = {} 79 for k in kwargs.keys()[:]: 80 if k in self.__class__.parms: 81 buildstep_kwargs[k] = kwargs[k] 82 del kwargs[k] 83 LoggingBuildStep.__init__(self, **buildstep_kwargs) 84 self.addFactoryArguments(workdir=workdir, 85 description=description, 86 descriptionDone=descriptionDone, 87 command=command) 88 89 # everything left over goes to the RemoteShellCommand 90 kwargs['workdir'] = workdir # including a copy of 'workdir' 91 kwargs['usePTY'] = usePTY 92 self.remote_kwargs = kwargs 93 # we need to stash the RemoteShellCommand's args too 94 self.addFactoryArguments(**kwargs)

95

96 - def setDefaultWorkdir(self, workdir):

97 rkw = self.remote_kwargs 98 rkw['workdir'] = rkw['workdir'] or workdir

99

100 - def setCommand(self, command):

101 self.command = command

102

103 - def describe(self, done=False):

104 """Return a list of short strings to describe this step, for the 105 status display. This uses the first few words of the shell command. 106 You can replace this by setting .description in your subclass, or by 107 overriding this method to describe the step better. 108 109 @type done: boolean 110 @param done: whether the command is complete or not, to improve the 111 way the command is described. C{done=False} is used 112 while the command is still running, so a single 113 imperfect-tense verb is appropriate ('compiling', 114 'testing', ...) C{done=True} is used when the command 115 has finished, and the default getText() method adds some 116 text, so a simple noun is appropriate ('compile', 117 'tests' ...) 118 """ 119 120 if done and self.descriptionDone is not None: 121 return list(self.descriptionDone) 122 if self.description is not None: 123 return list(self.description) 124 125 properties = self.build.getProperties() 126 words = self.command 127 if isinstance(words, (str, unicode)): 128 words = words.split() 129 # render() each word to handle WithProperties objects 130 words = properties.render(words) 131 if len(words) < 1: 132 return ["???"] 133 if len(words) == 1: 134 return ["'%s'" % words[0]] 135 if len(words) == 2: 136 return ["'%s" % words[0], "%s'" % words[1]] 137 return ["'%s" % words[0], "%s" % words[1], "...'"]

138

139 - def setupEnvironment(self, cmd):

140 # merge in anything from Build.slaveEnvironment 141 # This can be set from a Builder-level environment, or from earlier 142 # BuildSteps. The latter method is deprecated and superceded by 143 # BuildProperties. 144 # Environment variables passed in by a BuildStep override 145 # those passed in at the Builder level. 146 properties = self.build.getProperties() 147 slaveEnv = self.build.slaveEnvironment 148 if slaveEnv: 149 if cmd.args['env'] is None: 150 cmd.args['env'] = {} 151 fullSlaveEnv = slaveEnv.copy() 152 fullSlaveEnv.update(cmd.args['env']) 153 cmd.args['env'] = properties.render(fullSlaveEnv)

154 # note that each RemoteShellCommand gets its own copy of the 155 # dictionary, so we shouldn't be affecting anyone but ourselves. 156

157 - def checkForOldSlaveAndLogfiles(self):

158 if not self.logfiles: 159 return # doesn't matter 160 if not self.slaveVersionIsOlderThan("shell", "2.1"): 161 return # slave is new enough 162 # this buildslave is too old and will ignore the 'logfiles' 163 # argument. You'll either have to pull the logfiles manually 164 # (say, by using 'cat' in a separate RemoteShellCommand) or 165 # upgrade the buildslave. 166 msg1 = ("Warning: buildslave %s is too old " 167 "to understand logfiles=, ignoring it." 168 % self.getSlaveName()) 169 msg2 = "You will have to pull this logfile (%s) manually." 170 log.msg(msg1) 171 for logname,remotefilevalue in self.logfiles.items(): 172 remotefilename = remotefilevalue 173 # check for a dictionary of options 174 if type(remotefilevalue) == dict: 175 remotefilename = remotefilevalue['filename'] 176 177 newlog = self.addLog(logname) 178 newlog.addHeader(msg1 + "\n") 179 newlog.addHeader(msg2 % remotefilename + "\n") 180 newlog.finish() 181 # now prevent setupLogfiles() from adding them 182 self.logfiles = {}

183

184 - def start(self):

185 # this block is specific to ShellCommands. subclasses that don't need 186 # to set up an argv array, an environment, or extra logfiles= (like 187 # the Source subclasses) can just skip straight to startCommand() 188 properties = self.build.getProperties() 189 190 warnings = [] 191 192 # create the actual RemoteShellCommand instance now 193 kwargs = properties.render(self.remote_kwargs) 194 kwargs['command'] = properties.render(self.command) 195 kwargs['logfiles'] = self.logfiles 196 197 # check for the usePTY flag 198 if kwargs.has_key('usePTY') and kwargs['usePTY'] != 'slave-config': 199 slavever = self.slaveVersion("shell", "old") 200 if self.slaveVersionIsOlderThan("svn", "2.7"): 201 warnings.append("NOTE: slave does not allow master to override usePTY\n") 202 203 cmd = RemoteShellCommand(**kwargs) 204 self.setupEnvironment(cmd) 205 self.checkForOldSlaveAndLogfiles() 206 207 self.startCommand(cmd, warnings)

208 209 210

211 -class TreeSize(ShellCommand):

212 name = "treesize" 213 command = ["du", "-s", "-k", "."] 214 kib = None 215

216 - def commandComplete(self, cmd):

217 out = cmd.logs['stdio'].getText() 218 m = re.search(r'^(\d+)', out) 219 if m: 220 self.kib = int(m.group(1)) 221 self.setProperty("tree-size-KiB", self.kib, "treesize")

222

223 - def evaluateCommand(self, cmd):

224 if cmd.rc != 0: 225 return FAILURE 226 if self.kib is None: 227 return WARNINGS # not sure how 'du' could fail, but whatever 228 return SUCCESS

229

230 - def getText(self, cmd, results):

231 if self.kib is not None: 232 return ["treesize", "%d KiB" % self.kib] 233 return ["treesize", "unknown"]

234

235 -class SetProperty(ShellCommand):

236 name = "setproperty" 237

238 - def __init__(self, **kwargs):

239 self.property = None 240 self.extract_fn = None 241 self.strip = True 242 243 if kwargs.has_key('property'): 244 self.property = kwargs['property'] 245 del kwargs['property'] 246 if kwargs.has_key('extract_fn'): 247 self.extract_fn = kwargs['extract_fn'] 248 del kwargs['extract_fn'] 249 if kwargs.has_key('strip'): 250 self.strip = kwargs['strip'] 251 del kwargs['strip'] 252 253 ShellCommand.__init__(self, **kwargs) 254 255 self.addFactoryArguments(property=self.property) 256 self.addFactoryArguments(extract_fn=self.extract_fn) 257 self.addFactoryArguments(strip=self.strip) 258 259 assert self.property or self.extract_fn, \ 260 "SetProperty step needs either property= or extract_fn=" 261 262 self.property_changes = {}

263

264 - def commandComplete(self, cmd):

265 if self.property: 266 result = cmd.logs['stdio'].getText() 267 if self.strip: result = result.strip() 268 propname = self.build.getProperties().render(self.property) 269 self.setProperty(propname, result, "SetProperty Step") 270 self.property_changes[propname] = result 271 else: 272 log = cmd.logs['stdio'] 273 new_props = self.extract_fn(cmd.rc, 274 ''.join(log.getChunks([STDOUT], onlyText=True)), 275 ''.join(log.getChunks([STDERR], onlyText=True))) 276 for k,v in new_props.items(): 277 self.setProperty(k, v, "SetProperty Step") 278 self.property_changes = new_props

279

280 - def createSummary(self, log):

281 props_set = [ "%s: %r" % (k,v) for k,v in self.property_changes.items() ] 282 self.addCompleteLog('property changes', "\n".join(props_set))

283

284 - def getText(self, cmd, results):

285 if self.property_changes: 286 return [ "set props:" ] + self.property_changes.keys() 287 else: 288 return [ "no change" ]

289

290 -class Configure(ShellCommand):

291 292 name = "configure" 293 haltOnFailure = 1 294 flunkOnFailure = 1 295 description = ["configuring"] 296 descriptionDone = ["configure"] 297 command = ["./configure"]

298

299 -class WarningCountingShellCommand(ShellCommand):

300 warnCount = 0 301 warningPattern = '.*warning[: ].*' 302

303 - def __init__(self, warningPattern=None, **kwargs):

304 # See if we've been given a regular expression to use to match 305 # warnings. If not, use a default that assumes any line with "warning" 306 # present is a warning. This may lead to false positives in some cases. 307 if warningPattern: 308 self.warningPattern = warningPattern 309 310 # And upcall to let the base class do its work 311 ShellCommand.__init__(self, **kwargs) 312 313 self.addFactoryArguments(warningPattern=warningPattern)

314

315 - def createSummary(self, log):

316 self.warnCount = 0 317 318 # Now compile a regular expression from whichever warning pattern we're 319 # using 320 if not self.warningPattern: 321 return 322 323 wre = self.warningPattern 324 if isinstance(wre, str): 325 wre = re.compile(wre) 326 327 # Check if each line in the output from this command matched our 328 # warnings regular expressions. If did, bump the warnings count and 329 # add the line to the collection of lines with warnings 330 warnings = [] 331 # TODO: use log.readlines(), except we need to decide about stdout vs 332 # stderr 333 for line in log.getText().split("\n"): 334 if wre.match(line): 335 warnings.append(line) 336 self.warnCount += 1 337 338 # If there were any warnings, make the log if lines with warnings 339 # available 340 if self.warnCount: 341 self.addCompleteLog("warnings", "\n".join(warnings) + "\n") 342 343 warnings_stat = self.step_status.getStatistic('warnings', 0) 344 self.step_status.setStatistic('warnings', warnings_stat + self.warnCount) 345 346 try: 347 old_count = self.getProperty("warnings-count") 348 except KeyError: 349 old_count = 0 350 self.setProperty("warnings-count", old_count + self.warnCount, "WarningCountingShellCommand")

351 352

353 - def evaluateCommand(self, cmd):

354 if cmd.rc != 0: 355 return FAILURE 356 if self.warnCount: 357 return WARNINGS 358 return SUCCESS

359 360

361 -class Compile(WarningCountingShellCommand):

362 363 name = "compile" 364 haltOnFailure = 1 365 flunkOnFailure = 1 366 description = ["compiling"] 367 descriptionDone = ["compile"] 368 command = ["make", "all"] 369 370 OFFprogressMetrics = ('output',) 371 # things to track: number of files compiled, number of directories 372 # traversed (assuming 'make' is being used) 373

374 - def createSummary(self, log):

375 # TODO: grep for the characteristic GCC error lines and 376 # assemble them into a pair of buffers 377 WarningCountingShellCommand.createSummary(self, log)

378

379 -class Test(WarningCountingShellCommand):

380 381 name = "test" 382 warnOnFailure = 1 383 description = ["testing"] 384 descriptionDone = ["test"] 385 command = ["make", "test"] 386

387 - def setTestResults(self, total=0, failed=0, passed=0, warnings=0):

388 """ 389 Called by subclasses to set the relevant statistics; this actually 390 adds to any statistics already present 391 """ 392 total += self.step_status.getStatistic('tests-total', 0) 393 self.step_status.setStatistic('tests-total', total) 394 failed += self.step_status.getStatistic('tests-failed', 0) 395 self.step_status.setStatistic('tests-failed', failed) 396 warnings += self.step_status.getStatistic('tests-warnings', 0) 397 self.step_status.setStatistic('tests-warnings', warnings) 398 passed += self.step_status.getStatistic('tests-passed', 0) 399 self.step_status.setStatistic('tests-passed', passed)

400

401 - def describe(self, done=False):

402 description = WarningCountingShellCommand.describe(self, done) 403 if done: 404 if self.step_status.hasStatistic('tests-total'): 405 total = self.step_status.getStatistic("tests-total", 0) 406 failed = self.step_status.getStatistic("tests-failed", 0) 407 passed = self.step_status.getStatistic("tests-passed", 0) 408 warnings = self.step_status.getStatistic("tests-warnings", 0) 409 if not total: 410 total = failed + passed + warnings 411 412 if total: 413 description.append('%d tests' % total) 414 if passed: 415 description.append('%d passed' % passed) 416 if warnings: 417 description.append('%d warnings' % warnings) 418 if failed: 419 description.append('%d failed' % failed) 420 return description

421

422 -class PerlModuleTest(Test):

423 command=["prove", "--lib", "lib", "-r", "t"] 424 total = 0 425

426 - def evaluateCommand(self, cmd):

427 # Get stdio, stripping pesky newlines etc. 428 lines = map( 429 lambda line : line.replace('\r\n','').replace('\r','').replace('\n',''), 430 self.getLog('stdio').readlines() 431 ) 432 433 total = 0 434 passed = 0 435 failed = 0 436 rc = cmd.rc 437 438 # New version of Test::Harness? 439 try: 440 test_summary_report_index = lines.index("Test Summary Report") 441 442 del lines[0:test_summary_report_index + 2] 443 444 re_test_result = re.compile("^Result: (PASS|FAIL)$|Tests: \d+ Failed: (\d+)\)|Files=\d+, Tests=(\d+)") 445 446 mos = map(lambda line: re_test_result.search(line), lines) 447 test_result_lines = [mo.groups() for mo in mos if mo] 448 449 for line in test_result_lines: 450 if line[0] == 'PASS': 451 rc = SUCCESS 452 elif line[0] == 'FAIL': 453 rc = FAILURE 454 elif line[1]: 455 failed += int(line[1]) 456 elif line[2]: 457 total = int(line[2]) 458 459 except ValueError: # Nope, it's the old version 460 re_test_result = re.compile("^(All tests successful)|(\d+)/(\d+) subtests failed|Files=\d+, Tests=(\d+),") 461 462 mos = map(lambda line: re_test_result.search(line), lines) 463 test_result_lines = [mo.groups() for mo in mos if mo] 464 465 if test_result_lines: 466 test_result_line = test_result_lines[0] 467 468 success = test_result_line[0] 469 470 if success: 471 failed = 0 472 473 test_totals_line = test_result_lines[1] 474 total_str = test_totals_line[3] 475 476 rc = SUCCESS 477 else: 478 failed_str = test_result_line[1] 479 failed = int(failed_str) 480 481 total_str = test_result_line[2] 482 483 rc = FAILURE 484 485 total = int(total_str) 486 487 if total: 488 passed = total - failed 489 490 self.setTestResults(total=total, failed=failed, passed=passed) 491 492 return rc

493

Source Code for Module buildbot.steps.shell