buildbot.steps.transfer

1 # -*- test-case-name: buildbot.test.test_transfer -*- 2 3 import os.path, tarfile, tempfile 4 from twisted.internet import reactor 5 from twisted.spread import pb 6 from twisted.python import log 7 from buildbot.process.buildstep import RemoteCommand, BuildStep 8 from buildbot.process.buildstep import SUCCESS, FAILURE, SKIPPED 9 from buildbot.interfaces import BuildSlaveTooOldError 10 11

12 -class _FileWriter(pb.Referenceable):

13 """ 14 Helper class that acts as a file-object with write access 15 """ 16

17 - def __init__(self, destfile, maxsize, mode):

18 # Create missing directories. 19 destfile = os.path.abspath(destfile) 20 dirname = os.path.dirname(destfile) 21 if not os.path.exists(dirname): 22 os.makedirs(dirname) 23 24 self.destfile = destfile 25 self.fp = open(destfile, "wb") 26 if mode is not None: 27 os.chmod(destfile, mode) 28 self.remaining = maxsize

29

30 - def remote_write(self, data):

31 """ 32 Called from remote slave to write L{data} to L{fp} within boundaries 33 of L{maxsize} 34 35 @type data: C{string} 36 @param data: String of data to write 37 """ 38 if self.remaining is not None: 39 if len(data) > self.remaining: 40 data = data[:self.remaining] 41 self.fp.write(data) 42 self.remaining = self.remaining - len(data) 43 else: 44 self.fp.write(data)

45

46 - def remote_close(self):

47 """ 48 Called by remote slave to state that no more data will be transfered 49 """ 50 self.fp.close() 51 self.fp = None

52

53 - def __del__(self):

54 # unclean shutdown, the file is probably truncated, so delete it 55 # altogether rather than deliver a corrupted file 56 fp = getattr(self, "fp", None) 57 if fp: 58 fp.close() 59 os.unlink(self.destfile)

60 61

62 -def _extractall(self, path=".", members=None):

63 """Fallback extractall method for TarFile, in case it doesn't have its own.""" 64 65 import copy 66 import operator 67 68 directories = [] 69 70 if members is None: 71 members = self 72 73 for tarinfo in members: 74 if tarinfo.isdir(): 75 # Extract directories with a safe mode. 76 directories.append(tarinfo) 77 tarinfo = copy.copy(tarinfo) 78 tarinfo.mode = 0700 79 self.extract(tarinfo, path) 80 81 # Reverse sort directories. 82 directories.sort(lambda a, b: cmp(a.name, b.name)) 83 directories.reverse() 84 85 # Set correct owner, mtime and filemode on directories. 86 for tarinfo in directories: 87 dirpath = os.path.join(path, tarinfo.name) 88 try: 89 self.chown(tarinfo, dirpath) 90 self.utime(tarinfo, dirpath) 91 self.chmod(tarinfo, dirpath) 92 except tarfile.ExtractError, e: 93 if self.errorlevel > 1: 94 raise 95 else: 96 self._dbg(1, "tarfile: %s" % e)

97

98 -class _DirectoryWriter(_FileWriter):

99 """ 100 A DirectoryWriter is implemented as a FileWriter, with an added post-processing 101 step to unpack the archive, once the transfer has completed. 102 """ 103

104 - def __init__(self, destroot, maxsize, compress, mode):

105 self.destroot = destroot 106 107 self.fd, self.tarname = tempfile.mkstemp() 108 self.compress = compress 109 _FileWriter.__init__(self, self.tarname, maxsize, mode)

110

111 - def remote_unpack(self):

112 """ 113 Called by remote slave to state that no more data will be transfered 114 """ 115 if self.fp: 116 self.fp.close() 117 self.fp = None 118 fileobj = os.fdopen(self.fd, 'r') 119 if self.compress == 'bz2': 120 mode='r|bz2' 121 elif self.compress == 'gz': 122 mode='r|gz' 123 else: 124 mode = 'r' 125 if not hasattr(tarfile.TarFile, 'extractall'): 126 tarfile.TarFile.extractall = _extractall 127 archive = tarfile.open(name=self.tarname, mode=mode, fileobj=fileobj) 128 archive.extractall(path=self.destroot) 129 os.remove(self.tarname)

130 131

132 -class StatusRemoteCommand(RemoteCommand):

133 - def __init__(self, remote_command, args):

134 RemoteCommand.__init__(self, remote_command, args) 135 136 self.rc = None 137 self.stderr = ''

138

139 - def remoteUpdate(self, update):

140 #log.msg('StatusRemoteCommand: update=%r' % update) 141 if 'rc' in update: 142 self.rc = update['rc'] 143 if 'stderr' in update: 144 self.stderr = self.stderr + update['stderr'] + '\n'

145

146 -class _TransferBuildStep(BuildStep):

147 """ 148 Base class for FileUpload and FileDownload to factor out common 149 functionality. 150 """ 151 DEFAULT_WORKDIR = "build" # is this redundant? 152

153 - def setDefaultWorkdir(self, workdir):

154 if self.workdir is None: 155 self.workdir = workdir

156

157 - def _getWorkdir(self):

158 properties = self.build.getProperties() 159 if self.workdir is None: 160 workdir = self.DEFAULT_WORKDIR 161 else: 162 workdir = self.workdir 163 return properties.render(workdir)

164

165 - def finished(self, result):

166 # Subclasses may choose to skip a transfer. In those cases, self.cmd 167 # will be None, and we should just let BuildStep.finished() handle 168 # the rest 169 if result == SKIPPED: 170 return BuildStep.finished(self, SKIPPED) 171 if self.cmd.stderr != '': 172 self.addCompleteLog('stderr', self.cmd.stderr) 173 174 if self.cmd.rc is None or self.cmd.rc == 0: 175 return BuildStep.finished(self, SUCCESS) 176 return BuildStep.finished(self, FAILURE)

177 178

179 -class FileUpload(_TransferBuildStep):

180 """ 181 Build step to transfer a file from the slave to the master. 182 183 arguments: 184 185 - ['slavesrc'] filename of source file at slave, relative to workdir 186 - ['masterdest'] filename of destination file at master 187 - ['workdir'] string with slave working directory relative to builder 188 base dir, default 'build' 189 - ['maxsize'] maximum size of the file, default None (=unlimited) 190 - ['blocksize'] maximum size of each block being transfered 191 - ['mode'] file access mode for the resulting master-side file. 192 The default (=None) is to leave it up to the umask of 193 the buildmaster process. 194 195 """ 196 197 name = 'upload' 198

199 - def __init__(self, slavesrc, masterdest, 200 workdir=None, maxsize=None, blocksize=16*1024, mode=None, 201 **buildstep_kwargs):

202 BuildStep.__init__(self, **buildstep_kwargs) 203 self.addFactoryArguments(slavesrc=slavesrc, 204 masterdest=masterdest, 205 workdir=workdir, 206 maxsize=maxsize, 207 blocksize=blocksize, 208 mode=mode, 209 ) 210 211 self.slavesrc = slavesrc 212 self.masterdest = masterdest 213 self.workdir = workdir 214 self.maxsize = maxsize 215 self.blocksize = blocksize 216 assert isinstance(mode, (int, type(None))) 217 self.mode = mode

218

219 - def start(self):

220 version = self.slaveVersion("uploadFile") 221 properties = self.build.getProperties() 222 223 if not version: 224 m = "slave is too old, does not know about uploadFile" 225 raise BuildSlaveTooOldError(m) 226 227 source = properties.render(self.slavesrc) 228 masterdest = properties.render(self.masterdest) 229 # we rely upon the fact that the buildmaster runs chdir'ed into its 230 # basedir to make sure that relative paths in masterdest are expanded 231 # properly. TODO: maybe pass the master's basedir all the way down 232 # into the BuildStep so we can do this better. 233 masterdest = os.path.expanduser(masterdest) 234 log.msg("FileUpload started, from slave %r to master %r" 235 % (source, masterdest)) 236 237 self.step_status.setText(['uploading', os.path.basename(source)]) 238 239 # we use maxsize to limit the amount of data on both sides 240 fileWriter = _FileWriter(masterdest, self.maxsize, self.mode) 241 242 # default arguments 243 args = { 244 'slavesrc': source, 245 'workdir': self._getWorkdir(), 246 'writer': fileWriter, 247 'maxsize': self.maxsize, 248 'blocksize': self.blocksize, 249 } 250 251 self.cmd = StatusRemoteCommand('uploadFile', args) 252 d = self.runCommand(self.cmd) 253 d.addCallback(self.finished).addErrback(self.failed)

254 255

256 -class DirectoryUpload(BuildStep):

257 """ 258 Build step to transfer a directory from the slave to the master. 259 260 arguments: 261 262 - ['slavesrc'] name of source directory at slave, relative to workdir 263 - ['masterdest'] name of destination directory at master 264 - ['workdir'] string with slave working directory relative to builder 265 base dir, default 'build' 266 - ['maxsize'] maximum size of each file, default None (=unlimited) 267 - ['blocksize'] maximum size of each block being transfered 268 - ['compress'] compression type to use: one of [None, 'gz', 'bz2'] 269 - ['mode'] file access mode for the resulting master-side file. 270 The default (=None) is to leave it up to the umask of 271 the buildmaster process. 272 273 """ 274 275 name = 'upload' 276

277 - def __init__(self, slavesrc, masterdest, 278 workdir="build", maxsize=None, blocksize=16*1024, mode=None, 279 compress=None, **buildstep_kwargs):

280 BuildStep.__init__(self, **buildstep_kwargs) 281 self.addFactoryArguments(slavesrc=slavesrc, 282 masterdest=masterdest, 283 workdir=workdir, 284 maxsize=maxsize, 285 blocksize=blocksize, 286 compress=compress, 287 mode=mode, 288 ) 289 290 self.slavesrc = slavesrc 291 self.masterdest = masterdest 292 self.workdir = workdir 293 self.maxsize = maxsize 294 self.blocksize = blocksize 295 assert compress in (None, 'gz', 'bz2') 296 self.compress = compress 297 assert isinstance(mode, (int, type(None))) 298 self.mode = mode

299

300 - def start(self):

301 version = self.slaveVersion("uploadDirectory") 302 properties = self.build.getProperties() 303 304 if not version: 305 m = "slave is too old, does not know about uploadDirectory" 306 raise BuildSlaveTooOldError(m) 307 308 source = properties.render(self.slavesrc) 309 masterdest = properties.render(self.masterdest) 310 # we rely upon the fact that the buildmaster runs chdir'ed into its 311 # basedir to make sure that relative paths in masterdest are expanded 312 # properly. TODO: maybe pass the master's basedir all the way down 313 # into the BuildStep so we can do this better. 314 masterdest = os.path.expanduser(masterdest) 315 log.msg("DirectoryUpload started, from slave %r to master %r" 316 % (source, masterdest)) 317 318 self.step_status.setText(['uploading', os.path.basename(source)]) 319 320 # we use maxsize to limit the amount of data on both sides 321 dirWriter = _DirectoryWriter(masterdest, self.maxsize, self.compress, self.mode) 322 323 # default arguments 324 args = { 325 'slavesrc': source, 326 'workdir': self.workdir, 327 'writer': dirWriter, 328 'maxsize': self.maxsize, 329 'blocksize': self.blocksize, 330 'compress': self.compress 331 } 332 333 self.cmd = StatusRemoteCommand('uploadDirectory', args) 334 d = self.runCommand(self.cmd) 335 d.addCallback(self.finished).addErrback(self.failed)

336

337 - def finished(self, result):

338 # Subclasses may choose to skip a transfer. In those cases, self.cmd 339 # will be None, and we should just let BuildStep.finished() handle 340 # the rest 341 if result == SKIPPED: 342 return BuildStep.finished(self, SKIPPED) 343 if self.cmd.stderr != '': 344 self.addCompleteLog('stderr', self.cmd.stderr) 345 346 if self.cmd.rc is None or self.cmd.rc == 0: 347 return BuildStep.finished(self, SUCCESS) 348 return BuildStep.finished(self, FAILURE)

349 350 351 352

353 -class _FileReader(pb.Referenceable):

354 """ 355 Helper class that acts as a file-object with read access 356 """ 357

358 - def __init__(self, fp):

359 self.fp = fp

360

361 - def remote_read(self, maxlength):

362 """ 363 Called from remote slave to read at most L{maxlength} bytes of data 364 365 @type maxlength: C{integer} 366 @param maxlength: Maximum number of data bytes that can be returned 367 368 @return: Data read from L{fp} 369 @rtype: C{string} of bytes read from file 370 """ 371 if self.fp is None: 372 return '' 373 374 data = self.fp.read(maxlength) 375 return data

376

377 - def remote_close(self):

378 """ 379 Called by remote slave to state that no more data will be transfered 380 """ 381 if self.fp is not None: 382 self.fp.close() 383 self.fp = None

384 385

386 -class FileDownload(_TransferBuildStep):

387 """ 388 Download the first 'maxsize' bytes of a file, from the buildmaster to the 389 buildslave. Set the mode of the file 390 391 Arguments:: 392 393 ['mastersrc'] filename of source file at master 394 ['slavedest'] filename of destination file at slave 395 ['workdir'] string with slave working directory relative to builder 396 base dir, default 'build' 397 ['maxsize'] maximum size of the file, default None (=unlimited) 398 ['blocksize'] maximum size of each block being transfered 399 ['mode'] use this to set the access permissions of the resulting 400 buildslave-side file. This is traditionally an octal 401 integer, like 0644 to be world-readable (but not 402 world-writable), or 0600 to only be readable by 403 the buildslave account, or 0755 to be world-executable. 404 The default (=None) is to leave it up to the umask of 405 the buildslave process. 406 407 """ 408 name = 'download' 409

410 - def __init__(self, mastersrc, slavedest, 411 workdir=None, maxsize=None, blocksize=16*1024, mode=None, 412 **buildstep_kwargs):

413 BuildStep.__init__(self, **buildstep_kwargs) 414 self.addFactoryArguments(mastersrc=mastersrc, 415 slavedest=slavedest, 416 workdir=workdir, 417 maxsize=maxsize, 418 blocksize=blocksize, 419 mode=mode, 420 ) 421 422 self.mastersrc = mastersrc 423 self.slavedest = slavedest 424 self.workdir = workdir 425 self.maxsize = maxsize 426 self.blocksize = blocksize 427 assert isinstance(mode, (int, type(None))) 428 self.mode = mode

429

430 - def start(self):

431 properties = self.build.getProperties() 432 433 version = self.slaveVersion("downloadFile") 434 if not version: 435 m = "slave is too old, does not know about downloadFile" 436 raise BuildSlaveTooOldError(m) 437 438 # we are currently in the buildmaster's basedir, so any non-absolute 439 # paths will be interpreted relative to that 440 source = os.path.expanduser(properties.render(self.mastersrc)) 441 slavedest = properties.render(self.slavedest) 442 log.msg("FileDownload started, from master %r to slave %r" % 443 (source, slavedest)) 444 445 self.step_status.setText(['downloading', "to", 446 os.path.basename(slavedest)]) 447 448 # setup structures for reading the file 449 try: 450 fp = open(source, 'rb') 451 except IOError: 452 # if file does not exist, bail out with an error 453 self.addCompleteLog('stderr', 454 'File %r not available at master' % source) 455 # TODO: once BuildStep.start() gets rewritten to use 456 # maybeDeferred, just re-raise the exception here. 457 reactor.callLater(0, BuildStep.finished, self, FAILURE) 458 return 459 fileReader = _FileReader(fp) 460 461 # default arguments 462 args = { 463 'slavedest': slavedest, 464 'maxsize': self.maxsize, 465 'reader': fileReader, 466 'blocksize': self.blocksize, 467 'workdir': self._getWorkdir(), 468 'mode': self.mode, 469 } 470 471 self.cmd = StatusRemoteCommand('downloadFile', args) 472 d = self.runCommand(self.cmd) 473 d.addCallback(self.finished).addErrback(self.failed)

474

Source Code for Module buildbot.steps.transfer