source: trunk/moap/vcs/vcs.py @ 403

# -*- Mode: Python -*-
# vi:si:et:sw=4:sts=4:ts=4

"""
Version Control System functionality.
"""

import re
import os
import sys
import glob
import tarfile
import tempfile
import commands

from moap.util import util, log

def getNames():
    """
    Returns a sorted list of VCS names that moap can work with.
    """
    moduleNames = util.getPackageModules('moap.vcs', ignore=['vcs', ])
    modules = [util.namedModule('moap.vcs.%s' % s) for s in moduleNames]
    names = [m.VCSClass.name for m in modules]
    names.sort()
    return names

def detect(path=None):
    """
    Detect which version control system is being used in the source tree.

    @returns: an instance of a subclass of L{VCS}, or None.
    """
    log.debug('vcs', 'detecting VCS in %s' % path)
    if not path:
        path = os.getcwd()
    systems = util.getPackageModules('moap.vcs', ignore=['vcs', ])
    log.debug('vcs', 'trying vcs modules %r' % systems)

    for s in systems:
        m = util.namedModule('moap.vcs.%s' % s)

        try:
            ret = m.detect(path)
        except AttributeError:
                sys.stderr.write('moap.vcs.%s is missing detect()\n' % s)
                continue

        if ret:
            try:
                o = m.VCSClass(path)
            except AttributeError:
                sys.stderr.write('moap.vcs.%s is missing VCSClass()\n' % s)
                continue

            log.debug('vcs', 'detected VCS %s' % s)

            return o
        log.debug('vcs', 'did not find %s' % s)

    return None
    
# FIXME: add stdout and stderr, so all spawned commands output there instead
class VCS(log.Loggable):
    """
    @ivar path: the path to the top of the source tree
    @ivar meta: paths that contain VCS metadata
    @type meta: list of str
    """
    name = 'Some Version Control System'
    logCategory = 'VCS'

    path = None
    meta = None

    def __init__(self, path=None):
        self.path = path
        if not path:
            self.path = os.getcwd()

    def getAdded(self, path):
        """
        Get a list of paths newly added under the given path and relative to it.

        @param path: the path under which to check for files
        @type  path: str

        @returns: list of paths
        @rtype:   list of str
        """
        log.info('vcs', 
            "subclass %r should implement getAdded" % self.__class__)

    def getDeleted(self, path):
        """
        Get a list of deleted paths under the given path and relative to it.

        @param path: the path under which to check for files
        @type  path: str

        @returns: list of paths
        @rtype:   list of str
        """
        log.info('vcs', 
            "subclass %r should implement getDeleted" % self.__class__)

    def getIgnored(self, path):
        """
        Get a list of ignored paths under the given path and relative to it.

        @param path: the path under which to check for files
        @type  path: str

        @returns: list of paths
        @rtype:   list of str
        """
        raise NotImplementedError, \
            'subclass %s should implement getIgnored' % self.__class__

    def getUnknown(self, path):
        """
        Get a list of unknown paths under the given path and relative to it.

        @param path: the path under which to check for files
        @type  path: str

        @returns: list of paths
        @rtype:   list of str
        """
        raise NotImplementedError, \
            'subclass %s should implement getUnknown' % self.__class__

    def ignore(self, paths, commit=True):
        """
        Make the VCS ignore the given list of paths.

        @param paths:  list of paths, relative to the checkout directory
        @type  paths:  list of str
        @param commit: if True, commit the ignore updates.
        @type  commit: boolean
        """
        raise NotImplementedError, \
            'subclass %s should implement ignore' % self.__class__

    def commit(self, paths, message):
        """
        Commit the given list of paths, with the given message.
        Note that depending on the VCS, parents that were just added
        may need to be commited as well.

        @type paths:   list
        @type message: str

        @rtype: bool
        """

    def createTree(self, paths):
        """
        Given the list of paths, create a dict of parentPath -> [child, ...]
        If the path is in the root of the repository, parentPath will be ''

        @rtype: dict of str -> list of str
        """
        result = {}

        if not paths:
            return result

        for p in paths:
            # os.path.basename('test/') returns '', so strip possibly trailing /
            if p.endswith(os.path.sep): p = p[:-1]
            base = os.path.basename(p)
            dirname = os.path.dirname(p)
            if not dirname in result.keys():
                result[dirname] = []
            result[dirname].append(base)

        return result

    def diff(self, path):
        """
        Return a diff for the given path.

        The diff should not end in a newline; an empty diff should
        be an empty string.

        The diff should also be relative to the working directory; no
        absolute paths.

        @rtype:   str
        @returns: the diff
        """
        raise NotImplementedError, \
            'subclass %s should implement diff' % self.__class__

    def getFileMatcher(self):
        """
        Return an re matcher object that will expand to the file being
        changed.

        The default implementation works for CVS and SVN.
        """
        return re.compile('^Index: (\S+)$')

    def getChanges(self, path, diff=None):
        """
        Get a list of changes for the given path and subpaths.

        @type  diff: str
        @param diff: the diff to use instead of a local vcs diff
                     (only useful for testing)

        @returns: dict of path -> list of (oldLine, oldCount, newLine, newCount)
        """
        if not diff:
            self.debug('getting changes from diff in %s' % path)
            diff = self.diff(path)

        changes = {}
        fileMatcher = self.getFileMatcher()

        # cvs diff can put a function name after the final @@ pair
        # svn diff on a one-line change in a one-line file looks like this:
        # @@ -1 +1 @@
        changeMatcher = re.compile(
            '^\@\@\s+'         # start of line
            '(-)(\d+),?(\d*)'  # -x,y or -x
            '\s+'
            '(\+)(\d+),?(\d*)'
            '\s+\@\@'          # end of line
        )
        # We rstrip so that we don't end up with a dangling '' line
        lines = diff.rstrip('\n').split("\n")
        self.debug('diff is %d lines' % len(lines))
        for i in range(len(lines)):
            fm = fileMatcher.search(lines[i])
            if fm:
                # found a file being diffed, now get changes
                path = fm.expand('\\1')
                self.debug('Found file %s with deltas on line %d' % (
                    path, i + 1))
                changes[path] = []
                i += 1
                while i < len(lines) and not fileMatcher.search(lines[i]):
                    self.log('Looking at line %d for file match' % (i + 1))
                    m = changeMatcher.search(lines[i])
                    if m:
                        self.debug('Found change on line %d' % (i + 1))
                        oldLine = int(m.expand('\\2'))
                        # oldCount can be missing, which means it's 1
                        c = m.expand('\\3')
                        if not c: c = '1'
                        oldCount = int(c)
                        newLine = int(m.expand('\\5'))
                        c = m.expand('\\6')
                        if not c: c = '1'
                        newCount = int(c)
                        i += 1

                        # the diff has 3 lines of context by default
                        # if a line was added/removed at the beginning or end,
                        # that context is not always there
                        # so we need to parse each non-changeMatcher line
                        block = []
                        while i < len(lines) \
                            and not changeMatcher.search(lines[i]) \
                            and not fileMatcher.search(lines[i]):
                            block.append(lines[i])
                            i += 1

                        # now we have the whole block
                        self.log('Found change block of %d lines at line %d' % (
                            len(block), i - len(block) + 1))

                        for line in block:
                            # starting non-change lines add to Line and
                            # subtract from Count
                            if line[0] == ' ':
                                oldLine += 1
                                newLine += 1
                                oldCount -= 1
                                newCount -= 1
                            else:
                                break

                        block.reverse()
                        for line in block:
                            # trailing non-change lines subtract from Count
                            # line can be empty
                            if line and line[0] == ' ':
                                oldCount -= 1
                                newCount -= 1
                            else:
                                break

                        changes[path].append(
                            (oldLine, oldCount, newLine, newCount))

                        # we're at a change line, so go back
                        i -= 1

                    i += 1

        log.debug('vcs', '%d files changed' % len(changes.keys()))
        return changes

    def getPropertyChanges(self, path):
        """
        Get a list of property changes for the given path and subpaths.
        These are metadata changes to files, not content changes.

        @rtype: dict of str -> list of str
        @returns: dict of path -> list of property names
        """
        log.info('vcs', 
            "subclass %r should implement getPropertyChanges" % self.__class__)
 
    def update(self, path):
        """
        Update the given path to the latest version.
        """
        raise NotImplementedError, \
            'subclass %s should implement update' % self.__class__

    def getCheckoutCommands(self):
        """
        Return shell commands necessary to do a fresh checkout of the current
        checkout into a directory called 'checkout'.

        @returns: newline-terminated string of commands.
        @rtype:   str
        """
        raise NotImplementedError, \
            'subclass %s should implement getCheckoutCommands' % self.__class__

    def backup(self, archive):
        """
        Back up the given VCS checkout into an archive.

        This stores all unignored files, as well as a checkout command and
        a diff, so the working directory can be fully restored.

        The archive will contain:
         - a subdirectory called unignored
         - a file called diff
         - an executable file called checkout.sh

        @raises VCSBackupException: if for some reason it can't guarantee
                                    a correct backup
        """
        mode = 'w:'
        suffix = '.tar'
        if archive.endswith('.gz'):
            mode = 'w:gz'
            suffix = '.tar.gz'
        if archive.endswith('.bz2'):
            mode = 'w:bw2'
            suffix = '.tar.bz2'

        # P2.4
        # Pre-2.5, tarfile has a bug, creating hardlinks for temporary files
        # if the temporary files get deleted right after adding.
        # See http://mail.python.org/pipermail/python-bugs-list/2005-October/030793.html
        # the workaround chosen is to keep the temporary files until after
        # closing
        tar = tarfile.TarFile.open(name=archive, mode=mode)

        # store the diff
        (fd, diffpath) = tempfile.mkstemp(prefix='moap.backup.diff.')
        diff = self.diff('')
        if diff:
            os.write(fd, diff + '\n')
        os.close(fd)
        tar.add(diffpath, arcname='diff')

        # store the checkout commands
        (fd, checkoutpath) = tempfile.mkstemp(prefix='moap.backup.checkout.')
        os.write(fd, "#!/bin/sh\n" + self.getCheckoutCommands())
        os.close(fd)
        os.chmod(checkoutpath, 0755)
        tar.add(checkoutpath, arcname='checkout.sh')

        # store the unignored files
        tar.add(self.path, 'unignored', recursive=False)

        unignoreds = self.getUnknown(self.path)

        for rel in unignoreds:
            abspath = os.path.join(self.path, rel)
            self.debug('Adding unignored path %s', rel)
            tar.add(abspath, 'unignored/' + rel)
        tar.close()
        os.unlink(diffpath)
        os.unlink(checkoutpath)

        # now verify the backup
        restoreDir = tempfile.mkdtemp(prefix="moap.test.restore.")
        os.rmdir(restoreDir)
        self.restore(archive, restoreDir)

        diff = self.diffCheckout(restoreDir)
        if diff:
            msg = "Unexpected diff output between %s and %s:\n%s" % (
                self.path, restoreDir, diff)
            self.debug(msg)
            raise VCSBackupException(msg)
        else:
            self.debug('No important difference between '
                'extracted archive and original directory')

        os.system('rm -rf %s' % restoreDir)
        
    def restore(self, archive, path):
        """
        Restore from the given archive to the given path.
        """
        self.debug('Restoring from archive %s to path %s' % (
            archive, path))

        if os.path.exists(path):
            raise VCSException('path %s already exists')

        oldPath = os.getcwd()

        # P2.3: tarfile.extractall only exists since 2.5
        tar = tarfile.TarFile.open(name=archive)
        try:
            tar.extractall(path)
        except AttributeError:
            # do it the shell way
            self.debug('Restoring by using tar directly')
            os.system('mkdir -p %s' % path)
            if archive.endswith('.gz'):
                os.system('cd %s; tar xzf %s' % (path, archive))
            elif archive.endswith('.bz2'):
                os.system('cd %s; tar xjf %s' % (path, archive))
            else:
                raise AssertionError("Don't know how to handle %s" % archive)

        # start with the checkout
        os.chdir(path)
        status, output = commands.getstatusoutput('./checkout.sh')
        if status:
            raise VCSException('checkout failed with status %r: %r' % (
                status, output))
        os.unlink('checkout.sh')

        # apply the diff
        os.chdir('checkout')
        # FIXME: check errors ?
        os.system('patch -p0 < ../diff > /dev/null')
        os.chdir('..')
        os.unlink('diff')

        # move to parent directory
        # FIXME: make sure we handle . directories (like .svn)
        for path in glob.glob('checkout/*') + glob.glob('checkout/.*'):
            os.rename(path, os.path.basename(path))
        os.rmdir('checkout')

        # move all unignored files to parent directory
        # FIXME: make sure we handle . directories (like .svn)
        for path in glob.glob('unignored/*') + glob.glob('unignored/.*'):
            # there is no good equivalent to mv;
            # os.rename doesn't work on trees
            # shutil.move actually invokes copytree
            cmd = 'mv %s %s' % (path, os.path.basename(path))
            self.debug(cmd)
            os.system(cmd)
        os.rmdir('unignored')

        os.chdir(oldPath)

    def diffCheckout(self, checkoutDir):
        """
        Diff our checkout to the given checkout directory.

        Only complains about diffs in files we're interested in, which are
        tracked or unignored files.
        """
        options = ""
        if self.meta:
            metaPattern = [s.replace('.', '\.') for s in self.meta]
            options = "-x ".join([''] + metaPattern)
        cmd = 'diff -aur %s %s %s 2>&1' % (options, self.path, checkoutDir)
        self.debug('diffCheckout: running %s' % cmd)
        output = commands.getoutput(cmd)
        lines = output.split('\n')

        # we can't use diff -x for excluding ignored files, because -x
        # takes a pattern for basename only, so we need to scrub output
        d = {}
        for path in self.getIgnored(self.path):
            d[path] = True

        matcher = re.compile('Only in (.*): (.*)$')

        def isIgnored(line):
            # filter out lines for ignored files
            m = matcher.search(line)
            if m:
                path = os.path.join(m.expand("\\1"), m.expand("\\2"))
                if path in d.keys():
                    self.debug('Removing ignored path %s from diff' % path)
                    return True

            return False

        lines = [l for l in lines if isIgnored(l)]

        return "\n".join(lines)

class VCSException(Exception):
    """
    Generic exception for a failed VCS operation.
    """
    pass

class VCSBackupException(VCSException):
    'The VCS cannot back up the working directory.'