build/cygtar.py

#!/usr/bin/python
# Copyright (c) 2012 The Native Client Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import glob
import hashlib
import optparse
import os
import posixpath
import shutil
import subprocess
import stat
import sys
import tarfile

"""A Cygwin aware version compress/extract object.

This module supports creating and unpacking a tarfile on all platforms.  For
Cygwin, Mac, and Linux, it will use the standard tarfile implementation.  For
Win32 it will detect Cygwin style symlinks as it archives and convert them to
symlinks.

For Win32, it is unfortunate that os.stat does not return a FileID in the ino
field which would allow us to correctly determine which files are hardlinks, so
instead we assume that any files in the archive that are an exact match are
hardlinks to the same data.

We know they are not Symlinks because we are using Cygwin style symlinks only,
which appear to Win32 a normal file.

All paths stored and retrieved from a TAR file are expected to be POSIX style,
Win32 style paths will be rejected.

NOTE:
  All paths represent by the tarfile and all API functions are POSIX style paths
  except for CygTar.Add which assumes a Native path.
"""


def ToNativePath(native_path):
  """Convert to a posix style path if this is win32."""
  if sys.platform == 'win32':
    return native_path.replace('/', '\\')
  return native_path


def IsCygwinSymlink(symtext):
  """Return true if the provided text looks like a Cygwin symlink."""
  return symtext[:12] == '!<symlink>\xff\xfe'


def SymDatToPath(symtext):
  """Convert a Cygwin style symlink data to a relative path."""
  return ''.join([ch for ch in symtext[12:] if ch != '\x00'])


def PathToSymDat(filepath):
  """Convert a filepath to cygwin style symlink data."""
  symtag = '!<symlink>\xff\xfe'
  unipath = ''.join([ch + '\x00' for ch in filepath])
  strterm = '\x00\x00'
  return symtag + unipath + strterm


def CreateWin32Hardlink(filepath, targpath, try_mklink, verbose):
  """Create a hardlink on Win32 if possible

  Uses mklink to create a hardlink if possible.  On failure, it will
  assume mklink is unavailible and copy the file instead, returning False
  to indicate future calls should not attempt to use mklink."""

  # Assume an error, if subprocess succeeds, then it should return 0
  err = 1
  if try_mklink:
    dst_src = ToNativePath(filepath) + ' ' + ToNativePath(targpath)
    try:
      err = subprocess.call(['cmd', '/C', 'mklink /H ' + dst_src],
                            stdout = open(os.devnull, 'wb'),
                            stderr = open(os.devnull, 'wb'))
    except EnvironmentError:
      try_mklink = False

  # If we failed to create a hardlink, then just copy it.  We wrap this in a
  # retry for Windows which often has stale file lock issues.
  if err or not os.path.isfile(filepath):
    for cnt in range(1,4):
      try:
        shutil.copyfile(targpath, filepath)
        return False
      except EnvironmentError:
        if verbose:
          print 'Try %d: Failed hardlink %s -> %s\n' % (cnt, filepath, targpath)
    if verbose:
      print 'Giving up.'
  return try_mklink


def ComputeFileHash(filepath):
  """Generate a sha1 hash for the file at the given path."""
  sha1 = hashlib.sha1()
  with open(filepath, 'rb') as fp:
    sha1.update(fp.read())
  return sha1.hexdigest()


def ReadableSizeOf(num):
  """Convert to a human readable number."""
  if num < 1024.0:
    return '[%5dB]' % num
  for x in ['B','K','M','G','T']:
     if num < 1024.0:
       return '[%5.1f%s]' % (num, x)
     num /= 1024.0
  return '[%dT]' % int(num)


class CygTar(object):
  """ CygTar is an object which represents a Win32 and Cygwin aware tarball."""
  def __init__(self, filename, mode='r', verbose=False):
    self.size_map = {}
    self.file_hashes = {}
    # Set errorlevel=1 so that fatal errors actually raise!
    if 'r' in mode:
      self.read_file = open(filename, 'rb')
      self.read_filesize = os.path.getsize(filename)
      self.tar = tarfile.open(mode=mode, fileobj=self.read_file, errorlevel=1)
    else:
      self.read_file = None
      self.read_filesize = 0
      self.tar = tarfile.open(filename, mode=mode, errorlevel=1)
    self.verbose = verbose

  def __DumpInfo(self, tarinfo):
    """Prints information on a single object in the tarball."""
    typeinfo = '?'
    lnk = ''
    if tarinfo.issym():
      typeinfo = 'S'
      lnk = '-> ' + tarinfo.linkname
    if tarinfo.islnk():
      typeinfo = 'H'
      lnk = '-> ' + tarinfo.linkname
    if tarinfo.isdir():
      typeinfo = 'D'
    if tarinfo.isfile():
      typeinfo = 'F'
    reable_size = ReadableSizeOf(tarinfo.size)
    print '%s %s : %s %s' % (reable_size, typeinfo, tarinfo.name, lnk)
    return tarinfo

  def __AddFile(self, tarinfo, fileobj=None):
    """Add a file to the archive."""
    if self.verbose:
      self.__DumpInfo(tarinfo)
    self.tar.addfile(tarinfo, fileobj)

  def __AddLink(self, tarinfo, linktype, linkpath):
    """Add a Win32 symlink or hardlink to the archive."""
    tarinfo.linkname = linkpath
    tarinfo.type = linktype
    tarinfo.size = 0
    self.__AddFile(tarinfo)

  def Add(self, filepath, prefix=None):
    """Add path filepath to the archive which may be Native style.

    Add files individually recursing on directories.  For POSIX we use
    tarfile.addfile directly on symlinks and hardlinks.  For files, we
    must check if they are duplicates which we convert to hardlinks
    or Cygwin style symlinks which we convert form a file to a symlink
    in the tarfile.  All other files are added as a standard file.
    """

    # At this point tarinfo.name will contain a POSIX style path regardless
    # of the original filepath.
    tarinfo = self.tar.gettarinfo(filepath)
    if prefix:
      tarinfo.name = posixpath.join(prefix, tarinfo.name)

    if sys.platform == 'win32':
      # On win32 os.stat() always claims that files are world writable
      # which means that unless we remove this bit here we end up with
      # world writables files in the archive, which is almost certainly
      # not intended.
      tarinfo.mode &= ~stat.S_IWOTH
      tarinfo.mode &= ~stat.S_IWGRP

      # If we want cygwin to be able to extract this archive and use
      # executables and dll files we need to mark all the archive members as
      # executable.  This is essentially what happens anyway when the
      # archive is extracted on win32.
      tarinfo.mode |= stat.S_IXUSR | stat.S_IXOTH | stat.S_IXGRP

    # If this a symlink or hardlink, add it
    if tarinfo.issym() or tarinfo.islnk():
      tarinfo.size = 0
      self.__AddFile(tarinfo)
      return True

    # If it's a directory, then you want to recurse into it
    if tarinfo.isdir():
      self.__AddFile(tarinfo)
      native_files = glob.glob(os.path.join(filepath, '*'))
      for native_file in native_files:
        if not self.Add(native_file, prefix): return False
      return True

    # At this point we only allow addition of "FILES"
    if not tarinfo.isfile():
      print 'Failed to add non real file: %s' % filepath
      return False

    # Now check if it is a Cygwin style link disguised as a file.
    # We go ahead and check on all platforms just in case we are tar'ing a
    # mount shared with windows.
    if tarinfo.size <= 524:
      with open(filepath) as fp:
        symtext = fp.read()
      if IsCygwinSymlink(symtext):
        self.__AddLink(tarinfo, tarfile.SYMTYPE, SymDatToPath(symtext))
        return True

    # Otherwise, check if its a hardlink by seeing if it matches any unique
    # hash within the list of hashed files for that file size.
    nodelist = self.size_map.get(tarinfo.size, [])

    # If that size bucket is empty, add this file, no need to get the hash until
    # we get a bucket collision for the first time..
    if not nodelist:
      self.size_map[tarinfo.size] = [filepath]
      with open(filepath, 'rb') as fp:
        self.__AddFile(tarinfo, fp)
      return True

    # If the size collides with anything, we'll need to check hashes.  We assume
    # no hash collisions for SHA1 on a given bucket, since the number of files
    # in a bucket over possible SHA1 values is near zero.
    newhash = ComputeFileHash(filepath)
    self.file_hashes[filepath] = newhash

    for oldname in nodelist:
      oldhash = self.file_hashes.get(oldname, None)
      if not oldhash:
        oldhash = ComputeFileHash(oldname)
        self.file_hashes[oldname] = oldhash

      if oldhash == newhash:
        self.__AddLink(tarinfo, tarfile.LNKTYPE, oldname)
        return True

    # Otherwise, we missed, so add it to the bucket for this size
    self.size_map[tarinfo.size].append(filepath)
    with open(filepath, 'rb') as fp:
      self.__AddFile(tarinfo, fp)
    return True

  def Extract(self):
    """Extract the tarfile to the current directory."""
    try_mklink = True

    if self.verbose:
      sys.stdout.write('|' + ('-' * 48) + '|\n')
      sys.stdout.flush()
      dots_outputted = 0

    win32_symlinks = {}
    for m in self.tar:
      if self.verbose:
        cnt = self.read_file.tell()
        curdots = cnt * 50 / self.read_filesize
        if dots_outputted < curdots:
          for dot in xrange(dots_outputted, curdots):
            sys.stdout.write('.')
          sys.stdout.flush()
          dots_outputted = curdots

      # For hardlinks in Windows, we try to use mklink, and instead copy on
      # failure.
      if m.islnk() and sys.platform == 'win32':
        try_mklink = CreateWin32Hardlink(
            m.name, m.linkname, try_mklink, self.verbose)
      # On Windows we treat symlinks as if they were hard links.
      # Proper Windows symlinks supported by everything can be made with
      # mklink, but only by an Administrator.  The older toolchains are
      # built with Cygwin, so they could use Cygwin-style symlinks; but
      # newer toolchains do not use Cygwin, and nothing else on the system
      # understands Cygwin-style symlinks, so avoid them.
      elif m.issym() and sys.platform == 'win32':
        # For a hard link, the link target (m.linkname) always appears
        # in the archive before the link itself (m.name), so the links
        # can just be made on the fly.  However, a symlink might well
        # appear in the archive before its target file, so there would
        # not yet be any file to hard-link to.  Hence, we have to collect
        # all the symlinks and create them in dependency order at the end.
        linkname = m.linkname
        if not posixpath.isabs(linkname):
          linkname = posixpath.join(posixpath.dirname(m.name), linkname)
        linkname = posixpath.normpath(linkname)
        win32_symlinks[posixpath.normpath(m.name)] = linkname
      # Otherwise, extract normally.
      else:
        self.tar.extract(m)

    win32_symlinks_left = win32_symlinks.items()
    while win32_symlinks_left:
      this_symlink = win32_symlinks_left.pop(0)
      name, linkname = this_symlink
      if linkname in win32_symlinks:
        # The target is itself a symlink not yet created.
        # Wait for it to come 'round on the guitar.
        win32_symlinks_left.append(this_symlink)
      else:
        del win32_symlinks[name]
        try_mklink = CreateWin32Hardlink(
            name, linkname, try_mklink, self.verbose)

    if self.verbose:
      sys.stdout.write('\n')
      sys.stdout.flush()

  def List(self):
    """List the set of objects in the tarball."""
    for tarinfo in self.tar:
      self.__DumpInfo(tarinfo)

  def Close(self):
    self.tar.close()
    if self.read_file is not None:
      self.read_file.close()
      self.read_file = None
      self.read_filesize = 0


def Main(args):
  parser = optparse.OptionParser()
  # Modes
  parser.add_option('-c', '--create', help='Create a tarball.',
      action='store_const', const='c', dest='action', default='')
  parser.add_option('-x', '--extract', help='Extract a tarball.',
      action='store_const', const='x', dest='action')
  parser.add_option('-t', '--list', help='List sources in tarball.',
      action='store_const', const='t', dest='action')

  # Compression formats
  parser.add_option('-j', '--bzip2', help='Create a bz2 tarball.',
      action='store_const', const=':bz2', dest='format', default='')
  parser.add_option('-z', '--gzip', help='Create a gzip tarball.',
      action='store_const', const=':gz', dest='format', )
  # Misc
  parser.add_option('-v', '--verbose', help='Use verbose output.',
      action='store_true', dest='verbose', default=False)
  parser.add_option('-f', '--file', help='Name of tarball.',
      dest='filename', default='')
  parser.add_option('-C', '--directory', help='Change directory.',
      dest='cd', default='')
  parser.add_option('--prefix', help='Subdirectory prefix for all paths')

  options, args = parser.parse_args(args[1:])
  if not options.action:
    parser.error('Expecting compress or extract')
  if not options.filename:
    parser.error('Expecting a filename')

  if options.action in ['c'] and not args:
    parser.error('Expecting list of sources to add')
  if options.action in ['x', 't'] and args:
    parser.error('Unexpected source list on extract')

  if options.action == 'c':
    mode = 'w' + options.format
  else:
    mode = 'r'+ options.format

  tar = CygTar(options.filename, mode, verbose=options.verbose)
  if options.cd:
    os.chdir(options.cd)

  if options.action == 't':
    tar.List()
    return 0

  if options.action == 'x':
    tar.Extract()
    return 0

  if options.action == 'c':
    for filepath in args:
      if not tar.Add(filepath, options.prefix):
        return -1
    tar.Close()
    return 0

  parser.error('Missing action c, t, or x.')
  return -1


if __name__ == '__main__':
  sys.exit(Main(sys.argv))