Google Git
Sign in
chromium / chromium / src / c17fbc86ef53cb21dc03cfab09d12be3e1cb1586 / . / tools / clang / scripts / update.py
blob: aa8941754aecdfe777cfe72ffb5e59e0cef87ee7 [file] [log] [blame]
#!/usr/bin/env python
# Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Windows can't run .sh files, so this is a Python implementation of
update.sh. This script should replace update.sh on all platforms eventually."""
import argparse
import contextlib
import cStringIO
import glob
import os
import pipes
import re
import shutil
import subprocess
import stat
import sys
import tarfile
import time
import urllib2
import zipfile
# Do NOT CHANGE this if you don't know what you're doing -- see
# https://code.google.com/p/chromium/wiki/UpdatingClang
# Reverting problematic clang rolls is safe, though.
# Note: this revision is only used for Windows. Other platforms use update.sh.
# TODO(thakis): Use the same revision on Windows and non-Windows.
# TODO(thakis): Remove update.sh, use update.py everywhere.
LLVM_WIN_REVISION = '239674'
use_head_revision = 'LLVM_FORCE_HEAD_REVISION' in os.environ
if use_head_revision:
LLVM_WIN_REVISION = 'HEAD'
# This is incremented when pushing a new build of Clang at the same revision.
CLANG_SUB_REVISION=1
PACKAGE_VERSION = "%s-%s" % (LLVM_WIN_REVISION, CLANG_SUB_REVISION)
# Path constants. (All of these should be absolute paths.)
THIS_DIR = os.path.abspath(os.path.dirname(__file__))
CHROMIUM_DIR = os.path.abspath(os.path.join(THIS_DIR, '..', '..', '..'))
THIRD_PARTY_DIR = os.path.join(CHROMIUM_DIR, 'third_party')
LLVM_DIR = os.path.join(THIRD_PARTY_DIR, 'llvm')
LLVM_BOOTSTRAP_DIR = os.path.join(THIRD_PARTY_DIR, 'llvm-bootstrap')
LLVM_BOOTSTRAP_INSTALL_DIR = os.path.join(THIRD_PARTY_DIR,
'llvm-bootstrap-install')
CHROME_TOOLS_SHIM_DIR = os.path.join(LLVM_DIR, 'tools', 'chrometools')
LLVM_BUILD_DIR = os.path.join(CHROMIUM_DIR, 'third_party', 'llvm-build',
'Release+Asserts')
COMPILER_RT_BUILD_DIR = os.path.join(LLVM_BUILD_DIR, '32bit-compiler-rt')
CLANG_DIR = os.path.join(LLVM_DIR, 'tools', 'clang')
LLD_DIR = os.path.join(LLVM_DIR, 'tools', 'lld')
COMPILER_RT_DIR = os.path.join(LLVM_DIR, 'projects', 'compiler-rt')
LIBCXX_DIR = os.path.join(LLVM_DIR, 'projects', 'libcxx')
LIBCXXABI_DIR = os.path.join(LLVM_DIR, 'projects', 'libcxxabi')
LLVM_BUILD_TOOLS_DIR = os.path.abspath(
os.path.join(LLVM_DIR, '..', 'llvm-build-tools'))
STAMP_FILE = os.path.join(LLVM_DIR, '..', 'llvm-build', 'cr_build_revision')
BINUTILS_DIR = os.path.join(THIRD_PARTY_DIR, 'binutils')
VERSION = '3.7.0'
# URL for pre-built binaries.
CDS_URL = 'https://commondatastorage.googleapis.com/chromium-browser-clang'
LLVM_REPO_URL='https://llvm.org/svn/llvm-project'
if 'LLVM_REPO_URL' in os.environ:
LLVM_REPO_URL = os.environ['LLVM_REPO_URL']
def DownloadUrl(url, output_file):
"""Download url into output_file."""
CHUNK_SIZE = 4096
TOTAL_DOTS = 10
sys.stdout.write('Downloading %s ' % url)
sys.stdout.flush()
response = urllib2.urlopen(url)
total_size = int(response.info().getheader('Content-Length').strip())
bytes_done = 0
dots_printed = 0
while True:
chunk = response.read(CHUNK_SIZE)
if not chunk:
break
output_file.write(chunk)
bytes_done += len(chunk)
num_dots = TOTAL_DOTS * bytes_done / total_size
sys.stdout.write('.' * (num_dots - dots_printed))
sys.stdout.flush()
dots_printed = num_dots
print ' Done.'
def ReadStampFile():
"""Return the contents of the stamp file, or '' if it doesn't exist."""
try:
with open(STAMP_FILE, 'r') as f:
return f.read()
except IOError:
return ''
def WriteStampFile(s):
"""Write s to the stamp file."""
if not os.path.exists(os.path.dirname(STAMP_FILE)):
os.makedirs(os.path.dirname(STAMP_FILE))
with open(STAMP_FILE, 'w') as f:
f.write(s)
def GetSvnRevision(svn_repo):
"""Returns current revision of the svn repo at svn_repo."""
svn_info = subprocess.check_output('svn info ' + svn_repo, shell=True)
m = re.search(r'Revision: (\d+)', svn_info)
return m.group(1)
def RmTree(dir):
"""Delete dir."""
def ChmodAndRetry(func, path, _):
# Subversion can leave read-only files around.
if not os.access(path, os.W_OK):
os.chmod(path, stat.S_IWUSR)
return func(path)
raise
shutil.rmtree(dir, onerror=ChmodAndRetry)
def RunCommand(command, msvc_arch=None, env=None, fail_hard=True):
"""Run command and return success (True) or failure; or if fail_hard is
True, exit on failure. If msvc_arch is set, runs the command in a
shell with the msvc tools for that architecture."""
if msvc_arch and sys.platform == 'win32':
command = GetVSVersion().SetupScript(msvc_arch) + ['&&'] + command
# https://docs.python.org/2/library/subprocess.html:
# "On Unix with shell=True [...] if args is a sequence, the first item
# specifies the command string, and any additional items will be treated as
# additional arguments to the shell itself. That is to say, Popen does the
# equivalent of:
# Popen(['/bin/sh', '-c', args[0], args[1], ...])"
#
# We want to pass additional arguments to command[0], not to the shell,
# so manually join everything into a single string.
# Annoyingly, for "svn co url c:\path", pipes.quote() thinks that it should
# quote c:\path but svn can't handle quoted paths on Windows. Since on
# Windows follow-on args are passed to args[0] instead of the shell, don't
# do the single-string transformation there.
if sys.platform != 'win32':
command = ' '.join([pipes.quote(c) for c in command])
print 'Running', command
if subprocess.call(command, env=env, shell=True) == 0:
return True
print 'Failed.'
if fail_hard:
sys.exit(1)
return False
def CopyFile(src, dst):
"""Copy a file from src to dst."""
shutil.copy(src, dst)
print "Copying %s to %s" % (src, dst)
def CopyDirectoryContents(src, dst, filename_filter=None):
"""Copy the files from directory src to dst
with an optional filename filter."""
if not os.path.exists(dst):
os.makedirs(dst)
for root, _, files in os.walk(src):
for f in files:
if filename_filter and not re.match(filename_filter, f):
continue
CopyFile(os.path.join(root, f), dst)
def Checkout(name, url, dir):
"""Checkout the SVN module at url into dir. Use name for the log message."""
print "Checking out %s r%s into '%s'" % (name, LLVM_WIN_REVISION, dir)
command = ['svn', 'checkout', '--force', url + '@' + LLVM_WIN_REVISION, dir]
if RunCommand(command, fail_hard=False):
return
if os.path.isdir(dir):
print "Removing %s." % (dir)
RmTree(dir)
print "Retrying."
RunCommand(command)
def RevertPreviouslyPatchedFiles():
print 'Reverting previously patched files'
files = [
'%(clang)s/test/Index/crash-recovery-modules.m',
'%(clang)s/unittests/libclang/LibclangTest.cpp',
'%(compiler_rt)s/lib/asan/asan_rtl.cc',
'%(compiler_rt)s/test/asan/TestCases/Linux/new_array_cookie_test.cc',
'%(llvm)s/test/DebugInfo/gmlt.ll',
'%(llvm)s/lib/CodeGen/SpillPlacement.cpp',
'%(llvm)s/lib/CodeGen/SpillPlacement.h',
'%(llvm)s/lib/Transforms/Instrumentation/MemorySanitizer.cpp',
'%(clang)s/test/Driver/env.c',
'%(clang)s/lib/Frontend/InitPreprocessor.cpp',
'%(clang)s/test/Frontend/exceptions.c',
'%(clang)s/test/Preprocessor/predefined-exceptions.m',
'%(llvm)s/test/Bindings/Go/go.test',
'%(clang)s/lib/Parse/ParseExpr.cpp',
'%(clang)s/lib/Parse/ParseTemplate.cpp',
'%(clang)s/lib/Sema/SemaDeclCXX.cpp',
'%(clang)s/lib/Sema/SemaExprCXX.cpp',
'%(clang)s/test/SemaCXX/default2.cpp',
'%(clang)s/test/SemaCXX/typo-correction-delayed.cpp',
'%(compiler_rt)s/lib/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cc',
'%(compiler_rt)s/test/tsan/signal_segv_handler.cc',
'%(compiler_rt)s/lib/sanitizer_common/sanitizer_coverage_libcdep.cc',
'%(compiler_rt)s/cmake/config-ix.cmake',
'%(compiler_rt)s/CMakeLists.txt',
'%(compiler_rt)s/lib/ubsan/ubsan_platform.h',
]
for f in files:
f = f % {
'clang': CLANG_DIR,
'compiler_rt': COMPILER_RT_DIR,
'llvm': LLVM_DIR,
}
if os.path.exists(f):
os.remove(f) # For unversioned files.
RunCommand(['svn', 'revert', f])
def ApplyLocalPatches():
# There's no patch program on Windows by default. We don't need patches on
# Windows yet, and maybe this not working on Windows will motivate us to
# remove patches over time.
assert sys.platform != 'win32'
# Apply patch for tests failing with --disable-pthreads (llvm.org/PR11974)
clang_patches = [ r"""\
--- test/Index/crash-recovery-modules.m (revision 202554)
+++ test/Index/crash-recovery-modules.m (working copy)
@@ -12,6 +12,8 @@
// REQUIRES: crash-recovery
// REQUIRES: shell
+// XFAIL: *
+// (PR11974)
@import Crash;
""", r"""\
--- unittests/libclang/LibclangTest.cpp (revision 215949)
+++ unittests/libclang/LibclangTest.cpp (working copy)
@@ -431,7 +431,7 @@
EXPECT_EQ(0U, clang_getNumDiagnostics(ClangTU));
}
-TEST_F(LibclangReparseTest, ReparseWithModule) {
+TEST_F(LibclangReparseTest, DISABLED_ReparseWithModule) {
const char *HeaderTop = "#ifndef H\n#define H\nstruct Foo { int bar;";
const char *HeaderBottom = "\n};\n#endif\n";
const char *MFile = "#include \"HeaderFile.h\"\nint main() {"
"""
]
# This Go bindings test doesn't work after bootstrap on Linux, PR21552.
llvm_patches = [ r"""\
--- test/Bindings/Go/go.test (revision 223109)
+++ test/Bindings/Go/go.test (working copy)
@@ -1,3 +1,3 @@
-; RUN: llvm-go test llvm.org/llvm/bindings/go/llvm
+; RUN: true
; REQUIRES: shell
"""
]
# The UBSan run-time, which is now bundled with the ASan run-time, doesn't
# work on Mac OS X 10.8 (PR23539).
compiler_rt_patches = [ r"""\
--- CMakeLists.txt (revision 241602)
+++ CMakeLists.txt (working copy)
@@ -305,6 +305,7 @@
list(APPEND SANITIZER_COMMON_SUPPORTED_OS iossim)
endif()
endif()
+ set(SANITIZER_MIN_OSX_VERSION "10.7")
if(SANITIZER_MIN_OSX_VERSION VERSION_LESS "10.7")
message(FATAL_ERROR "Too old OS X version: ${SANITIZER_MIN_OSX_VERSION}")
endif()
"""
]
for path, patches in [(LLVM_DIR, llvm_patches),
(CLANG_DIR, clang_patches),
(COMPILER_RT_DIR, compiler_rt_patches)]:
print 'Applying patches in', path
for patch in patches:
print patch
p = subprocess.Popen( ['patch', '-p0', '-d', path], stdin=subprocess.PIPE)
(stdout, stderr) = p.communicate(input=patch)
if p.returncode != 0:
raise RuntimeError('stdout %s, stderr %s' % (stdout, stderr))
def DeleteChromeToolsShim():
shutil.rmtree(CHROME_TOOLS_SHIM_DIR, ignore_errors=True)
def CreateChromeToolsShim():
"""Hooks the Chrome tools into the LLVM build.
Several Chrome tools have dependencies on LLVM/Clang libraries. The LLVM build
detects implicit tools in the tools subdirectory, so this helper install a
shim CMakeLists.txt that forwards to the real directory for the Chrome tools.
Note that the shim directory name intentionally has no - or _. The implicit
tool detection logic munges them in a weird way."""
assert not any(i in os.path.basename(CHROME_TOOLS_SHIM_DIR) for i in '-_')
os.mkdir(CHROME_TOOLS_SHIM_DIR)
with file(os.path.join(CHROME_TOOLS_SHIM_DIR, 'CMakeLists.txt'), 'w') as f:
f.write('# Automatically generated by tools/clang/scripts/update.py. ' +
'Do not edit.\n')
f.write('# Since tools/clang is located in another directory, use the \n')
f.write('# two arg version to specify where build artifacts go. CMake\n')
f.write('# disallows reuse of the same binary dir for multiple source\n')
f.write('# dirs, so the build artifacts need to go into a subdirectory.\n')
f.write('# dirs, so the build artifacts need to go into a subdirectory.\n')
f.write('if (CHROMIUM_TOOLS_SRC)\n')
f.write(' add_subdirectory(${CHROMIUM_TOOLS_SRC} ' +
'${CMAKE_CURRENT_BINARY_DIR}/a)\n')
f.write('endif (CHROMIUM_TOOLS_SRC)\n')
def AddCMakeToPath():
"""Download CMake and add it to PATH."""
if sys.platform == 'win32':
zip_name = 'cmake-3.2.2-win32-x86.zip'
cmake_dir = os.path.join(LLVM_BUILD_TOOLS_DIR,
'cmake-3.2.2-win32-x86', 'bin')
else:
suffix = 'Darwin' if sys.platform == 'darwin' else 'Linux'
zip_name = 'cmake310_%s.tgz' % suffix
cmake_dir = os.path.join(LLVM_BUILD_TOOLS_DIR, 'cmake310', 'bin')
if not os.path.exists(cmake_dir):
if not os.path.exists(LLVM_BUILD_TOOLS_DIR):
os.makedirs(LLVM_BUILD_TOOLS_DIR)
# The cmake archive is smaller than 20 MB, small enough to keep in memory:
with contextlib.closing(cStringIO.StringIO()) as f:
DownloadUrl(CDS_URL + '/tools/' + zip_name, f)
f.seek(0)
if zip_name.endswith('.zip'):
zipfile.ZipFile(f).extractall(path=LLVM_BUILD_TOOLS_DIR)
else:
tarfile.open(mode='r:gz', fileobj=f).extractall(path=
LLVM_BUILD_TOOLS_DIR)
os.environ['PATH'] = cmake_dir + os.pathsep + os.environ.get('PATH', '')
vs_version = None
def GetVSVersion():
global vs_version
if vs_version:
return vs_version
# Try using the toolchain in depot_tools.
# This sets environment variables used by SelectVisualStudioVersion below.
sys.path.append(os.path.join(CHROMIUM_DIR, 'build'))
import vs_toolchain
vs_toolchain.SetEnvironmentAndGetRuntimeDllDirs()
# Use gyp to find the MSVS installation, either in depot_tools as per above,
# or a system-wide installation otherwise.
sys.path.append(os.path.join(CHROMIUM_DIR, 'tools', 'gyp', 'pylib'))
import gyp.MSVSVersion
vs_version = gyp.MSVSVersion.SelectVisualStudioVersion('2013')
return vs_version
def UpdateClang(args):
print 'Updating Clang to %s...' % PACKAGE_VERSION
if ReadStampFile() == PACKAGE_VERSION:
print 'Already up to date.'
return 0
# Reset the stamp file in case the build is unsuccessful.
WriteStampFile('')
if not args.force_local_build:
cds_file = "clang-%s.tgz" % PACKAGE_VERSION
cds_full_url = CDS_URL + '/Win/' + cds_file
# Check if there's a prebuilt binary and if so just fetch that. That's
# faster, and goma relies on having matching binary hashes on client and
# server too.
print 'Trying to download prebuilt clang'
# clang packages are smaller than 50 MB, small enough to keep in memory.
with contextlib.closing(cStringIO.StringIO()) as f:
try:
DownloadUrl(cds_full_url, f)
f.seek(0)
tarfile.open(mode='r:gz', fileobj=f).extractall(path=LLVM_BUILD_DIR)
print 'clang %s unpacked' % PACKAGE_VERSION
WriteStampFile(PACKAGE_VERSION)
return 0
except urllib2.HTTPError:
print 'Did not find prebuilt clang %s, building locally' % cds_file
AddCMakeToPath()
RevertPreviouslyPatchedFiles()
DeleteChromeToolsShim()
Checkout('LLVM', LLVM_REPO_URL + '/llvm/trunk', LLVM_DIR)
Checkout('Clang', LLVM_REPO_URL + '/cfe/trunk', CLANG_DIR)
if sys.platform == 'win32':
Checkout('LLD', LLVM_REPO_URL + '/lld/trunk', LLD_DIR)
Checkout('compiler-rt', LLVM_REPO_URL + '/compiler-rt/trunk', COMPILER_RT_DIR)
if sys.platform == 'darwin':
# clang needs a libc++ checkout, else -stdlib=libc++ won't find includes
# (i.e. this is needed for bootstrap builds).
Checkout('libcxx', LLVM_REPO_URL + '/libcxx/trunk', LIBCXX_DIR)
# While we're bundling our own libc++ on OS X, we need to compile libc++abi
# into it too (since OS X 10.6 doesn't have libc++abi.dylib either).
Checkout('libcxxabi', LLVM_REPO_URL + '/libcxxabi/trunk', LIBCXXABI_DIR)
if args.with_patches and sys.platform != 'win32':
ApplyLocalPatches()
cc, cxx = None, None
cflags = cxxflags = ldflags = []
# LLVM uses C++11 starting in llvm 3.5. On Linux, this means libstdc++4.7+ is
# needed, on OS X it requires libc++. clang only automatically links to libc++
# when targeting OS X 10.9+, so add stdlib=libc++ explicitly so clang can run
# on OS X versions as old as 10.7.
# TODO(thakis): Some bots are still on 10.6 (nacl...), so for now bundle
# libc++.dylib. Remove this once all bots are on 10.7+, then use
# -DLLVM_ENABLE_LIBCXX=ON and change deployment_target to 10.7.
deployment_target = ''
if sys.platform == 'darwin':
# When building on 10.9, /usr/include usually doesn't exist, and while
# Xcode's clang automatically sets a sysroot, self-built clangs don't.
cflags = ['-isysroot', subprocess.check_output(
['xcrun', '--show-sdk-path']).rstrip()]
cxxflags = ['-stdlib=libc++', '-nostdinc++',
'-I' + os.path.join(LIBCXX_DIR, 'include')] + cflags
if args.bootstrap:
deployment_target = '10.6'
base_cmake_args = ['-GNinja',
'-DCMAKE_BUILD_TYPE=Release',
'-DLLVM_ENABLE_ASSERTIONS=ON',
'-DLLVM_ENABLE_THREADS=OFF',
]
if args.bootstrap:
print 'Building bootstrap compiler'
if not os.path.exists(LLVM_BOOTSTRAP_DIR):
os.makedirs(LLVM_BOOTSTRAP_DIR)
os.chdir(LLVM_BOOTSTRAP_DIR)
bootstrap_args = base_cmake_args + [
'-DLLVM_TARGETS_TO_BUILD=host',
'-DCMAKE_INSTALL_PREFIX=' + LLVM_BOOTSTRAP_INSTALL_DIR,
'-DCMAKE_C_FLAGS=' + ' '.join(cflags),
'-DCMAKE_CXX_FLAGS=' + ' '.join(cxxflags),
]
if cc is not None: bootstrap_args.append('-DCMAKE_C_COMPILER=' + cc)
if cxx is not None: bootstrap_args.append('-DCMAKE_CXX_COMPILER=' + cxx)
RunCommand(['cmake'] + bootstrap_args + [LLVM_DIR], msvc_arch='x64')
RunCommand(['ninja'], msvc_arch='x64')
if args.run_tests:
RunCommand(['ninja', 'check-all'], msvc_arch='x64')
RunCommand(['ninja', 'install'], msvc_arch='x64')
if sys.platform == 'win32':
cc = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'clang-cl.exe')
cxx = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'clang-cl.exe')
# CMake has a hard time with backslashes in compiler paths:
# https://stackoverflow.com/questions/13050827
cc = cc.replace('\\', '/')
cxx = cxx.replace('\\', '/')
else:
cc = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'clang')
cxx = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'clang++')
print 'Building final compiler'
if sys.platform == 'darwin':
# Build libc++.dylib while some bots are still on OS X 10.6.
libcxxbuild = os.path.join(LLVM_BUILD_DIR, 'libcxxbuild')
if os.path.isdir(libcxxbuild):
RmTree(libcxxbuild)
libcxxflags = ['-O3', '-std=c++11', '-fstrict-aliasing']
# libcxx and libcxxabi both have a file stdexcept.cpp, so put their .o files
# into different subdirectories.
os.makedirs(os.path.join(libcxxbuild, 'libcxx'))
os.chdir(os.path.join(libcxxbuild, 'libcxx'))
RunCommand(['c++', '-c'] + cxxflags + libcxxflags +
glob.glob(os.path.join(LIBCXX_DIR, 'src', '*.cpp')))
os.makedirs(os.path.join(libcxxbuild, 'libcxxabi'))
os.chdir(os.path.join(libcxxbuild, 'libcxxabi'))
RunCommand(['c++', '-c'] + cxxflags + libcxxflags +
glob.glob(os.path.join(LIBCXXABI_DIR, 'src', '*.cpp')) +
['-I' + os.path.join(LIBCXXABI_DIR, 'include')])
os.chdir(libcxxbuild)
libdir = os.path.join(LIBCXX_DIR, 'lib')
RunCommand(['cc'] + glob.glob('libcxx/*.o') + glob.glob('libcxxabi/*.o') +
['-o', 'libc++.1.dylib', '-dynamiclib', '-nodefaultlibs',
'-current_version', '1', '-compatibility_version', '1', '-lSystem',
'-install_name', '@executable_path/libc++.dylib',
'-Wl,-unexported_symbols_list,' + libdir + '/libc++unexp.exp',
'-Wl,-force_symbols_not_weak_list,' + libdir + '/notweak.exp',
'-Wl,-force_symbols_weak_list,' + libdir + '/weak.exp'])
if os.path.exists('libc++.dylib'):
os.remove('libc++.dylib')
os.symlink('libc++.1.dylib', 'libc++.dylib')
ldflags += ['-stdlib=libc++', '-L' + libcxxbuild]
if args.bootstrap:
# Now that the libc++ headers have been installed and libc++.dylib is
# built, delete the libc++ checkout again so that it's not part of the
# main build below -- the libc++(abi) tests don't pass on OS X in
# bootstrap builds (http://llvm.org/PR24068)
RmTree(LIBCXX_DIR)
RmTree(LIBCXXABI_DIR)
cxxflags = ['-stdlib=libc++', '-nostdinc++',
'-I' + os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR,
'include/c++/v1')
] + cflags
# Build clang.
binutils_incdir = ''
if sys.platform.startswith('linux'):
binutils_incdir = os.path.join(BINUTILS_DIR, 'Linux_x64/Release/include')
# If building at head, define a macro that plugins can use for #ifdefing
# out code that builds at head, but not at LLVM_WIN_REVISION or vice versa.
if use_head_revision:
cflags += ['-DLLVM_FORCE_HEAD_REVISION']
cxxflags += ['-DLLVM_FORCE_HEAD_REVISION']
CreateChromeToolsShim()
deployment_env = None
if deployment_target:
deployment_env = os.environ.copy()
deployment_env['MACOSX_DEPLOYMENT_TARGET'] = deployment_target
cmake_args = base_cmake_args + [
'-DLLVM_BINUTILS_INCDIR=' + binutils_incdir,
'-DCMAKE_C_FLAGS=' + ' '.join(cflags),
'-DCMAKE_CXX_FLAGS=' + ' '.join(cxxflags),
'-DCMAKE_EXE_LINKER_FLAGS=' + ' '.join(ldflags),
'-DCMAKE_SHARED_LINKER_FLAGS=' + ' '.join(ldflags),
'-DCMAKE_MODULE_LINKER_FLAGS=' + ' '.join(ldflags),
'-DCMAKE_INSTALL_PREFIX=' + LLVM_BUILD_DIR,
'-DCHROMIUM_TOOLS_SRC=%s' % os.path.join(CHROMIUM_DIR, 'tools', 'clang'),
'-DCHROMIUM_TOOLS=%s' % ';'.join(args.tools)]
# TODO(thakis): Unconditionally append this to base_cmake_args instead once
# compiler-rt can build with clang-cl on Windows (http://llvm.org/PR23698)
cc_args = base_cmake_args if sys.platform != 'win32' else cmake_args
if cc is not None: cc_args.append('-DCMAKE_C_COMPILER=' + cc)
if cxx is not None: cc_args.append('-DCMAKE_CXX_COMPILER=' + cxx)
if not os.path.exists(LLVM_BUILD_DIR):
os.makedirs(LLVM_BUILD_DIR)
os.chdir(LLVM_BUILD_DIR)
RunCommand(['cmake'] + cmake_args + [LLVM_DIR],
msvc_arch='x64', env=deployment_env)
RunCommand(['ninja'], msvc_arch='x64')
if args.tools:
# If any Chromium tools were built, install those now.
RunCommand(['ninja', 'cr-install'], msvc_arch='x64')
if sys.platform == 'darwin':
CopyFile(os.path.join(LLVM_BUILD_DIR, 'libc++.1.dylib'),
os.path.join(LLVM_BUILD_DIR, 'bin'))
# See http://crbug.com/256342
RunCommand(['strip', '-x', os.path.join(LLVM_BUILD_DIR, 'bin', 'clang')])
elif sys.platform.startswith('linux'):
RunCommand(['strip', os.path.join(LLVM_BUILD_DIR, 'bin', 'clang')])
# Do an x86 build of compiler-rt to get the 32-bit ASan run-time.
# TODO(hans): Remove once the regular build above produces this.
if not os.path.exists(COMPILER_RT_BUILD_DIR):
os.makedirs(COMPILER_RT_BUILD_DIR)
os.chdir(COMPILER_RT_BUILD_DIR)
# TODO(thakis): Add this once compiler-rt can build with clang-cl (see
# above).
#if args.bootstrap and sys.platform == 'win32':
# The bootstrap compiler produces 64-bit binaries by default.
#cflags += ['-m32']
#cxxflags += ['-m32']
compiler_rt_args = base_cmake_args + [
'-DCMAKE_C_FLAGS=' + ' '.join(cflags),
'-DCMAKE_CXX_FLAGS=' + ' '.join(cxxflags)]
if sys.platform != 'win32':
compiler_rt_args += ['-DLLVM_CONFIG_PATH=' +
os.path.join(LLVM_BUILD_DIR, 'bin', 'llvm-config')]
RunCommand(['cmake'] + compiler_rt_args + [LLVM_DIR],
msvc_arch='x86', env=deployment_env)
RunCommand(['ninja', 'compiler-rt'], msvc_arch='x86')
# TODO(hans): Make this (and the .gypi and .isolate files) version number
# independent.
if sys.platform == 'win32':
platform = 'windows'
elif sys.platform == 'darwin':
platform = 'darwin'
else:
assert sys.platform.startswith('linux')
platform = 'linux'
asan_rt_lib_src_dir = os.path.join(COMPILER_RT_BUILD_DIR, 'lib', 'clang',
VERSION, 'lib', platform)
asan_rt_lib_dst_dir = os.path.join(LLVM_BUILD_DIR, 'lib', 'clang',
VERSION, 'lib', platform)
CopyDirectoryContents(asan_rt_lib_src_dir, asan_rt_lib_dst_dir,
r'^.*-i386\.lib$')
CopyDirectoryContents(asan_rt_lib_src_dir, asan_rt_lib_dst_dir,
r'^.*-i386\.dll$')
CopyFile(os.path.join(asan_rt_lib_src_dir, '..', '..', 'asan_blacklist.txt'),
os.path.join(asan_rt_lib_dst_dir, '..', '..'))
if sys.platform == 'win32':
# Make an extra copy of the sanitizer headers, to be put on the include path
# of the fallback compiler.
sanitizer_include_dir = os.path.join(LLVM_BUILD_DIR, 'lib', 'clang',
VERSION, 'include', 'sanitizer')
aux_sanitizer_include_dir = os.path.join(LLVM_BUILD_DIR, 'lib', 'clang',
VERSION, 'include_sanitizer',
'sanitizer')
if not os.path.exists(aux_sanitizer_include_dir):
os.makedirs(aux_sanitizer_include_dir)
for _, _, files in os.walk(sanitizer_include_dir):
for f in files:
CopyFile(os.path.join(sanitizer_include_dir, f),
aux_sanitizer_include_dir)
# Run tests.
if args.run_tests or use_head_revision:
os.chdir(LLVM_BUILD_DIR)
RunCommand(GetVSVersion().SetupScript('x64') +
['&&', 'ninja', 'cr-check-all'])
if args.run_tests:
os.chdir(LLVM_BUILD_DIR)
RunCommand(GetVSVersion().SetupScript('x64') +
['&&', 'ninja', 'check-all'])
WriteStampFile(PACKAGE_VERSION)
print 'Clang update was successful.'
return 0
def main():
if not sys.platform in ['win32', 'cygwin']:
# For non-Windows, fall back to update.sh.
# TODO(hans): Make update.py replace update.sh completely.
# This script is called by gclient. gclient opens its hooks subprocesses
# with (stdout=subprocess.PIPE, stderr=subprocess.STDOUT) and then does
# custom output processing that breaks printing '\r' characters for
# single-line updating status messages as printed by curl and wget.
# Work around this by setting stderr of the update.sh process to stdin (!):
# gclient doesn't redirect stdin, and while stdin itself is read-only, a
# dup()ed sys.stdin is writable, try
# fd2 = os.dup(sys.stdin.fileno()); os.write(fd2, 'hi')
# TODO: Fix gclient instead, http://crbug.com/95350
if '--no-stdin-hack' in sys.argv:
sys.argv.remove('--no-stdin-hack')
stderr = None
else:
try:
stderr = os.fdopen(os.dup(sys.stdin.fileno()))
except:
stderr = sys.stderr
return subprocess.call(
[os.path.join(os.path.dirname(__file__), 'update.sh')] + sys.argv[1:],
stderr=stderr)
parser = argparse.ArgumentParser(description='Build Clang.')
parser.add_argument('--bootstrap', action='store_true',
help='first build clang with CC, then with itself.')
parser.add_argument('--if-needed', action='store_true',
help="run only if the script thinks clang is needed")
parser.add_argument('--force-local-build', action='store_true',
help="don't try to download prebuild binaries")
parser.add_argument('--print-revision', action='store_true',
help='print current clang revision and exit.')
parser.add_argument('--run-tests', action='store_true',
help='run tests after building; only for local builds')
parser.add_argument('--tools', nargs='*',
help='select which chrome tools to build',
default=['plugins', 'blink_gc_plugin'])
parser.add_argument('--without-patches', action='store_false',
help="don't apply patches (default)", dest='with_patches',
default=True)
# For now, these flags are only used for the non-Windows flow, but argparser
# gets mad if it sees a flag it doesn't recognize.
parser.add_argument('--no-stdin-hack', action='store_true')
args = parser.parse_args()
if args.if_needed:
is_clang_required = False
# clang is always used on Mac and Linux.
if sys.platform == 'darwin' or sys.platform.startswith('linux'):
is_clang_required = True
# clang requested via $GYP_DEFINES.
if re.search(r'\b(clang|asan|lsan|msan|tsan)=1',
os.environ.get('GYP_DEFINES', '')):
is_clang_required = True
# clang previously downloaded, keep it up-to-date.
# If you don't want this, delete third_party/llvm-build on your machine.
if os.path.isdir(LLVM_BUILD_DIR):
is_clang_required = True
if not is_clang_required:
return 0
global LLVM_WIN_REVISION, PACKAGE_VERSION
if args.print_revision:
if use_head_revision:
print GetSvnRevision(LLVM_DIR)
else:
print PACKAGE_VERSION
return 0
if re.search(r'\b(make_clang_dir)=', os.environ.get('GYP_DEFINES', '')):
print 'Skipping Clang update (make_clang_dir= was set in GYP_DEFINES).'
return 0
# Don't buffer stdout, so that print statements are immediately flushed.
# Do this only after --print-revision has been handled, else we'll get
# an error message when this script is run from gn for some reason.
sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
if use_head_revision:
# Use a real revision number rather than HEAD to make sure that the stamp
# file logic works.
LLVM_WIN_REVISION = GetSvnRevision(LLVM_REPO_URL)
PACKAGE_VERSION = LLVM_WIN_REVISION + '-0'
args.force_local_build = True
# Skip local patches when using HEAD: they probably don't apply anymore.
args.with_patches = False
return UpdateClang(args)
if __name__ == '__main__':
sys.exit(main())