# Copyright (c) 2014 The Chromium Embedded Framework Authors. All rights
# reserved. Use of this source code is governed by a BSD-style license that
# can be found in the LICENSE file.
from __future__ import absolute_import
from __future__ import print_function
from datetime import datetime
from io import open
from optparse import OptionParser
import os
import re
import shlex
import shutil
import subprocess
import sys
import tempfile
import zipfile
is_python2 = sys.version_info.major == 2
if is_python2:
from urllib import FancyURLopener
from urllib2 import urlopen
else:
from urllib.request import FancyURLopener, urlopen
##
# Default URLs.
##
depot_tools_url = 'https://chromium.googlesource.com/chromium/tools/depot_tools.git'
depot_tools_archive_url = 'https://storage.googleapis.com/chrome-infra/depot_tools.zip'
cef_git_url = 'https://bitbucket.org/chromiumembedded/cef.git'
##
# Global system variables.
##
# Script directory.
script_dir = os.path.dirname(__file__)
##
# Helper functions.
##
def msg(message):
""" Output a message. """
sys.stdout.write('--> ' + message + "\n")
def run(command_line, working_dir, depot_tools_dir=None, output_file=None):
""" Runs the specified command. """
# add depot_tools to the path
env = os.environ
if not depot_tools_dir is None:
env['PATH'] = depot_tools_dir + os.pathsep + env['PATH']
sys.stdout.write('-------- Running "'+command_line+'" in "'+\
working_dir+'"...'+"\n")
if not options.dryrun:
args = shlex.split(command_line.replace('\\', '\\\\'))
if not output_file:
return subprocess.check_call(
args, cwd=working_dir, env=env, shell=(sys.platform == 'win32'))
try:
msg('Writing %s' % output_file)
with open(output_file, 'w', encoding='utf-8') as fp:
return subprocess.check_call(
args,
cwd=working_dir,
env=env,
shell=(sys.platform == 'win32'),
stderr=subprocess.STDOUT,
stdout=fp)
except subprocess.CalledProcessError:
msg('ERROR Run failed. See %s for output.' % output_file)
raise
def create_directory(path):
""" Creates a directory if it doesn't already exist. """
if not os.path.exists(path):
msg("Creating directory %s" % (path))
if not options.dryrun:
os.makedirs(path)
def delete_directory(path):
""" Removes an existing directory. """
if os.path.exists(path):
msg("Removing directory %s" % (path))
if not options.dryrun:
shutil.rmtree(path, onerror=onerror)
def copy_directory(source, target, allow_overwrite=False):
""" Copies a directory from source to target. """
if not options.dryrun and os.path.exists(target):
if not allow_overwrite:
raise Exception("Directory %s already exists" % (target))
remove_directory(target)
if os.path.exists(source):
msg("Copying directory %s to %s" % (source, target))
if not options.dryrun:
shutil.copytree(source, target)
def move_directory(source, target, allow_overwrite=False):
""" Copies a directory from source to target. """
if not options.dryrun and os.path.exists(target):
if not allow_overwrite:
raise Exception("Directory %s already exists" % (target))
remove_directory(target)
if os.path.exists(source):
msg("Moving directory %s to %s" % (source, target))
if not options.dryrun:
shutil.move(source, target)
def is_git_checkout(path):
""" Returns true if the path represents a git checkout. """
return os.path.exists(os.path.join(path, '.git'))
def exec_cmd(cmd, path):
""" Execute the specified command and return the result. """
out = ''
err = ''
sys.stdout.write("-------- Running \"%s\" in \"%s\"...\n" % (cmd, path))
parts = cmd.split()
try:
process = subprocess.Popen(
parts,
cwd=path,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=(sys.platform == 'win32'))
out, err = process.communicate()
except IOError as e:
(errno, strerror) = e.args
raise
except:
raise
return {'out': out.decode('utf-8'), 'err': err.decode('utf-8')}
def get_git_hash(path, branch):
""" Returns the git hash for the specified branch/tag/hash. """
cmd = "%s rev-parse %s" % (git_exe, branch)
result = exec_cmd(cmd, path)
if result['out'] != '':
return result['out'].strip()
return 'Unknown'
def get_git_url(path):
""" Returns the origin url for the specified path. """
cmd = "%s config --get remote.origin.url" % (git_exe)
result = exec_cmd(cmd, path)
if result['out'] != '':
return result['out'].strip()
return 'Unknown'
def download_and_extract(src, target):
""" Extracts the contents of src, which may be a URL or local file, to the
target directory. """
temporary = False
if src[:4] == 'http':
# Attempt to download a URL.
opener = FancyURLopener({})
response = opener.open(src)
temporary = True
handle, archive_path = tempfile.mkstemp(suffix='.zip')
os.write(handle, response.read())
os.close(handle)
elif os.path.exists(src):
# Use a local file.
archive_path = src
else:
raise Exception('Path type is unsupported or does not exist: ' + src)
if not zipfile.is_zipfile(archive_path):
raise Exception('Not a valid zip archive: ' + src)
# Attempt to extract the archive file.
try:
os.makedirs(target)
zf = zipfile.ZipFile(archive_path, 'r')
zf.extractall(target)
except:
shutil.rmtree(target, onerror=onerror)
raise
zf.close()
# Delete the archive file if temporary.
if temporary and os.path.exists(archive_path):
os.remove(archive_path)
def read_file(path):
""" Read a file. """
if os.path.exists(path):
with open(path, 'r', encoding='utf-8') as fp:
return fp.read()
else:
raise Exception("Path does not exist: %s" % (path))
def write_fp(fp, data):
if is_python2:
fp.write(data.decode('utf-8'))
else:
fp.write(data)
def write_file(path, data):
""" Write a file. """
msg('Writing %s' % path)
if not options.dryrun:
with open(path, 'w', encoding='utf-8') as fp:
write_fp(fp, data)
def read_config_file(path):
""" Read a configuration file. """
# Parse the contents.
return eval(read_file(path), {'__builtins__': None}, None)
def write_config_file(path, contents):
""" Write a configuration file. """
data = "{\n"
for key in sorted(contents.keys()):
data += " '%s': '%s',\n" % (key, contents[key])
data += "}\n"
write_file(path, data)
def read_branch_config_file(path):
""" Read the CEF branch from the specified path. """
config_file = os.path.join(path, 'cef.branch')
if os.path.isfile(config_file):
contents = read_config_file(config_file)
if 'branch' in contents:
return contents['branch']
return ''
def write_branch_config_file(path, branch):
""" Write the CEF branch to the specified path. """
config_file = os.path.join(path, 'cef.branch')
if not os.path.isfile(config_file):
write_config_file(config_file, {'branch': branch})
def apply_patch(name):
patch_file = os.path.join(cef_dir, 'patch', 'patches', name)
if os.path.exists(patch_file + ".patch"):
# Attempt to apply the patch file.
patch_tool = os.path.join(cef_dir, 'tools', 'patcher.py')
run('%s %s --patch-file "%s" --patch-dir "%s"' %
(python_exe, patch_tool, patch_file,
chromium_src_dir), chromium_src_dir, depot_tools_dir)
def apply_deps_patch():
""" Patch the Chromium DEPS file before `gclient sync` if necessary. """
deps_path = os.path.join(chromium_src_dir, deps_file)
if os.path.isfile(deps_path):
msg("Chromium DEPS file: %s" % (deps_path))
apply_patch(deps_file)
else:
raise Exception("Path does not exist: %s" % (deps_path))
def apply_runhooks_patch():
""" Patch the Chromium runhooks files before `gclient runhooks` if necessary. """
apply_patch('runhooks')
def run_patch_updater(args='', output_file=None):
""" Run the patch updater script. """
tool = os.path.join(cef_src_dir, 'tools', 'patch_updater.py')
if len(args) > 0:
args = ' ' + args
run('%s %s%s' % (python_exe, tool, args), cef_src_dir, depot_tools_dir,
output_file)
def onerror(func, path, exc_info):
"""
Error handler for ``shutil.rmtree``.
If the error is due to an access error (read only file)
it attempts to add write permission and then retries.
If the error is for another reason it re-raises the error.
Usage : ``shutil.rmtree(path, onerror=onerror)``
"""
import stat
if not os.access(path, os.W_OK):
# Is the error an access error ?
os.chmod(path, stat.S_IWUSR)
func(path)
else:
raise
def get_chromium_main_position(commit):
""" Returns the closest main position for the specified Chromium commit. """
# Using -2 because a "Publish DEPS" commit which does not have a master
# position may be first.
cmd = "%s log -2 %s" % (git_exe, commit)
result = exec_cmd(cmd, chromium_src_dir)
if result['out'] != '':
match = re.search(r'refs/heads/(?:master|main)@{#([\d]+)}', result['out'])
assert match != None, 'Failed to find position'
return int(match.groups()[0])
return None
def get_chromium_main_commit(position):
""" Returns the main commit for the specified Chromium commit position. """
cmd = '%s log -1 --grep=refs/heads/master@{#%s} --grep=refs/heads/main@{#%s} origin/main' % (
git_exe, str(position), str(position))
result = exec_cmd(cmd, chromium_src_dir)
if result['out'] != '':
match = re.search(r'^commit ([a-f0-9]+)', result['out'])
assert match != None, 'Failed to find commit'
return match.groups()[0]
return None
def get_build_compat_versions():
""" Returns the compatible Chromium and (optionally) depot_tools versions
specified by the CEF checkout. """
compat_path = os.path.join(cef_dir, 'CHROMIUM_BUILD_COMPATIBILITY.txt')
msg("Reading %s" % compat_path)
config = read_config_file(compat_path)
if not 'chromium_checkout' in config:
raise Exception("Missing chromium_checkout value in %s" % (compat_path))
return config
def get_build_directory_name(is_debug):
build_dir = ('Debug' if is_debug else 'Release') + '_'
# CEF uses a consistent directory naming scheme for GN via
# GetAllPlatformConfigs in tools/gn_args.py.
if options.x64build:
build_dir += 'GN_x64'
elif options.armbuild:
build_dir += 'GN_arm'
elif options.arm64build:
build_dir += 'GN_arm64'
else:
build_dir += 'GN_x86'
return build_dir
def read_update_file():
update_path = os.path.join(cef_src_dir, 'CHROMIUM_UPDATE.txt')
if not os.path.exists(update_path):
msg("Missing file: %s" % update_path)
return None
msg("Reading %s" % update_path)
return read_config_file(update_path)
def log_chromium_changes():
""" Evaluate the Chromium checkout for changes. """
config = read_update_file()
if config is None:
msg("Skipping Chromium changes log.")
return
if 'files' in config:
out_file = os.path.join(download_dir, 'chromium_update_changes.diff')
if os.path.exists(out_file):
os.remove(out_file)
old_commit = get_chromium_main_commit(
get_chromium_main_position(chromium_compat_version))
new_commit = get_chromium_main_commit(
get_chromium_main_position(chromium_checkout))
cmd = '%s diff --relative --no-prefix %s..%s -- %s' % (
git_exe, old_commit, new_commit, ' '.join(config['files']))
result = exec_cmd(cmd, chromium_src_dir)
if result['out'] != '':
write_file(out_file, result['out'])
def check_pattern_matches(output_file=None):
""" Evaluate the Chromium checkout for pattern matches. """
config = read_update_file()
if config is None:
msg("Skipping Chromium pattern matching.")
return
if 'patterns' in config:
if output_file is None:
fp = sys.stdout
else:
msg('Writing %s' % output_file)
fp = open(output_file, 'w', encoding='utf-8')
has_output = False
for entry in config['patterns']:
msg("Evaluating pattern: %s" % entry['pattern'])
# Read patterns from a file to avoid formatting problems.
pattern_handle, pattern_file = tempfile.mkstemp()
os.write(pattern_handle, entry['pattern'])
os.close(pattern_handle)
cmd = '%s grep -n -f %s' % (git_exe, pattern_file)
result = exec_cmd(cmd, chromium_src_dir)
os.remove(pattern_file)
if result['out'] != '':
write_msg = True
re_exclude = re.compile(
entry['exclude_matches']) if 'exclude_matches' in entry else None
for line in result['out'].split('\n'):
line = line.strip()
if len(line) == 0:
continue
skip = not re_exclude is None and re_exclude.match(line) != None
if not skip:
if write_msg:
if has_output:
write_fp(fp, '\n')
write_fp(fp,
'!!!! WARNING: FOUND PATTERN: %s\n' % entry['pattern'])
if 'message' in entry:
write_fp(fp, entry['message'] + '\n')
write_fp(fp, '\n')
write_msg = False
write_fp(fp, line + '\n')
has_output = True
if not output_file is None:
if has_output:
msg('ERROR Matches found. See %s for output.' % out_file)
else:
write_fp(fp, 'Good news! No matches.\n')
fp.close()
if has_output:
# Don't continue when we know the build will be wrong.
sys.exit(1)
##
# Program entry point.
##
# Cannot be loaded as a module.
if __name__ != "__main__":
sys.stderr.write('This file cannot be loaded as a module!')
sys.exit(1)
# Parse command-line options.
disc = """
This utility implements automation for the download, update, build and
distribution of CEF.
"""
parser = OptionParser(description=disc)
# Setup options.
parser.add_option(
'--download-dir',
dest='downloaddir',
metavar='DIR',
help='Download directory with no spaces [required].')
parser.add_option(
'--depot-tools-dir',
dest='depottoolsdir',
metavar='DIR',
help='Download directory for depot_tools.',
default='')
parser.add_option('--depot-tools-archive', dest='depottoolsarchive',
help='Zip archive file that contains a single top-level '+\
'depot_tools directory.', default='')
parser.add_option('--branch', dest='branch',
help='Branch of CEF to build (master, 3987, ...). This '+\
'will be used to name the CEF download directory and '+\
'to identify the correct URL if --url is not '+\
'specified. The default value is master.',
default='master')
parser.add_option('--url', dest='url',
help='CEF download URL. If not specified the default URL '+\
'will be used.',
default='')
parser.add_option('--chromium-url', dest='chromiumurl',
help='Chromium download URL. If not specified the default '+\
'URL will be used.',
default='')
parser.add_option('--checkout', dest='checkout',
help='Version of CEF to checkout. If not specified the '+\
'most recent remote version of the branch will be used.',
default='')
parser.add_option('--chromium-checkout', dest='chromiumcheckout',
help='Version of Chromium to checkout (Git '+\
'branch/hash/tag). This overrides the value specified '+\
'by CEF in CHROMIUM_BUILD_COMPATIBILITY.txt.',
default='')
# Miscellaneous options.
parser.add_option(
'--force-config',
action='store_true',
dest='forceconfig',
default=False,
help='Force creation of a new gclient config file.')
parser.add_option('--force-clean',
action='store_true', dest='forceclean', default=False,
help='Force a clean checkout of Chromium and CEF. This will'+\
' trigger a new update, build and distribution.')
parser.add_option('--force-clean-deps',
action='store_true', dest='forcecleandeps', default=False,
help='Force a clean checkout of Chromium dependencies. Used'+\
' in combination with --force-clean.')
parser.add_option(
'--dry-run',
action='store_true',
dest='dryrun',
default=False,
help="Output commands without executing them.")
parser.add_option('--dry-run-platform', dest='dryrunplatform', default=None,
help='Simulate a dry run on the specified platform '+\
'(windows, mac, linux). Must be used in combination'+\
' with the --dry-run flag.')
# Update-related options.
parser.add_option('--force-update',
action='store_true', dest='forceupdate', default=False,
help='Force a Chromium and CEF update. This will trigger a '+\
'new build and distribution.')
parser.add_option('--no-update',
action='store_true', dest='noupdate', default=False,
help='Do not update Chromium or CEF. Pass --force-build or '+\
'--force-distrib if you desire a new build or '+\
'distribution.')
parser.add_option('--no-cef-update',
action='store_true', dest='nocefupdate', default=False,
help='Do not update CEF. Pass --force-build or '+\
'--force-distrib if you desire a new build or '+\
'distribution.')
parser.add_option('--force-cef-update',
action='store_true', dest='forcecefupdate', default=False,
help='Force a CEF update. This will cause local changes in '+\
'the CEF checkout to be discarded and patch files to '+\
'be reapplied.')
parser.add_option(
'--no-chromium-update',
action='store_true',
dest='nochromiumupdate',
default=False,
help='Do not update Chromium.')
parser.add_option(
'--no-depot-tools-update',
action='store_true',
dest='nodepottoolsupdate',
default=False,
help='Do not update depot_tools.')
parser.add_option('--fast-update',
action='store_true', dest='fastupdate', default=False,
help='Update existing Chromium/CEF checkouts for fast incremental '+\
'builds by attempting to minimize the number of modified files. '+\
'The update will fail if there are unstaged CEF changes or if '+\
'Chromium changes are not included in a patch file.')
parser.add_option(
'--force-patch-update',
action='store_true',
dest='forcepatchupdate',
default=False,
help='Force update of patch files.')
parser.add_option(
'--resave',
action='store_true',
dest='resave',
default=False,
help='Resave patch files.')
parser.add_option(
'--log-chromium-changes',
action='store_true',
dest='logchromiumchanges',
default=False,
help='Create a log of the Chromium changes.')
# Build-related options.
parser.add_option('--force-build',
action='store_true', dest='forcebuild', default=False,
help='Force CEF debug and release builds. This builds '+\
'[build-target] on all platforms and chrome_sandbox '+\
'on Linux.')
parser.add_option(
'--no-build',
action='store_true',
dest='nobuild',
default=False,
help='Do not build CEF.')
parser.add_option(
'--build-target',
dest='buildtarget',
default='cefclient',
help='Target name(s) to build (defaults to "cefclient").')
parser.add_option(
'--build-tests',
action='store_true',
dest='buildtests',
default=False,
help='Also build the test target specified via --test-target.')
parser.add_option(
'--no-debug-build',
action='store_true',
dest='nodebugbuild',
default=False,
help="Don't perform the CEF debug build.")
parser.add_option(
'--no-release-build',
action='store_true',
dest='noreleasebuild',
default=False,
help="Don't perform the CEF release build.")
parser.add_option(
'--verbose-build',
action='store_true',
dest='verbosebuild',
default=False,
help='Show all command lines while building.')
parser.add_option(
'--build-failure-limit',
dest='buildfailurelimit',
default=1,
type="int",
help='Keep going until N jobs fail.')
parser.add_option('--build-log-file',
action='store_true', dest='buildlogfile', default=False,
help='Write build logs to file. The file will be named '+\
'"build-[branch]-[debug|release].log" in the download '+\
'directory.')
parser.add_option(
'--x64-build',
action='store_true',
dest='x64build',
default=False,
help='Create a 64-bit build.')
parser.add_option(
'--arm-build',
action='store_true',
dest='armbuild',
default=False,
help='Create an ARM build.')
parser.add_option(
'--arm64-build',
action='store_true',
dest='arm64build',
default=False,
help='Create an ARM64 build.')
parser.add_option(
'--with-pgo-profiles',
action='store_true',
dest='withpgoprofiles',
default=False,
help='Download PGO profiles for the build.')
# Test-related options.
parser.add_option(
'--run-tests',
action='store_true',
dest='runtests',
default=False,
help='Run the ceftests target.')
parser.add_option(
'--no-debug-tests',
action='store_true',
dest='nodebugtests',
default=False,
help="Don't run debug build tests.")
parser.add_option(
'--no-release-tests',
action='store_true',
dest='noreleasetests',
default=False,
help="Don't run release build tests.")
parser.add_option(
'--test-target',
dest='testtarget',
default='ceftests',
help='Test target name to build (defaults to "ceftests").')
parser.add_option(
'--test-prefix',
dest='testprefix',
default='',
help='Prefix for running the test executable (e.g. `xvfb-run` on Linux).')
parser.add_option(
'--test-args',
dest='testargs',
default='',
help='Arguments that will be passed to the test executable.')
# Distribution-related options.
parser.add_option(
'--force-distrib',
action='store_true',
dest='forcedistrib',
default=False,
help='Force creation of a CEF binary distribution.')
parser.add_option(
'--no-distrib',
action='store_true',
dest='nodistrib',
default=False,
help="Don't create a CEF binary distribution.")
parser.add_option(
'--minimal-distrib',
action='store_true',
dest='minimaldistrib',
default=False,
help='Create a minimal CEF binary distribution.')
parser.add_option(
'--minimal-distrib-only',
action='store_true',
dest='minimaldistribonly',
default=False,
help='Create a minimal CEF binary distribution only.')
parser.add_option(
'--client-distrib',
action='store_true',
dest='clientdistrib',
default=False,
help='Create a client CEF binary distribution.')
parser.add_option(
'--client-distrib-only',
action='store_true',
dest='clientdistribonly',
default=False,
help='Create a client CEF binary distribution only.')
parser.add_option(
'--sandbox-distrib',
action='store_true',
dest='sandboxdistrib',
default=False,
help='Create a cef_sandbox static library distribution.')
parser.add_option(
'--sandbox-distrib-only',
action='store_true',
dest='sandboxdistribonly',
default=False,
help='Create a cef_sandbox static library distribution only.')
parser.add_option(
'--no-distrib-docs',
action='store_true',
dest='nodistribdocs',
default=False,
help="Don't create CEF documentation.")
parser.add_option(
'--no-distrib-archive',
action='store_true',
dest='nodistribarchive',
default=False,
help="Don't create archives for output directories.")
parser.add_option(
'--clean-artifacts',
action='store_true',
dest='cleanartifacts',
default=False,
help='Clean the artifacts output directory.')
parser.add_option(
'--distrib-subdir',
dest='distribsubdir',
default='',
help='CEF distrib dir name, child of chromium/src/cef/binary_distrib')
parser.add_option(
'--distrib-subdir-suffix',
dest='distribsubdirsuffix',
default='',
help='CEF distrib dir name suffix, child of chromium/src/cef/binary_distrib'
)
(options, args) = parser.parse_args()
if options.downloaddir is None:
print("The --download-dir option is required.")
parser.print_help(sys.stderr)
sys.exit(1)
# Opt into component-specific flags for later use.
if options.noupdate:
options.nocefupdate = True
options.nochromiumupdate = True
options.nodepottoolsupdate = True
if options.runtests:
options.buildtests = True
if (options.nochromiumupdate and options.forceupdate) or \
(options.nocefupdate and options.forceupdate) or \
(options.nobuild and options.forcebuild) or \
(options.nodistrib and options.forcedistrib) or \
((options.forceclean or options.forcecleandeps) and options.fastupdate):
print("Invalid combination of options.")
parser.print_help(sys.stderr)
sys.exit(1)
if (options.noreleasebuild and \
(options.minimaldistrib or options.minimaldistribonly or \
options.clientdistrib or options.clientdistribonly)) or \
(options.minimaldistribonly + options.clientdistribonly + options.sandboxdistribonly > 1):
print('Invalid combination of options.')
parser.print_help(sys.stderr)
sys.exit(1)
if options.x64build + options.armbuild + options.arm64build > 1:
print('Invalid combination of options.')
parser.print_help(sys.stderr)
sys.exit(1)
if (options.buildtests or options.runtests) and len(options.testtarget) == 0:
print("A test target must be specified via --test-target.")
parser.print_help(sys.stderr)
sys.exit(1)
# Operating system.
if options.dryrun and options.dryrunplatform is not None:
platform = options.dryrunplatform
if not platform in ['windows', 'mac', 'linux']:
print('Invalid dry-run-platform value: %s' % (platform))
sys.exit(1)
elif sys.platform == 'win32':
platform = 'windows'
elif sys.platform == 'darwin':
platform = 'mac'
elif sys.platform.startswith('linux'):
platform = 'linux'
else:
print('Unknown operating system platform')
sys.exit(1)
if options.clientdistrib or options.clientdistribonly:
if platform == 'linux' or (platform == 'windows' and options.arm64build):
client_app = 'cefsimple'
else:
client_app = 'cefclient'
if options.buildtarget.find(client_app) == -1:
print('A client distribution cannot be generated if --build-target ' +
'excludes %s.' % client_app)
parser.print_help(sys.stderr)
sys.exit(1)
# CEF branch.
cef_branch = options.branch
branch_is_master = (cef_branch == 'master' or cef_branch == 'trunk')
if not branch_is_master:
# Verify that the branch value is numeric.
if not cef_branch.isdigit():
print('Invalid branch value: %s' % cef_branch)
sys.exit(1)
# Verify the minimum supported branch number.
if int(cef_branch) < 3071:
print('The requested branch (%s) is too old to build using this tool. ' +
'The minimum supported branch is 3071.' % cef_branch)
sys.exit(1)
# True if the requested branch is 3538 or newer.
branch_is_3538_or_newer = (branch_is_master or int(cef_branch) >= 3538)
# True if the requested branch is 3945 or newer.
branch_is_3945_or_newer = (branch_is_master or int(cef_branch) >= 3945)
# Enable Python 3 usage in Chromium for branches 3945 and newer.
if branch_is_3945_or_newer and not is_python2 and \
not 'GCLIENT_PY3' in os.environ.keys():
os.environ['GCLIENT_PY3'] = '1'
if not branch_is_3945_or_newer and \
(not is_python2 or bool(int(os.environ.get('GCLIENT_PY3', '0')))):
print('Python 3 is not supported with branch 3904 and older ' +
'(set GCLIENT_PY3=0 and run with Python 2 executable).')
sys.exit(1)
if options.armbuild:
if platform != 'linux':
print('The ARM build option is only supported on Linux.')
sys.exit(1)
deps_file = 'DEPS'
if platform == 'mac' and not (options.x64build or options.arm64build):
print('32-bit MacOS builds are not supported. ' +
'Add --x64-build or --arm64-build flag to generate a 64-bit build.')
sys.exit(1)
# Platforms that build a cef_sandbox library.
sandbox_lib_platforms = ['windows']
if branch_is_3538_or_newer:
sandbox_lib_platforms.append('mac')
if not platform in sandbox_lib_platforms and (options.sandboxdistrib or
options.sandboxdistribonly):
print('The sandbox distribution is not supported on this platform.')
sys.exit(1)
# Options that force the sources to change.
force_change = options.forceclean or options.forceupdate
# Options that cause local changes to be discarded.
discard_local_changes = force_change or options.forcecefupdate
if options.resave and (options.forcepatchupdate or discard_local_changes):
print('--resave cannot be combined with options that modify or discard ' +
'patches.')
parser.print_help(sys.stderr)
sys.exit(1)
if platform == 'windows':
# Avoid errors when the "vs_toolchain.py update" Chromium hook runs.
os.environ['DEPOT_TOOLS_WIN_TOOLCHAIN'] = '0'
download_dir = os.path.abspath(options.downloaddir)
chromium_dir = os.path.join(download_dir, 'chromium')
chromium_src_dir = os.path.join(chromium_dir, 'src')
out_src_dir = os.path.join(chromium_src_dir, 'out')
cef_src_dir = os.path.join(chromium_src_dir, 'cef')
if options.fastupdate and os.path.exists(cef_src_dir):
cef_dir = cef_src_dir
else:
cef_dir = os.path.join(download_dir, 'cef')
##
# Manage the download directory.
##
# Create the download directory if necessary.
create_directory(download_dir)
msg("Download Directory: %s" % (download_dir))
##
# Manage the depot_tools directory.
##
# Check if the depot_tools directory exists.
if options.depottoolsdir != '':
depot_tools_dir = os.path.abspath(options.depottoolsdir)
else:
depot_tools_dir = os.path.join(download_dir, 'depot_tools')
msg("Depot Tools Directory: %s" % (depot_tools_dir))
if not os.path.exists(depot_tools_dir):
if platform == 'windows' and options.depottoolsarchive == '':
# On Windows download depot_tools as an archive file since we can't assume
# that git is already installed.
options.depottoolsarchive = depot_tools_archive_url
if options.depottoolsarchive != '':
# Extract depot_tools from an archive file.
msg('Extracting %s to %s.' % \
(options.depottoolsarchive, depot_tools_dir))
if not options.dryrun:
download_and_extract(options.depottoolsarchive, depot_tools_dir)
else:
# On Linux and OS X check out depot_tools using Git.
run('git clone ' + depot_tools_url + ' ' + depot_tools_dir, download_dir)
if not options.nodepottoolsupdate:
# Update depot_tools.
# On Windows this will download required python and git binaries.
msg('Updating depot_tools')
if platform == 'windows':
run('update_depot_tools.bat', depot_tools_dir, depot_tools_dir)
else:
run('update_depot_tools', depot_tools_dir, depot_tools_dir)
# Determine the executables to use.
if platform == 'windows':
# Force use of the version bundled with depot_tools.
git_exe = os.path.join(depot_tools_dir, 'git.bat')
python_bat = 'python.bat' if is_python2 else 'python3.bat'
python_exe = os.path.join(depot_tools_dir, python_bat)
if options.dryrun and not os.path.exists(git_exe):
sys.stdout.write("WARNING: --dry-run assumes that depot_tools" \
" is already in your PATH. If it isn't\nplease" \
" specify a --depot-tools-dir value.\n")
git_exe = 'git.bat'
python_exe = python_bat
else:
git_exe = 'git'
python_exe = sys.executable
##
# Manage the cef directory.
##
# Delete the existing CEF directory if requested.
if options.forceclean and os.path.exists(cef_dir):
delete_directory(cef_dir)
# Determine the type of CEF checkout to use.
if os.path.exists(cef_dir) and not is_git_checkout(cef_dir):
raise Exception("Not a valid CEF Git checkout: %s" % (cef_dir))
# Determine the CEF download URL to use.
cef_url = options.url.strip()
if cef_url == '':
cef_url = cef_git_url
# Verify that the requested CEF URL matches the existing checkout.
if not options.nocefupdate and os.path.exists(cef_dir):
cef_existing_url = get_git_url(cef_dir)
if cef_url != cef_existing_url:
raise Exception(
'Requested CEF checkout URL %s does not match existing URL %s' %
(cef_url, cef_existing_url))
msg("CEF Branch: %s" % (cef_branch))
msg("CEF URL: %s" % (cef_url))
msg("CEF Source Directory: %s" % (cef_dir))
# Determine the CEF Git branch to use.
if options.checkout == '':
# Target the most recent branch commit from the remote repo.
if branch_is_master:
cef_checkout = 'origin/master'
else:
cef_checkout = 'origin/' + cef_branch
else:
cef_checkout = options.checkout
# Create the CEF checkout if necessary.
if not options.nocefupdate and not os.path.exists(cef_dir):
cef_checkout_new = True
run('%s clone %s %s' % (git_exe, cef_url, cef_dir), download_dir,
depot_tools_dir)
else:
cef_checkout_new = False
# Determine if the CEF checkout needs to change.
if not options.nocefupdate and os.path.exists(cef_dir):
cef_current_hash = get_git_hash(cef_dir, 'HEAD')
if not cef_checkout_new:
# Fetch updated sources.
run('%s fetch' % (git_exe), cef_dir, depot_tools_dir)
cef_desired_hash = get_git_hash(cef_dir, cef_checkout)
cef_checkout_changed = cef_checkout_new or force_change or \
options.forcecefupdate or \
cef_current_hash != cef_desired_hash
msg("CEF Current Checkout: %s" % (cef_current_hash))
msg("CEF Desired Checkout: %s (%s)" % (cef_desired_hash, cef_checkout))
if cef_checkout_changed:
if cef_dir == cef_src_dir:
# Running in fast update mode. Backup and revert the patched files before
# changing the CEF checkout.
run_patch_updater("--backup --revert")
# Update the CEF checkout.
run('%s checkout %s%s' %
(git_exe, '--force ' if discard_local_changes else '', cef_checkout), \
cef_dir, depot_tools_dir)
else:
cef_checkout_changed = False
build_compat_versions = get_build_compat_versions()
if not options.nodepottoolsupdate and \
'depot_tools_checkout' in build_compat_versions:
# Update the depot_tools checkout.
depot_tools_compat_version = build_compat_versions['depot_tools_checkout']
run('%s checkout %s%s' %
(git_exe, '--force ' if discard_local_changes else '', depot_tools_compat_version), \
depot_tools_dir, depot_tools_dir)
# Disable further depot_tools updates.
os.environ['DEPOT_TOOLS_UPDATE'] = '0'
##
# Manage the out directory.
##
out_dir = os.path.join(download_dir, 'out_' + cef_branch)
# Delete the existing out directory if requested.
if options.forceclean and os.path.exists(out_dir):
delete_directory(out_dir)
msg("CEF Output Directory: %s" % (out_dir))
##
# Manage the chromium directory.
##
# Create the chromium directory if necessary.
create_directory(chromium_dir)
if options.chromiumurl != '':
chromium_url = options.chromiumurl
else:
chromium_url = 'https://chromium.googlesource.com/chromium/src.git'
# Create gclient configuration file.
gclient_file = os.path.join(chromium_dir, '.gclient')
if not os.path.exists(gclient_file) or options.forceconfig:
# Exclude unnecessary directories. Intentionally written without newlines.
gclient_spec = \
"solutions = [{"+\
"'managed': False,"+\
"'name': 'src', "+\
"'url': '" + chromium_url + "', "+\
"'custom_vars': {"+\
"'checkout_pgo_profiles': " + ('True' if options.withpgoprofiles else 'False') + ", "+\
"}, "+\
"'custom_deps': {"+\
"'build': None, "+\
"'build/scripts/command_wrapper/bin': None, "+\
"'build/scripts/gsd_generate_index': None, "+\
"'build/scripts/private/data/reliability': None, "+\
"'build/scripts/tools/deps2git': None, "+\
"'build/third_party/lighttpd': None, "+\
"'commit-queue': None, "+\
"'depot_tools': None, "+\
"'src/chrome_frame/tools/test/reference_build/chrome': None, "+\
"'src/chrome/tools/test/reference_build/chrome_linux': None, "+\
"'src/chrome/tools/test/reference_build/chrome_mac': None, "+\
"'src/chrome/tools/test/reference_build/chrome_win': None, "+\
"}, "+\
"'deps_file': '" + deps_file + "', "+\
"'safesync_url': ''"+\
"}]"
msg('Writing %s' % gclient_file)
if not options.dryrun:
with open(gclient_file, 'w', encoding='utf-8') as fp:
write_fp(fp, gclient_spec)
# Initial Chromium checkout.
if not options.nochromiumupdate and not os.path.exists(chromium_src_dir):
chromium_checkout_new = True
run("gclient sync --nohooks --with_branch_heads --jobs 16", \
chromium_dir, depot_tools_dir)
else:
chromium_checkout_new = False
# Verify the Chromium checkout.
if not options.dryrun and not is_git_checkout(chromium_src_dir):
raise Exception('Not a valid git checkout: %s' % (chromium_src_dir))
if os.path.exists(chromium_src_dir):
msg("Chromium URL: %s" % (get_git_url(chromium_src_dir)))
# Fetch Chromium changes so that we can perform the necessary calculations using
# local history.
if not options.nochromiumupdate and os.path.exists(chromium_src_dir):
# Fetch updated sources.
run("%s fetch" % (git_exe), chromium_src_dir, depot_tools_dir)
# Also fetch tags, which are required for release branch builds.
run("%s fetch --tags" % (git_exe), chromium_src_dir, depot_tools_dir)
# Determine the Chromium checkout options required by CEF.
chromium_compat_version = build_compat_versions['chromium_checkout']
if len(options.chromiumcheckout) > 0:
chromium_checkout = options.chromiumcheckout
else:
chromium_checkout = chromium_compat_version
# Determine if the Chromium checkout needs to change.
if not options.nochromiumupdate and os.path.exists(chromium_src_dir):
chromium_current_hash = get_git_hash(chromium_src_dir, 'HEAD')
chromium_desired_hash = get_git_hash(chromium_src_dir, chromium_checkout)
chromium_checkout_changed = chromium_checkout_new or force_change or \
chromium_current_hash != chromium_desired_hash
msg("Chromium Current Checkout: %s" % (chromium_current_hash))
msg("Chromium Desired Checkout: %s (%s)" % \
(chromium_desired_hash, chromium_checkout))
else:
chromium_checkout_changed = options.dryrun
if cef_checkout_changed:
if cef_dir != cef_src_dir and os.path.exists(cef_src_dir):
# Delete the existing src/cef directory. It will be re-copied from the
# download directory later.
delete_directory(cef_src_dir)
elif chromium_checkout_changed and cef_dir == cef_src_dir:
# Running in fast update mode. Backup and revert the patched files before
# changing the Chromium checkout.
run_patch_updater("--backup --revert")
# Delete the existing src/out directory if requested.
if options.forceclean and os.path.exists(out_src_dir):
delete_directory(out_src_dir)
# Move the existing src/out directory to the correct location in the download
# directory. It will be moved back from the download directory later.
if os.path.exists(out_src_dir):
old_branch = read_branch_config_file(out_src_dir)
if old_branch != '' and (chromium_checkout_changed or
old_branch != cef_branch):
old_out_dir = os.path.join(download_dir, 'out_' + old_branch)
move_directory(out_src_dir, old_out_dir)
# Update the Chromium checkout.
if chromium_checkout_changed:
if not chromium_checkout_new and not options.fastupdate:
if options.forceclean and options.forcecleandeps:
# Remove all local changes including third-party git checkouts managed by
# gclient.
run("%s clean -dffx" % (git_exe), chromium_src_dir, depot_tools_dir)
else:
# Revert all changes in the Chromium checkout.
run("gclient revert --nohooks", chromium_dir, depot_tools_dir)
# Checkout the requested branch.
run("%s checkout %s%s" % \
(git_exe, '--force ' if discard_local_changes else '', chromium_checkout), \
chromium_src_dir, depot_tools_dir)
# Patch the Chromium DEPS file if necessary.
apply_deps_patch()
# Update third-party dependencies including branch/tag information.
run("gclient sync %s--nohooks --with_branch_heads --jobs 16" % \
('--reset ' if discard_local_changes else ''), chromium_dir, depot_tools_dir)
# Patch the Chromium runhooks scripts if necessary.
apply_runhooks_patch()
# Runs hooks for files that have been modified in the local working copy.
run("gclient runhooks --jobs 16", chromium_dir, depot_tools_dir)
# Delete the src/out directory created by `gclient sync`.
delete_directory(out_src_dir)
if cef_dir == cef_src_dir:
# Running in fast update mode.
if cef_checkout_changed or chromium_checkout_changed:
# Check and restore the patched files.
run_patch_updater("--reapply --restore")
elif os.path.exists(cef_dir) and not os.path.exists(cef_src_dir):
# Restore the src/cef directory.
copy_directory(cef_dir, cef_src_dir)
# Restore the src/out directory.
out_src_dir_exists = os.path.exists(out_src_dir)
if os.path.exists(out_dir) and not out_src_dir_exists:
move_directory(out_dir, out_src_dir)
out_src_dir_exists = True
elif not out_src_dir_exists:
create_directory(out_src_dir)
# Write the config file for identifying the branch.
write_branch_config_file(out_src_dir, cef_branch)
if options.logchromiumchanges and chromium_checkout != chromium_compat_version:
log_chromium_changes()
if options.forcepatchupdate or ((chromium_checkout_new or not options.fastupdate) and \
chromium_checkout_changed and \
chromium_checkout != chromium_compat_version):
# Not using the known-compatible Chromium version. Try to update patch files.
if options.logchromiumchanges:
out_file = os.path.join(download_dir, 'chromium_update_patches.txt')
if os.path.exists(out_file):
os.remove(out_file)
else:
out_file = None
run_patch_updater(output_file=out_file)
elif options.resave:
# Resave patch files.
run_patch_updater("--resave")
if chromium_checkout != chromium_compat_version:
if options.logchromiumchanges:
out_file = os.path.join(download_dir, 'chromium_update_patterns.txt')
if os.path.exists(out_file):
os.remove(out_file)
else:
out_file = None
check_pattern_matches(output_file=out_file)
##
# Build CEF.
##
if not options.nobuild and (chromium_checkout_changed or \
cef_checkout_changed or options.forcebuild or \
not out_src_dir_exists):
# Building should also force a distribution.
options.forcedistrib = True
# Make sure the GN configuration exists.
if not options.dryrun and \
not os.path.exists(os.path.join(cef_src_dir, 'BUILD.gn')):
raise Exception('GN configuration does not exist.')
# Print all build-related environment variables including any that were set
# previously.
for key in os.environ.keys():
if key.startswith('CEF_') or key.startswith('GCLIENT_') or \
key.startswith('GN_') or key.startswith('GYP_') or \
key.startswith('DEPOT_TOOLS_'):
msg('%s=%s' % (key, os.environ[key]))
# Generate project files.
tool = os.path.join(cef_src_dir, 'tools', 'gclient_hook.py')
run('%s %s' % (python_exe, tool), cef_src_dir, depot_tools_dir)
# Build using Ninja.
command = 'ninja '
if options.verbosebuild:
command += '-v '
if options.buildfailurelimit != 1:
command += '-k %d ' % options.buildfailurelimit
command += '-C '
target = ' ' + options.buildtarget
if options.buildtests:
target += ' ' + options.testtarget
if platform == 'linux':
target += ' chrome_sandbox'
# Make a CEF Debug build.
if not options.nodebugbuild:
build_path = os.path.join('out', get_build_directory_name(True))
args_path = os.path.join(chromium_src_dir, build_path, 'args.gn')
msg(args_path + ' contents:\n' + read_file(args_path))
run(command + build_path + target, chromium_src_dir, depot_tools_dir,
os.path.join(download_dir, 'build-%s-debug.log' % (cef_branch)) \
if options.buildlogfile else None)
if platform in sandbox_lib_platforms:
# Make the separate cef_sandbox build when GN is_official_build=true.
build_path += '_sandbox'
if os.path.exists(os.path.join(chromium_src_dir, build_path)):
args_path = os.path.join(chromium_src_dir, build_path, 'args.gn')
msg(args_path + ' contents:\n' + read_file(args_path))
run(command + build_path + ' cef_sandbox', chromium_src_dir, depot_tools_dir,
os.path.join(download_dir, 'build-%s-debug-sandbox.log' % (cef_branch)) \
if options.buildlogfile else None)
# Make a CEF Release build.
if not options.noreleasebuild:
build_path = os.path.join('out', get_build_directory_name(False))
args_path = os.path.join(chromium_src_dir, build_path, 'args.gn')
msg(args_path + ' contents:\n' + read_file(args_path))
run(command + build_path + target, chromium_src_dir, depot_tools_dir,
os.path.join(download_dir, 'build-%s-release.log' % (cef_branch)) \
if options.buildlogfile else None)
if platform in sandbox_lib_platforms:
# Make the separate cef_sandbox build when GN is_official_build=true.
build_path += '_sandbox'
if os.path.exists(os.path.join(chromium_src_dir, build_path)):
args_path = os.path.join(chromium_src_dir, build_path, 'args.gn')
msg(args_path + ' contents:\n' + read_file(args_path))
run(command + build_path + ' cef_sandbox', chromium_src_dir, depot_tools_dir,
os.path.join(download_dir, 'build-%s-release-sandbox.log' % (cef_branch)) \
if options.buildlogfile else None)
elif not options.nobuild:
msg('Not building. The source hashes have not changed and ' +
'the output folder "%s" already exists' % (out_src_dir))
##
# Run CEF tests.
##
if options.runtests:
if platform == 'windows':
test_exe = '%s.exe' % options.testtarget
elif platform == 'mac':
test_exe = '%s.app/Contents/MacOS/%s' % (options.testtarget,
options.testtarget)
elif platform == 'linux':
test_exe = options.testtarget
test_prefix = options.testprefix
if len(test_prefix) > 0:
test_prefix += ' '
test_args = options.testargs
if len(test_args) > 0:
test_args = ' ' + test_args
if not options.nodebugtests:
build_path = os.path.join(out_src_dir, get_build_directory_name(True))
test_path = os.path.join(build_path, test_exe)
if os.path.exists(test_path):
run(test_prefix + test_path + test_args, build_path, depot_tools_dir)
else:
msg('Not running debug tests. Missing executable: %s' % test_path)
if not options.noreleasetests:
build_path = os.path.join(out_src_dir, get_build_directory_name(False))
test_path = os.path.join(build_path, test_exe)
if os.path.exists(test_path):
run(test_prefix + test_path + test_args, build_path, depot_tools_dir)
else:
msg('Not running release tests. Missing executable: %s' % test_path)
##
# Create the CEF binary distribution.
##
if not options.nodistrib and (chromium_checkout_changed or \
cef_checkout_changed or options.forcedistrib):
if not options.forceclean and options.cleanartifacts:
# Clean the artifacts output directory.
artifacts_path = os.path.join(cef_src_dir, 'binary_distrib')
delete_directory(artifacts_path)
# Determine the requested distribution types.
distrib_types = []
if options.minimaldistribonly:
distrib_types.append('minimal')
elif options.clientdistribonly:
distrib_types.append('client')
elif options.sandboxdistribonly:
distrib_types.append('sandbox')
else:
distrib_types.append('standard')
if options.minimaldistrib:
distrib_types.append('minimal')
if options.clientdistrib:
distrib_types.append('client')
if options.sandboxdistrib:
distrib_types.append('sandbox')
cef_tools_dir = os.path.join(cef_src_dir, 'tools')
# Create the requested distribution types.
first_type = True
for type in distrib_types:
path = '%s make_distrib.py --output-dir=../binary_distrib/' % python_exe
if options.nodebugbuild or options.noreleasebuild or type != 'standard':
path += ' --allow-partial'
path = path + ' --ninja-build'
if options.x64build:
path += ' --x64-build'
elif options.armbuild:
path += ' --arm-build'
elif options.arm64build:
path += ' --arm64-build'
if type == 'minimal':
path += ' --minimal'
elif type == 'client':
path += ' --client'
elif type == 'sandbox':
path += ' --sandbox'
if first_type:
if options.nodistribdocs:
path += ' --no-docs'
if options.nodistribarchive:
path += ' --no-archive'
first_type = False
else:
# Don't create the symbol archives or documentation more than once.
path += ' --no-symbols --no-docs'
# Override the subdirectory name of binary_distrib if the caller requested.
if options.distribsubdir != '':
path += ' --distrib-subdir=' + options.distribsubdir
if options.distribsubdirsuffix != '':
path += ' --distrib-subdir-suffix=' + options.distribsubdirsuffix
# Create the distribution.
run(path, cef_tools_dir, depot_tools_dir)
