| # Copyright 2015 The Chromium Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| import copy |
| import re |
| |
| from recipe_engine import recipe_api |
| |
| |
| def cache_name(build_version): |
| """Returns a name for a named cache for the `build_version`.""" |
| return 'xcode_ios_%s' % build_version |
| |
| |
| class iOSApi(recipe_api.RecipeApi): |
| |
| # Mapping of common names of supported iOS devices to product types |
| # exposed by the Swarming server. |
| PRODUCT_TYPES = { |
| 'iPad 4 GSM CDMA': 'iPad3,6', |
| 'iPad 5th Gen': 'iPad6,11', |
| 'iPad Air': 'iPad4,1', |
| 'iPad Air 2': 'iPad5,3', |
| 'iPhone 5': 'iPhone5,1', |
| 'iPhone 5s': 'iPhone6,1', |
| 'iPhone 6s': 'iPhone8,1', |
| 'iPhone 7': 'iPhone9,1', |
| 'iPhone X': 'iPhone10,3', |
| } |
| # Service account to use in swarming tasks. |
| SWARMING_SERVICE_ACCOUNT = \ |
| 'ios-isolated-tester@chops-service-accounts.iam.gserviceaccount.com' |
| CIPD_CREDENTIALS = \ |
| '/creds/service_accounts/service-account-xcode-cipd-access.json' |
| |
| # Map Xcode short version to Xcode build version. |
| XCODE_BUILD_VERSIONS = { |
| '8.0': '8a218a', |
| '8.3.3': '8e3004b', |
| '9.0': '9a235', |
| '9.2': '9c40b', |
| } |
| XCODE_BUILD_VERSION_DEFAULT = '9c40b' |
| |
| # Pinned version of |
| # https://chromium.googlesource.com/infra/infra/+/master/go/src/infra/cmd/mac_toolchain |
| MAC_TOOLCHAIN_PACKAGE = 'infra/tools/mac_toolchain/${platform}' |
| MAC_TOOLCHAIN_VERSION = ( |
| 'git_revision:796d2b92cff93fc2059623ce0a66284373ceea0a') |
| MAC_TOOLCHAIN_ROOT = '.' |
| XCODE_APP_PATH = 'Xcode.app' |
| |
| # CIPD package containing various static test utilities and binaries for WPR testing. |
| # Used with WprProxySimulatorTestRunner. |
| WPR_TOOLS_PACKAGE = 'chromium/ios/autofill/wpr-ios-tools' |
| WPR_TOOLS_VERSION = 'version:1.0' |
| WPR_TOOLS_ROOT = 'wpr-ios-tools' |
| |
| WPR_REPLAY_DATA_ROOT = 'wpr-replay-data' |
| |
| DASHBOARD_UPLOAD_URL = 'https://chromeperf.appspot.com' |
| UPLOAD_DASHBOARD_API_PROPERTIES = [ |
| 'got_revision_cp', |
| 'git_revision', |
| ] |
| |
| def __init__(self, *args, **kwargs): |
| super(iOSApi, self).__init__(*args, **kwargs) |
| self.__config = None |
| self._include_cache = {} |
| self.compilation_targets = None |
| self._checkout_dir = None |
| self._swarming_service_account = self.SWARMING_SERVICE_ACCOUNT |
| self._xcode_build_version = None |
| |
| @property |
| def bucket(self): |
| assert self.__config is not None |
| return self.__config.get('bucket') |
| |
| @property |
| def configuration(self): |
| assert self.__config is not None |
| if 'is_debug=true' in self.__config['gn_args']: |
| return 'Debug' |
| if 'is_debug=false' in self.__config['gn_args']: |
| return 'Release' |
| raise self.m.step.StepFailure('Missing required gn_arg: is_debug') |
| |
| @property |
| def platform(self): |
| assert self.__config is not None |
| if 'target_cpu="arm"' in self.__config['gn_args']: |
| return 'device' |
| if 'target_cpu="arm64"' in self.__config['gn_args']: |
| return 'device' |
| if 'target_cpu="x86"' in self.__config['gn_args']: |
| return 'simulator' |
| if 'target_cpu="x64"' in self.__config['gn_args']: |
| return 'simulator' |
| raise self.m.step.StepFailure('Missing required gn_arg: target_cpu') |
| |
| @property |
| def swarming_service_account(self): |
| return self._swarming_service_account |
| |
| @swarming_service_account.setter |
| def swarming_service_account(self, val): |
| self._swarming_service_account = val |
| |
| @property |
| def use_goma(self): |
| assert self.__config is not None |
| return 'use_goma=true' in self.__config['gn_args'] |
| |
| @property |
| def xcode_build_version(self): |
| if not self._xcode_build_version: |
| self._xcode_build_version = self.__config.get('xcode build version') |
| if not self._xcode_build_version: |
| self._xcode_build_version = self._deprecate_xcode_version( |
| self.__config.get('xcode version')) |
| if not self._xcode_build_version: # pragma: no cover |
| raise self.m.step.StepFailure('Missing required "xcode build version"') |
| return self._xcode_build_version |
| |
| def _deprecate_xcode_version(self, xcode_version, location='top level'): |
| # Let the caller handle the missing "xcode version". |
| if not xcode_version: # pragma: no cover |
| return None |
| xcode_build_version = self.XCODE_BUILD_VERSIONS.get( |
| xcode_version, self.XCODE_BUILD_VERSION_DEFAULT) |
| step_result = self.m.step('"xcode version" is DEPRECATED', None) |
| step_result.presentation.status = self.m.step.FAILURE |
| step_result.presentation.step_text = ( |
| 'Implicitly using "xcode build version": "%s" at %s.<br />' |
| 'Please update your configs.') % ( |
| xcode_build_version, location) |
| return xcode_build_version |
| |
| def _ensure_checkout_dir(self): |
| if not self._checkout_dir: |
| self._checkout_dir = self.m.chromium_checkout.get_checkout_dir({}) |
| return self._checkout_dir |
| |
| def checkout(self, gclient_apply_config=None, **kwargs): |
| """Checks out Chromium.""" |
| self.m.gclient.set_config('ios') |
| |
| gclient_apply_config = gclient_apply_config or [] |
| for config in gclient_apply_config: |
| self.m.gclient.apply_config(config) |
| |
| checkout_dir = self._ensure_checkout_dir() |
| |
| # Support for legacy buildbot clobber. If the "clobber" property is |
| # present at all with any value, clobber the whole checkout. |
| if 'clobber' in self.m.properties: |
| self.m.file.rmcontents('rmcontents checkout', checkout_dir) |
| |
| with self.m.context(cwd=kwargs.get('cwd', checkout_dir)): |
| return self.m.bot_update.ensure_checkout(**kwargs) |
| |
| def parse_tests(self, tests, include_dir, start_index=0): |
| """Parses the tests dict, reading necessary includes. |
| |
| Args: |
| tests: A list of test dicts. |
| """ |
| # Elements of the "tests" list are dicts. There are two types of elements, |
| # determined by the presence of one of these mutually exclusive keys: |
| # "app": This says to run a particular app. |
| # "include": This says to include a common set of tests from include_dir. |
| # So now we go through the "tests" list replacing any "include" keys. |
| # The value of an "include" key is the name of a set of tests to include, |
| # which can be found as a .json file in include_dir. Read the contents |
| # lazily as needed into includes. |
| |
| # expanded_tests_list will be the list of test dicts, with |
| # any "include" replaced with the tests from that include. |
| expanded_tests_list = [] |
| |
| # Generate a unique ID we can use to refer to each test, since the config |
| # may specify to run the exact same test multiple times. |
| i = start_index |
| |
| for element in tests: |
| if element.get('include'): |
| # This is an include dict. |
| include = str(element.pop('include')) |
| |
| # Lazily read the include if we haven't already. |
| if include not in self._include_cache: |
| self._include_cache[include] = self.m.json.read( |
| 'include %s' % include, |
| include_dir.join(include), |
| step_test_data=lambda: self.m.json.test_api.output({ |
| 'tests': [ |
| { |
| 'app': 'fake included test 1', |
| }, |
| { |
| 'app': 'fake included test 2', |
| }, |
| ], |
| }), |
| ).json.output |
| |
| # Now take each test dict from the include, update it with the |
| # extra keys (e.g. device, OS), and append to the list of tests. |
| for included_test in self._include_cache[include]['tests']: |
| expanded_tests_list.append(copy.deepcopy(included_test)) |
| expanded_tests_list[-1].update(element) |
| expanded_tests_list[-1]['id'] = str(i) |
| i += 1 |
| |
| else: |
| # This is a test dict. |
| expanded_tests_list.append(copy.deepcopy(element)) |
| expanded_tests_list[-1]['id'] = str(i) |
| i += 1 |
| |
| return expanded_tests_list |
| |
| def read_build_config( |
| self, |
| master_name=None, |
| build_config_base_dir=None, |
| buildername=None, |
| ): |
| """Reads the iOS build config for this bot. |
| |
| Args: |
| master_name: Name of a master to read the build config from, or None |
| to read from buildbot properties at run-time. |
| build_config_base_dir: Directory to search for build config master and |
| test include directories. |
| """ |
| buildername = buildername or self.m.properties['buildername'] |
| master_name = master_name or self.m.properties['mastername'] |
| build_config_base_dir = build_config_base_dir or ( |
| self.m.path['checkout'].join('ios', 'build', 'bots')) |
| build_config_dir = build_config_base_dir.join(master_name) |
| include_dir = build_config_base_dir.join('tests') |
| |
| self.__config = self.m.json.read( |
| 'read build config', |
| build_config_dir.join('%s.json' % buildername), |
| step_test_data=lambda: self.m.json.test_api.output( |
| self._test_data['build_config'] |
| ), |
| ).json.output |
| |
| # If this bot is triggered by another bot, then the build configuration |
| # has to be read from the parent's build config. A triggered bot only |
| # specifies the tests. |
| parent = str(self.__config.get('triggered by', '')) |
| |
| if parent: |
| parent_config = self.m.json.read( |
| 'read parent build config (%s)' % parent, |
| build_config_dir.join('%s.json' % parent), |
| step_test_data=lambda: self.m.json.test_api.output( |
| self._test_data['parent_build_config'], |
| ), |
| ).json.output |
| |
| for key, value in parent_config.iteritems(): |
| # Inherit the config of the parent, except for triggered bots. |
| # Otherwise this builder will infinitely trigger itself. |
| if key != 'triggered bots': |
| self.__config[key] = value |
| |
| # In order to simplify the code that uses the values of self.__config, here |
| # we default to empty values of their respective types, so in other places |
| # we can iterate over them without having to check if they are in the dict |
| # at all. |
| self.__config.setdefault('additional_compile_targets', []) |
| self.__config.setdefault('clobber', False) |
| self.__config.setdefault('compiler flags', []) |
| self.__config.setdefault('device check', True) |
| self.__config.setdefault('env', {}) |
| self.__config.setdefault('explain', False) |
| self.__config.setdefault('gn_args', []) |
| self.__config.setdefault('tests', []) |
| self.__config.setdefault('triggered bots', {}) |
| self.__config.setdefault('upload', []) |
| |
| self.__config['mastername'] = master_name |
| |
| self.__config['tests'] = self.parse_tests( |
| self.__config['tests'], include_dir) |
| next_index = len(self.__config['tests']) |
| self.__config['triggered tests'] = {} |
| for i, bot in enumerate(self.__config['triggered bots']): |
| bot = str(bot) |
| child_config = self.m.json.read( |
| 'read build config (%s)' % bot, |
| build_config_dir.join('%s.json' % bot), |
| step_test_data=lambda: self.m.json.test_api.output( |
| self._test_data['child_build_configs'][i], |
| ), |
| ).json.output |
| self.__config['triggered tests'][bot] = self.parse_tests( |
| child_config.get('tests', []), include_dir, start_index=next_index) |
| next_index += len(self.__config['triggered tests'][bot]) |
| |
| cfg = self.m.chromium.make_config() |
| |
| self.m.chromium.c = cfg |
| |
| if self.use_goma: |
| # Make sure these chromium configs are applied consistently for the |
| # rest of the recipe; they are needed in order for m.chromium.compile() |
| # to work correctly. |
| self.m.chromium.apply_config('ninja') |
| self.m.chromium.apply_config('default_compiler') |
| self.m.chromium.apply_config('goma') |
| |
| # apply_config('goma') sets the old (wrong) directory for goma in |
| # chromium.c.compile_py.goma_dir, but calling ensure_goma() after |
| # that fixes things, and makes sure that goma is actually |
| # available as well. |
| self.m.chromium.ensure_goma( |
| client_type=self.__config.get('goma_client_type')) |
| |
| return copy.deepcopy(self.__config) |
| |
| def get_mac_toolchain_cmd(self): |
| cipd_root = self.m.path['start_dir'] |
| self.m.cipd.ensure(cipd_root, { |
| self.MAC_TOOLCHAIN_PACKAGE: self.MAC_TOOLCHAIN_VERSION}) |
| return cipd_root.join('mac_toolchain') |
| |
| def ensure_xcode(self, xcode_build_version): |
| xcode_build_version = xcode_build_version.lower() |
| |
| # TODO(sergeyberezin): for LUCI migration, this must be a requested named |
| # cache. Make sure it exists, to avoid installing Xcode on every build. |
| xcode_app_path = self.m.path['cache'].join( |
| 'xcode_ios_%s.app' % xcode_build_version) |
| with self.m.step.nest('ensure xcode') as step_result: |
| step_result.presentation.step_text = ( |
| 'Ensuring Xcode version %s in %s' % ( |
| xcode_build_version, xcode_app_path)) |
| |
| mac_toolchain_cmd = self.get_mac_toolchain_cmd() |
| install_xcode_cmd = [ |
| mac_toolchain_cmd, 'install', |
| '-kind', 'ios', |
| '-xcode-version', xcode_build_version, |
| '-output-dir', xcode_app_path, |
| ] |
| if not self.m.runtime.is_luci: |
| install_xcode_cmd.extend([ |
| '-service-account-json', self.CIPD_CREDENTIALS, |
| ]) |
| self.m.step('install xcode', install_xcode_cmd, infra_step=True) |
| self.m.step('select xcode', |
| ['sudo', 'xcode-select', '-switch', xcode_app_path], |
| infra_step=True) |
| |
| def build( |
| self, |
| analyze=False, |
| mb_path=None, |
| suffix=None, |
| use_mb=True, |
| ): |
| """Builds from this bot's build config. |
| |
| Args: |
| analyze: Whether to use the gyp_chromium analyzer to only build affected |
| targets and filter out unaffected tests. |
| mb_path: Custom path to MB. Uses the default if unspecified. |
| suffix: Suffix to use at the end of step names. |
| use_mb: Whether or not to use mb to generate build files. |
| """ |
| assert self.__config is not None |
| |
| suffix = ' (%s)' % suffix if suffix else '' |
| |
| env = { |
| 'LANDMINES_VERBOSE': '1', |
| } |
| self.ensure_xcode(self.xcode_build_version) |
| self.m.chromium.c.env.FORCE_MAC_TOOLCHAIN = 0 |
| env['FORCE_MAC_TOOLCHAIN'] = '' |
| |
| env.update(self.__config['env']) |
| |
| build_sub_path = '%s-%s' % (self.configuration, { |
| 'simulator': 'iphonesimulator', |
| 'device': 'iphoneos', |
| }[self.platform]) |
| cwd = self.m.path['checkout'].join('out', build_sub_path) |
| |
| if self.__config['clobber']: |
| self.m.file.rmcontents('rmcontents out', cwd) |
| |
| with self.m.context(cwd=self.m.path['checkout'], env=env): |
| self.m.gclient.runhooks(name='runhooks' + suffix) |
| |
| if use_mb: |
| with self.m.context(env=env): |
| self.m.chromium.mb_gen( |
| self.__config['mastername'], |
| self.m.properties['buildername'], |
| build_dir='//out/%s' % build_sub_path, |
| mb_path=mb_path, |
| name='generate build files (mb)' + suffix, |
| use_goma=self.use_goma, |
| ) |
| else: |
| # Ensure the directory containing args.gn exists before creating the file. |
| self.m.file.ensure_directory( |
| 'ensure_directory //out/%s' % build_sub_path, |
| self.m.path['checkout'].join('out', build_sub_path)) |
| |
| # If mb is not being used, set goma_dir before generating build files. |
| if self.use_goma: |
| self.__config['gn_args'].append('goma_dir="%s"' % self.m.goma.goma_dir) |
| |
| self.m.file.write_text( |
| 'write args.gn' + suffix, |
| self.m.path['checkout'].join('out', build_sub_path, 'args.gn'), |
| '%s\n' % '\n'.join(self.__config['gn_args']), |
| ) |
| self.m.step.active_result.presentation.step_text = ( |
| '<br />%s' % '<br />'.join(self.__config['gn_args'])) |
| with self.m.context( |
| cwd=self.m.path['checkout'].join('out', build_sub_path), |
| env=env): |
| gn_path = self.m.path['checkout'].join('third_party', 'gn', 'gn') |
| |
| # TODO(jbudorick): Remove this once the gn move has fully rolled |
| # downstream. |
| if not self.m.path.exists(gn_path): |
| gn_path = self.m.path['checkout'].join('buildtools', 'mac', 'gn') |
| |
| self.m.step('generate build files (gn)' + suffix, [ |
| gn_path, |
| 'gen', |
| '--check', |
| '//out/%s' % build_sub_path, |
| ]) |
| |
| # The same test may be configured to run on multiple platforms. |
| tests = sorted(set(test['app'] for test in self.__config['tests'])) |
| |
| if self.compilation_targets is None: |
| if analyze: |
| with self.m.context(cwd=self.m.path['checkout']): |
| affected_files = ( |
| self.m.chromium_checkout.get_files_affected_by_patch()) |
| |
| test_targets, self.compilation_targets = ( |
| self.m.filter.analyze( |
| affected_files, |
| tests, |
| self.__config['additional_compile_targets'], |
| 'trybot_analyze_config.json', |
| additional_names=['chromium', 'ios'], |
| mb_mastername=self.__config['mastername'], |
| ) |
| ) |
| |
| test_targets = set(test_targets) |
| |
| for test in self.__config['tests']: |
| if test['app'] not in test_targets: |
| test['skip'] = True |
| |
| if not self.compilation_targets: |
| return |
| else: |
| self.compilation_targets = [] |
| self.compilation_targets.extend(tests) |
| self.compilation_targets.extend( |
| self.__config['additional_compile_targets']) |
| |
| self.compilation_targets.sort() |
| |
| cmd = [str(self.m.depot_tools.ninja_path), '-C', cwd] |
| cmd.extend(self.__config['compiler flags']) |
| |
| if self.use_goma: |
| cmd.extend(['-j', '50']) |
| self.m.goma.start() |
| |
| cmd.extend(self.compilation_targets) |
| exit_status = -1 |
| try: |
| with self.m.context(cwd=cwd, env=env): |
| if self.__config['explain']: |
| self.m.step('explain compile' + suffix, cmd + ['-d', 'explain', '-n']) |
| self.m.step('compile' + suffix, cmd) |
| exit_status = 0 |
| except self.m.step.StepFailure as e: |
| exit_status = e.retcode |
| raise e |
| finally: |
| if self.use_goma: |
| self.m.goma.stop( |
| ninja_log_outdir=cwd, |
| ninja_log_compiler='goma', |
| ninja_log_command=cmd, |
| build_exit_status=exit_status) |
| |
| def symupload(self, artifact, url): |
| """Uploads the given symbols file. |
| |
| Args: |
| artifact: Name of the artifact to upload. Will be found relative to the |
| out directory, so must have already been compiled. |
| url: URL of the symbol server to upload to. |
| """ |
| cmd = [ |
| self.most_recent_app_path.join('symupload'), |
| self.most_recent_app_path.join(artifact), |
| url, |
| ] |
| self.m.step('symupload %s' % artifact, cmd) |
| |
| def upload_tgz(self, artifact, bucket, path): |
| """Tar gzips and uploads the given artifact to Google Storage. |
| |
| Args: |
| artifact: Name of the artifact to upload. Will be found relative to the |
| out directory, so must have already been compiled. |
| bucket: Name of the Google Storage bucket to upload to. |
| path: Path to upload the artifact to relative to the bucket. |
| """ |
| tgz = self.m.path.basename(path) |
| archive = self.m.path.mkdtemp('tgz').join(tgz) |
| cwd = self.most_recent_app_path |
| cmd = [ |
| 'tar', |
| '--create', |
| '--directory', cwd, |
| '--file', archive, |
| '--gzip', |
| '--verbose', |
| artifact, |
| ] |
| with self.m.context(cwd=cwd): |
| self.m.step('tar %s' % tgz, cmd) |
| self.m.gsutil.upload( |
| archive, |
| bucket, |
| path, |
| link_name=tgz, |
| name='upload %s' % tgz, |
| ) |
| |
| def upload(self, base_path=None): |
| """Uploads built artifacts as instructed by this bot's build config.""" |
| assert self.__config |
| |
| if not base_path: |
| base_path = '%s/%s' % ( |
| self.m.properties['buildername'], |
| str(self.m.time.utcnow().strftime('%Y%m%d%H%M%S')), |
| ) |
| |
| for artifact in self.__config['upload']: |
| name = str(artifact['artifact']) |
| if artifact.get('symupload'): |
| self.symupload(name, artifact['symupload']) |
| elif artifact.get('compress'): |
| with self.m.step.nest('upload %s' % name): |
| self.upload_tgz( |
| name, |
| artifact.get('bucket', self.bucket), |
| '%s/%s' % (base_path, '%s.tar.gz' % (name.split('.', 1)[0])), |
| ) |
| else: |
| self.m.gsutil.upload( |
| self.most_recent_app_path.join(name), |
| artifact.get('bucket', self.bucket), |
| '%s/%s' % (base_path, name), |
| link_name=name, |
| name='upload %s' % name, |
| ) |
| |
| def bootstrap_swarming(self): |
| """Bootstraps Swarming.""" |
| self.m.swarming.show_outputs_ref_in_collect_step = False |
| self.m.swarming.show_shards_in_collect_step = True |
| self.m.swarming_client.query_script_version('swarming.py') |
| |
| @staticmethod |
| def get_step_name(test): |
| return str('%s (%s iOS %s)' % ( |
| test['app'], test['device type'], test['os'])) |
| |
| def _ensure_xcode_version(self, task): |
| """Update task with xcode version if needed.""" |
| if task.get('xcode build version'): |
| task['xcode build version'] = task['xcode build version'].lower() |
| return |
| if task.get('xcode version'): |
| task['xcode build version'] = self._deprecate_xcode_version( |
| task['xcode version'], location=task['step name']) |
| # Keep task['xcode version'] for backwards compatibility. |
| return |
| # If there is build-global "xcode version", add it here for backwards |
| # compatibility. |
| if self.__config.get('xcode version'): |
| task['xcode version'] = self.__config.get('xcode version') |
| task['xcode build version'] = self.xcode_build_version |
| |
| def isolate_test(self, test, tmp_dir, isolate_template, |
| test_cases=None, shard_num=None): |
| """Isolates a single test.""" |
| test_cases = test_cases or [] |
| step_name = self.get_step_name(test) |
| test_id = test['id'] |
| if test_cases and shard_num is not None: |
| test_id = '%s_%s' % (test_id, shard_num) |
| step_name = '%s shard %s' % (step_name, shard_num) |
| task = { |
| 'bot id': test.get('bot id'), |
| 'dimensions': test.get('dimensions'), |
| 'isolated.gen': None, |
| 'isolated hash': None, |
| 'pool': test.get('pool'), |
| 'skip': 'skip' in test, |
| 'step name': step_name, |
| 'task': None, |
| 'task_id': test_id, |
| 'test': copy.deepcopy(test), |
| 'tmp dir': None, |
| 'xcode version': test.get('xcode version'), |
| 'xcode build version': test.get('xcode build version', ''), |
| } |
| self._ensure_xcode_version(task) |
| |
| if task['skip']: |
| return task |
| |
| app_path = self.m.path.join(self.most_recent_app_dir, |
| '%s.app' % test['app']) |
| task['isolated.gen'] = tmp_dir.join('%s.isolated.gen.json' % test_id) |
| |
| args = [ |
| '--config-variable', 'OS', 'ios', |
| '--config-variable', 'app_path', app_path, |
| '--config-variable', 'restart', ( |
| 'true' if test.get('restart') else 'false'), |
| '--config-variable', 'shards', self.m.json.dumps(test.get('shards') or 1), |
| '--config-variable', 'test_args', self.m.json.dumps( |
| test.get('test args') or []), |
| '--config-variable', 'test_cases', self.m.json.dumps(test_cases or []), |
| '--config-variable', 'xctest', ( |
| 'true' if test.get('xctest') else 'false'), |
| '--isolate', isolate_template, |
| '--isolated', tmp_dir.join('%s.isolated' % test_id), |
| '--path-variable', 'app_path', app_path, |
| ] |
| |
| args.extend([ |
| '--config-variable', 'wpr_tools_path', ( |
| self.WPR_TOOLS_ROOT if test.get('replay package name') else 'NO_PATH'), |
| ]) |
| |
| args.extend([ |
| '--config-variable', 'replay_path', ( |
| self.WPR_REPLAY_DATA_ROOT if test.get('replay package name') else 'NO_PATH'), |
| ]) |
| |
| args.extend([ |
| '--config-variable', 'xcode_arg_name', 'xcode-build-version', |
| '--config-variable', 'xcode_version', task['xcode build version'], |
| ]) |
| |
| if self.platform == 'simulator': |
| args.extend([ |
| '--config-variable', 'platform', test['device type'], |
| '--config-variable', 'version', test['os'], |
| ]) |
| isolate_gen_file_contents = self.m.json.dumps({ |
| 'args': args, |
| 'dir': self._ensure_checkout_dir(), |
| 'version': 1, |
| }, indent=2) |
| try: |
| self.m.file.write_text( |
| 'generate %s.isolated.gen.json' % test_id, |
| task['isolated.gen'], |
| isolate_gen_file_contents, |
| ) |
| pres = self.m.step.active_result.presentation |
| pres.logs['%s.isolated.gen.json' % test_id] = ( |
| isolate_gen_file_contents.splitlines()) |
| pres.step_text = task['step name'] |
| except self.m.step.StepFailure as f: |
| f.result.presentation.status = self.m.step.EXCEPTION |
| task['isolated.gen'] = None |
| |
| return task |
| |
| def isolate_earlgrey_test(self, test, shard_size, tmp_dir, isolate_template): |
| """Isolate earlgrey test into small shards""" |
| cmd = ['otool', '-ov', '%s/%s' % |
| (self.m.path.join(self.most_recent_app_path, '%s.app' % test['app']), |
| test['app'])] |
| step_result = self.m.step( |
| 'shard EarlGrey test', |
| cmd, |
| stdout=self.m.raw_io.output(), |
| step_test_data=( |
| lambda: self.m.raw_io.test_api.stream_output( |
| 'name 0x1064b8438 CacheTestCase' \ |
| 'baseMethods 0x1068586d8 (struct method_list_t *)' \ |
| 'imp 0x1075e6887 -[CacheTestCase testA]' \ |
| 'types 0x1064cc3e1' \ |
| 'imp 0x1075e6887 -[CacheTestCase testB]' \ |
| 'imp 0x1075e6887 -[CacheTestCase testc]' \ |
| 'name 0x1064b8438 TabUITestCase' \ |
| 'baseMethods 0x1068586d8 (struct method_list_t *)' \ |
| 'imp 0x1075e6887 -[TabUITestCase testD]' \ |
| 'types 0x1064cc3e1 v16@0:8' \ |
| 'imp 0x1075e6887 -[TabUITestCase testE]' \ |
| 'name 0x1064b8438 KeyboardTestCase' \ |
| 'imp 0x1075e6887 -[KeyboardTestCase testF]' \ |
| 'name 0x1064b8438 PasswordsTestCase' \ |
| 'imp 0x1075e6887 -[PasswordsTestCase testG]' \ |
| 'name 0x1064b8438 ToolBarTestCase' \ |
| 'imp 0x1075e6887 -[ToolBarTestCase testH]' \ |
| ) |
| ) |
| ) |
| |
| # Shard tests by testSuites first. Get the information of testMethods |
| # as well in case we want to shard tests more evenly. |
| test_pattern = re.compile( |
| 'imp (?:0[xX][0-9a-fA-F]+ )?-\[(?P<testSuite>[A-Za-z_][A-Za-z0-9_]' |
| '*Test[Case]*) (?P<testMethod>test[A-Za-z0-9_]*)\]') |
| test_names = test_pattern.findall(step_result.stdout) |
| tests_set = set() |
| for test_name in test_names: |
| # 'ChromeTestCase' is the parent class of all EarlGrey test classes. It |
| # has no real tests. |
| if 'ChromeTestCase' != test_name[0]: |
| tests_set.add('%s' % test_name[0]) |
| testcases = sorted(tests_set) |
| |
| sublists = [testcases[i : i + shard_size] |
| for i in range(0, len(testcases), shard_size)] |
| tasks = [] |
| for i, sublist in enumerate(sublists): |
| tasks.append(self.isolate_test( |
| test, tmp_dir, isolate_template, sublist, i)) |
| tasks[-1]['buildername'] = self.m.properties['buildername'] |
| return tasks |
| |
| def isolate(self, scripts_dir='src/ios/build/bots/scripts'): |
| """Isolates the tests specified in this bot's build config.""" |
| assert self.__config |
| |
| tasks = [] |
| |
| cmd = [ |
| '%s/run.py' % scripts_dir, |
| '--app', '<(app_path)', |
| '--args-json', |
| '{"test_args": <(test_args), \ |
| "xctest": <(xctest), \ |
| "test_cases": <(test_cases), \ |
| "restart": <(restart)}', |
| '--out-dir', '${ISOLATED_OUTDIR}', |
| '--retries', '3', |
| '--shards', '<(shards)', |
| '--<(xcode_arg_name)', '<(xcode_version)', |
| '--mac-toolchain-cmd', '%s/mac_toolchain' % self.MAC_TOOLCHAIN_ROOT, |
| '--xcode-path', self.XCODE_APP_PATH, |
| '--wpr-tools-path', '<(wpr_tools_path)', |
| '--replay-path', '<(replay_path)' |
| ] |
| if self.__config.get('xcode parallelization', False): |
| cmd.append('--xcode-parallelization') |
| |
| files = [ |
| # .apps are directories. Need the trailing slash to isolate the |
| # contents of a directory. |
| '<(app_path)/', |
| '%s/' % scripts_dir, |
| ] |
| if self.__config.get('additional files'): |
| files.extend(self.__config.get('additional files')) |
| if self.platform == 'simulator': |
| iossim = self.most_recent_iossim |
| cmd.extend([ |
| '--iossim', iossim, |
| '--platform', '<(platform)', |
| '--version', '<(version)', |
| ]) |
| files.append(iossim) |
| isolate_template_contents = { |
| 'conditions': [ |
| ['OS == "ios"', { |
| 'variables': { |
| 'command': cmd, |
| 'files': files, |
| }, |
| }], |
| ], |
| } |
| |
| isolate_template = self._ensure_checkout_dir().join('template.isolate') |
| self.m.file.write_text( |
| 'generate template.isolate', |
| isolate_template, |
| str(isolate_template_contents), |
| ) |
| self.m.step.active_result.presentation.logs['template.isolate'] = ( |
| self.m.json.dumps(isolate_template_contents, indent=2).splitlines()) |
| |
| tmp_dir = self.m.path.mkdtemp('isolate') |
| |
| for test in self.__config['tests']: |
| if test.get('shard size') and 'skip' not in test: |
| tasks += self.isolate_earlgrey_test(test, test['shard size'], |
| tmp_dir, isolate_template) |
| else: |
| tasks.append(self.isolate_test(test, tmp_dir, isolate_template)) |
| tasks[-1]['buildername'] = self.m.properties['buildername'] |
| for bot, tests in self.__config['triggered tests'].iteritems(): |
| for test in tests: |
| tasks.append(self.isolate_test(test, tmp_dir, isolate_template)) |
| tasks[-1]['buildername'] = bot |
| |
| targets_to_isolate = [ |
| t['task_id'] for t in tasks |
| if t['isolated.gen'] and not t['skip']] |
| if targets_to_isolate: |
| step_result = self.m.isolate.isolate_tests( |
| tmp_dir, targets=targets_to_isolate, verbose=True) |
| for task in tasks: |
| if task['task_id'] in step_result.json.output: |
| task['isolated hash'] = step_result.json.output[task['task_id']] |
| |
| return tasks |
| |
| def trigger(self, tasks): |
| """Triggers the given Swarming tasks.""" |
| for task in tasks: |
| if not task['isolated hash']: # pragma: no cover |
| continue |
| if task['buildername'] != self.m.properties['buildername']: |
| continue |
| |
| self._ensure_xcode_version(task) |
| |
| task['tmp_dir'] = self.m.path.mkdtemp(task['task_id']) |
| |
| # TODO(huangml): Move all dimensions into configuration files. |
| trigger_script = None |
| if task.get('dimensions'): |
| trigger_script = { |
| 'script': self.m.path['checkout'].join( |
| 'testing', 'trigger_scripts', 'trigger_multiple_dimensions.py'), |
| 'args': [ |
| '--multiple-trigger-configs', self.m.json.dumps(task['dimensions']), |
| '--multiple-dimension-script-verbose', 'True', |
| ], |
| } |
| |
| cipd_packages = [( |
| self.MAC_TOOLCHAIN_ROOT, |
| self.MAC_TOOLCHAIN_PACKAGE, |
| self.MAC_TOOLCHAIN_VERSION, |
| )] |
| |
| replay_package_name = task['test'].get('replay package name') |
| replay_package_version = task['test'].get('replay package version') |
| if replay_package_name and replay_package_version: |
| cipd_packages.append(( |
| self.WPR_TOOLS_ROOT, |
| self.WPR_TOOLS_PACKAGE, |
| self.WPR_TOOLS_VERSION, |
| )) |
| cipd_packages.append(( |
| self.WPR_REPLAY_DATA_ROOT, |
| replay_package_name, |
| replay_package_version, |
| )) |
| |
| swarming_task = self.m.swarming.task( |
| task['step name'], |
| task['isolated hash'], |
| task_output_dir=task['tmp_dir'], |
| trigger_script=trigger_script, |
| service_account=self.swarming_service_account, |
| cipd_packages=cipd_packages, |
| ) |
| swarming_task.dimensions = { |
| 'pool': 'Chrome', |
| } |
| |
| # TODO(crbug.com/835036): remove this when all configs are migrated to |
| # "xcode build version". Otherwise keep it for backwards compatibility; |
| # otherwise we may receive an older Mac OS which does not support the |
| # requested Xcode version. |
| if task.get('xcode version'): |
| swarming_task.dimensions['xcode_version'] = task['xcode version'] |
| |
| assert task.get('xcode build version') |
| named_cache = cache_name(task['xcode build version']) |
| swarming_task.named_caches[named_cache] = self.XCODE_APP_PATH |
| |
| if self.platform == 'simulator': |
| swarming_task.dimensions['os'] = task['test'].get('host os') or 'Mac' |
| elif self.platform == 'device': |
| swarming_task.dimensions['os'] = 'iOS-%s' % str(task['test']['os']) |
| if self.__config.get('device check'): |
| swarming_task.dimensions['device_status'] = 'available' |
| swarming_task.wait_for_capacity = True |
| swarming_task.dimensions['device'] = self.PRODUCT_TYPES.get( |
| task['test']['device type']) |
| if not swarming_task.dimensions['device']: |
| # Create a dummy step so we can annotate it to explain what |
| # went wrong. |
| step_result = self.m.step('[trigger] %s' % task['step name'], []) |
| step_result.presentation.status = self.m.step.EXCEPTION |
| step_result.presentation.logs['supported devices'] = sorted( |
| self.PRODUCT_TYPES.keys()) |
| step_result.presentation.step_text = ( |
| 'Requested unsupported device type.') |
| continue |
| if task['bot id']: |
| swarming_task.dimensions['id'] = task['bot id'] |
| if task['pool']: |
| swarming_task.dimensions['pool'] = task['pool'] |
| |
| swarming_task.priority = task['test'].get('priority', 200) |
| |
| spec = [ |
| self.m.properties['mastername'], |
| self.m.properties['buildername'], |
| task['test']['app'], |
| self.platform, |
| task['test']['device type'], |
| task['test']['os'], |
| task['xcode build version'], |
| ] |
| |
| # e.g. |
| # chromium.mac:ios-simulator:base_unittests:simulator:iPad Air:10.0:8.0 |
| swarming_task.tags.add('spec_name:%s' % str(':'.join(spec))) |
| |
| swarming_task.tags.add( |
| 'device_type:%s' % str(task['test']['device type'])) |
| swarming_task.tags.add('ios_version:%s' % str(task['test']['os'])) |
| swarming_task.tags.add('platform:%s' % self.platform) |
| swarming_task.tags.add('test:%s' % str(task['test']['app'])) |
| |
| expiration = task['test'].get('expiration_time') or self.__config.get( |
| 'expiration_time') |
| if expiration: |
| swarming_task.expiration = expiration |
| |
| hard_timeout = task['test'].get('max runtime seconds') or self.__config.get( |
| 'max runtime seconds') |
| if hard_timeout: |
| swarming_task.hard_timeout = hard_timeout |
| |
| try: |
| self.m.swarming.trigger_task(swarming_task) |
| task['task'] = swarming_task |
| except self.m.step.StepFailure as f: |
| f.result.presentation.status = self.m.step.EXCEPTION |
| |
| return tasks |
| |
| def collect(self, tasks, upload_test_results=True): |
| """Collects the given Swarming task results.""" |
| failures = set() |
| infra_failure = False |
| |
| for task in tasks: |
| if task['buildername'] != self.m.properties['buildername']: |
| # This task isn't for this builder to collect. |
| continue |
| |
| if task['skip']: |
| # Create a dummy step to indicate we skipped this test. |
| step_result = self.m.step('[skipped] %s' % task['step name'], []) |
| step_result.presentation.step_text = ( |
| 'This test was skipped because it was not affected.') |
| continue |
| |
| if not task['task']: |
| # We failed to trigger this test. |
| # Create a dummy step for it and mark it as failed. |
| step_result = self.m.step(task['step name'], []) |
| step_result.presentation.status = self.m.step.EXCEPTION |
| if not task['isolated.gen']: |
| step_result.presentation.step_text = 'Failed to isolate the test.' |
| else: |
| step_result.presentation.step_text = 'Failed to trigger the test.' |
| failures.add(task['step name']) |
| infra_failure = True |
| continue |
| |
| try: |
| step_result = self.m.swarming.collect_task(task['task']) |
| except self.m.step.StepFailure as f: |
| step_result = f.result |
| |
| # We only run one shard, so the results we're interested in will |
| # always be shard 0. |
| swarming_summary = step_result.swarming.summary['shards'][0] |
| state = swarming_summary['state'] |
| |
| exit_code = None |
| if state == 'COMPLETED': |
| exit_code = 0 |
| exit_code = swarming_summary.get('exit_code', exit_code) |
| |
| if isinstance(exit_code, basestring): |
| try: |
| exit_code = int(exit_code) |
| except ValueError: |
| self.m.python.infra_failing_step( |
| 'Unrecognized exit_code from swarming', |
| 'Cannot handle non-integer exit_code "%s"' % exit_code) |
| |
| # Link to isolate file browser for files emitted by the test. |
| if swarming_summary.get('outputs_ref', None): |
| outputs_ref = swarming_summary['outputs_ref'] |
| step_result.presentation.links['test data'] = ( |
| '%s/browse?namespace=%s&hash=%s' % ( |
| outputs_ref['isolatedserver'], outputs_ref['namespace'], |
| outputs_ref['isolated'])) |
| |
| # Interpret the result and set the display appropriately. |
| if state == 'COMPLETED': |
| # Task completed and we got an exit code from the iOS test runner. |
| if exit_code == 1: |
| step_result.presentation.status = self.m.step.FAILURE |
| failures.add(task['step name']) |
| elif exit_code == 2: |
| # The iOS test runner exits 2 to indicate an infrastructure failure. |
| step_result.presentation.status = self.m.step.EXCEPTION |
| failures.add(task['step name']) |
| infra_failure = True |
| elif state == 'TIMED_OUT': |
| # The task was killed for taking too long. This is a test failure |
| # because the test itself hung. |
| step_result.presentation.status = self.m.step.FAILURE |
| step_result.presentation.step_text = 'Test timed out.' |
| failures.add(task['step name']) |
| elif state == 'EXPIRED': |
| # No Swarming bot accepted the task in time. |
| step_result.presentation.status = self.m.step.EXCEPTION |
| step_result.presentation.step_text = ( |
| 'No suitable Swarming bot found in time.' |
| ) |
| failures.add(task['step name']) |
| infra_failure = True |
| else: |
| step_result.presentation.status = self.m.step.EXCEPTION |
| step_result.presentation.step_text = ( |
| 'Unexpected infrastructure failure.' |
| ) |
| failures.add(task['step name']) |
| infra_failure = True |
| |
| # Add any iOS test runner results to the display. |
| shard_output_dir = self.m.path.join( |
| task['task'].task_output_dir, |
| task['task'].get_task_shard_output_dirs()[0]) |
| test_summary = self.m.path.join(shard_output_dir, 'summary.json') |
| if self.m.path.exists(test_summary): # pragma: no cover |
| with open(test_summary) as f: |
| test_summary_json = self.m.json.loads(f.read()) |
| step_result.presentation.logs['test_summary.json'] = self.m.json.dumps( |
| test_summary_json, indent=2).splitlines() |
| step_result.presentation.logs.update(test_summary_json.get('logs', {})) |
| step_result.presentation.links.update( |
| test_summary_json.get('links', {})) |
| if test_summary_json.get('step_text'): |
| step_result.presentation.step_text = '%s<br />%s' % ( |
| step_result.presentation.step_text, test_summary_json['step_text']) |
| |
| # Upload test results JSON to the flakiness dashboard. |
| if self.m.bot_update.last_returned_properties and upload_test_results: |
| test_results = self.m.path.join(shard_output_dir, 'full_results.json') |
| test_type = task['step name'] |
| if self.m.path.exists(test_results): |
| self.m.test_results.upload( |
| test_results, |
| test_type, |
| self.m.bot_update.last_returned_properties.get( |
| 'got_revision_cp', 'x@{#0}'), |
| builder_name_suffix='%s-%s' % ( |
| task['test']['device type'], task['test']['os']), |
| test_results_server='test-results.appspot.com', |
| ) |
| |
| # Upload performance data result to the perf dashboard. |
| perf_results = self.m.path.join( |
| shard_output_dir, 'Documents', 'perf_result.json') |
| if self.m.path.exists(perf_results): |
| data = self.get_perftest_data(perf_results) |
| if 'Perf Data' in data: |
| data_decode = data['Perf Data'] |
| data_result = [] |
| for testcase in data_decode: |
| for trace in data_decode[testcase]['value']: |
| data_point = self.m.perf_dashboard.get_skeleton_point( |
| 'chrome_ios_perf/%s/%s' % (testcase, trace), |
| # TODO(huangml): Use revision. |
| int(self.m.time.time()), |
| data_decode[testcase]['value'][trace] |
| ) |
| data_point['units'] = data_decode[testcase]['unit'] |
| data_result.extend([data_point]) |
| self.m.perf_dashboard.set_default_config() |
| self.m.perf_dashboard.add_point(data_result) |
| else: |
| data['benchmark_name'] = task['test']['app'] |
| args = [ |
| '--build-dir', self.m.path['checkout'].join('out'), |
| '--buildername', self.m.properties['buildername'], |
| '--buildnumber', self.m.properties['buildnumber'], |
| '--name', task['test']['app'], |
| '--perf-id', self.m.properties['buildername'], |
| '--results-file', self.m.json.input(data), |
| '--results-url', self.DASHBOARD_UPLOAD_URL, |
| ] |
| |
| for revision_name in self.UPLOAD_DASHBOARD_API_PROPERTIES: |
| if revision_name in self.m.properties: |
| args.extend(('--%s' % revision_name.replace('_', '-'), |
| self.m.properties[revision_name])) |
| |
| args.append('--output-json-dashboard-url') |
| args.append(self.m.json.output( |
| add_json_log=False, name='dashboard_url')) |
| |
| step_result = self.m.build.python( |
| '%s Dashboard Upload' % task['test']['app'], |
| self.m.chromium.package_repo_resource( |
| 'scripts', 'slave', 'upload_perf_dashboard_results.py'), |
| args, |
| step_test_data=( |
| lambda: self.m.json.test_api.output('chromeperf.appspot.com', |
| name='dashboard_url') + |
| self.m.json.test_api.output({}))) |
| |
| step_result.presentation.links['Results Dashboard'] = ( |
| step_result.json.outputs.get('dashboard_url', '')) |
| |
| self.m.swarming.report_stats() |
| |
| if failures: |
| failure = self.m.step.StepFailure |
| if infra_failure: |
| failure = self.m.step.InfraFailure |
| raise failure('Failed %s.' % ', '.join(sorted(failures))) |
| |
| def get_perftest_data(self, path): |
| # Use fake data for recipe testing. |
| if self._test_data.enabled: |
| data = None |
| if 'got_revision_cp' not in self.m.properties: |
| data = { |
| 'Perf Data' : { |
| 'startup test' : { |
| 'unit' : 'seconds', |
| 'value' : { |
| 'finish_launching' : 0.55, |
| 'become_active' : 0.68, |
| } |
| } |
| } |
| } |
| else: |
| data = { |
| "format_version": "1.0", |
| "charts": { |
| "warm_times": { |
| "http://www.google.com/": { |
| "type": "list_of_scalar_values", |
| "values": [9, 9, 8, 9], |
| "units": "sec" |
| }, |
| }, |
| "html_size": { |
| "http://www.google.com/": { |
| "type": "scalar", |
| "value": 13579, |
| "units": "bytes" |
| } |
| }, |
| "load_times": { |
| "http://www.google.com/": { |
| "type": "list_of_scalar_values", |
| "value": [4.2], |
| "std": 1.25, |
| "units": "sec" |
| } |
| } |
| } |
| } |
| else: |
| with open(path) as f: # pragma: no cover |
| data = self.m.json.loads(f.read()) |
| return data |
| |
| def test_swarming(self, scripts_dir='src/ios/build/bots/scripts', |
| upload_test_results=True): |
| """Runs tests on Swarming as instructed by this bot's build config.""" |
| assert self.__config |
| |
| with self.m.context(cwd=self.m.path['checkout']): |
| with self.m.step.nest('bootstrap swarming'): |
| self.bootstrap_swarming() |
| |
| with self.m.step.nest('isolate'): |
| tasks = self.isolate(scripts_dir=scripts_dir) |
| if self.__config['triggered bots']: |
| self.m.file.write_text( |
| 'generate isolated_tasks.json', |
| self._ensure_checkout_dir().join('isolated_tasks.json'), |
| self.m.json.dumps(tasks), |
| ) |
| |
| with self.m.step.nest('trigger'): |
| self.trigger(tasks) |
| |
| self.collect(tasks, upload_test_results) |
| |
| @property |
| def most_recent_app_path(self): |
| """Returns the Path to the directory of the most recently compiled apps.""" |
| platform = { |
| 'device': 'iphoneos', |
| 'simulator': 'iphonesimulator', |
| }[self.platform] |
| |
| return self.m.path['checkout'].join( |
| 'out', |
| '%s-%s' % (self.configuration, platform), |
| ) |
| |
| @property |
| def most_recent_app_dir(self): |
| """Returns the path (relative to checkout working dir) of the most recently |
| compiled apps.""" |
| platform = { |
| 'device': 'iphoneos', |
| 'simulator': 'iphonesimulator', |
| }[self.platform] |
| |
| return self.m.path.join( |
| 'src', |
| 'out', |
| '%s-%s' % (self.configuration, platform), |
| ) |
| |
| @property |
| def most_recent_iossim(self): |
| """Returns the path to the most recently compiled iossim.""" |
| return self.m.path.join(self.most_recent_app_dir, 'iossim') |
