| # Copyright 2018 The Chromium Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| from recipe_engine import recipe_api |
| from collections import OrderedDict |
| import json, re |
| |
| BLACKLIST = ( |
| r'(^(out|xcodebuild)[/\\](Release|Debug|Product)\w*[/\\]' + |
| r'(clang_x64[/\\])?(generated_tests|obj)[/\\])' + |
| r'|(^tools[/\\]sdks)') |
| # TODO(athom): move to third_party when swarming_client.path has a setter |
| SWARMING_CLIENT_PATH = 'tools/swarming_client' |
| SWARMING_CLIENT_REPO = ( |
| 'https://chromium.googlesource.com/infra/luci/client-py.git') |
| SWARMING_CLIENT_REV = '88229872dd17e71658fe96763feaa77915d8cbd6' |
| |
| TEST_PY_PATH = 'tools/test.py' |
| |
| CHROME_PATH_ARGUMENT = { |
| 'linux': '--chrome=browsers/chrome/google-chrome', |
| 'mac': '--chrome=browsers/Google Chrome.app/Contents/MacOS/Google Chrome', |
| 'win7': '--chrome=browsers\\Chrome\\Application\\chrome.exe', |
| 'win10': '--chrome=browsers\\Chrome\\Application\\chrome.exe', |
| 'win': '--chrome=browsers\\Chrome\\Application\\chrome.exe' |
| } |
| |
| FIREFOX_PATH_ARGUMENT = { |
| 'linux': '--firefox=browsers/firefox/firefox', |
| 'mac': '--firefox=browsers/Firefox.app/Contents/MacOS/firefox', |
| 'win7': '--firefox=browsers\\firefox\\firefox.exe', |
| 'win10': '--firefox=browsers\\firefox\\firefox.exe', |
| 'win': '--firefox=browsers\\firefox\\firefox.exe' |
| } |
| |
| |
| class DartApi(recipe_api.RecipeApi): |
| """Recipe module for code commonly used in dart recipes. |
| |
| Shouldn't be used elsewhere.""" |
| def checkout(self, clobber=False, revision=None): |
| """Checks out the dart code and prepares it for building.""" |
| self.m.gclient.set_config('dart') |
| sdk = self.m.gclient.c.solutions[0] |
| sdk.custom_deps['sdk/%s' % SWARMING_CLIENT_PATH] = \ |
| '%s@%s' % (SWARMING_CLIENT_REPO, SWARMING_CLIENT_REV) |
| self.m.goma.ensure_goma() |
| |
| with self.m.context(cwd=self.m.path['cache'].join('builder'), |
| env={'GOMA_DIR':self.m.goma.goma_dir, |
| 'GIT_TRACE_CURL': '1', |
| 'GIT_TRACE_CURL_NO_DATA': '1', |
| 'GIT_REDACT_COOKIES': 'o,SSO,GSSO_UberProxy'}): |
| try: |
| self.m.bot_update.ensure_checkout(root_solution_revision=revision) |
| except self.m.step.InfraFailure: |
| if not revision: |
| raise |
| # Maybe we failed to because of the revision, try the latest instead. |
| self.m.bot_update.ensure_checkout() |
| |
| with self.m.context(cwd=self.m.path['checkout']): |
| if clobber: |
| self.m.python('clobber', |
| self.m.path['checkout'].join( |
| 'tools', 'clean_output_directory.py')) |
| self.m.gclient.runhooks() |
| |
| |
| def get_secret(self, name): |
| """Decrypts the specified secret and returns the location of the result""" |
| cloudkms_dir = self.m.path['start_dir'].join('cloudkms') |
| self.m.cipd.ensure(cloudkms_dir, |
| {'infra/tools/luci/cloudkms/${platform}': 'latest'}) |
| |
| with self.m.context(cwd=self.m.path['cleanup']): |
| file_name = '%s.encrypted' % name |
| self.m.gsutil.download('dart-ci-credentials', file_name, file_name) |
| |
| executable_suffix = '.exe' if self.m.platform.name == 'win' else '' |
| secret_key = self.m.path['cleanup'].join('%s.key' % name) |
| self.m.step('cloudkms get key', |
| [cloudkms_dir.join('cloudkms%s' % executable_suffix), 'decrypt', |
| '-input', file_name, |
| '-output', secret_key, |
| 'projects/dart-ci/locations/' |
| 'us-central1/keyRings/dart-ci/cryptoKeys/dart-ci']) |
| return secret_key |
| |
| |
| def kill_tasks(self, ok_ret='any'): |
| """Kills leftover tasks from previous runs or steps.""" |
| self.m.python('kill processes', |
| self.m.path['checkout'].join('tools', 'task_kill.py'), |
| args=['--kill_browsers=True', '--kill_vsbuild=True'], |
| ok_ret=ok_ret) |
| |
| |
| def dart_executable(self): |
| """Returns the path to the checked-in SDK dart executable.""" |
| executable = 'dart.exe' if self.m.platform.name == 'win' else 'dart' |
| return self.m.path['checkout'].join( |
| 'tools','sdks', 'dart-sdk', 'bin', executable) |
| |
| |
| def build(self, build_args=None, name='build dart'): |
| """Builds dart using the specified build_args |
| and optionally isolates the sdk for testing using the specified isolate. |
| If an isolate is specified, it returns the hash of the isolated archive. |
| """ |
| if not build_args: # pragma: no cover |
| build_args = [] |
| build_args = build_args + ['--no-start-goma', '-j200'] |
| with self.m.context(cwd=self.m.path['checkout'], |
| env={'GOMA_DIR':self.m.goma.goma_dir}): |
| with self.m.depot_tools.on_path(): |
| self.kill_tasks() |
| build_exit_status = None |
| try: |
| self.m.goma.start() |
| self.m.python(name, |
| self.m.path['checkout'].join('tools', 'build.py'), |
| args=build_args, |
| timeout=20 * 60) |
| build_exit_status = 0 |
| except self.m.step.StepTimeout as e: |
| raise self.m.step.StepFailure( |
| 'Step "%s" timed out after 20 minutes' % name) |
| except self.m.step.StepFailure as e: |
| build_exit_status = e.retcode |
| raise e |
| finally: |
| self.m.goma.stop(build_exit_status=build_exit_status) |
| |
| |
| def upload_isolate(self, isolate_fileset): |
| """Builds an isolate""" |
| if isolate_fileset == self.m.properties.get('parent_fileset_name', None): |
| return self.m.properties.get('parent_fileset') |
| step_result = self.m.python( |
| 'upload testing fileset %s' % isolate_fileset, |
| self.m.swarming_client.path.join('isolate.py'), |
| args= ['archive', |
| '--blacklist=%s' % BLACKLIST, |
| '--ignore_broken_items', # TODO(athom) find a way to avoid that |
| '-Ihttps://isolateserver.appspot.com', |
| '-i%s' % self.m.path['checkout'].join('%s' % isolate_fileset), |
| '-s%s' % self.m.path['checkout'].join( |
| '%s.isolated' % isolate_fileset)], |
| stdout=self.m.raw_io.output('out')) |
| isolate_hash = step_result.stdout.strip()[:40] |
| step_result.presentation.step_text = 'swarming fileset hash: %s' % ( |
| isolate_hash) |
| return isolate_hash |
| |
| |
| def download_parent_isolate(self): |
| self.m.path['checkout'] = self.m.path['cleanup'] |
| isolate_hash = self.m.properties['parent_fileset'] |
| fileset_name = self.m.properties['parent_fileset_name'] |
| with self.m.context(cwd=self.m.path['cleanup']): |
| self.m.python( |
| 'downloading fileset %s' % fileset_name, |
| self.m.swarming_client.path.join('isolateserver.py'), |
| args= ['download', |
| '-Ihttps://isolateserver.appspot.com', |
| '-s%s' % isolate_hash, |
| '--target=.'], |
| stdout=self.m.raw_io.output('out')) |
| |
| |
| def shard(self, title, isolate_hash, test_args, os=None, cpu='x86-64', |
| pool='dart.tests', num_shards=0, last_shard_is_local=False, |
| cipd_packages=None, ignore_failure=False): |
| """Runs test.py in the given isolate, sharded over several swarming tasks. |
| Requires the 'shards' build property to be set to the number of tasks. |
| Returns the created task(s), which are meant to be passed into collect(). |
| """ |
| if not cipd_packages: # pragma: no cover |
| cipd_packages = [] |
| if 'shards' in self.m.properties: |
| num_shards = int(self.m.properties['shards']) |
| assert(num_shards > 0) |
| tasks = [] |
| if os is None: |
| os = self.m.platform.name |
| for shard in range(num_shards): |
| # TODO(athom) collect all the triggers, and present as a single step |
| if last_shard_is_local and shard == num_shards - 1: |
| break |
| task = self.m.swarming.task('%s_shard_%s' % (title, (shard + 1)), |
| isolate_hash, |
| cipd_packages=cipd_packages, |
| raw_cmd=test_args + |
| ['--shards=%s' % num_shards, |
| '--shard=%s' % (shard + 1), |
| '--output_directory=${ISOLATED_OUTDIR}'], |
| ignore_task_failure=ignore_failure) |
| os_names = { |
| 'win': 'Windows', |
| 'linux': 'Linux', |
| 'mac': 'Mac' |
| } |
| task.dimensions['os'] = os_names.get(os, os) |
| task.dimensions['cpu'] = cpu |
| task.dimensions['pool'] = pool |
| task.dimensions.pop('gpu', None) |
| if 'shard_timeout' in self.m.properties: |
| task.hard_timeout = int(self.m.properties['shard_timeout']) |
| # Set a priority lower than any builder, to prioritize shards. |
| task.priority = 25 |
| self.m.swarming.trigger_task(task) |
| tasks.append(task) |
| return tasks |
| |
| |
| def _report_new_results(self): |
| """Boolean that controls whether to report status of |
| deflaked tests measured against |
| latest or approved results as buildbot status, |
| instead of the status-file based exit code of test.py.""" |
| return 'new_workflow_enabled' in self.m.properties |
| |
| |
| def _run_new_steps(self): |
| """Boolean that controls whether to run the new |
| workflow that uploads test results to cloud |
| storage and deflakes changed tests.""" |
| return (self._report_new_results or |
| not self.m.buildbucket.builder_name.endswith('-try')) |
| |
| |
| def collect_all(self, tasks, commit, all_results): |
| """Collects the results of a sharded test run.""" |
| # Defer results in case one of the shards has a non-zero exit code. |
| with self.m.step.defer_results(): |
| # TODO(athom) collect all the output, and present as a single step |
| for task in tasks: |
| shards = task['shards'] |
| # All shards share the same step name, so just use the first |
| step_name = shards[0].title.split('_shard')[0] |
| results = StepResults(step_name, self.m, commit) |
| for shard in shards: |
| shard.task_output_dir = self.m.raw_io.output_dir() |
| self.m.swarming.collect_task(shard) |
| active_result = self.m.step.active_result |
| # Every shard is only a single task in swarming |
| bot_name = active_result.swarming.summary['shards'][0]['bot_id'] |
| self._add_results_and_links(bot_name, results) |
| if results.runs or results.results: |
| self._add_to_all_results(all_results, results) |
| |
| |
| def _add_results_and_links(self, bot_name, results): |
| output_dir = self.m.step.active_result.raw_io.output_dir |
| for filepath in output_dir: |
| filename = filepath.split('/')[-1] |
| filename = filename.split('\\')[-1] |
| if filename in ( |
| 'logs.json', 'results.json', 'run.json'): |
| contents = output_dir[filepath] |
| self.m.step.active_result.presentation.logs[ |
| filename] = [contents] |
| if filename == 'results.json': |
| results.add_results(bot_name, contents) |
| if filename == 'logs.json': |
| results.logs += contents |
| if filename == 'run.json': |
| results.add_run(bot_name, contents) |
| |
| |
| def get_latest_tested_commit(self): |
| builder = self._get_builder_dir() |
| latest_result = self.m.gsutil.download( |
| 'dart-test-results', |
| 'builders/%s/latest' % builder, |
| self.m.raw_io.output_text(name='latest', add_output_log=True), |
| name='find latest build', |
| ok_ret='any') # todo(athom): succeed only if file does not exist |
| latest = latest_result.raw_io.output_texts.get('latest') |
| revision = None |
| if latest: |
| latest = latest.strip() |
| revision_result = self.m.gsutil.download( |
| 'dart-test-results', |
| 'builders/%s/%s/revision' % (builder, latest), |
| self.m.raw_io.output_text(name='revision', add_output_log=True), |
| name='get revision for latest build', |
| ok_ret='any') # todo(athom): succeed only if file does not exist |
| revision = revision_result.raw_io.output_texts.get('revision') |
| |
| return (latest, revision) |
| |
| |
| def _get_builder_dir(self): |
| builder = self.m.buildbucket.builder_name |
| if builder.endswith('-try'): |
| builder = builder[:-4] |
| return builder |
| |
| |
| def _download_results(self, latest): |
| filenames = ['results.json', 'flaky.json'] |
| builder = self._get_builder_dir() |
| results_path = self.m.path['checkout'].join('LATEST') |
| self.m.file.ensure_directory('ensure LATEST dir', results_path) |
| for filename in filenames + ['approved_results.json']: |
| self.m.file.write_text( |
| 'ensure %s exists' % filename, results_path.join(filename), '') |
| if latest: |
| self.m.gsutil.download( |
| 'dart-test-results', |
| 'builders/%s/%s/*.json' % (builder, latest), |
| results_path, |
| name='download previous results', |
| ok_ret='any' if self._report_new_results() else {0}) |
| self.m.gsutil.download( |
| 'dart-test-results-approved-results', |
| 'builders/%s/approved_results.json' % builder, |
| 'LATEST/approved_results.json', |
| name='download approved results', |
| ok_ret='any') |
| |
| |
| def _deflake_results(self, results): |
| step_name = results.step_name |
| environment = results.environment |
| deflake_list = self.m.step('list tests to deflake (%s)' % step_name, |
| [self.dart_executable(), |
| 'tools/bots/compare_results.dart', |
| '--flakiness-data', |
| 'LATEST/flaky.json', |
| '--changed', |
| '--passing', |
| '--failing', |
| '--count', |
| '50', |
| 'LATEST/results.json', |
| self.m.raw_io.input_text(results.results)], |
| stdout=self.m.raw_io.output_text(add_output_log=True), |
| ok_ret='any' if self._report_new_results() else {0}).stdout |
| args = results.args + ['--repeat=5', '--test-list', |
| self.m.raw_io.input_text(deflake_list)] |
| self.run_script( |
| 'deflake %s' % step_name, TEST_PY_PATH, |
| args, |
| None, None, |
| False, environment, None, ignore_failure=True) |
| self._add_results_and_links(self.m.properties.get('bot_id'), results) |
| |
| |
| def _update_flakiness_information(self, results_str): |
| flaky_json = self.m.step('update flakiness information', |
| [self.dart_executable(), |
| 'tools/bots/update_flakiness.dart', |
| '-i', |
| 'LATEST/flaky.json', |
| '-o', |
| self.m.raw_io.output_text(name='flaky.json', add_output_log=True), |
| self.m.raw_io.input_text(results_str, name='results.json')], |
| ok_ret='any' if self._report_new_results() else {0}) |
| return flaky_json.raw_io.output_texts.get('flaky.json') |
| |
| |
| def _upload_results(self, flaky_json_str, logs_str, results_str, runs_str): |
| # Try builders do not upload results. |
| builder = str(self.m.buildbucket.builder_name) |
| if builder.endswith('try'): |
| return # pragma: no cover |
| |
| build_revision = str(self.m.buildbucket.gitiles_commit.id) |
| build_number = str(self.m.buildbucket.build.number) |
| |
| |
| self._upload_result(builder, build_number, 'revision', build_revision) |
| self._upload_result(builder, build_number, 'logs.json', logs_str) |
| self._upload_result(builder, build_number, 'results.json', results_str) |
| self._upload_result(builder, build_number, 'flaky.json', flaky_json_str) |
| self._upload_result('current_flakiness', 'single_directory', |
| 'flaky_current_%s.json' % builder, flaky_json_str) |
| self.m.gsutil.upload( |
| 'LATEST/approved_results.json', |
| 'dart-test-results', |
| 'builders/%s/%s/approved_results.json' % (builder, build_number), |
| ok_ret='any' if self._report_new_results() else {0}) |
| self._upload_result(builder, build_number, 'runs.json', runs_str) |
| # Update "latest" file |
| new_latest = self.m.raw_io.input_text(build_number, name='latest') |
| self.m.gsutil.upload( |
| new_latest, |
| 'dart-test-results', |
| 'builders/%s/%s' % (builder, 'latest'), |
| name='update "latest" reference', |
| ok_ret='any' if self._report_new_results() else {0}) |
| self._upload_results_to_bq(results_str) |
| |
| |
| def _upload_result(self, builder, build_number, filename, result_str): |
| self.m.gsutil.upload( |
| self.m.raw_io.input_text(result_str, name=filename), |
| 'dart-test-results', |
| 'builders/%s/%s/%s' % (builder, build_number, filename), |
| name='upload %s' % filename, ok_ret='any' |
| if self._report_new_results() else {0}) |
| |
| |
| def _present_results(self, logs_str, results_str, flaky_json_str): |
| # CQ: |
| # unapproved new test failures |
| # unapproved new test failures (logs) |
| # approved new test failures |
| # approved new test failures (logs) |
| # tests that began passing |
| # CI: |
| # tests that began failing |
| # tests that began failing (logs) |
| # tests that began passing |
| # unapproved failing tests |
| # unapproved failing tests (logs) |
| # unapproved passing tests |
| args = [self.dart_executable(), |
| 'tools/bots/compare_results.dart', |
| '--flakiness-data', |
| self.m.raw_io.input_text(flaky_json_str, name='flaky.json'), |
| '--human', |
| '--verbose', |
| self.m.path['checkout'].join('LATEST', 'results.json'), |
| self.m.raw_io.input_text(results_str), |
| self.m.path['checkout'].join('LATEST', 'approved_results.json'), |
| ] |
| args_logs = ["--logs", |
| self.m.raw_io.input_text(logs_str, name='logs.json'), |
| "--logs-only"] |
| links = OrderedDict() |
| judgement_args = list(args) |
| if self._report_new_results(): |
| judgement_args.append('--judgement') |
| builder_name = self.m.buildbucket.builder_name |
| if builder_name.endswith('-try'): |
| links["unapproved new test failures"] = self.m.step( |
| 'find unapproved new test failures', |
| args + ["--changed", "--failing", "--unapproved"], |
| stdout=self.m.raw_io.output_text(add_output_log=True)).stdout |
| links["unapproved new test failures (logs)"] = self.m.step( |
| 'find unapproved new test failures (logs)', |
| args + args_logs + ["--changed", "--failing", "--unapproved"], |
| stdout=self.m.raw_io.output_text(add_output_log=True)).stdout |
| links["approved new test failures"] = self.m.step( |
| 'find approved new test failures', |
| args + ["--changed", "--failing", "--approved"], |
| stdout=self.m.raw_io.output_text(add_output_log=True)).stdout |
| links["approved new test failures (logs)"] = self.m.step( |
| 'find approved new test failures (logs)', |
| args + args_logs + ["--changed", "--failing", "--approved"], |
| stdout=self.m.raw_io.output_text(add_output_log=True)).stdout |
| links["tests that began passing"] = self.m.step( |
| 'find tests that began passing', |
| args + ["--changed", "--passing", "--approved", "--unapproved"], |
| stdout=self.m.raw_io.output_text(add_output_log=True)).stdout |
| judgement_args += ["--changed", "--passing", "--failing", "--approved", |
| "--unapproved"] |
| else: |
| links["tests that began failing"] = self.m.step( |
| 'find tests that began failing', |
| args + ["--changed", "--failing", "--approved", "--unapproved"], |
| stdout=self.m.raw_io.output_text(add_output_log=True)).stdout |
| links["tests that began failing (logs)"] = self.m.step( |
| 'find tests that began failing (logs)', |
| args + args_logs + ["--changed", "--failing"], |
| stdout=self.m.raw_io.output_text(add_output_log=True)).stdout |
| links["tests that began passing"] = self.m.step( |
| 'find tests that began passing', |
| args + ["--changed", "--passing", "--approved", "--unapproved"], |
| stdout=self.m.raw_io.output_text(add_output_log=True)).stdout |
| links["unapproved failing tests"] = self.m.step( |
| 'find unapproved failing tests', |
| args + ["--unapproved", "--failing"], |
| stdout=self.m.raw_io.output_text(add_output_log=True)).stdout |
| links["unapproved failing tests (logs)"] = self.m.step( |
| 'find unapproved failing tests (logs)', |
| args + args_logs + ["--unapproved", "--failing"], |
| stdout=self.m.raw_io.output_text(add_output_log=True)).stdout |
| links["unapproved passing tests"] = self.m.step( |
| 'find unapproved passing tests', |
| args + ["--unapproved", "--passing"], |
| stdout=self.m.raw_io.output_text(add_output_log=True)).stdout |
| judgement_args += ["--failing", "--unapproved"] |
| with self.m.step.defer_results(): |
| self.m.step('test results', judgement_args) |
| # Show only the links with non-empty output (something happened). |
| for link, contents in links.iteritems(): |
| if contents != '': # pragma: no cover |
| self.m.step.active_result.presentation.logs[link] = [contents] |
| self.m.step.active_result.presentation.logs['results.json'] = [ |
| results_str] |
| |
| |
| def read_debug_log(self): |
| """Reads the debug log file""" |
| if self.m.platform.name == 'win': |
| self.m.step('debug log', |
| ['cmd.exe', '/c', 'type', '.debug.log'], |
| ok_ret='any') |
| else: |
| self.m.step('debug log', |
| ['cat', '.debug.log'], |
| ok_ret='any') |
| |
| |
| def delete_debug_log(self): |
| """Deletes the debug log file""" |
| self.m.file.remove('delete debug log', |
| self.m.path['checkout'].join('.debug.log')) |
| |
| |
| def test(self, latest, test_data): |
| """Reads the test-matrix.json file in checkout and performs each step listed |
| in the file |
| |
| Raises StepFailure. |
| """ |
| test_matrix_path = self.m.path['checkout'].join('tools', |
| 'bots', |
| 'test_matrix.json') |
| read_json = self.m.json.read( |
| 'read test-matrix.json', |
| test_matrix_path, |
| step_test_data=lambda: self.m.json.test_api.output(test_data)) |
| test_matrix = read_json.json.output |
| builder = self.m.properties["buildername"] |
| if builder.endswith(('-be', '-try', '-stable', 'dev')): |
| builder = builder[0:builder.rfind('-')] |
| isolate_hashes = {} |
| global_config = test_matrix['global'] |
| config = None |
| for c in test_matrix['builder_configurations']: |
| if builder in c['builders']: |
| config = c |
| break |
| if config is None: |
| raise self.m.step.StepFailure( |
| 'Error, could not find builder by name %s in test-matrix' % builder) |
| self.delete_debug_log() |
| self._write_file_sets(test_matrix['filesets']) |
| self._run_steps(config, isolate_hashes, builder, global_config, latest) |
| |
| |
| def _write_file_sets(self, filesets): |
| """Writes the fileset to the root of the sdk to allow for swarming to pick |
| up the files and isolate the files. |
| Args: |
| * filesets - Filesets from the test-matrix |
| """ |
| for fileset,files in filesets.iteritems(): |
| isolate_fileset = { 'variables': { 'files': files } } |
| destination_path = self.m.path['checkout'].join(fileset) |
| self.m.file.write_text('write fileset %s to sdk root' % fileset, |
| destination_path, |
| str(isolate_fileset)) |
| |
| |
| def _build_isolates(self, config, isolate_hashes): |
| """Isolate filesets from all steps in config and returns a dictionary with a |
| mapping from fileset to isolate_hash. |
| Args: |
| * config (dict) - Configuration of the builder, including the steps |
| |
| Returns (dict) - A mapping from fileset to isolate_hashes |
| """ |
| for step in config['steps']: |
| if 'fileset' in step and step['fileset'] not in isolate_hashes: |
| isolate_hash = self.upload_isolate(step['fileset']) |
| isolate_hashes[step['fileset']] = isolate_hash |
| |
| |
| def _get_option(self, builder_fragments, options, default_value): |
| """Gets an option from builder_fragments in options, or returns the default |
| value.""" |
| intersection = set(builder_fragments) & set(options) |
| if len(intersection) == 1: |
| return intersection.pop() |
| return default_value |
| |
| |
| def _get_specific_argument(self, arguments, options): |
| for arg in arguments: |
| if isinstance(arg, basestring): |
| for option in options: |
| if arg.startswith(option): |
| return arg[len(option):] |
| return None |
| |
| |
| def _has_specific_argument(self, arguments, options): |
| return self._get_specific_argument(arguments, options) is not None |
| |
| |
| def _replace_specific_argument(self, arguments, options, replacement): |
| for index,arg in enumerate(arguments): |
| for option in options: |
| if arg.startswith(option): |
| arguments[index] = replacement |
| return None |
| |
| |
| def _run_steps(self, config, isolate_hashes, builder_name, global_config, latest): |
| """Executes all steps from a json test-matrix builder entry""" |
| # Find information from the builder name. It should be in the form |
| # <info>-<os>-<mode>-<arch>-<runtime> or <info>-<os>-<mode>-<arch>. |
| builder_fragments = builder_name.split('-') |
| system = self._get_option( |
| builder_fragments, |
| ['linux', 'mac', 'win7', 'win8', 'win10', 'win'], |
| 'linux') |
| mode = self._get_option( |
| builder_fragments, |
| ['debug', 'release', 'product'], |
| 'release') |
| arch = self._get_option( |
| builder_fragments, |
| ['ia32', 'x64', 'arm', 'armv6', 'armv5te', 'arm64', 'simarm', 'simarmv6', |
| 'simarmv5te', 'simarm64', 'simdbc', 'simdbc64'], |
| 'x64') |
| runtime = self._get_option( |
| builder_fragments, |
| ['none', 'd8', 'jsshell', 'edge', 'ie11', 'firefox', 'safari', 'chrome'], |
| None) |
| with self.m.context(cwd=self.m.path['checkout']): |
| result = self.m.gclient('get checked-in SDK version', |
| ['getdep', '-r', 'sdk/tools/sdks:dart/dart-sdk/${platform}'], |
| stdout=self.m.raw_io.output_text(add_output_log=True)) |
| environment = {'system': system, |
| 'mode': mode, |
| 'arch': arch, |
| 'copy-coredumps': False, |
| 'checked_in_sdk_version': result.stdout} |
| environment['commit'] = { |
| 'commit_hash': self.m.buildbucket.gitiles_commit.id, |
| 'commit_time': self.m.git.get_timestamp(test_data='1234567') |
| } |
| # Linux and vm-*-win builders should use copy-coredumps |
| environment['copy-coredumps'] = (system == 'linux' or system == 'mac' or |
| (system.startswith('win') and builder_name.startswith('vm-'))) |
| |
| if runtime is not None: |
| if runtime == 'ff': |
| runtime = 'firefox' # pragma: no cover |
| environment['runtime'] = runtime |
| if runtime == 'chrome' or runtime == 'firefox': |
| self._download_browser(runtime, global_config[runtime]) |
| channel = 'try' |
| if 'branch' in self.m.properties: |
| channels = { |
| "refs/heads/master": "be", |
| "refs/heads/stable": "stable", |
| "refs/heads/dev": "dev" |
| } |
| channel = channels.get(self.m.properties['branch'], 'try') |
| build_py_path = 'tools/build.py' |
| tasks = [] |
| all_results = {} |
| with self.m.step.defer_results(): |
| for index,step in enumerate(config['steps']): |
| step_name = step['name'] |
| # If script is not defined, use test.py. |
| script = step.get('script', TEST_PY_PATH) |
| args = step.get('arguments', []) |
| is_build_step = script.endswith(build_py_path) |
| is_trigger = 'trigger' in step |
| |
| if self.m.platform.name == 'mac' and script.startswith('out/'): |
| script = script.replace('out/', 'xcodebuild/', 1) |
| |
| is_dart = script.endswith('/dart') |
| if is_dart: |
| executable_suffix = '.exe' if self.m.platform.name == 'win' else '' |
| script += executable_suffix |
| |
| isolate_hash = None |
| shards = step.get('shards', 0) |
| local_shard = shards > 1 and index == len(config['steps']) - 1 |
| if 'fileset' in step: |
| # We build isolates here, every time we see fileset, to wait for the |
| # building of Dart, which may be included in the fileset. |
| self._build_isolates(config, isolate_hashes) |
| isolate_hash = isolate_hashes[step['fileset']] |
| |
| environment_variables = step.get('environment', {}) |
| environment_variables['BUILDBOT_BUILDERNAME'] = ( |
| builder_name + "-%s" % channel) |
| |
| # Enable Crashpad integration if a Windows bot wants to copy coredumps. |
| if environment['copy-coredumps'] and self.m.platform.name == 'win': |
| environment_variables['DART_USE_CRASHPAD'] = '1' |
| |
| with self.m.context(cwd=self.m.path['checkout'], |
| env=environment_variables): |
| with self.m.depot_tools.on_path(): |
| if is_build_step: |
| if not self._has_specific_argument(args, ['-m', '--mode']): |
| args = ['-m%s' % mode] + args |
| if not self._has_specific_argument(args, ['-a', '--arch']): |
| args = ['-a%s' % arch] + args |
| deferred_result = self.build(name=step_name, build_args=args) |
| deferred_result.get_result() # raises build errors |
| elif is_trigger: |
| self.run_trigger(step_name, step, isolate_hash) |
| elif self._is_test_py_step(script): |
| self.run_test_py(step_name, step, isolate_hash, |
| shards, local_shard, environment, |
| tasks, global_config, all_results) |
| else: |
| self.run_script(step_name, script, args, isolate_hash, shards, |
| local_shard, environment, tasks) |
| self.collect_all(tasks, environment['commit'], all_results) |
| # todo(athom): remove this when the new workflow is fully rolled out. |
| # Old workflow test.py steps throw on failing tests and results need to |
| # be processed in the defer_results block. |
| if not self._report_new_results(): |
| self._process_test_results(all_results, latest) |
| if self._report_new_results(): |
| self._process_test_results(all_results, latest) |
| |
| |
| def _process_test_results(self, all_results, latest): |
| # If there are no test steps, all_results will be empty. |
| if self._run_new_steps() and all_results: |
| with self.m.context(cwd=self.m.path['checkout']): |
| with self.m.step.nest('download previous results'): |
| self._download_results(latest) |
| for results in all_results.itervalues(): |
| self._deflake_results(results) |
| logs_str = ''.join( |
| (results.logs for results in all_results.itervalues())) |
| results_str = ''.join( |
| (results.results for results in all_results.itervalues())) |
| flaky_json_str = self._update_flakiness_information(results_str) |
| try: |
| self._present_results(logs_str, results_str, flaky_json_str) |
| finally: |
| # Upload even if present_results fails the build |
| with self.m.step.nest('upload new results'): |
| runs_str = ''.join( |
| (results.runs for results in all_results.itervalues())) |
| self._upload_results( |
| flaky_json_str, logs_str, results_str, runs_str) |
| |
| |
| def _is_test_py_step(self, script): |
| return script.endswith(TEST_PY_PATH) |
| |
| |
| def _add_to_all_results(self, all_results, results): |
| step_name = results.step_name |
| if all_results.get(step_name) is None: |
| all_results[step_name] = results |
| else: |
| all_results[step_name].merge(results) |
| |
| |
| def _copy_property(self, src, dest, key): |
| if key in src: |
| dest[key] = src[key] |
| |
| |
| def _download_browser(self, runtime, version): |
| # Download CIPD package |
| # dart/browsers/<runtime>/${platform} <version> |
| # to directory |
| # [sdk root]/browsers |
| # Shards must install this CIPD package to the same location - |
| # there is an argument to the swarming module task creation api for this. |
| browser_path = self.m.path['checkout'].join('browsers') |
| self.m.file.ensure_directory('create browser cache', browser_path) |
| self.m.cipd.set_service_account_credentials(None) |
| version_tag = 'version:%s' % version |
| package = 'dart/browsers/%s/${platform}' % runtime |
| self.m.cipd.ensure(browser_path, { package: version_tag }) |
| |
| |
| def _upload_results_to_bq(self, results_str): |
| if not results_str: |
| return |
| assert(results_str[-1] == '\n') |
| |
| bqupload_path = self.m.path['checkout'].join('bqupload') |
| package = r'infra/tools/bqupload/${platform}' |
| self.m.cipd.ensure(bqupload_path, {package: 'latest'}) |
| |
| bqupload = bqupload_path.join('bqupload') |
| cmd = [bqupload , 'dart-ci.results.results'] |
| with self.m.step.nest('upload test results to big query'): |
| # Upload at most 1000 lines at once |
| for match in re.finditer(r'(?:.*\n){1,1000}', results_str): |
| chunk = match.group(0) |
| self.m.step('upload results chunk to big query', cmd, |
| stdin=self.m.raw_io.input_text(chunk), infra_step=True, |
| ok_ret='any' if self._report_new_results() else {0}) |
| |
| |
| def run_trigger(self, step_name, step, isolate_hash): |
| trigger_props = {} |
| self._copy_property(self.m.properties, trigger_props, 'git_revision') |
| self._copy_property(self.m.properties, trigger_props, 'revision') |
| trigger_props['parent_buildername'] = self.m.buildbucket.builder_name |
| trigger_props['parent_build_id'] = self.m.properties.get('build_id', '') |
| if isolate_hash: |
| trigger_props['parent_fileset'] = isolate_hash |
| trigger_props['parent_fileset_name'] = step['fileset'] |
| put_result = self.m.buildbucket.put( |
| [ |
| { |
| 'bucket': 'luci.dart.ci', |
| 'parameters': { |
| 'builder_name': builder_name, |
| 'properties': trigger_props, |
| 'changes': [ |
| { |
| 'author': { |
| 'email': author, |
| }, |
| } |
| for author in self.m.properties.get('blamelist', []) |
| ], |
| }, |
| } |
| for builder_name in step['trigger'] |
| ]) |
| self.m.step.active_result.presentation.step_text = step_name |
| for build in put_result.stdout['results']: |
| builder_tag = ( |
| x for x in build['build']['tags'] if x.startswith('builder:')).next() |
| builder_name = builder_tag[len('builder:'):] |
| self.m.step.active_result.presentation.links[builder_name] = ( |
| build['build']['url']) |
| |
| |
| def run_test_py(self, step_name, step, isolate_hash, shards, local_shard, |
| environment, tasks, global_config, all_results): |
| """Runs test.py with default arguments, based on configuration from. |
| Args: |
| * step_name (str) - Name of the step |
| * step (dict) - Test-matrix step |
| * isolate_hash (String) - Hash of uploadet fileset/isolate if the |
| process is to be sharded |
| * shards (int) - The number of shards |
| * local_shard (bool) - Should the current builder be one of the shards. |
| * environment (dict) - Environment with runtime, arch, system etc |
| * tasks ([task]) - placeholder to put all swarming tasks in |
| * global_config (dict) - The global section from test_matrix.json. |
| Contains version tags for the pinned browsers Firefox and Chrome. |
| """ |
| args = step.get('arguments', []) |
| test_args = ['--progress=status', |
| '--report', |
| '--time', |
| '--write-debug-log', |
| '--write-results', |
| '--write-logs'] |
| if self._report_new_results(): |
| test_args.append("--clean-exit") |
| template = self._get_specific_argument(args, ['-n']) |
| if template is not None: |
| for term in ['runtime', 'system', 'mode', 'arch']: |
| if '${%s}' % term in template: |
| template = template.replace('${%s}' % term, environment.get(term, '')) |
| self._replace_specific_argument(args, ['-n'], "-n%s" % template) |
| else: |
| if not self._has_specific_argument(args, ['-m', '--mode']): |
| test_args = ['-m%s' % environment['mode']] + test_args |
| if not self._has_specific_argument(args, ['-a', '--arch']): |
| test_args = ['-a%s' % environment['arch']] + test_args |
| if 'runtime' in environment and not self._has_specific_argument( |
| args, ['-r', '--runtime']): |
| test_args = test_args + ['-r%s' % environment['runtime']] |
| args = test_args + args |
| if environment['system'] in ['win7', 'win8', 'win10']: |
| args = args + ['--builder-tag=%s' % environment['system']] |
| if environment['copy-coredumps']: |
| args = args + ['--copy-coredumps'] |
| # The --chrome flag is added here if the runtime for the bot is |
| # chrome. This also catches the case where there is a specific |
| # argument -r or --runtime. It misses the case where |
| # a recipe calls run_script directly with a test.py command. |
| # The download of the browser from CIPD should also be moved |
| # here (perhaps checking if it is already done) so we catch |
| # specific test steps with runtime chrome in a bot without that |
| # global runtime. |
| cipd_packages = [] |
| if any('chrome' in arg for arg in args): |
| version_tag = 'version:%s' % global_config['chrome'] |
| cipd_packages.append(('browsers', |
| 'dart/browsers/chrome/${platform}', |
| version_tag)) |
| args = args + [CHROME_PATH_ARGUMENT[environment['system']]] |
| if any('firefox' in arg for arg in args): |
| version_tag = 'version:%s' % global_config['firefox'] |
| cipd_packages.append(('browsers', |
| 'dart/browsers/firefox/${platform}', |
| version_tag)) |
| args = args + [FIREFOX_PATH_ARGUMENT[environment['system']]] |
| if 'exclude_tests' in step: |
| args = args + ['--exclude_suite=' + ','.join(step['exclude_tests'])] |
| if 'tests' in step: |
| args = args + step['tests'] |
| |
| with self.m.step.defer_results(): |
| # TODO(https://github.com/dart-lang/sdk/issues/34158): The Firefox |
| # processes are not exiting properly, causing the step to fail at the end, |
| # despite the results having been written out. Temporarily ignore failures |
| # on firefox. |
| firefox_bug = 'firefox' in self.m.buildbucket.builder_name |
| self.run_script(step_name, TEST_PY_PATH, args, isolate_hash, shards, |
| local_shard, environment, tasks, |
| cipd_packages=cipd_packages, |
| ignore_failure=firefox_bug) |
| results = StepResults(step_name, self.m, environment['commit']) |
| results.args = args |
| results.environment = environment |
| if shards == 0 or local_shard: |
| self._add_results_and_links(self.m.properties.get('bot_id'), results) |
| # Add even if we don't have a local shard to record args and environment |
| self._add_to_all_results(all_results, results) |
| |
| |
| def run_script(self, step_name, script, args, isolate_hash, shards, |
| local_shard, environment, tasks, |
| cipd_packages=None, ignore_failure=False): |
| """Runs a specific script with current working directory to be checkout. If |
| the runtime (passed in environment) is a browser, and the system is linux, |
| xvfb is used. If an isolate_hash is passed in, it will swarm the command. |
| Args: |
| * step_name (str) - Name of the step |
| * script (str) - The script to invoke |
| * args ([str]) - Additional arguments to test.py |
| * isolate_hash (str) - The isolate hash if the script should be swarmed |
| * shards (int) - The number of shards to invoke |
| * local_shard (bool) - Should the current builder be used as a shard |
| * environment (dict) - Environment with runtime, arch, system etc |
| * tasks ([task]) - placeholder to hold swarming tasks |
| * cipd_packages ([tuple]) - list of 3-tuples specifying a cipd package |
| to be downloaded |
| * ignore_failure - Do not turn step red if this script fails. |
| """ |
| if not cipd_packages: # pragma: no cover |
| cipd_packages = [] |
| cipd_packages.append(('tools/sdks', |
| 'dart/dart-sdk/${platform}', |
| environment['checked_in_sdk_version'])) |
| ok_ret = 'any' if ignore_failure else {0} |
| runtime = self._get_specific_argument(args, ['-r', '--runtime']) |
| if runtime is None: |
| runtime = environment.get('runtime', None) |
| use_xvfb = (runtime in ['chrome', 'firefox'] and |
| environment['system'] == 'linux') |
| is_python = str(script).endswith('.py') |
| xvfb_cmd = [] |
| if use_xvfb: |
| xvfb_cmd = [ |
| '/usr/bin/xvfb-run', |
| '-a', |
| '--server-args=-screen 0 1024x768x24'] |
| if is_python: |
| xvfb_cmd += ['python', '-u'] |
| |
| if isolate_hash: |
| with self.m.step.nest('trigger shards for %s' % step_name): |
| cpu = 'arm64' if environment['arch'] == 'arm64' else 'x86-64' |
| tasks.append({ |
| 'shards': self.shard(step_name, isolate_hash, |
| xvfb_cmd + [script] + args, |
| num_shards=shards, |
| last_shard_is_local=local_shard, |
| cipd_packages=cipd_packages, |
| ignore_failure=ignore_failure, |
| cpu=cpu), |
| 'args': args, |
| 'environment': environment, |
| 'step_name': step_name}) |
| if local_shard: |
| args = args + [ |
| '--shards=%s' % shards, |
| '--shard=%s' % shards, |
| ] |
| step_name = '%s_shard_%s' % (step_name, shards) |
| else: |
| return # Shards have been triggered, no local shard to run. |
| |
| if self._is_test_py_step(script): |
| args = args + [ |
| '--output_directory', |
| self.m.raw_io.output_dir(), |
| ] |
| |
| cmd = self.m.path['checkout'].join(*script.split('/')) |
| if is_python and not use_xvfb: |
| self.m.python(step_name, cmd, args=args, ok_ret=ok_ret) |
| else: |
| self.m.step(step_name, xvfb_cmd + [cmd] + args, ok_ret=ok_ret) |
| |
| |
| class StepResults: |
| def __init__(self, step_name, m, commit): |
| self.step_name = step_name |
| self.logs = '' |
| self.results = '' |
| self.runs = '' |
| self.args = None |
| self.environment = None |
| self.commit = commit |
| self.builder_name = str(m.buildbucket.builder_name) |
| self.build_number = str(m.buildbucket.build.number) |
| |
| |
| TEMPLATE = (',"commit_time":%s,"commit_hash":"%s",' + |
| '"build_number":%s,"builder_name":"%s","bot_name":"%s"}\n') |
| |
| |
| def add_results(self, bot_name, results_str): |
| extra = self.TEMPLATE % (str(self.commit['commit_time']), |
| str(self.commit['commit_hash']), self.build_number, |
| self.builder_name, bot_name) |
| all_matches = re.finditer(r'(^{.*)(?:})', results_str, flags=re.MULTILINE) |
| all_chunks = (chunk for match in all_matches for chunk in (match.group(1), extra)) |
| self.results += ''.join(all_chunks) |
| |
| |
| def add_run(self, bot_name, run_json): |
| run = json.loads(run_json) |
| run['commit_time'] = self.commit['commit_time'] |
| run['commit_hash'] = self.commit['commit_hash'] |
| run['builder_name'] = self.builder_name |
| run['bot_name'] = bot_name |
| self.runs += json.dumps(run) + '\n' |
| |
| |
| def merge(self, other): |
| self.logs += other.logs |
| self.results += other.results |
| self.runs += other.runs |
| # Environment and Args are the same for all shards |
| self.args = self.args or other.args |
| self.environment = self.environment or other.environment |
