| # Copyright 2018 The Chromium Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| import json |
| import re |
| |
| from recipe_engine import recipe_api |
| |
| _BUCKET_NAME = 'code-coverage-data' |
| |
| # Name of the file to store the component map. |
| _COMPONENT_MAPPING_FILE_NAME = 'component_mapping_path.json' |
| |
| # Name of the file to store local diff. |
| _LOCAL_DIFF_FILE_NAME = 'local_diff.txt' |
| |
| # Name of the file to store the diff fetched from Gerrit. |
| _GERRIT_DIFF_FILE_NAME = 'gerrit_diff.txt' |
| |
| # Name of the file to store the diff mapping from local to Gerrit. |
| _LOCAL_TO_GERRIT_DIFF_MAPPING_FILE_NAME = 'local_to_gerrit_diff_mapping.json' |
| |
| # Set of valid extensions for source files that use Clang. |
| _EXTENTIONS_OF_SOURCE_FILES_SUPPORTED_BY_CLANG = set([ |
| '.mm', '.S', '.c', '.hh', '.cxx', '.hpp', '.cc', '.cpp', '.ipp', '.h', '.m', |
| '.hxx' |
| ]) |
| |
| |
| class ClangCoverageApi(recipe_api.RecipeApi): |
| """This module contains apis to interact with llvm-cov and llvm-profdata.""" |
| |
| def __init__(self, *args, **kwargs): |
| super(ClangCoverageApi, self).__init__(*args, **kwargs) |
| # A single temporary directory to contain the profile data for all targets |
| # in the build. |
| self._base_profdata_dir = None |
| # Temp dir for report. |
| self._report_dir = None |
| # Temp dir for metadata |
| self._metadata_dir = None |
| # Temp dir for source and html report. |
| self._src_and_report_dir = None |
| # Maps step names to subdirectories of the above. |
| self._profdata_dirs = {} |
| # When set, subset of source files to include in the coverage report. |
| self._affected_source_files = None |
| # When set, indicates that current context is per-cl coverage for try jobs. |
| self._is_per_cl_coverage = False |
| |
| @staticmethod |
| def _dir_name_for_step(step_name): |
| """Normalizes string, converts to lowercase, removes non-alpha characters, |
| and converts spaces to underscores. |
| |
| Adapted from: |
| https://stackoverflow.com/questions/295135/turn-a-string-into-a-valid-filename |
| |
| Args: |
| step_name (str): the name of the step to use. |
| """ |
| value = re.sub('[^\w\s]', '', step_name).strip().lower() |
| value = re.sub('[-\s]+', '_', value) |
| return value |
| |
| @property |
| def raw_profile_merge_script(self): |
| """Returns the location of a script that merges raw profiles from shards. |
| |
| This is intended to be passed to the swarming recipe module to be called |
| upon completion of the shards. |
| """ |
| return self.resource('merge_profiles.py') |
| |
| def _llvm_exec(self, name): |
| return self.m.path['checkout'].join('third_party', 'llvm-build', |
| 'Release+Asserts', 'bin', name) |
| |
| @property |
| def profdata_executable(self): |
| """Returns the path to the llvm-profdata executable.""" |
| return self._llvm_exec('llvm-profdata') |
| |
| @property |
| def cov_executable(self): |
| """Returns the path to the llvm-cov executable.""" |
| return self._llvm_exec('llvm-cov') |
| |
| @property |
| def report_dir(self): |
| """A temporary directory to save a report to. Created on first access.""" |
| if not self._report_dir: |
| self._report_dir = self.m.path.mkdtemp() |
| return self._report_dir |
| |
| @property |
| def metadata_dir(self): |
| """A temporary directory for the metadata. Created on first access.""" |
| if not self._metadata_dir: |
| self._metadata_dir = self.m.path.mkdtemp() |
| return self._metadata_dir |
| |
| @property |
| def src_and_report_dir(self): |
| """A temporary directory to copy source and html report report to.""" |
| if not self._src_and_report_dir: |
| self._src_and_report_dir = self.m.path.mkdtemp() |
| return self._src_and_report_dir |
| |
| def profdata_dir(self, step_name=None): |
| """Ensures a directory exists for writing the step-level merged profdata. |
| |
| Args: |
| step_name (str): The name of the step for the target whose profile we'll |
| save in in this dir. None for getting the parent directory to contain |
| the dirs for all steps. |
| """ |
| # Create the parent directory when first needed. |
| if not self._base_profdata_dir: |
| self._base_profdata_dir = self.m.path.mkdtemp() |
| |
| if not step_name: |
| return self._base_profdata_dir |
| |
| if step_name in self._profdata_dirs: |
| return self._profdata_dirs[step_name] |
| |
| new_dir = self._base_profdata_dir.join(self._dir_name_for_step(step_name)) |
| self.m.file.ensure_directory('ensure profdata dir for %s' % step_name, |
| new_dir) |
| self._profdata_dirs[step_name] = new_dir |
| return new_dir |
| |
| @property |
| def using_coverage(self): |
| """Checks if the current build is running coverage-instrumented targets.""" |
| # TODO(crbug.com/896751): Implement a cleaner way to determine if the recipe |
| # is using code coverage instrumentation. |
| return (self.m.gclient.c and self.m.gclient.c.solutions and |
| 'checkout_clang_coverage_tools' in self.m.gclient.c.solutions[0] |
| .custom_vars) |
| |
| def _get_binaries(self, tests): |
| """Returns a path to the binary for the given test object.""" |
| # TODO(crbug.com/899974): Implement a sturdier approach that also works in |
| # separate builder-tester setup. |
| |
| # This naive approach relies on the test binary sharing a name with the test |
| # target. Also, this only works for builder_tester on linux. |
| binaries = [] |
| for t in tests: |
| if t.is_gtest and t.runs_on_swarming: |
| binaries.append(self.m.chromium.output_dir.join(t.isolate_target)) |
| # TODO(crbug.com/914213): Remove webkit_layout_tests reference. |
| elif 'webkit_layout_tests' in t.isolate_target or ( |
| 'blink_web_tests' in t.isolate_target): |
| binaries.append(self.m.chromium.output_dir.join('content_shell')) |
| elif t.isolate_target.endswith('_fuzzer'): |
| binaries.append(self.m.chromium.output_dir.join(t.isolate_target)) |
| return list(set(binaries)) |
| |
| def _filter_source_file(self, file_paths): |
| """Fitlers source files with valid extensions. |
| |
| Set of valid extensions is defined in: |
| _EXTENTIONS_OF_SOURCE_FILES_SUPPORTED_BY_CLANG. |
| |
| Args: |
| file_paths: A list of file paths relative to the checkout path. |
| |
| Returns: |
| A sub-list of the input with valid extensions. |
| """ |
| source_files = [] |
| for file_path in file_paths: |
| if any([ |
| file_path.endswith(extension) |
| for extension in _EXTENTIONS_OF_SOURCE_FILES_SUPPORTED_BY_CLANG |
| ]): |
| source_files.append(file_path) |
| |
| return source_files |
| |
| def instrument(self, affected_files): |
| """Saves source paths to generate coverage instrumentation for to a file. |
| |
| Args: |
| affected_files (list of str): paths to the files we want to instrument, |
| relative to the checkout path. |
| """ |
| self._is_per_cl_coverage = True |
| |
| self.m.file.ensure_directory( |
| 'create .clang-coverage', |
| self.m.path['checkout'].join('.clang-coverage')) |
| self._affected_source_files = self._filter_source_file(affected_files) |
| return self.m.python( |
| 'save paths of affected files', |
| self.resource('write_paths_to_instrument.py'), |
| args=[ |
| '--write-to', |
| self.m.path['checkout'].join('.clang-coverage', |
| 'files_to_instrument.txt'), |
| '--src-path', |
| self.m.path['checkout'], |
| '--build-path', |
| self.m.chromium.c.build_dir.join(self.m.chromium.c.build_config_fs), |
| ] + self._affected_source_files, |
| stdout=self.m.raw_io.output_text(add_output_log=True)) |
| |
| def process_coverage_data(self, tests): |
| """Processes the coverage data for html report or metadata. |
| |
| Args: |
| tests (list of self.m.chromium_tests.stepsl.Test): A list of test objects |
| whose binaries we are to create a coverage report for. |
| """ |
| if not self._profdata_dirs: # pragma: no cover. |
| return |
| |
| if self._is_per_cl_coverage and not self._affected_source_files: |
| self.m.python.succeeding_step( |
| 'skip collecting coverage data because no source file is changed', '') |
| return |
| |
| merged_profdata = self._merge_profdata() |
| self._surface_merging_errors() |
| binaries = self._get_binaries(tests) |
| |
| self._generate_metadata(binaries, merged_profdata) |
| self._generate_html_report(binaries, merged_profdata) |
| |
| # Put source file and html report file side-by-side. |
| self.m.python( |
| 'copy source and html report files', |
| self.resource('copy_src_and_html_report.py'), |
| args=[ |
| '--src-path', |
| self.m.path['checkout'], |
| '--html-report-dir', |
| self.report_dir, |
| '--output-dir', |
| self.src_and_report_dir, |
| ]) |
| |
| source_and_report_gs_path = self._compose_gs_path_for_coverage_data( |
| 'source_and_report') |
| upload_step = self.m.gsutil.upload( |
| self.src_and_report_dir, |
| _BUCKET_NAME, |
| source_and_report_gs_path, |
| link_name=None, |
| args=['-r'], |
| multithreaded=True, |
| name='upload source and html report files') |
| upload_step.presentation.links['html report'] = ( |
| 'https://storage.cloud.google.com/%s/%s/index.html' % |
| (_BUCKET_NAME, source_and_report_gs_path)) |
| upload_step.presentation.properties[ |
| 'coverage_source_and_report_gs_path'] = source_and_report_gs_path |
| |
| def _merge_profdata(self): |
| """Merges the profdata generated by each step to a single profdata.""" |
| merged_profdata = self.profdata_dir().join('merged.profdata') |
| self.m.python( |
| 'merge profile data for %d targets' % len(self._profdata_dirs), |
| self.resource('merge_steps.py'), |
| args=[ |
| '--input-dir', |
| self.profdata_dir(), |
| '--output-file', |
| merged_profdata, |
| '--llvm-profdata', |
| self.profdata_executable, |
| ]) |
| |
| gs_path = self._compose_gs_path_for_coverage_data('merged.profdata') |
| upload_step = self.m.gsutil.upload( |
| merged_profdata, |
| _BUCKET_NAME, |
| gs_path, |
| link_name=None, |
| name='upload merged.profdata') |
| upload_step.presentation.links['merged.profdata'] = ( |
| 'https://storage.cloud.google.com/%s/%s' % ( |
| _BUCKET_NAME, gs_path)) |
| upload_step.presentation.properties['merged_profdata_gs_path'] = gs_path |
| |
| return merged_profdata |
| |
| def _generate_html_report(self, binaries, profdata_path): |
| """Generate html coverage report for the given binaries. |
| |
| Produce a coverage report for the instrumented test targets and upload to |
| the appropriate bucket. |
| """ |
| args = [ |
| '--report-directory', self.report_dir, '--profdata-path', profdata_path, |
| '--llvm-cov', self.cov_executable, '--binaries' |
| ] |
| args.extend(binaries) |
| if self._is_per_cl_coverage: |
| args.append('--sources') |
| args.extend([ |
| self.m.path['checkout'].join(s) for s in self._affected_source_files |
| ]) |
| |
| self.m.python( |
| 'generate html report for %d targets' % len(self._profdata_dirs), |
| self.resource('make_report.py'), |
| args=args) |
| |
| def shard_merge(self, step_name): |
| """Returns a merge object understood by the swarming module. |
| |
| See the docstring for the `merge` parameter of api.swarming.task. |
| """ |
| return { |
| 'script': |
| self.raw_profile_merge_script, |
| 'args': [ |
| '--profdata-dir', |
| self.profdata_dir(step_name), |
| '--llvm-profdata', |
| self.profdata_executable, |
| ], |
| } |
| |
| def _compose_gs_path_for_coverage_data(self, data_type): |
| build = self.m.buildbucket.build |
| if build.input.gerrit_changes: |
| # Assume that there is only one gerrit patchset which is true for |
| # Chromium CQ in practice. |
| gerrit_change = build.input.gerrit_changes[0] |
| return 'presubmit/%s/%s/%s/%s/%s/%s/%s' % ( |
| gerrit_change.host, |
| gerrit_change.change, # Change id is unique in a Gerrit host. |
| gerrit_change.patchset, |
| build.builder.bucket, |
| build.builder.builder, |
| build.id, |
| data_type, |
| ) |
| else: |
| commit = build.input.gitiles_commit |
| assert commit is not None, 'No gitiles commit' |
| return 'postsubmit/%s/%s/%s/%s/%s/%s/%s' % ( |
| commit.host, |
| commit.project, |
| commit.id, # A commit HEX SHA1 is unique in a Gitiles project. |
| build.builder.bucket, |
| build.builder.builder, |
| build.id, |
| data_type, |
| ) |
| |
| def _generate_component_mapping(self): |
| """Generates the mapping from crbug components to directories.""" |
| component_mapping = self.m.path.mkdtemp().join(_COMPONENT_MAPPING_FILE_NAME) |
| command_path = self.m.path['checkout'].join('tools', 'checkteamtags', |
| 'extract_components.py') |
| command_parts = [command_path, '-o', component_mapping] |
| self.m.step( |
| 'Run component extraction script to generate mapping', |
| command_parts, |
| stdout=self.m.raw_io.output_text(add_output_log=True)) |
| return component_mapping |
| |
| def _generate_metadata(self, binaries, profdata_path): |
| """Generates the coverage info in metadata format.""" |
| llvm_cov = self.cov_executable |
| if not self._is_per_cl_coverage: |
| # Download the version with multi-thread support. |
| # Assume that this is running on Linux. |
| temp_dir = self.m.path.mkdtemp() |
| self.m.gsutil.download( |
| _BUCKET_NAME, |
| 'llvm_cov_multithread', |
| temp_dir, |
| name='download llvm-cov') |
| llvm_cov = temp_dir.join('llvm_cov_multithread') |
| |
| args = [ |
| '--src-path', |
| self.m.path['checkout'], |
| '--output-dir', |
| self.metadata_dir, |
| '--profdata-path', |
| profdata_path, |
| '--llvm-cov', |
| llvm_cov, |
| '--binaries', |
| ] |
| args.extend(binaries) |
| if self._is_per_cl_coverage: |
| args.append('--sources') |
| args.extend([ |
| self.m.path['checkout'].join(s) for s in self._affected_source_files |
| ]) |
| |
| # In order to correctly display the (un)covered line numbers on Gerrit. |
| # Per-cl metadata's line numbers need to be rebased because the base |
| # revision of the change in this build is different from the one on Gerrit. |
| self._generate_and_save_local_git_diff() |
| self._fetch_and_save_gerrit_git_diff() |
| self._generate_diff_mapping_from_local_to_gerrit( |
| self._affected_source_files) |
| args.extend([ |
| '--diff-mapping-path', |
| self.metadata_dir.join(_LOCAL_TO_GERRIT_DIFF_MAPPING_FILE_NAME) |
| ]) |
| else: |
| args.extend( |
| ['--component-mapping-path', |
| self._generate_component_mapping()]) |
| |
| try: |
| self.m.python( |
| 'generate metadata for %d targets' % len(self._profdata_dirs), |
| self.resource('generate_coverage_metadata.py'), |
| args=args, |
| venv=True) |
| finally: |
| gs_path = self._compose_gs_path_for_coverage_data('metadata') |
| upload_step = self.m.gsutil.upload( |
| self.metadata_dir, |
| _BUCKET_NAME, |
| gs_path, |
| link_name=None, |
| args=['-r'], |
| multithreaded=True, |
| name='upload metadata') |
| upload_step.presentation.links['metadata report'] = ( |
| 'https://storage.cloud.google.com/%s/%s/index.html' % (_BUCKET_NAME, |
| gs_path)) |
| upload_step.presentation.properties['coverage_metadata_gs_path'] = gs_path |
| upload_step.presentation.properties['coverage_gs_bucket'] = _BUCKET_NAME |
| |
| def _generate_and_save_local_git_diff(self): |
| """Generates the 'git diff' output of the patch relative to the builder.""" |
| test_output = ('diff --git a/path/test.txt b/path/test.txt\n' |
| 'index 0719398930..4a2b716881 100644\n' |
| '--- a/path/test.txt\n' |
| '+++ b/path/test.txt\n' |
| '@@ -15,2 +15,3 @@\n' |
| ' Line 10\n' |
| '-Line 11\n' |
| '+A different line 11\n' |
| '+A newly added line 12\n') |
| local_diff_file = self.metadata_dir.join(_LOCAL_DIFF_FILE_NAME) |
| |
| with self.m.context(cwd=self.m.path['checkout']): |
| self.m.git( |
| '-c', |
| 'core.quotePath=false', |
| 'diff', |
| '--cached', |
| name='generate git diff locally', |
| stdout=self.m.raw_io.output_text( |
| leak_to=local_diff_file, add_output_log=True), |
| step_test_data= |
| lambda: self.m.raw_io.test_api.stream_output(test_output)) |
| |
| def _fetch_and_save_gerrit_git_diff(self): |
| """Fetches the 'git diff' output of the patch from Gerrit.""" |
| test_output = ('diff --git a/path/test.txt b/path/test.txt\n' |
| 'index 0719398930..4a2b716881 100644\n' |
| '--- a/path/test.txt\n' |
| '+++ b/path/test.txt\n' |
| '@@ -10,2 +10,3 @@\n' |
| ' Line 10\n' |
| '-Line 11\n' |
| '+A different line 11\n' |
| '+A newly added line 12\n') |
| gerrit_diff_file = self.metadata_dir.join(_GERRIT_DIFF_FILE_NAME) |
| gerrit_change = self.m.buildbucket.build.input.gerrit_changes[0] |
| |
| self.m.python( |
| 'fetch git diff from Gerrit', |
| self.resource('fetch_diff_from_gerrit.py'), |
| args=[ |
| '--host', gerrit_change.host, '--project', gerrit_change.project, |
| '--change', gerrit_change.change, '--patchset', |
| gerrit_change.patchset |
| ], |
| stdout=self.m.raw_io.output_text( |
| leak_to=gerrit_diff_file, add_output_log=True), |
| step_test_data=lambda: self.m.raw_io.test_api.stream_output(test_output) |
| ) |
| |
| def _generate_diff_mapping_from_local_to_gerrit(self, source_files): |
| """Generates the diff mapping from local to Gerrit. |
| |
| Args: |
| source_files: List of source files to generate diff mapping for, the paths |
| are relative to the checkout path. |
| |
| So that the coverage data produced locally by the builder can be correctly |
| displayed on Gerrit. |
| """ |
| local_diff_file = self.metadata_dir.join(_LOCAL_DIFF_FILE_NAME) |
| gerrit_diff_file = self.metadata_dir.join(_GERRIT_DIFF_FILE_NAME) |
| local_to_gerrit_diff_mapping_file = self.metadata_dir.join( |
| _LOCAL_TO_GERRIT_DIFF_MAPPING_FILE_NAME) |
| |
| self.m.python( |
| 'generate diff mapping from local to Gerrit', |
| self.resource('rebase_git_diff.py'), |
| args=[ |
| '--local-diff-file', local_diff_file, '--gerrit-diff-file', |
| gerrit_diff_file, '--output-file', local_to_gerrit_diff_mapping_file |
| ] + source_files, |
| stdout=self.m.json.output()) |
| |
| def _surface_merging_errors(self): |
| test_data = { |
| "failed profiles": { |
| "browser_tests": ["/tmp/1/default-123.profraw"] |
| }, |
| "total": 1 |
| } |
| step_result = self.m.python( |
| 'Finding merging errors', |
| self.resource('load_merge_errors.py'), |
| args=['--root-dir', self.profdata_dir()], |
| step_test_data=lambda: self.m.json.test_api.output_stream(test_data), |
| stdout=self.m.json.output()) |
| |
| if step_result.stdout: |
| step_result.presentation.status = self.m.step.FAILURE |
| step_result.presentation.properties[ |
| 'bad_coverage_profiles'] = step_result.stdout |
| |
| def get_local_isolated_coverage(self, step_name, local_run_isolate_step): |
| """Collect coverage data from local isolated run. |
| |
| Analogous to the merge script that we pass for the swarming collect step, |
| this api finds the output isolate from the stdout of the step that ran the |
| isolated test locally, downloads it and merges the raw profile(s) into a |
| single profdata.""" |
| output_isolated = _find_isolated_json(local_run_isolate_step.stdout) |
| profraw_dir = self.m.path.mkdtemp() |
| if output_isolated: |
| with self.m.step.nest('generate coverage profdata for %s' % step_name): |
| self.m.python( |
| 'retrieve raw profiles for %s' % step_name, |
| self.m.swarming_client.path.join('isolateserver.py'), |
| args=[ |
| 'download', |
| '-I%s' % output_isolated['storage'], |
| '-s%s' % output_isolated['hash'], |
| '--target=%s' % profraw_dir |
| ]) |
| self.m.python( |
| 'index raw profiles for %s' % step_name, |
| self.raw_profile_merge_script, |
| args=[ |
| '--profdata-dir', |
| self.profdata_dir(step_name), |
| '--task-output-dir', |
| profraw_dir, |
| '--llvm-profdata', |
| self.profdata_executable, |
| '--output-json', |
| self.profdata_dir(step_name).join('output.json'), |
| ]) |
| |
| |
| def _find_isolated_json(stdout): |
| isolated_re = re.compile( |
| r'\[run_isolated_out_hack\](.*)\[\/run_isolated_out_hack\]') |
| match = isolated_re.search(stdout) |
| if match: |
| return json.loads(match.group(1)) |
| return None |
