| #!/usr/bin/env python |
| # Copyright 2014 The Chromium Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """Compare the artifacts from two builds.""" |
| |
| import difflib |
| import json |
| import optparse |
| import os |
| import struct |
| import sys |
| import time |
| |
| from infra.libs.infra_types import freeze |
| |
| BASE_DIR = os.path.dirname(os.path.abspath(__file__)) |
| |
| |
| # List of files that are known to be non deterministic. This is a "temporary" |
| # workaround to find regression on the deterministic builders. |
| # |
| # PNaCl general bug: http://crbug.com/429358 |
| # |
| # TODO(sebmarchand): Remove this once all the files are deterministic. |
| WHITELIST = freeze({ |
| # http://crbug.com/383340 |
| 'android': { |
| # Completed. |
| }, |
| |
| # http://crbug.com/330263 |
| 'linux': { |
| # Completed. |
| }, |
| |
| # http://crbug.com/330262 |
| 'mac': { |
| 'base_unittests', |
| 'base_unittests.isolated', |
| 'browser_tests', |
| 'browser_tests.isolated', |
| 'content_browsertests', |
| 'content_browsertests.isolated', |
| 'content_unittests', |
| 'content_unittests.isolated', |
| 'crash_inspector', |
| 'crashpad_handler', |
| 'd8', |
| 'exif.so', |
| 'ffmpegsumo.so', |
| 'genmacro', |
| 'genmodule', |
| 'genperf', |
| 'genstring', |
| 'genversion', |
| 'image_diff', |
| 'infoplist_strings_tool', |
| 'interactive_ui_tests', |
| 'interactive_ui_tests.isolated', |
| 'ipc_mojo_perftests', |
| 'ipc_mojo_unittests', |
| 'libclearkeycdm.dylib', |
| 'mksnapshot', |
| 'net_unittests', |
| 'net_unittests.isolated', |
| 'osmesa.so', |
| 'pdfsqueeze', |
| 'peerconnection_server', |
| 'protoc', |
| 're2c', |
| 'sync_integration_tests', |
| 'sync_integration_tests.isolated', |
| 'tls_edit', |
| 'unit_tests', |
| 'unit_tests.isolated', |
| 'yasm', |
| }, |
| |
| # http://crbug.com/330260 |
| 'win': { |
| 'base_unittests.exe', |
| 'base_unittests.isolated', |
| 'browser_tests.exe', |
| 'browser_tests.isolated', |
| 'chrome.dll', |
| 'chrome.exe', |
| 'chrome_child.dll', |
| 'chrome_watcher.dll', |
| 'clearkeycdm.dll', |
| 'content_browsertests.exe', |
| 'content_browsertests.isolated', |
| 'content_unittests.exe', |
| 'content_unittests.isolated', |
| 'd8.exe', |
| 'delegate_execute.exe', |
| 'delegate_execute_unittests.exe', |
| 'ipc_mojo_perftests.exe', |
| 'ipc_mojo_unittests.exe', |
| 'interactive_ui_tests.exe', |
| 'interactive_ui_tests.isolated', |
| 'metro_driver.dll', |
| 'mksnapshot.exe', |
| 'mock_nacl_gdb.exe', |
| 'net_unittests.exe', |
| 'net_unittests.isolated', |
| 'np_test_netscape_plugin.dll', |
| 'npapi_test_plugin.dll', |
| 'peerconnection_server.exe', |
| 'sync_integration_tests.exe', |
| 'sync_integration_tests.isolated', |
| 'test_registrar.exe', |
| 'unit_tests.exe', |
| 'unit_tests.isolated', |
| }, |
| }) |
| |
| def get_files_to_compare(build_dir, recursive=False): |
| """Get the list of files to compare.""" |
| allowed = frozenset( |
| ('', '.apk', '.app', '.dll', '.dylib', '.exe', '.nexe', '.so')) |
| non_x_ok_exts = frozenset(('.apk', '.isolated')) |
| def check(f): |
| if not os.path.isfile(f): |
| return False |
| if os.path.basename(f).startswith('.'): |
| return False |
| ext = os.path.splitext(f)[1] |
| if ext in non_x_ok_exts: |
| return True |
| return ext in allowed and os.access(f, os.X_OK) |
| |
| ret_files = set() |
| for root, dirs, files in os.walk(build_dir): |
| if not recursive: |
| dirs[:] = [d for d in dirs if d.endswith('_apk')] |
| for f in (f for f in files if check(os.path.join(root, f))): |
| ret_files.add(os.path.relpath(os.path.join(root, f), build_dir)) |
| return ret_files |
| |
| |
| def diff_dict(a, b): |
| """Returns a yaml-like textural diff of two dict. |
| |
| It is currently optimized for the .isolated format. |
| """ |
| out = '' |
| for key in set(a) | set(b): |
| va = a.get(key) |
| vb = b.get(key) |
| if va.__class__ != vb.__class__: |
| out += '- %s: %r != %r\n' % (key, va, vb) |
| elif isinstance(va, dict): |
| c = diff_dict(va, vb) |
| if c: |
| out += '- %s:\n%s\n' % ( |
| key, '\n'.join(' ' + l for l in c.splitlines())) |
| elif va != vb: |
| out += '- %s: %s != %s\n' % (key, va, vb) |
| return out.rstrip() |
| |
| |
| def diff_binary(first_filepath, second_filepath, file_len): |
| """Returns a compact binary diff if the diff is small enough.""" |
| CHUNK_SIZE = 32 |
| MAX_STREAMS = 10 |
| diffs = 0 |
| streams = [] |
| offset = 0 |
| with open(first_filepath, 'rb') as lhs: |
| with open(second_filepath, 'rb') as rhs: |
| while True: |
| lhs_data = lhs.read(CHUNK_SIZE) |
| rhs_data = rhs.read(CHUNK_SIZE) |
| if not lhs_data: |
| break |
| if lhs_data != rhs_data: |
| diffs += sum(l != r for l, r in zip(lhs_data, rhs_data)) |
| if streams is not None: |
| if len(streams) < MAX_STREAMS: |
| streams.append((offset, lhs_data, rhs_data)) |
| else: |
| streams = None |
| offset += len(lhs_data) |
| del lhs_data |
| del rhs_data |
| if not diffs: |
| return None |
| result = '%d out of %d bytes are different (%.2f%%)' % ( |
| diffs, file_len, 100.0 * diffs / file_len) |
| if streams: |
| encode = lambda text: ''.join(i if 31 < ord(i) < 128 else '.' for i in text) |
| for offset, lhs_data, rhs_data in streams: |
| lhs_line = '%s \'%s\'' % (lhs_data.encode('hex'), encode(lhs_data)) |
| rhs_line = '%s \'%s\'' % (rhs_data.encode('hex'), encode(rhs_data)) |
| diff = list(difflib.Differ().compare([lhs_line], [rhs_line]))[-1][2:-1] |
| result += '\n 0x%-8x: %s\n %s\n %s' % ( |
| offset, lhs_line, rhs_line, diff) |
| return result |
| |
| |
| def compare_files(first_filepath, second_filepath): |
| """Compares two binaries and return the number of differences between them. |
| |
| Returns None if the files are equal, a string otherwise. |
| """ |
| if first_filepath.endswith('.isolated'): |
| with open(first_filepath, 'rb') as f: |
| lhs = json.load(f) |
| with open(second_filepath, 'rb') as f: |
| rhs = json.load(f) |
| diff = diff_dict(lhs, rhs) |
| if diff: |
| return '\n' + '\n'.join(' ' + line for line in diff.splitlines()) |
| # else, falls through binary comparison, it must be binary equal too. |
| |
| file_len = os.stat(first_filepath).st_size |
| if file_len != os.stat(second_filepath).st_size: |
| return 'different size: %d != %d' % ( |
| file_len, os.stat(second_filepath).st_size) |
| |
| return diff_binary(first_filepath, second_filepath, file_len) |
| |
| |
| def compare_build_artifacts(first_dir, second_dir, target_platform, |
| recursive=False): |
| """Compares the artifacts from two distinct builds.""" |
| if not os.path.isdir(first_dir): |
| print >> sys.stderr, '%s isn\'t a valid directory.' % first_dir |
| return 1 |
| if not os.path.isdir(second_dir): |
| print >> sys.stderr, '%s isn\'t a valid directory.' % second_dir |
| return 1 |
| |
| epoch_hex = struct.pack('<I', int(time.time())).encode('hex') |
| print('Epoch: %s' % |
| ' '.join(epoch_hex[i:i+2] for i in xrange(0, len(epoch_hex), 2))) |
| |
| with open(os.path.join(BASE_DIR, 'deterministic_build_blacklist.json')) as f: |
| blacklist = frozenset(json.load(f)) |
| whitelist = WHITELIST[target_platform] |
| |
| # The two directories. |
| first_list = get_files_to_compare(first_dir, recursive) - blacklist |
| second_list = get_files_to_compare(second_dir, recursive) - blacklist |
| |
| equals = [] |
| expected_diffs = [] |
| unexpected_diffs = [] |
| unexpected_equals = [] |
| all_files = sorted(first_list & second_list) |
| missing_files = sorted(first_list.symmetric_difference(second_list)) |
| if missing_files: |
| print >> sys.stderr, 'Different list of files in both directories:' |
| print >> sys.stderr, '\n'.join(' ' + i for i in missing_files) |
| unexpected_diffs.extend(missing_files) |
| |
| max_filepath_len = max(len(n) for n in all_files) |
| for f in all_files: |
| first_file = os.path.join(first_dir, f) |
| second_file = os.path.join(second_dir, f) |
| result = compare_files(first_file, second_file) |
| if not result: |
| tag = 'equal' |
| equals.append(f) |
| if f in whitelist: |
| unexpected_equals.append(f) |
| else: |
| if f in whitelist: |
| expected_diffs.append(f) |
| tag = 'expected' |
| else: |
| unexpected_diffs.append(f) |
| tag = 'unexpected' |
| result = 'DIFFERENT (%s): %s' % (tag, result) |
| print('%-*s: %s' % (max_filepath_len, f, result)) |
| unexpected_diffs.sort() |
| |
| print('Equals: %d' % len(equals)) |
| print('Expected diffs: %d' % len(expected_diffs)) |
| print('Unexpected diffs: %d' % len(unexpected_diffs)) |
| if unexpected_diffs: |
| print('Unexpected files with diffs:\n') |
| for u in unexpected_diffs: |
| print(' %s\n' % u) |
| if unexpected_equals: |
| print('Unexpected files with no diffs:\n') |
| for u in unexpected_equals: |
| print(' %s\n' % u) |
| |
| return int(bool(unexpected_diffs)) |
| |
| |
| def main(): |
| parser = optparse.OptionParser(usage='%prog [options]') |
| parser.add_option( |
| '-f', '--first-build-dir', help='The first build directory.') |
| parser.add_option( |
| '-s', '--second-build-dir', help='The second build directory.') |
| parser.add_option('-r', '--recursive', action='store_true', default=False, |
| help='Indicates if the comparison should be recursive.') |
| parser.add_option('-t', '--target-platform', help='The target platform.') |
| options, _ = parser.parse_args() |
| |
| if not options.first_build_dir: |
| parser.error('--first-build-dir is required') |
| if not options.second_build_dir: |
| parser.error('--second-build-dir is required') |
| if not options.target_platform: |
| parser.error('--target-platform is required') |
| |
| return compare_build_artifacts(os.path.abspath(options.first_build_dir), |
| os.path.abspath(options.second_build_dir), |
| options.target_platform, |
| options.recursive) |
| |
| |
| if __name__ == '__main__': |
| sys.exit(main()) |
