| # Copyright 2017 The Chromium Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| import json |
| |
| from recipe_engine import post_process |
| |
| DEPS = [ |
| 'chromium_tests', |
| 'recipe_engine/properties', |
| 'swarming', |
| 'test_utils', |
| ] |
| |
| |
| def RunSteps(api): |
| bot_config = api.chromium_tests.trybots[ |
| api.properties['mastername']]['builders'][api.properties['buildername']] |
| bot_config_object = api.chromium_tests.create_bot_config_object( |
| bot_config['bot_ids']) |
| api.chromium_tests.configure_build(bot_config_object) |
| update_step, _bot_db = api.chromium_tests.prepare_checkout(bot_config_object) |
| |
| if api.properties.get('use_gtest', True): |
| kwargs = {} |
| if api.properties.get('shards'): |
| kwargs['shards'] = api.properties['shards'] |
| |
| test = api.chromium_tests.steps.SwarmingGTestTest( |
| 'base_unittests', **kwargs) |
| else: |
| test = api.chromium_tests.steps.BlinkTest() |
| |
| if api.properties.get('disable_retry_with_patch'): |
| test._should_retry_with_patch = False |
| |
| api.chromium_tests._run_tests_on_tryserver( |
| bot_config_object, |
| tests=[test], |
| bot_update_step=update_step, |
| affected_files=api.properties.get('affected_files', [])) |
| |
| |
| def GenTests(api): |
| yield ( |
| api.test('basic') + |
| api.properties.tryserver( |
| mastername='tryserver.chromium.linux', |
| buildername='linux_chromium_rel_ng', |
| swarm_hashes={ |
| 'base_unittests': 'ffffffffffffffffffffffffffffffffffffffff', |
| }) + |
| api.override_step_data( |
| 'base_unittests (with patch)', |
| api.swarming.canned_summary_output(failure=True) + |
| api.test_utils.canned_gtest_output(False)) |
| ) |
| |
| retry_with_tests_filter = post_process.Filter().include_re( |
| r'.*retry with patch.*') |
| yield ( |
| api.test('enable_retry_with_patch_recipes') + |
| api.properties.tryserver( |
| mastername='tryserver.chromium.linux', |
| buildername='linux_chromium_rel_ng', |
| swarm_hashes={ |
| 'base_unittests': 'ffffffffffffffffffffffffffffffffffffffff', |
| }) + |
| api.override_step_data( |
| 'base_unittests (with patch)', |
| api.swarming.canned_summary_output(failure=True) + |
| api.test_utils.canned_gtest_output(passing=False)) + |
| api.override_step_data( |
| 'base_unittests (without patch)', |
| api.swarming.canned_summary_output(failure=False) + |
| api.test_utils.canned_gtest_output(passing=True)) + |
| api.post_process(retry_with_tests_filter) |
| ) |
| |
| yield ( |
| api.test('disable_retry_with_patch_and_deapply') + |
| api.properties.tryserver( |
| mastername='tryserver.chromium.linux', |
| buildername='linux_chromium_rel_ng', |
| disable_retry_with_patch=True, |
| affected_files=['testing/buildbot/chromium.linux.json'], |
| swarm_hashes={ |
| 'base_unittests': 'ffffffffffffffffffffffffffffffffffffffff', |
| }) + |
| api.override_step_data( |
| 'base_unittests (with patch)', |
| api.swarming.canned_summary_output(failure=True) + |
| api.test_utils.canned_gtest_output(passing=False)) + |
| api.post_process(post_process.DoesNotRun, |
| 'base_unittests (without patch)') + |
| api.post_process(post_process.DoesNotRun, |
| 'base_unittests (retry with patch)') + |
| api.post_process(post_process.StatusFailure) + |
| api.post_process(post_process.DropExpectation) |
| ) |
| |
| yield ( |
| api.test('disable_retry_with_patch_and_deapply_invalid_results') + |
| api.properties.tryserver( |
| mastername='tryserver.chromium.linux', |
| buildername='linux_chromium_rel_ng', |
| disable_retry_with_patch=True, |
| affected_files=['testing/buildbot/chromium.linux.json'], |
| swarm_hashes={ |
| 'base_unittests': 'ffffffffffffffffffffffffffffffffffffffff', |
| }) + |
| api.override_step_data( |
| 'base_unittests (with patch)', |
| api.test_utils.canned_isolated_script_output(passing=False, |
| valid=False)) + |
| api.post_process(post_process.DoesNotRun, |
| 'base_unittests (without patch)') + |
| api.post_process(post_process.DoesNotRun, |
| 'base_unittests (retry with patch)') + |
| api.post_process(post_process.StatusFailure) + |
| api.post_process(post_process.DropExpectation) |
| ) |
| |
| yield ( |
| api.test('disable_retry_with_patch') + |
| api.properties.tryserver( |
| mastername='tryserver.chromium.linux', |
| buildername='linux_chromium_rel_ng', |
| disable_retry_with_patch=True, |
| swarm_hashes={ |
| 'base_unittests': 'ffffffffffffffffffffffffffffffffffffffff', |
| }) + |
| api.override_step_data( |
| 'base_unittests (with patch)', |
| api.swarming.canned_summary_output(failure=True) + |
| api.test_utils.canned_gtest_output(passing=False)) + |
| api.override_step_data( |
| 'base_unittests (without patch)', |
| api.swarming.canned_summary_output(failure=False) + |
| api.test_utils.canned_gtest_output(passing=True)) + |
| api.post_process(post_process.DoesNotRun, |
| 'base_unittests (retry with patch)') + |
| api.post_process(post_process.StatusFailure) + |
| api.post_process(post_process.DropExpectation) |
| ) |
| |
| yield ( |
| api.test('enable_retry_with_patch_succeed_after_deapply') + |
| api.properties.tryserver( |
| mastername='tryserver.chromium.linux', |
| buildername='linux_chromium_rel_ng', |
| swarm_hashes={ |
| 'base_unittests': 'ffffffffffffffffffffffffffffffffffffffff', |
| }) + |
| api.override_step_data( |
| 'base_unittests (with patch)', |
| api.swarming.canned_summary_output(failure=True) + |
| api.test_utils.canned_gtest_output(passing=False)) + |
| api.override_step_data( |
| 'base_unittests (without patch)', |
| api.swarming.canned_summary_output(failure=True) + |
| api.test_utils.canned_gtest_output(passing=False)) |
| ) |
| |
| |
| yield ( |
| api.test('retry_with_patch_failure') + |
| api.properties.tryserver( |
| mastername='tryserver.chromium.linux', |
| buildername='linux_chromium_rel_ng', |
| use_gtest=False, |
| swarm_hashes={ |
| 'base_unittests': 'ffffffffffffffffffffffffffffffffffffffff', |
| }) + |
| api.override_step_data( |
| 'blink_web_tests (with patch)', |
| api.test_utils.canned_test_output(passing=False)) + |
| api.override_step_data( |
| 'blink_web_tests (without patch)', |
| api.test_utils.canned_test_output(passing=True)) + |
| api.override_step_data( |
| 'blink_web_tests (retry with patch)', |
| api.test_utils.canned_test_output(passing=False)) + |
| api.post_process(post_process.MustRun, |
| 'blink_web_tests (retry with patch summary)') + |
| api.post_process(post_process.AnnotationContains, |
| 'blink_web_tests (retry with patch summary)', ['STEP_FAILURE']) + |
| api.post_process(post_process.DropExpectation) |
| ) |
| |
| # To simulate a real test suite, we create results for 100 tests, 3 of |
| # which fail. We rerun failing tests 10 times, so the equivalent load is 3*10 |
| # = 30 tests, which is 30% of the original load of 100 tests. We start with 20 |
| # shards, so we want 30% * 20 = 6 shards on rerun. However, it doesn't make |
| # sense to use more shards than there are tests, so we limit down to 3 shards. |
| def generate_results_for_failure_many_shards(): |
| success_dict = [ |
| { |
| 'elapsed_time_ms': 0, |
| 'output_snippet': ':)', |
| 'status': 'SUCCESS', |
| }, |
| ] |
| failure_dict = [ |
| { |
| 'elapsed_time_ms': 0, |
| 'output_snippet': ':(', |
| 'status': 'FAILURE' |
| }, |
| ] |
| failure_count = 3 |
| all_results = {} |
| for i in xrange(100): |
| result_dict = failure_dict if i < failure_count else success_dict |
| name = 'Test{}'.format(i) |
| all_results[name] = result_dict |
| canned_jsonish = { |
| 'per_iteration_data': [all_results] |
| } |
| return json.dumps(canned_jsonish) |
| |
| yield ( |
| api.test('retry_with_patch_failure_many_shards') + |
| api.properties.tryserver( |
| mastername='tryserver.chromium.linux', |
| buildername='linux_chromium_rel_ng', |
| shards=20, |
| swarm_hashes={ |
| 'base_unittests': 'ffffffffffffffffffffffffffffffffffffffff', |
| }) + |
| api.override_step_data( |
| 'base_unittests (with patch)', |
| api.swarming.canned_summary_output(failure=True, shards=20) + |
| api.test_utils.gtest_results(generate_results_for_failure_many_shards(), retcode=1)) + |
| api.override_step_data( |
| 'base_unittests (without patch)', |
| api.swarming.canned_summary_output(failure=False) + |
| api.test_utils.canned_gtest_output(passing=True)) + |
| api.override_step_data( |
| 'base_unittests (retry with patch)', |
| api.swarming.canned_summary_output(failure=True) + |
| api.test_utils.canned_gtest_output(passing=False)) + |
| api.post_process( |
| post_process.StepCommandContains, |
| 'test_pre_run (without patch)' + |
| '.[trigger] base_unittests (without patch)', |
| ['--shards', '3']) + |
| api.post_process(post_process.DropExpectation) |
| ) |
| |
| yield ( |
| api.test('enable_retry_with_patch_invalid_test_results') + |
| api.properties.tryserver( |
| mastername='tryserver.chromium.linux', |
| buildername='linux_chromium_rel_ng', |
| swarm_hashes={ |
| 'base_unittests': 'ffffffffffffffffffffffffffffffffffffffff', |
| }) + |
| api.override_step_data( |
| 'base_unittests (with patch)', |
| api.swarming.canned_summary_output(failure=True) + |
| api.test_utils.canned_gtest_output(passing=False)) + |
| api.override_step_data( |
| 'base_unittests (without patch)', |
| api.swarming.canned_summary_output(failure=False) + |
| api.test_utils.canned_gtest_output(passing=True)) + |
| api.override_step_data( |
| 'base_unittests (retry with patch)', |
| api.test_utils.canned_isolated_script_output(passing=False, |
| valid=False)) + |
| api.post_process(retry_with_tests_filter) |
| ) |
| |
| yield ( |
| api.test('enable_retry_with_patch_invalid_initial_test_results') + |
| api.properties.tryserver( |
| mastername='tryserver.chromium.linux', |
| buildername='linux_chromium_rel_ng', |
| swarm_hashes={ |
| 'base_unittests': 'ffffffffffffffffffffffffffffffffffffffff', |
| }) + |
| api.override_step_data( |
| 'base_unittests (with patch)', |
| api.test_utils.canned_isolated_script_output(passing=False, |
| valid=False)) + |
| api.override_step_data( |
| 'base_unittests (without patch)', |
| api.test_utils.canned_isolated_script_output(passing=False, |
| valid=False)) + |
| api.override_step_data( |
| 'base_unittests (retry with patch)', |
| api.swarming.canned_summary_output() + |
| api.test_utils.canned_gtest_output(passing=True)) + |
| api.post_process( |
| post_process.MustRun, |
| 'test_pre_run (retry with patch)' + |
| '.[trigger] base_unittests (retry with patch)') + |
| api.post_process(post_process.DropExpectation) |
| ) |
| |
| yield ( |
| api.test('enable_retry_with_patch_invalid_without_patch_results') + |
| api.properties.tryserver( |
| mastername='tryserver.chromium.linux', |
| buildername='linux_chromium_rel_ng', |
| swarm_hashes={ |
| 'base_unittests': 'ffffffffffffffffffffffffffffffffffffffff', |
| }) + |
| api.override_step_data( |
| 'base_unittests (with patch)', |
| api.swarming.canned_summary_output(failure=True) + |
| api.test_utils.canned_gtest_output(passing=False)) + |
| api.override_step_data( |
| 'base_unittests (without patch)', |
| api.test_utils.canned_isolated_script_output(passing=False, |
| valid=False)) + |
| api.override_step_data( |
| 'base_unittests (retry with patch)', |
| api.swarming.canned_summary_output() + |
| api.test_utils.canned_gtest_output(passing=True)) + |
| api.post_process( |
| post_process.MustRun, |
| 'test_pre_run (retry with patch)' + |
| '.[trigger] base_unittests (retry with patch)') + |
| api.post_process(post_process.DropExpectation) |
| ) |
| |
| FAILURE_THEN_SUCCESS_DATA = ( |
| """ |
| { |
| "per_iteration_data": [ |
| { |
| "Test.One": [{"status": "SUCCESS", "output_snippet": ""}], |
| "Test.Two": [ |
| {"status": "FAILURE", "output_snippet": ""}, |
| {"status": "SUCCESS", "output_snippet": ""} |
| ] |
| } |
| ] |
| } |
| """) |
| # Any failure in 'retry without patch' should cause the test to be considered |
| # flaky on tip of tree, and failures should be ignored. |
| yield ( |
| api.test('retry_without_patch_any_failure') + |
| api.properties.tryserver( |
| mastername='tryserver.chromium.linux', |
| buildername='linux_chromium_rel_ng', |
| swarm_hashes={ |
| 'base_unittests': 'ffffffffffffffffffffffffffffffffffffffff', |
| }) + |
| |
| # canned_gtest_output(passing=False)) marks Test.One as a success, |
| # Test.Two as a failure. |
| api.override_step_data( |
| 'base_unittests (with patch)', |
| api.swarming.canned_summary_output(failure=False) + |
| api.test_utils.canned_gtest_output(passing=False)) + |
| |
| # When running the test without patch, it first fails, then succeeds. This |
| # indicates that the test is flaky on tip of tree. |
| api.override_step_data( |
| 'base_unittests (without patch)', |
| api.swarming.canned_summary_output(failure=False) + |
| api.test_utils.gtest_results(FAILURE_THEN_SUCCESS_DATA, retcode=0)) + |
| api.post_process(post_process.AnnotationContains, |
| 'base_unittests (retry summary)', ['ignored']) + |
| api.post_process(post_process.AnnotationContains, |
| 'base_unittests (retry summary)', ['Test.Two']) + |
| api.post_process(post_process.DropExpectation) |
| ) |
| |
| yield ( |
| api.test('disable_deapply_patch_affected_files') + |
| api.properties.tryserver( |
| mastername='tryserver.chromium.linux', |
| buildername='linux_chromium_rel_ng', |
| affected_files=['testing/buildbot/chromium.linux.json'], |
| swarm_hashes={ |
| 'base_unittests': 'ffffffffffffffffffffffffffffffffffffffff', |
| }) + |
| api.override_step_data( |
| 'base_unittests (with patch)', |
| api.swarming.canned_summary_output(failure=True) + |
| api.test_utils.canned_gtest_output(False)) |
| ) |
| |
| yield ( |
| api.test('nonzero_exit_code_no_gtest_output') + |
| api.properties.tryserver( |
| mastername='tryserver.chromium.linux', |
| buildername='linux_chromium_rel_ng', |
| affected_files=['testing/buildbot/chromium.linux.json'], |
| swarm_hashes={ |
| 'base_unittests': 'ffffffffffffffffffffffffffffffffffffffff', |
| }) + |
| api.override_step_data( |
| 'base_unittests (with patch)', |
| api.swarming.canned_summary_output(failure=True) + |
| api.test_utils.gtest_results(json.dumps({'per_iteration_data': []}), |
| retcode=1)) |
| ) |
