# Copyright 2015 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

from collections import defaultdict
import json

from recipe_engine.config import Dict
from recipe_engine.config import List
from recipe_engine.config import Single
from recipe_engine.recipe_api import Property


DEPS = [
    'adb',
    'depot_tools/bot_update',
    'chromium',
    'chromium_android',
    'chromium_checkout',
    'chromium_swarming',
    'chromium_tests',
    'commit_position',
    'depot_tools/gclient',
    'depot_tools/git',
    'filter',
    'findit',
    'isolate',
    'recipe_engine/buildbucket',
    'recipe_engine/context',
    'recipe_engine/json',
    'recipe_engine/path',
    'recipe_engine/platform',
    'recipe_engine/properties',
    'recipe_engine/python',
    'recipe_engine/raw_io',
    'recipe_engine/step',
    'swarming',
    'test_results',
    'test_utils',
]

import re

from recipe_engine import post_process


PROPERTIES = {
    'target_mastername': Property(
        kind=str, help='The target master to match compile config to.'),
    'target_testername': Property(
        kind=str,
        help='The target tester to match test config to. If the tests are run '
             'on a builder, just treat the builder as a tester.'),
    'good_revision': Property(
        kind=str, help='The last known good revision.'),
    'bad_revision': Property(
        kind=str, help='The first known good revision.'),
    'tests': Property(
        kind=Dict(value_type=list),
        default={},
        help='The failed tests, the test name should be full name, e.g.: {'
             '  "browser_tests": ['
             '    "suite.test1", "suite.test2"'
             '  ]'
             '}'),
    'buildbucket': Property(
        default=None,
        help='The buildbucket property in which we can find build id.'
             'We need to use build id to get tests.'),
    'use_analyze': Property(
        kind=Single(bool, empty_val=False, required=False), default=True,
        help='Use analyze to skip commits that do not affect tests.'),
    'suspected_revisions': Property(
        kind=List(basestring), default=[],
        help='A list of suspected revisions from heuristic analysis.'),
    'test_on_good_revision': Property(
        kind=Single(bool, empty_val=False, required=False), default=True,
        help='Run test on good revision as well if the first revision '
             'in range is suspected.'),
    'test_repeat_count': Property(
        kind=Single(int, required=False), default=20,
        help='How many times to repeat the tests.'),
}


def _get_reduced_test_dict(original_test_dict, failed_tests_dict):
  # Remove tests that are in both dicts from the original test dict.
  if not failed_tests_dict:
    return original_test_dict
  reduced_dict = defaultdict(list)
  for step, tests in original_test_dict.iteritems():
    remain_tests = list(set(tests) - set(failed_tests_dict.get(step, [])))
    if remain_tests:
      reduced_dict[step] = remain_tests
  return reduced_dict


def _get_flaky_tests(test_results):
  # Uses pass_fail_count to get flaky tests.
  flaky_tests = defaultdict(list)
  if not test_results:
    return flaky_tests

  for step, result in test_results.iteritems():
    pass_fail_counts = result.get('pass_fail_counts')
    if not pass_fail_counts:
      continue
    for test, test_counts in pass_fail_counts.iteritems():
      if test_counts.get('pass_count') and test_counts.get('fail_count'):
        flaky_tests[step].append(test)

  return flaky_tests


def _consolidate_flaky_tests(all_flakes, new_flakes):
  for step, tests in new_flakes.iteritems():
    all_flakes[step] = list(set(all_flakes[step]) | set(tests))


def RunSteps(api, target_mastername, target_testername, good_revision,
             bad_revision, tests, buildbucket, use_analyze,
             suspected_revisions, test_on_good_revision, test_repeat_count):
  if not tests:
    # Retrieve the failed tests from the build request.
    # If the recipe is run by swarmbucket, the property 'buildbucket' will be
    # a dict instead of a string containing json.
    if isinstance(buildbucket, dict):
      buildbucket_json = buildbucket  # pragma: no cover. To be deprecated.
    else:
      buildbucket_json = json.loads(buildbucket)
    build_id = buildbucket_json['build']['id']
    build_request = api.buildbucket.get_build(build_id)
    tests = json.loads(
        build_request.stdout['build']['parameters_json']).get(
            'additional_build_parameters', {}).get('tests')
  assert tests, 'No failed tests were specified.'

  target_buildername, checked_out_revision, cached_revision = (
      api.findit.configure_and_sync(api, target_mastername, target_testername,
                                    bad_revision))

  # retrieve revisions in the regression range.
  revisions_to_check = api.findit.revisions_between(good_revision, bad_revision)

  suspected_revision_index = [
      revisions_to_check.index(r)
          for r in set(suspected_revisions) if r in revisions_to_check]

  # Segments revisions_to_check by suspected_revisions.
  # Each sub_range will contain following elements:
  # 1. Revision before a suspected_revision or None as a placeholder
  #    when no such revision
  # 2. Suspected_revision
  # 3. Revisions between a suspected_revision and the revision before next
  #    suspected_revision, or remaining revisions before all suspect_revisions.
  # For example, if revisions_to_check are [r0, r1, ..., r6] and
  # suspected_revisions are [r2, r5], sub_ranges will be:
  # [[None, r0], [r1, r2, r3], [r4, r5, r6]]
  if suspected_revision_index:
    # If there are consecutive revisions being suspected, include them
    # in the same sub_range by only saving the oldest revision.
    suspected_revision_index = [i for i in suspected_revision_index
                                if i - 1 not in suspected_revision_index]
    sub_ranges = []
    remaining_revisions = revisions_to_check[:]
    for index in sorted(suspected_revision_index, reverse=True):
      if index > 0:
        # try job will not run linearly, sets use_analyze to False.
        # TODO(https://crbug.com/874292): filter out irrelevant commits first.
        use_analyze = False
        sub_ranges.append(remaining_revisions[index - 1:])
        remaining_revisions = remaining_revisions[:index - 1]
    # None is a placeholder for the last known good revision.
    sub_ranges.append([None] + remaining_revisions)
  else:
    # Treats the entire regression range as a single sub-range.
    sub_ranges = [[None] + revisions_to_check]

  test_results = {}
  try_job_metadata = {
      'regression_range_size': len(revisions_to_check)
  }
  report = {
      'result': test_results,
      'metadata': try_job_metadata,
      'previously_checked_out_revision': checked_out_revision,
      'previously_cached_revision': cached_revision
  }

  revision_being_checked = None
  found = False
  flakes = defaultdict(list)
  try:
    culprits = defaultdict(dict)
    # Tests that haven't found culprits in tested revision(s).
    tests_have_not_found_culprit = tests
    # Iterates through sub_ranges and find culprits for each failed test.
    # Sub-ranges with newer revisions are tested first so we have better chance
    # that try job will reproduce exactly the same failure as in waterfall.
    for sub_range in sub_ranges:
      if not tests_have_not_found_culprit:  # All tests have found culprits.
        break

      # The revision right before the suspected revision provided by
      # the heuristic result.
      first_revision = sub_range[0]
      following_revisions = sub_range[1:]
      if first_revision:
        revision_being_checked = first_revision
        # Should not use "analyze" to skip tests at the first revision, because
        # if a skipped test fails at the second revision in the sub range, the
        # second revision is not necessarily the culprit.
        test_results[first_revision], tests_failed_in_potential_green = (
            api.findit.compile_and_test_at_revision(
                api, target_mastername, target_buildername, target_testername,
                first_revision, tests_have_not_found_culprit,
                use_analyze=False, test_repeat_count=test_repeat_count))
      else:
        tests_failed_in_potential_green = {}

      # Looks for reliably failed tests.
      flaky_tests_in_potential_green = _get_flaky_tests(test_results.get(
          first_revision))
      _consolidate_flaky_tests(flakes, flaky_tests_in_potential_green)
      tests_passed_in_potential_green = _get_reduced_test_dict(
          tests_have_not_found_culprit, tests_failed_in_potential_green
      )

      # Culprits for tests that failed in potential green should be earlier, so
      # removes failed tests and only runs passed ones in following revisions.
      if tests_passed_in_potential_green:
        tests_to_run = tests_passed_in_potential_green
        for revision in following_revisions:
          revision_being_checked = revision
          # Since tests_to_run are tests that passed in previous revision,
          # whichever test that fails now will find current revision is the
          # culprit.
          test_results[revision], tests_failed_in_revision = (
              api.findit.compile_and_test_at_revision(
                  api, target_mastername, target_buildername, target_testername,
                  revision, tests_to_run, use_analyze, test_repeat_count))

          flaky_tests_in_revision = _get_flaky_tests(test_results[revision])
          reliable_failed_tests_in_revision = _get_reduced_test_dict(
            tests_failed_in_revision, flaky_tests_in_revision)
          _consolidate_flaky_tests(flakes, flaky_tests_in_revision)
          # Removes tests that passed in potential green and failed in
          # following revisions: culprits have been found for them.
          tests_have_not_found_culprit = _get_reduced_test_dict(
              tests_have_not_found_culprit, tests_failed_in_revision)

          # Only runs tests that have not found culprits in later revisions.
          tests_to_run = _get_reduced_test_dict(
              tests_to_run, tests_failed_in_revision)

          # Records found culprits.
          for step, test_list in reliable_failed_tests_in_revision.iteritems():
            for test in test_list:
              culprits[step][test] = revision

          if not tests_to_run:
            break

    if culprits and test_on_good_revision:
      # Need to deflake by running on good revision.
      tests_run_on_good_revision = defaultdict(list)
      for step, step_culprits in culprits.iteritems():
        for test, test_culprit in step_culprits.iteritems():
          if test_culprit == revisions_to_check[0]:
            tests_run_on_good_revision[step].append(test)

      if tests_run_on_good_revision:
        test_results[good_revision], tests_failed_in_revision = (
          api.findit.compile_and_test_at_revision(
            api, target_mastername, target_buildername, target_testername,
            good_revision, tests_run_on_good_revision, use_analyze,
            test_repeat_count))
        if tests_failed_in_revision:
          # Some tests also failed on good revision, they should be flaky.
          # Should remove them from culprits.
          new_culprits = defaultdict(dict)
          for step, step_culprits in culprits.iteritems():
            for test, test_culprit in step_culprits.iteritems():
              if test in tests_failed_in_revision.get(step, []):
                continue
              new_culprits[step][test] = test_culprit
          culprits = new_culprits
          _consolidate_flaky_tests(flakes, tests_failed_in_revision)

    found = bool(culprits)

  except api.step.InfraFailure:
    test_results[revision_being_checked] = api.findit.TestResult.INFRA_FAILED
    report['metadata']['infra_failure'] = True
    raise
  finally:
    report['last_checked_out_revision'] = api.properties.get('got_revision')
    if found:
      report['culprits'] = culprits

    if flakes:
      report['flakes'] = flakes

    # Give the full report including test results and metadata.
    api.python.succeeding_step(
        'report', [json.dumps(report, indent=2)], as_log='report')

  return report


def GenTests(api):
  def props(
      tests, platform_name, tester_name, use_analyze=False, good_revision=None,
      bad_revision=None, suspected_revisions=None, buildbucket=None,
      test_on_good_revision=True, test_repeat_count=20):
    properties = {
        'path_config': 'kitchen',
        'mastername': 'tryserver.chromium.%s' % platform_name,
        'buildername': '%s_chromium_variable' % platform_name,
        'bot_id': 'build1-a1',
        'buildnumber': 1,
        'target_mastername': 'chromium.%s' % platform_name,
        'target_testername': tester_name,
        'good_revision': good_revision or 'r0',
        'bad_revision': bad_revision or 'r1',
        'use_analyze': use_analyze,
        'test_on_good_revision': test_on_good_revision,
        'test_repeat_count': test_repeat_count,
    }
    if tests:
      properties['tests'] = tests
    if suspected_revisions:
      properties['suspected_revisions'] = suspected_revisions
    if buildbucket:
      properties['buildbucket'] = buildbucket
    return api.properties(**properties) + api.platform.name(platform_name)

  def verify_report_fields(check, step_odict, expected_report_fields):
    """Verifies fields in report are with expected values."""
    step = step_odict['report']
    followup_annotations = step['~followup_annotations']

    def get_report_dict():
      long_line_pattern = re.compile(r'@@@STEP_LOG_LINE@report@\s*(.*)@@@')

      report_json = ''
      for log_line in followup_annotations:
        match = long_line_pattern.match(log_line)
        if match:
          report_json += match.group(1)
      return json.loads(report_json)

    report_dict = get_report_dict()

    def check_fields(expected_fields, actual_fields):
      for key, value in expected_fields.iteritems():
        if value and isinstance(value, dict):
          check_fields(value, actual_fields.get(key) or {})
        else:
          check(actual_fields.get(key) == value)

    check_fields(expected_report_fields, report_dict)

    return step_odict

  yield (
      api.test('nonexistent_test_step_skipped') +
      props({'newly_added_tests': ['Test.One', 'Test.Two', 'Test.Three']},
            'win', 'Win7 Tests (1)') +
      api.override_step_data(
          'test r1.read test spec (chromium.win.json)',
          api.json.output({
              'Win7 Tests (1)': {
                  'gtest_tests': [
                      {
                          'test': 'gl_tests',
                          'swarming': {'can_use_on_swarming_builders': True},
                      },
                  ],
              },
          })
      )
  )

  yield (
      api.test('unaffected_test_skipped_by_analyze') +
      props({'affected_tests': ['Test.One'], 'unaffected_tests': ['Test.Two']},
            'win', 'Win7 Tests (1)', use_analyze=True) +
      api.override_step_data(
          'test r1.read test spec (chromium.win.json)',
          api.json.output({
              'Win7 Tests (1)': {
                  'gtest_tests': [
                      {
                        'test': 'affected_tests',
                        'swarming': {'can_use_on_swarming_builders': True},
                      },
                      {
                        'test': 'unaffected_tests',
                        'swarming': {'can_use_on_swarming_builders': True},
                      },
                  ],
              },
          })
      ) +
      api.override_step_data(
          'test r1.analyze',
          api.json.output({
              'status': 'Found dependency',
              'compile_targets': ['affected_tests', 'affected_tests_run'],
              'test_targets': ['affected_tests', 'affected_tests_run'],
          })
      ) +
      api.override_step_data(
          'test r1.affected_tests (r1)',
          api.swarming.canned_summary_output() +
          api.test_utils.simulated_gtest_output(passed_test_names=['Test.One'])
      )
  )

  yield (
      api.test('test_without_targets_not_skipped') +
      props({'unaffected_tests': ['Test.One'], 'checkperms': []},
            'win', 'Win7 Tests (1)', use_analyze=True) +
      api.override_step_data(
          'test r1.read test spec (chromium.win.json)',
          api.json.output({
              'Win7 Tests (1)': {
                  'gtest_tests': [
                      {
                        'test': 'unaffected_tests',
                        'swarming': {'can_use_on_swarming_builders': True},
                      },
                  ],
                  'scripts': [
                      {
                          'name': 'checkperms',
                          'script': 'checkperms.py'
                      },
                  ]
              },
          })
      ) +
      api.override_step_data(
          'test r1.analyze',
          api.json.output({
              'status': 'No dependencies',
              'compile_targets': [],
              'test_targets': [],
          })
      )
  )

  yield (
      api.test('all_test_failed') +
      props({'gl_tests': ['Test.One', 'Test.Two', 'Test.Three']},
            'win', 'Win7 Tests (1)', test_on_good_revision=False) +
      api.override_step_data(
          'test r1.read test spec (chromium.win.json)',
          api.json.output({
              'Win7 Tests (1)': {
                  'gtest_tests': [
                      {
                          'test': 'gl_tests',
                          'swarming': {'can_use_on_swarming_builders': True},
                      },
                  ],
              },
          })
      ) +
      api.override_step_data(
          'test r1.gl_tests (r1)',
          api.swarming.canned_summary_output(failure=True) +
          api.test_utils.simulated_gtest_output(
              failed_test_names=['Test.One', 'Test.Two', 'Test.Three'])
      )
  )

  yield (
      api.test('all_test_passed') +
      props({'gl_tests': ['Test.One', 'Test.Two', 'Test.Three']},
            'win', 'Win7 Tests (1)') +
      api.override_step_data(
          'test r1.read test spec (chromium.win.json)',
          api.json.output({
              'Win7 Tests (1)': {
                  'gtest_tests': [
                      {
                          'test': 'gl_tests',
                          'swarming': {'can_use_on_swarming_builders': True},
                      },
                  ],
              },
          })
      ) +
      api.override_step_data(
          'test r1.gl_tests (r1)',
          api.swarming.canned_summary_output() +
          api.test_utils.simulated_gtest_output(
              passed_test_names=['Test.One', 'Test.Two', 'Test.Three'])
      )
  )

  yield (
      api.test('only_one_test_passed') +
      props({'gl_tests': ['Test.One', 'Test.Two', 'Test.Three']},
            'win', 'Win7 Tests (1)') +
      api.override_step_data(
        'test r0.read test spec (chromium.win.json)',
        api.json.output({
          'Win7 Tests (1)': {
            'gtest_tests': [
              {
                'test': 'gl_tests',
                'swarming': {'can_use_on_swarming_builders': True},
              },
            ],
          },
        })
      ) +
      api.override_step_data(
        'test r0.gl_tests (r0)',
        api.swarming.canned_summary_output(failure=True) +
        api.test_utils.simulated_gtest_output(
          passed_test_names=['Test.One', 'Test.Two'])
      ) +
      api.override_step_data(
          'test r1.read test spec (chromium.win.json)',
          api.json.output({
              'Win7 Tests (1)': {
                  'gtest_tests': [
                      {
                          'test': 'gl_tests',
                          'swarming': {'can_use_on_swarming_builders': True},
                      },
                  ],
              },
          })
      ) +
      api.override_step_data(
          'test r1.gl_tests (r1)',
          api.swarming.canned_summary_output(failure=True) +
          api.test_utils.simulated_gtest_output(
              failed_test_names=['Test.One', 'Test.Two'],
              passed_test_names=['Test.Three'])
      )
  )

  yield (
      api.test('compile_skipped') +
      props({'checkperms': []}, 'win', 'Win7 Tests (1)') +
      api.override_step_data(
          'test r1.read test spec (chromium.win.json)',
          api.json.output({
              'Win7 Tests (1)': {
                  'scripts': [
                      {
                          'name': 'checkperms',
                          'script': 'checkperms.py'
                      },
                  ]
              },
          })
      )
  )

  yield (
      api.test('none_swarming_tests') +
      props({'gl_tests': ['Test.One', 'Test.Two', 'Test.Three']},
            'win', 'Win7 Tests (1)', test_on_good_revision=False) +
      api.override_step_data(
          'test r1.read test spec (chromium.win.json)',
          api.json.output({
              'Win7 Tests (1)': {
                  'gtest_tests': [
                      {
                          'test': 'gl_tests',
                          'swarming': {'can_use_on_swarming_builders': False},
                      },
                  ],
              },
          })
      ) +
      api.override_step_data(
          'test r1.gl_tests (r1)',
          api.swarming.canned_summary_output(failure=True) +
          api.test_utils.simulated_gtest_output(
              failed_test_names=['Test.One', 'Test.Two'],
              passed_test_names=['Test.Three'])
      )
  )

  yield (
      api.test('swarming_tests') +
      props({'gl_tests': ['Test.One']}, 'mac', 'Mac10.13 Tests') +
      api.override_step_data(
          'test r1.read test spec (chromium.mac.json)',
          api.json.output({
              'Mac10.13 Tests': {
                  'gtest_tests': [
                      {
                          'test': 'gl_tests',
                          'swarming': {'can_use_on_swarming_builders': True},
                      },
                  ],
              },
          })
      ) +
      api.override_step_data(
          'test r1.gl_tests (r1)',
          api.swarming.canned_summary_output() +
          api.test_utils.simulated_gtest_output(passed_test_names=['Test.One'])
      )
  )

  yield (
      api.test('findit_culprit_in_last_sub_range') +
      props(
          {'gl_tests': ['Test.One']}, 'mac', 'Mac10.13 Tests', use_analyze=False,
           good_revision='r0', bad_revision='r6', suspected_revisions=['r3']) +
      api.override_step_data(
          'test r2.read test spec (chromium.mac.json)',
          api.json.output({
              'Mac10.13 Tests': {
                  'gtest_tests': [
                      {
                          'test': 'gl_tests',
                          'swarming': {'can_use_on_swarming_builders': True},
                      },
                  ],
              },
          })
      ) +
      api.override_step_data(
          'test r3.read test spec (chromium.mac.json)',
          api.json.output({
              'Mac10.13 Tests': {
                  'gtest_tests': [
                      {
                          'test': 'gl_tests',
                          'swarming': {'can_use_on_swarming_builders': True},
                      },
                  ],
              },
          })
      ) +
      api.override_step_data(
          'git commits in range',
          api.raw_io.stream_output(
              '\n'.join('r%d' % i for i in reversed(range(1, 7))))) +
      api.override_step_data(
          'test r2.gl_tests (r2)',
          api.swarming.canned_summary_output() +
          api.test_utils.simulated_gtest_output(
              passed_test_names=['Test.One'])) +
      api.override_step_data(
          'test r3.gl_tests (r3)',
          api.swarming.canned_summary_output(failure=True) +
          api.test_utils.simulated_gtest_output(failed_test_names=['Test.One']))
  )

  yield (
      api.test('findit_culprit_in_middle_sub_range') +
      props(
          {'gl_tests': ['Test.One']}, 'mac', 'Mac10.13 Tests', use_analyze=False,
           good_revision='r0', bad_revision='r6',
           suspected_revisions=['r3', 'r6']) +
      api.override_step_data(
          'test r2.read test spec (chromium.mac.json)',
          api.json.output({
              'Mac10.13 Tests': {
                  'gtest_tests': [
                      {
                          'test': 'gl_tests',
                          'swarming': {'can_use_on_swarming_builders': True},
                      },
                  ],
              },
          })
      ) +
      api.override_step_data(
          'test r3.read test spec (chromium.mac.json)',
          api.json.output({
              'Mac10.13 Tests': {
                  'gtest_tests': [
                      {
                          'test': 'gl_tests',
                          'swarming': {'can_use_on_swarming_builders': True},
                      },
                  ],
              },
          })
      ) +
      api.override_step_data(
          'test r5.read test spec (chromium.mac.json)',
          api.json.output({
              'Mac10.13 Tests': {
                  'gtest_tests': [
                      {
                          'test': 'gl_tests',
                          'swarming': {'can_use_on_swarming_builders': True},
                      },
                  ],
              },
          })
      ) +
      api.override_step_data(
          'test r6.read test spec (chromium.mac.json)',
          api.json.output({
              'Mac10.13 Tests': {
                  'gtest_tests': [
                      {
                          'test': 'gl_tests',
                          'swarming': {'can_use_on_swarming_builders': True},
                      },
                  ],
              },
          })
      ) +
      api.override_step_data(
          'git commits in range',
          api.raw_io.stream_output(
              '\n'.join('r%d' % i for i in reversed(range(1, 7))))) +
      api.override_step_data(
          'test r2.gl_tests (r2)',
          api.swarming.canned_summary_output() +
          api.test_utils.simulated_gtest_output(
              passed_test_names=['Test.One'])) +
      api.override_step_data(
          'test r3.gl_tests (r3)',
          api.swarming.canned_summary_output(failure=True) +
          api.test_utils.simulated_gtest_output(
              failed_test_names=['Test.One'])) +
      api.override_step_data(
          'test r5.gl_tests (r5)',
          api.swarming.canned_summary_output(failure=True) +
          api.test_utils.simulated_gtest_output(
              passed_test_names=['Test.One'])) +
      api.override_step_data(
          'test r6.gl_tests (r6)',
          api.swarming.canned_summary_output() +
          api.test_utils.simulated_gtest_output(passed_test_names=['Test.One']))
  )

  yield (
      api.test('findit_culprit_in_first_sub_range') +
      props(
          {'gl_tests': ['Test.One']}, 'mac', 'Mac10.13 Tests', use_analyze=False,
           good_revision='r0', bad_revision='r6',
           suspected_revisions=['r6'], test_on_good_revision=False) +
      api.override_step_data(
          'test r1.read test spec (chromium.mac.json)',
          api.json.output({
              'Mac10.13 Tests': {
                  'gtest_tests': [
                      {
                          'test': 'gl_tests',
                          'swarming': {'can_use_on_swarming_builders': True},
                      },
                  ],
              },
          })
      ) +
      api.override_step_data(
          'test r5.read test spec (chromium.mac.json)',
          api.json.output({
              'Mac10.13 Tests': {
                  'gtest_tests': [
                      {
                          'test': 'gl_tests',
                          'swarming': {'can_use_on_swarming_builders': True},
                      },
                  ],
              },
          })
      ) +
      api.override_step_data(
          'test r6.read test spec (chromium.mac.json)',
          api.json.output({
              'Mac10.13 Tests': {
                  'gtest_tests': [
                      {
                          'test': 'gl_tests',
                          'swarming': {'can_use_on_swarming_builders': True},
                      },
                  ],
              },
          })
      ) +
      api.override_step_data(
          'git commits in range',
          api.raw_io.stream_output(
              '\n'.join('r%d' % i for i in reversed(range(1, 7))))) +
      api.override_step_data(
          'test r1.gl_tests (r1)',
          api.swarming.canned_summary_output(failure=True) +
          api.test_utils.simulated_gtest_output(
              failed_test_names=['Test.One'])) +
      api.override_step_data(
          'test r5.gl_tests (r5)',
          api.swarming.canned_summary_output() +
          api.test_utils.simulated_gtest_output(
              passed_test_names=['Test.One'])) +
      api.override_step_data(
          'test r6.gl_tests (r6)',
          api.swarming.canned_summary_output() +
          api.test_utils.simulated_gtest_output(passed_test_names=['Test.One']))
  )

  yield (
      api.test('findit_steps_multiple_culprits') +
      props(
          {'gl_tests': ['Test.gl_One'], 'browser_tests': ['Test.browser_One']},
          'mac', 'Mac10.13 Tests', use_analyze=False,
           good_revision='r0', bad_revision='r6',
           suspected_revisions=['r3', 'r6']) +
      api.override_step_data(
          'test r2.read test spec (chromium.mac.json)',
          api.json.output({
              'Mac10.13 Tests': {
                  'gtest_tests': [
                      {
                        'test': 'gl_tests',
                        'swarming': {'can_use_on_swarming_builders': True},
                      },
                      {
                          'test': 'browser_tests',
                          'swarming': {'can_use_on_swarming_builders': True},
                      },
                  ],
              },
          })
      ) +
      api.override_step_data(
          'test r3.read test spec (chromium.mac.json)',
          api.json.output({
              'Mac10.13 Tests': {
                  'gtest_tests': [
                      {
                          'test': 'gl_tests',
                          'swarming': {'can_use_on_swarming_builders': True},
                      },
                      {
                          'test': 'browser_tests',
                          'swarming': {'can_use_on_swarming_builders': True},
                      },
                  ],
              },
          })
      ) +
      api.override_step_data(
          'test r5.read test spec (chromium.mac.json)',
          api.json.output({
              'Mac10.13 Tests': {
                  'gtest_tests': [
                      {
                          'test': 'gl_tests',
                          'swarming': {'can_use_on_swarming_builders': True},
                      },
                      {
                          'test': 'browser_tests',
                          'swarming': {'can_use_on_swarming_builders': True},
                      },
                  ],
              },
          })
      ) +
      api.override_step_data(
          'test r6.read test spec (chromium.mac.json)',
          api.json.output({
              'Mac10.13 Tests': {
                  'gtest_tests': [
                      {
                          'test': 'gl_tests',
                          'swarming': {'can_use_on_swarming_builders': True},
                      },
                      {
                          'test': 'browser_tests',
                          'swarming': {'can_use_on_swarming_builders': True},
                      },
                  ],
              },
          })
      ) +
      api.override_step_data(
          'git commits in range',
          api.raw_io.stream_output(
              '\n'.join('r%d' % i for i in reversed(range(1, 7))))) +
      api.override_step_data(
          'test r5.gl_tests (r5)',
          api.swarming.canned_summary_output(failure=True) +
          api.test_utils.simulated_gtest_output(
              failed_test_names=['Test.gl_One'])) +
      api.override_step_data(
          'test r5.browser_tests (r5)',
          api.swarming.canned_summary_output() +
          api.test_utils.simulated_gtest_output(
              passed_test_names=['Test.browser_One'])) +
      api.override_step_data(
          'test r6.browser_tests (r6)',
          api.swarming.canned_summary_output(failure=True) +
          api.test_utils.simulated_gtest_output(
              failed_test_names=['Test.browser_One']))+
      api.override_step_data(
          'test r2.gl_tests (r2)',
          api.swarming.canned_summary_output() +
          api.test_utils.simulated_gtest_output(
              passed_test_names=['Test.gl_One'])) +
      api.override_step_data(
          'test r3.gl_tests (r3)',
          api.swarming.canned_summary_output(failure=True) +
          api.test_utils.simulated_gtest_output(
              failed_test_names=['Test.gl_One']))
  )

  yield (
      api.test('findit_tests_multiple_culprits') +
      props(
          {'gl_tests': ['Test.One', 'Test.Two', 'Test.Three']},
          'mac', 'Mac10.13 Tests', use_analyze=False,
           good_revision='r0', bad_revision='r6',
           suspected_revisions=['r3', 'r5']) +
      api.override_step_data(
          'test r2.read test spec (chromium.mac.json)',
          api.json.output({
              'Mac10.13 Tests': {
                  'gtest_tests': [
                      {
                          'test': 'gl_tests',
                          'swarming': {'can_use_on_swarming_builders': True},
                      },
                  ],
              },
          })
      ) +
      api.override_step_data(
          'test r3.read test spec (chromium.mac.json)',
          api.json.output({
              'Mac10.13 Tests': {
                  'gtest_tests': [
                      {
                          'test': 'gl_tests',
                          'swarming': {'can_use_on_swarming_builders': True},
                      },
                  ],
              },
          })
      ) +
      api.override_step_data(
          'test r4.read test spec (chromium.mac.json)',
          api.json.output({
              'Mac10.13 Tests': {
                  'gtest_tests': [
                      {
                          'test': 'gl_tests',
                          'swarming': {'can_use_on_swarming_builders': True},
                      },
                  ],
              },
          })
      ) +
      api.override_step_data(
          'test r5.read test spec (chromium.mac.json)', api.json.output({
              'Mac10.13 Tests': {
                  'gtest_tests': [
                      {
                          'test': 'gl_tests',
                          'swarming': {'can_use_on_swarming_builders': True},
                      },
                  ],
              },
          })
      ) +
      api.override_step_data(
          'test r6.read test spec (chromium.mac.json)',
          api.json.output({
              'Mac10.13 Tests': {
                  'gtest_tests': [
                      {
                          'test': 'gl_tests',
                          'swarming': {'can_use_on_swarming_builders': True},
                      },
                  ],
              },
          })
      ) +
      api.override_step_data(
          'git commits in range',
          api.raw_io.stream_output(
              '\n'.join('r%d' % i for i in reversed(range(1, 7))))) +
      api.override_step_data(
          'test r4.gl_tests (r4)',
          api.swarming.canned_summary_output(failure=True) +
          api.test_utils.simulated_gtest_output(
              passed_test_names=['Test.One', 'Test.Three'],
              failed_test_names=['Test.Two'])) +
      api.override_step_data(
          'test r5.gl_tests (r5)',
          api.swarming.canned_summary_output(failure=True) +
          api.test_utils.simulated_gtest_output(
              passed_test_names=['Test.One'],
              failed_test_names=['Test.Three'])) +
      api.override_step_data(
          'test r6.gl_tests (r6)',
          api.swarming.canned_summary_output(failure=True) +
          api.test_utils.simulated_gtest_output(
              failed_test_names=['Test.One']))+
      api.override_step_data(
          'test r2.gl_tests (r2)',
          api.swarming.canned_summary_output() +
          api.test_utils.simulated_gtest_output(
              passed_test_names=['Test.Two'])) +
      api.override_step_data(
          'test r3.gl_tests (r3)',
          api.swarming.canned_summary_output(failure=True) +
          api.test_utils.simulated_gtest_output(failed_test_names=['Test.Two']))
  )

  yield (
      api.test('findit_consecutive_culprits') +
      props(
          {'gl_tests': ['Test.One']},
          'mac', 'Mac10.13 Tests', use_analyze=False,
           good_revision='r0', bad_revision='r6',
           suspected_revisions=['r3', 'r4']) +
      api.override_step_data(
          'test r2.read test spec (chromium.mac.json)',
          api.json.output({
              'Mac10.13 Tests': {
                  'gtest_tests': [
                      {
                          'test': 'gl_tests',
                          'swarming': {'can_use_on_swarming_builders': True},
                      },
                  ],
              },
          })
      ) +
      api.override_step_data(
          'test r3.read test spec (chromium.mac.json)',
          api.json.output({
              'Mac10.13 Tests': {
                  'gtest_tests': [
                      {
                          'test': 'gl_tests',
                          'swarming': {'can_use_on_swarming_builders': True},
                      },
                  ],
              },
          })
      ) +
      api.override_step_data(
          'test r4.read test spec (chromium.mac.json)',
          api.json.output({
              'Mac10.13 Tests': {
                  'gtest_tests': [
                      {
                          'test': 'gl_tests',
                          'swarming': {'can_use_on_swarming_builders': True},
                      },
                  ],
              },
          })
      ) +
      api.override_step_data(
          'git commits in range',
          api.raw_io.stream_output(
              '\n'.join('r%d' % i for i in reversed(range(1, 7))))) +
      api.override_step_data(
          'test r4.gl_tests (r4)',
          api.swarming.canned_summary_output(failure=True) +
          api.test_utils.simulated_gtest_output(
              failed_test_names=['Test.One'])) +
      api.override_step_data(
          'test r2.gl_tests (r2)',
          api.swarming.canned_summary_output() +
          api.test_utils.simulated_gtest_output(
              passed_test_names=['Test.One'])) +
      api.override_step_data(
          'test r3.gl_tests (r3)',
          api.swarming.canned_summary_output() +
          api.test_utils.simulated_gtest_output(passed_test_names=['Test.One']))
  )

  yield (
      api.test('record_infra_failure') +
      props({'gl_tests': ['Test.One']}, 'mac', 'Mac10.13 Tests') +
      api.override_step_data(
          'test r1.read test spec (chromium.mac.json)',
          api.json.output({
              'Mac10.13 Tests': {
                  'gtest_tests': [
                      {
                          'test': 'gl_tests',
                          'swarming': {'can_use_on_swarming_builders': True},
                      },
                  ],
              },
          })
      ) +
      api.override_step_data(
          'test r1.preprocess_for_goma.start_goma', retcode=1) +
      api.step_data(
          'test r1.preprocess_for_goma.goma_jsonstatus',
          api.json.output(
              data={
                  'notice': [
                      {
                          'infra_status': {
                              'ping_status_code': 408,
                          },
                      },
                  ],
              }))
  )

  yield (
      api.test('use_build_parameter_for_tests') +
      props({}, 'mac', 'Mac10.13 Tests', use_analyze=False,
            good_revision='r0', bad_revision='r6',
            suspected_revisions=['r3', 'r4'],
            buildbucket=json.dumps({'build': {'id': 1}})) +
      api.buildbucket.simulated_buildbucket_output({
          'additional_build_parameters' : {
              'tests': {
                  'gl_tests': ['Test.One']
              }
          }}) +
      api.override_step_data(
          'test r2.read test spec (chromium.mac.json)',
          api.json.output({
              'Mac10.13 Tests': {
                  'gtest_tests': [
                      {
                          'test': 'gl_tests',
                          'swarming': {'can_use_on_swarming_builders': True},
                      },
                  ],
              },
          })
      ) +
      api.override_step_data(
          'test r3.read test spec (chromium.mac.json)',
          api.json.output({
              'Mac10.13 Tests': {
                  'gtest_tests': [
                      {
                          'test': 'gl_tests',
                          'swarming': {'can_use_on_swarming_builders': True},
                      },
                  ],
              },
          })
      ) +
      api.override_step_data(
          'test r4.read test spec (chromium.mac.json)',
          api.json.output({
              'Mac10.13 Tests': {
                  'gtest_tests': [
                      {
                          'test': 'gl_tests',
                          'swarming': {'can_use_on_swarming_builders': True},
                      },
                  ],
              },
          })
      ) +
      api.override_step_data(
          'git commits in range',
          api.raw_io.stream_output(
              '\n'.join('r%d' % i for i in reversed(range(1, 7))))) +
      api.override_step_data(
          'test r4.gl_tests (r4)',
          api.swarming.canned_summary_output(failure=True) +
          api.test_utils.simulated_gtest_output(
              failed_test_names=['Test.One'])) +
      api.override_step_data(
          'test r2.gl_tests (r2)',
          api.swarming.canned_summary_output() +
          api.test_utils.simulated_gtest_output(
              passed_test_names=['Test.One'])) +
      api.override_step_data(
          'test r3.gl_tests (r3)',
          api.swarming.canned_summary_output() +
          api.test_utils.simulated_gtest_output(passed_test_names=['Test.One']))
  )

  yield (
      api.test('use_build_parameter_for_tests_non_json_buildbucket') +
      props({}, 'mac', 'Mac10.13 Tests', use_analyze=False,
            good_revision='r0', bad_revision='r6',
            suspected_revisions=['r3', 'r4'],
            buildbucket={'build': {'id': 1}}) +
      api.buildbucket.simulated_buildbucket_output({
          'additional_build_parameters' : {
              'tests': {
                  'gl_tests': ['Test.One']
              }
          }}) +
      api.override_step_data(
          'test r2.read test spec (chromium.mac.json)',
          api.json.output({
              'Mac10.13 Tests': {
                  'gtest_tests': [
                      {
                          'test': 'gl_tests',
                          'swarming': {'can_use_on_swarming_builders': True},
                      },
                  ],
              },
          })
      ) +
      api.override_step_data(
          'test r3.read test spec (chromium.mac.json)',
          api.json.output({
              'Mac10.13 Tests': {
                  'gtest_tests': [
                      {
                          'test': 'gl_tests',
                          'swarming': {'can_use_on_swarming_builders': True},
                      },
                  ],
              },
          })
      ) +
      api.override_step_data(
          'test r4.read test spec (chromium.mac.json)',
          api.json.output({
              'Mac10.13 Tests': {
                  'gtest_tests': [
                      {
                          'test': 'gl_tests',
                          'swarming': {'can_use_on_swarming_builders': True},
                      },
                  ],
              },
          })
      ) +
      api.override_step_data(
          'git commits in range',
          api.raw_io.stream_output(
              '\n'.join('r%d' % i for i in reversed(range(1, 7))))) +
      api.override_step_data(
          'test r4.gl_tests (r4)',
          api.swarming.canned_summary_output(failure=True) +
          api.test_utils.simulated_gtest_output(
              failed_test_names=['Test.One'])) +
      api.override_step_data(
          'test r2.gl_tests (r2)',
          api.swarming.canned_summary_output() +
          api.test_utils.simulated_gtest_output(
              passed_test_names=['Test.One'])) +
      api.override_step_data(
          'test r3.gl_tests (r3)',
          api.swarming.canned_summary_output() +
          api.test_utils.simulated_gtest_output(passed_test_names=['Test.One']))
  )

  yield (
      api.test('use_analyze_set_to_False_for_non_linear_try_job') +
      props(
          {'gl_tests': ['Test.One']}, 'mac', 'Mac10.13 Tests', use_analyze=True,
           good_revision='r0', bad_revision='r6', suspected_revisions=['r3']) +
      api.override_step_data(
          'test r2.read test spec (chromium.mac.json)',
          api.json.output({
              'Mac10.13 Tests': {
                  'gtest_tests': [
                      {
                          'test': 'gl_tests',
                          'swarming': {'can_use_on_swarming_builders': True},
                      },
                  ],
              },
          })
      ) +
      api.override_step_data(
          'test r3.read test spec (chromium.mac.json)',
          api.json.output({
              'Mac10.13 Tests': {
                  'gtest_tests': [
                      {
                          'test': 'gl_tests',
                          'swarming': {'can_use_on_swarming_builders': True},
                      },
                  ],
              },
          })
      ) +
      api.override_step_data(
          'git commits in range',
          api.raw_io.stream_output(
              '\n'.join('r%d' % i for i in reversed(range(1, 7))))) +
      api.override_step_data(
          'test r2.gl_tests (r2)',
          api.swarming.canned_summary_output() +
          api.test_utils.simulated_gtest_output(
              passed_test_names=['Test.One'])) +
      api.override_step_data(
          'test r3.gl_tests (r3)',
          api.swarming.canned_summary_output(failure=True) +
          api.test_utils.simulated_gtest_output(failed_test_names=['Test.One']))
  )

  yield (
      api.test('flaky_tests') +
      props({'gl_tests': ['Test.One', 'Test.Two', 'Test.Three']},
            'win', 'Win7 Tests (1)') +
      api.override_step_data(
        'test r0.read test spec (chromium.win.json)',
        api.json.output({
          'Win7 Tests (1)': {
            'gtest_tests': [
              {
                'test': 'gl_tests',
                'swarming': {'can_use_on_swarming_builders': True},
              },
            ],
          },
        })
      ) +
      api.override_step_data(
        'test r0.gl_tests (r0)',
        api.swarming.canned_summary_output(failure=True) +
        api.test_utils.simulated_gtest_output(
              failed_test_names=['Test.One'],
              passed_test_names=['Test.Two'])
      ) +
      api.override_step_data(
          'test r1.read test spec (chromium.win.json)',
          api.json.output({
              'Win7 Tests (1)': {
                  'gtest_tests': [
                      {
                          'test': 'gl_tests',
                          'swarming': {'can_use_on_swarming_builders': True},
                      },
                  ],
              },
          })
      ) +
      api.override_step_data(
          'test r1.gl_tests (r1)',
          api.swarming.canned_summary_output(failure=True) +
          api.test_utils.simulated_gtest_output(
              failed_test_names=['Test.One', 'Test.Two'],
              passed_test_names=['Test.Three']))
  )

  yield (
      api.test('use_abbreviated_revision_in_step_name') +
      props(
          {'gl_tests': ['Test.One']}, 'mac', 'Mac10.13 Tests', use_analyze=False,
           good_revision='1234567890abcdefg', bad_revision='gfedcba0987654321',
           test_on_good_revision=False) +
      api.override_step_data(
          'test gfedcba.read test spec (chromium.mac.json)',
          api.json.output({
              'Mac10.13 Tests': {
                  'gtest_tests': [
                      {
                          'test': 'gl_tests',
                          'swarming': {'can_use_on_swarming_builders': True},
                      },
                  ],
              },
          })
      ) +
      api.override_step_data(
          'git commits in range',
          api.raw_io.stream_output('gfedcba0987654321')) +
      api.override_step_data(
          'test gfedcba.gl_tests (gfedcba)',
          api.swarming.canned_summary_output(failure=True) +
          api.test_utils.simulated_gtest_output(
              failed_test_names=['Test.One']))
  )

  yield (
      api.test('remove_culprits_for_flaky_failures') +
      props(
          {'gl_tests': ['Test.One', 'Test.Two']},
          'mac', 'Mac10.13 Tests', use_analyze=False,
           good_revision='r0', bad_revision='r6',
           suspected_revisions=['r4']) +
      api.override_step_data(
        'test r3.read test spec (chromium.mac.json)',
        api.json.output({
          'Mac10.13 Tests': {
            'gtest_tests': [
              {
                'test': 'gl_tests',
                'swarming': {'can_use_on_swarming_builders': True},
              },
            ],
          },
        })
      ) +
      api.override_step_data(
          'test r4.read test spec (chromium.mac.json)',
          api.json.output({
              'Mac10.13 Tests': {
                  'gtest_tests': [
                      {
                          'test': 'gl_tests',
                          'swarming': {'can_use_on_swarming_builders': True},
                      },
                  ],
              },
          })
      ) +
      api.override_step_data(
          'git commits in range',
          api.raw_io.stream_output(
              '\n'.join('r%d' % i for i in reversed(range(1, 7))))) +
      api.override_step_data(
          'test r3.gl_tests (r3)',
          api.swarming.canned_summary_output(failure=True) +
          api.test_utils.simulated_gtest_output(
              passed_test_names=['Test.One', 'Test.Two'])) +
      api.override_step_data(
          'test r4.gl_tests (r4)',
          api.swarming.canned_summary_output(failure=True) +
          api.test_utils.simulated_gtest_output(
              flaky_test_names=['Test.One'],
              failed_test_names=['Test.Two']))
  )

  yield (
      api.test('webkit_layout_tests') +
      props({'webkit_layout_tests': [
                'fast/Test/One.html', 'fast/Test/Two.html', 'dummy/Three.js']},
            'mac', 'Mac10.13 Tests') +
      api.override_step_data(
          'test r0.read test spec (chromium.mac.json)',
          api.json.output({
              'Mac10.13 Tests': {
                  'isolated_scripts': [
                    {
                      'isolate_name': 'webkit_layout_tests',
                      'name': 'webkit_layout_tests',
                      'swarming': {
                        'can_use_on_swarming_builders': True,
                        'shards': 1,
                      },
                    },
                  ],
              },
          })
      ) +
      api.override_step_data(
        'test r0.webkit_layout_tests (r0)',
        api.swarming.canned_summary_output(failure=True) +
        api.test_utils.simulated_isolated_script_output(
              failed_test_names=['fast/Test/One.html'],
              passed_test_names=['fast/Test/Two.html']),
              path_delimiter='/'
      ) +
      api.override_step_data(
          'test r1.read test spec (chromium.mac.json)',
          api.json.output({
              'Mac10.13 Tests': {
                  'isolated_scripts': [
                    {
                      'isolate_name': 'webkit_layout_tests',
                      'name': 'webkit_layout_tests',
                      'swarming': {
                        'can_use_on_swarming_builders': True,
                        'shards': 1,
                      },
                    },
                  ],
              },
          })
      ) +
      api.override_step_data(
          'test r1.webkit_layout_tests (r1)',
          api.swarming.canned_summary_output(failure=True) +
          api.test_utils.simulated_isolated_script_output(
              failed_test_names=['fast/Test/One.html', 'fast/Test/Two.html'],
              passed_test_names=['dummy/Three.js'],
              path_delimiter='/'))
  )

  yield (
      api.test('builder_as_tester') +
      props({'services_unittests': ['Test.One']}, 'linux', 'linux-ozone-rel') +
      api.override_step_data(
          'test r1.read test spec (chromium.linux.json)',
          api.json.output({
              'linux-ozone-rel': {
                  'gtest_tests': [
                      {
                          'test': 'services_unittests',
                          'swarming': {'can_use_on_swarming_builders': True},
                      },
                  ],
              },
          })
      ) +
      api.override_step_data(
          'test r1.services_unittests (r1)',
          api.swarming.canned_summary_output() +
          api.test_utils.simulated_gtest_output(passed_test_names=['Test.One'])
      )
  )

  yield (
      api.test('gtest_task_failed') +
      props({'services_unittests': ['Test.One']}, 'linux', 'linux-ozone-rel') +
      api.override_step_data(
          'test r1.read test spec (chromium.linux.json)',
          api.json.output({
              'linux-ozone-rel': {
                  'gtest_tests': [
                      {
                          'test': 'services_unittests',
                          'swarming': {'can_use_on_swarming_builders': True},
                      },
                  ],
              },
          })
      ) +
      api.override_step_data(
          'test r1.services_unittests (r1)',
          api.swarming.canned_summary_output() +
          api.test_utils.gtest_results(None, 255)) +
      api.post_process(
          verify_report_fields,
          {'result': {
                  'r1': {
                      'services_unittests': {
                          'pass_fail_counts': {}
                      }
                  }
              }
          }) +
      api.post_process(post_process.DropExpectation)
  )

  yield (
      api.test('isolated_script_test_task_failed') +
      props({'webkit_layout_tests': [
                'fast/Test/One.html', 'fast/Test/Two.html', 'dummy/Three.js']},
            'mac', 'Mac10.13 Tests') +
      api.override_step_data(
          'test r1.read test spec (chromium.mac.json)',
          api.json.output({
              'Mac10.13 Tests': {
                  'isolated_scripts': [
                    {
                      'isolate_name': 'webkit_layout_tests',
                      'name': 'webkit_layout_tests',
                      'swarming': {
                        'can_use_on_swarming_builders': True,
                        'shards': 1,
                      },
                    },
                  ],
              },
          })
      ) +
      api.override_step_data(
          'test r1.webkit_layout_tests (r1)',
          api.swarming.canned_summary_output(failure=True) +
          api.test_utils.m.json.output(None, 255)) +
      api.post_process(
          verify_report_fields,
          {'result': {
                  'r1': {
                      'webkit_layout_tests': {
                          'pass_fail_counts': {}
                      }
                  }
              }
          }) +
      api.post_process(post_process.DropExpectation)
  )