scripts/slave/recipes/blink_downstream.py - chromium/tools/build - Git at Google

 # Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.

 """
 This recipe can be used by components like v8 to verify blink tests with a
 low false positive rate. Similar to a trybot, this recipe compares test
 failures from a build with a current component revision with test failures
 from a build with a pinned component revision.

 Summary of the recipe flow:
 1. Sync chromium to HEAD
 2. Sync blink to HEAD
 3. Sync component X to revision Y
 4. Run blink tests
 -> In case of failures:
 5. Sync chromium to same revision as 1
 6. Sync blink to same revision as 2
 7. Sync component X to pinned revision from DEPS file
 8. Run blink tests
 -> If failures in 4 don't happen in 8, then revision Y reveals a problem not
    present in the pinned revision

 Revision Y will be the revision property as provided by buildbot or HEAD (i.e.
 in a forced build with no revision provided).
 """

 from recipe_engine.types import freeze

 DEPS = [
   'build/build',
   'build/chromium',
   'build/chromium_checkout',
   'build/chromium_tests',
   'build/test_utils',
   'depot_tools/bot_update',
   'depot_tools/gclient',
   'recipe_engine/context',
   'recipe_engine/path',
   'recipe_engine/platform',
   'recipe_engine/properties',
 ]


 def V8Builder(config, bits, platform):
   return {
     'gclient_apply_config': ['show_v8_revision'],
     'chromium_apply_config': [],
     'chromium_config_kwargs': {
       'BUILD_CONFIG': config,
       'TARGET_BITS': bits,
     },
     'additional_expectations': [
       'v8', 'tools', 'blink_tests', 'TestExpectations',
     ],
     'component': {'path': 'src/v8', 'revision': '%s'},
     'testing': {'platform': platform},
   }


 BUILDERS = freeze({
   'client.v8.fyi': {
     'builders': {
       'V8-Blink Win': V8Builder('Release', 32, 'win'),
       'V8-Blink Mac': V8Builder('Release', 64, 'mac'),
       'V8-Blink Linux 64': V8Builder('Release', 64, 'linux'),
       'V8-Blink Linux 64 - future': V8Builder('Release', 64, 'linux'),
       'V8-Blink Linux 64 (dbg)': V8Builder('Debug', 64, 'linux'),
     },
   },
 })


 def determine_new_future_failures(caller_api, extra_args):
   tests = [
     caller_api.chromium_tests.steps.BlinkTest(
         extra_args=extra_args + [
           '--additional-expectations',
           caller_api.path['checkout'].join(
               'v8', 'tools', 'blink_tests', 'TestExpectationsFuture'),
           '--additional-driver-flag',
           '--js-flags=--future',
         ],
     ),
   ]

   # Since we don't implement 'retry with patch', we also set the flag on
   # BlinkTest.
   for test in tests:
     test._should_retry_with_patch = False

   failing_tests = caller_api.test_utils.run_tests_with_patch(caller_api, tests)
   if not failing_tests:
     return

   try:
     # HACK(machenbach): Blink tests store state about failing tests. In order
     # to rerun without future, we need to remove the extra args from the
     # existing test object.
     failing_tests[0]._extra_args = extra_args
     caller_api.test_utils.run_tests(caller_api, failing_tests, 'without patch')
   finally:
     with caller_api.step.defer_results():
       for t in failing_tests:
         caller_api.test_utils.summarize_test_with_patch_deapplied(
             caller_api, t, failure_is_fatal=True)


 def determine_new_failures(caller_api, tests, deapply_patch_fn):
   """
   Utility function for running steps with a patch applied, and retrying
   failing steps without the patch. Failures from the run without the patch are
   ignored.

   Args:
     caller_api - caller's recipe API; this is needed because self.m here
                  is different than in the caller (different recipe modules
                  get injected depending on caller's DEPS vs. this module's
                  DEPS)
     tests - iterable of objects implementing the Test interface above
     deapply_patch_fn - function that takes a list of failing tests
                        and undoes any effect of the previously applied patch
   """
   # Convert iterable to list, since it is enumerated multiple times.
   tests = list(tests)

   # Since we don't implement 'retry with patch', we set the corresponding flag
   # on the Test instances.
   for test in tests:
     test._should_retry_with_patch = False

   failing_tests = caller_api.test_utils.run_tests_with_patch(caller_api, tests)
   if not failing_tests:
     return

   try:
     result = deapply_patch_fn(failing_tests)
     caller_api.test_utils.run_tests(caller_api, failing_tests, 'without patch')
     return result
   finally:
     with caller_api.step.defer_results():
       for t in failing_tests:
         caller_api.test_utils.summarize_test_with_patch_deapplied(
             caller_api, t, failure_is_fatal=True)

 def RunSteps(api):
   mastername = api.properties.get('mastername')
   buildername = api.properties.get('buildername')
   master_dict = BUILDERS.get(mastername, {})
   bot_config = master_dict.get('builders', {}).get(buildername)

   # Sync chromium to HEAD.
   api.gclient.set_config('chromium', GIT_MODE=True)
   api.gclient.c.revisions['src'] = 'HEAD'
   api.chromium.set_config('blink',
                           **bot_config.get('chromium_config_kwargs', {}))

   for c in bot_config.get('gclient_apply_config', []):
     api.gclient.apply_config(c)
   api.chromium_tests.set_config('chromium')

   # Sync component to current component revision.
   component_revision = api.properties.get('revision') or 'HEAD'
   api.gclient.c.revisions[bot_config['component']['path']] = (
       bot_config['component']['revision'] % component_revision)

   # Ensure we remember the chromium revision.
   api.gclient.c.got_revision_reverse_mapping['got_cr_revision'] = 'src'
   api.gclient.c.got_revision_mapping.pop('src', None)

   # Run all steps in the checkout dir (consistent with chromium_tests).
   with api.context(cwd=api.chromium_checkout.get_checkout_dir(bot_config)):
     step_result = api.bot_update.ensure_checkout()

     api.chromium.ensure_goma()

     with api.context(cwd=api.path['checkout']):
       api.chromium.runhooks()

     api.chromium_tests.run_mb_and_compile(
         ['blink_tests'], [],
         name_suffix=' (with patch)',
     )

     def component_pinned_fn(_failing_steps):
       bot_update_json = step_result.json.output
       api.gclient.c.revisions['src'] = str(
           bot_update_json['properties']['got_cr_revision'])
       # Reset component revision to the pinned revision from chromium's DEPS
       # for comparison.
       del api.gclient.c.revisions[bot_config['component']['path']]
       # Update without changing got_revision. The first sync is the revision
       # that is tested. The second is just for comparison. Setting got_revision
       # again confuses the waterfall's console view.
       api.bot_update.ensure_checkout(update_presentation=False)

       api.chromium_tests.run_mb_and_compile(
           ['blink_tests'], [],
           name_suffix=' (without patch)',
       )

     extra_args = []
     if bot_config.get('additional_expectations'):
       extra_args.extend([
         '--additional-expectations',
         api.path['checkout'].join(*bot_config['additional_expectations']),
       ])

     tests = [
       api.chromium_tests.steps.BlinkTest(extra_args=extra_args),
     ]

     if 'future' in buildername:
       determine_new_future_failures(api.chromium_tests.m, extra_args)
     else:
       determine_new_failures(api.chromium_tests.m, tests, component_pinned_fn)


 def _sanitize_nonalpha(text):
   return ''.join(c if c.isalnum() else '_' for c in text)


 def GenTests(api):
   canned_test = api.test_utils.canned_test_output
   with_patch = 'webkit_layout_tests (with patch)'
   without_patch = 'webkit_layout_tests (without patch)'

   def properties(mastername, buildername):
     return (
       api.properties.generic(mastername=mastername,
                              buildername=buildername,
                              revision='a' * 40,
                              path_config='kitchen')
     )

   for mastername, master_config in BUILDERS.iteritems():
     for buildername, bot_config in master_config['builders'].iteritems():
       test_name = 'full_%s_%s' % (_sanitize_nonalpha(mastername),
                                   _sanitize_nonalpha(buildername))
       tests = []
       for (pass_first, suffix) in ((True, '_pass'), (False, '_fail')):
         test = (
           properties(mastername, buildername) +
           api.platform(
               bot_config['testing']['platform'],
               bot_config.get(
                   'chromium_config_kwargs', {}).get('TARGET_BITS', 64)) +
           api.test(test_name + suffix) +
           api.override_step_data(with_patch, canned_test(passing=pass_first))
         )
         if not pass_first:
           test += api.override_step_data(
               without_patch, canned_test(passing=False, minimal=True))
         tests.append(test)

       for test in tests:
         yield test

   # This tests that if the first fails, but the second pass succeeds
   # that we fail the whole build.
   yield (
     api.test('minimal_pass_continues') +
     properties('client.v8.fyi', 'V8-Blink Linux 64') +
     api.override_step_data(with_patch, canned_test(passing=False)) +
     api.override_step_data(without_patch,
                            canned_test(passing=True, minimal=True))
   )


   # This tests what happens if something goes horribly wrong in
   # run_web_tests.py and we return an internal error; the step should
   # be considered a hard failure and we shouldn't try to compare the
   # lists of failing tests.
   # 255 == test_run_results.UNEXPECTED_ERROR_EXIT_STATUS in run_web_tests.py.
   yield (
     api.test('webkit_layout_tests_unexpected_error') +
     properties('client.v8.fyi', 'V8-Blink Linux 64') +
     api.override_step_data(with_patch, canned_test(passing=False, retcode=255))
   )

   # TODO(dpranke): crbug.com/357866 . This tests what happens if we exceed the
   # number of failures specified with --exit-after-n-crashes-or-times or
   # --exit-after-n-failures; the step should be considered a hard failure and
   # we shouldn't try to compare the lists of failing tests.
   # 130 == test_run_results.INTERRUPTED_EXIT_STATUS in run_web_tests.py.
   yield (
     api.test('webkit_layout_tests_interrupted') +
     properties('client.v8.fyi', 'V8-Blink Linux 64') +
     api.override_step_data(with_patch, canned_test(passing=False, retcode=130))
   )

   # This tests what happens if we don't trip the thresholds listed
   # above, but fail more tests than we can safely fit in a return code.
   # (this should be a soft failure and we can still retry w/o the patch
   # and compare the lists of failing tests).
   yield (
     api.test('too_many_failures_for_retcode') +
     properties('client.v8.fyi', 'V8-Blink Linux 64') +
     api.override_step_data(with_patch,
                            canned_test(passing=False,
                                        num_additional_failures=125)) +
     api.override_step_data(without_patch,
                            canned_test(passing=True, minimal=True))
   )
	# Copyright 2014 The Chromium Authors. All rights reserved.
	# Use of this source code is governed by a BSD-style license that can be
	# found in the LICENSE file.

	"""
	This recipe can be used by components like v8 to verify blink tests with a
	low false positive rate. Similar to a trybot, this recipe compares test
	failures from a build with a current component revision with test failures
	from a build with a pinned component revision.

	Summary of the recipe flow:
	1. Sync chromium to HEAD
	2. Sync blink to HEAD
	3. Sync component X to revision Y
	4. Run blink tests
	-> In case of failures:
	5. Sync chromium to same revision as 1
	6. Sync blink to same revision as 2
	7. Sync component X to pinned revision from DEPS file
	8. Run blink tests
	-> If failures in 4 don't happen in 8, then revision Y reveals a problem not
	present in the pinned revision

	Revision Y will be the revision property as provided by buildbot or HEAD (i.e.
	in a forced build with no revision provided).
	"""

	from recipe_engine.types import freeze

	DEPS = [
	'build/build',
	'build/chromium',
	'build/chromium_checkout',
	'build/chromium_tests',
	'build/test_utils',
	'depot_tools/bot_update',
	'depot_tools/gclient',
	'recipe_engine/context',
	'recipe_engine/path',
	'recipe_engine/platform',
	'recipe_engine/properties',
	]


	def V8Builder(config, bits, platform):
	return {
	'gclient_apply_config': ['show_v8_revision'],
	'chromium_apply_config': [],
	'chromium_config_kwargs': {
	'BUILD_CONFIG': config,
	'TARGET_BITS': bits,
	},
	'additional_expectations': [
	'v8', 'tools', 'blink_tests', 'TestExpectations',
	],
	'component': {'path': 'src/v8', 'revision': '%s'},
	'testing': {'platform': platform},
	}


	BUILDERS = freeze({
	'client.v8.fyi': {
	'builders': {
	'V8-Blink Win': V8Builder('Release', 32, 'win'),
	'V8-Blink Mac': V8Builder('Release', 64, 'mac'),
	'V8-Blink Linux 64': V8Builder('Release', 64, 'linux'),
	'V8-Blink Linux 64 - future': V8Builder('Release', 64, 'linux'),
	'V8-Blink Linux 64 (dbg)': V8Builder('Debug', 64, 'linux'),
	},
	},
	})


	def determine_new_future_failures(caller_api, extra_args):
	tests = [
	caller_api.chromium_tests.steps.BlinkTest(
	extra_args=extra_args + [
	'--additional-expectations',
	caller_api.path['checkout'].join(
	'v8', 'tools', 'blink_tests', 'TestExpectationsFuture'),
	'--additional-driver-flag',
	'--js-flags=--future',
	],
	),
	]

	# Since we don't implement 'retry with patch', we also set the flag on
	# BlinkTest.
	for test in tests:
	test._should_retry_with_patch = False

	failing_tests = caller_api.test_utils.run_tests_with_patch(caller_api, tests)
	if not failing_tests:
	return

	try:
	# HACK(machenbach): Blink tests store state about failing tests. In order
	# to rerun without future, we need to remove the extra args from the
	# existing test object.
	failing_tests[0]._extra_args = extra_args
	caller_api.test_utils.run_tests(caller_api, failing_tests, 'without patch')
	finally:
	with caller_api.step.defer_results():
	for t in failing_tests:
	caller_api.test_utils.summarize_test_with_patch_deapplied(
	caller_api, t, failure_is_fatal=True)


	def determine_new_failures(caller_api, tests, deapply_patch_fn):
	"""
	Utility function for running steps with a patch applied, and retrying
	failing steps without the patch. Failures from the run without the patch are
	ignored.

	Args:
	caller_api - caller's recipe API; this is needed because self.m here
	is different than in the caller (different recipe modules
	get injected depending on caller's DEPS vs. this module's
	DEPS)
	tests - iterable of objects implementing the Test interface above
	deapply_patch_fn - function that takes a list of failing tests
	and undoes any effect of the previously applied patch
	"""
	# Convert iterable to list, since it is enumerated multiple times.
	tests = list(tests)

	# Since we don't implement 'retry with patch', we set the corresponding flag
	# on the Test instances.
	for test in tests:
	test._should_retry_with_patch = False

	failing_tests = caller_api.test_utils.run_tests_with_patch(caller_api, tests)
	if not failing_tests:
	return

	try:
	result = deapply_patch_fn(failing_tests)
	caller_api.test_utils.run_tests(caller_api, failing_tests, 'without patch')
	return result
	finally:
	with caller_api.step.defer_results():
	for t in failing_tests:
	caller_api.test_utils.summarize_test_with_patch_deapplied(
	caller_api, t, failure_is_fatal=True)

	def RunSteps(api):
	mastername = api.properties.get('mastername')
	buildername = api.properties.get('buildername')
	master_dict = BUILDERS.get(mastername, {})
	bot_config = master_dict.get('builders', {}).get(buildername)

	# Sync chromium to HEAD.
	api.gclient.set_config('chromium', GIT_MODE=True)
	api.gclient.c.revisions['src'] = 'HEAD'
	api.chromium.set_config('blink',
	**bot_config.get('chromium_config_kwargs', {}))

	for c in bot_config.get('gclient_apply_config', []):
	api.gclient.apply_config(c)
	api.chromium_tests.set_config('chromium')

	# Sync component to current component revision.
	component_revision = api.properties.get('revision') or 'HEAD'
	api.gclient.c.revisions[bot_config['component']['path']] = (
	bot_config['component']['revision'] % component_revision)

	# Ensure we remember the chromium revision.
	api.gclient.c.got_revision_reverse_mapping['got_cr_revision'] = 'src'
	api.gclient.c.got_revision_mapping.pop('src', None)

	# Run all steps in the checkout dir (consistent with chromium_tests).
	with api.context(cwd=api.chromium_checkout.get_checkout_dir(bot_config)):
	step_result = api.bot_update.ensure_checkout()

	api.chromium.ensure_goma()

	with api.context(cwd=api.path['checkout']):
	api.chromium.runhooks()

	api.chromium_tests.run_mb_and_compile(
	['blink_tests'], [],
	name_suffix=' (with patch)',
	)

	def component_pinned_fn(_failing_steps):
	bot_update_json = step_result.json.output
	api.gclient.c.revisions['src'] = str(
	bot_update_json['properties']['got_cr_revision'])
	# Reset component revision to the pinned revision from chromium's DEPS
	# for comparison.
	del api.gclient.c.revisions[bot_config['component']['path']]
	# Update without changing got_revision. The first sync is the revision
	# that is tested. The second is just for comparison. Setting got_revision
	# again confuses the waterfall's console view.
	api.bot_update.ensure_checkout(update_presentation=False)

	api.chromium_tests.run_mb_and_compile(
	['blink_tests'], [],
	name_suffix=' (without patch)',
	)

	extra_args = []
	if bot_config.get('additional_expectations'):
	extra_args.extend([
	'--additional-expectations',
	api.path['checkout'].join(*bot_config['additional_expectations']),
	])

	tests = [
	api.chromium_tests.steps.BlinkTest(extra_args=extra_args),
	]

	if 'future' in buildername:
	determine_new_future_failures(api.chromium_tests.m, extra_args)
	else:
	determine_new_failures(api.chromium_tests.m, tests, component_pinned_fn)


	def _sanitize_nonalpha(text):
	return ''.join(c if c.isalnum() else '_' for c in text)


	def GenTests(api):
	canned_test = api.test_utils.canned_test_output
	with_patch = 'webkit_layout_tests (with patch)'
	without_patch = 'webkit_layout_tests (without patch)'

	def properties(mastername, buildername):
	return (
	api.properties.generic(mastername=mastername,
	buildername=buildername,
	revision='a' * 40,
	path_config='kitchen')
	)

	for mastername, master_config in BUILDERS.iteritems():
	for buildername, bot_config in master_config['builders'].iteritems():
	test_name = 'full_%s_%s' % (_sanitize_nonalpha(mastername),
	_sanitize_nonalpha(buildername))
	tests = []
	for (pass_first, suffix) in ((True, '_pass'), (False, '_fail')):
	test = (
	properties(mastername, buildername) +
	api.platform(
	bot_config['testing']['platform'],
	bot_config.get(
	'chromium_config_kwargs', {}).get('TARGET_BITS', 64)) +
	api.test(test_name + suffix) +
	api.override_step_data(with_patch, canned_test(passing=pass_first))
	)
	if not pass_first:
	test += api.override_step_data(
	without_patch, canned_test(passing=False, minimal=True))
	tests.append(test)

	for test in tests:
	yield test

	# This tests that if the first fails, but the second pass succeeds
	# that we fail the whole build.
	yield (
	api.test('minimal_pass_continues') +
	properties('client.v8.fyi', 'V8-Blink Linux 64') +
	api.override_step_data(with_patch, canned_test(passing=False)) +
	api.override_step_data(without_patch,
	canned_test(passing=True, minimal=True))
	)


	# This tests what happens if something goes horribly wrong in
	# run_web_tests.py and we return an internal error; the step should
	# be considered a hard failure and we shouldn't try to compare the
	# lists of failing tests.
	# 255 == test_run_results.UNEXPECTED_ERROR_EXIT_STATUS in run_web_tests.py.
	yield (
	api.test('webkit_layout_tests_unexpected_error') +
	properties('client.v8.fyi', 'V8-Blink Linux 64') +
	api.override_step_data(with_patch, canned_test(passing=False, retcode=255))
	)

	# TODO(dpranke): crbug.com/357866 . This tests what happens if we exceed the
	# number of failures specified with --exit-after-n-crashes-or-times or
	# --exit-after-n-failures; the step should be considered a hard failure and
	# we shouldn't try to compare the lists of failing tests.
	# 130 == test_run_results.INTERRUPTED_EXIT_STATUS in run_web_tests.py.
	yield (
	api.test('webkit_layout_tests_interrupted') +
	properties('client.v8.fyi', 'V8-Blink Linux 64') +
	api.override_step_data(with_patch, canned_test(passing=False, retcode=130))
	)

	# This tests what happens if we don't trip the thresholds listed
	# above, but fail more tests than we can safely fit in a return code.
	# (this should be a soft failure and we can still retry w/o the patch
	# and compare the lists of failing tests).
	yield (
	api.test('too_many_failures_for_retcode') +
	properties('client.v8.fyi', 'V8-Blink Linux 64') +
	api.override_step_data(with_patch,
	canned_test(passing=False,
	num_additional_failures=125)) +
	api.override_step_data(without_patch,
	canned_test(passing=True, minimal=True))
	)