masters/master.chromium.perf/master.cfg - chromium/tools/build - Git at Google

 # Copyright 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.

 # This is the buildmaster config file for the 'chromium.perf' bot. It must
 # be installed as 'master.cfg' in your buildmaster's base directory
 # (although the filename can be changed with the --basedir option to
 # 'mktap buildbot master').

 # It has one job: define a dictionary named BuildmasterConfig. This
 # dictionary has a variety of keys to control different aspects of the
 # buildmaster. They are documented in docs/config.xhtml .

 # This file follows this naming convention:
 # Factories: f_cr_rel_[type]
 # Builders:  b_chromium_rel_[os]_[type]
 # BuildDir:  chromium-rel-[os]-[type]
 #
 # os = xp/vista/linux/mac
 # type = perf

 from buildbot.scheduler import Scheduler
 from buildbot.scheduler import Triggerable

 from common import chromium_utils

 from master import chromium_notifier
 from master import gitiles_poller
 from master import master_config
 from master import master_utils
 from master import slaves_list
 from master.factory import annotator_factory

 import config
 import master_site_config

 ActiveMaster = master_site_config.ChromiumPerf


 # Enable MAIL_NOTIFIER in production to send perf sheriff alerts.
 MAIL_NOTIFIER = ActiveMaster.is_production_host

 # This is the dictionary that the buildmaster pays attention to. We also use
 # a shorter alias to save typing.
 c = BuildmasterConfig = {}

 # 'slavePortnum' defines the TCP port to listen on. This must match the value
 # configured into the buildslaves (with their --master option)
 c['slavePortnum'] = ActiveMaster.slave_port

 # Disable compression for the stdio files.
 c['logCompressionLimit'] = False

 # Load the list of slaves.
 slaves = slaves_list.SlavesList('slaves.cfg', 'ChromiumPerf')

 config.DatabaseSetup(c, require_dbconfig=ActiveMaster.is_production_host)

 def recipe_factory(recipe):
   f_annotations = annotator_factory.AnnotatorFactory(ActiveMaster)
   return f_annotations.BaseFactory(recipe=recipe)


 # ------------------------------------------------------------------------------
 # Change Sources.

 # Polls config.Master.trunk_url for changes
 master_poller = gitiles_poller.GitilesPoller(
     'https://chromium.googlesource.com/chromium/src')

 c['change_source'] = [master_poller]

 # End Change Sources.
 # ------------------------------------------------------------------------------


 # ----------------------------------------------------------------------------
 # Builders.

 # The 'builders' list defines the Builders. Each one is configured with a
 # dictionary, using the following keys:
 #  name (required): the name used to describe this bilder
 #  builddir (required): which subdirectory to run the builder in
 #  factory (required): a BuildFactory to define how the build is run
 #  periodicBuildTime (optional): if set, force a build every N seconds
 #  category (optional): it is not used in the normal 'buildbot' meaning. It is
 #                       used by gatekeeper to determine which steps it should
 #                       look for to close the tree.
 #

 def _TriggerName(platform, target_bits):
   return '%s_%d' % (platform, target_bits)


 _category_index = enumerate(('builders', 'android', 'win', 'mac', 'linux'))
 _category_index = {category: index + 1 for index, category in _category_index}
 def _CategoryIndex(category):
   return _category_index[category]


 _builder_names = []
 def _AddBuilder(name, platform, merge_requests=True, timeout=1200,
                 target_bits=64):
   if platform == 'android':
     recipe = 'android/builder'
   else:
     recipe = 'chromium'

   c['builders'].append({
       'name': name,
       'factory': annotator_factory.AnnotatorFactory().BaseFactory(
           recipe=recipe,
           triggers=[_TriggerName(platform, target_bits)],
           timeout=timeout),
       'category':
           '%d%s|%s' % (_CategoryIndex('builders'), 'builders', platform),
       'mergeRequests': merge_requests,
       'auto_reboot': False,
   })

   _builder_names.append(name)


 _trigger_testers = {}
 def _AddTesterShard(name, recipe, platform, target_bits):
   c['builders'].append({
       'name': name,
       'factory': recipe_factory(recipe),
       'category': '%d%s|builder_testers' % (_CategoryIndex(platform), platform),
   })

   trigger_name = _TriggerName(platform, target_bits)
   if trigger_name not in _trigger_testers:
     _trigger_testers[trigger_name] = []
   _trigger_testers[trigger_name].append(name)


 def _AddTester(name, platform, num_shards=1, target_bits=64):
   if platform == 'android':
     recipe = 'android/perf'
   else:
     recipe = 'chromium'

   if num_shards == 1:
     _AddTesterShard(name, recipe, platform, target_bits)
   else:
     for index in xrange(num_shards):
       indexed_name = '%s (%d)' % (name, index + 1)
       _AddTesterShard(indexed_name, recipe, platform, target_bits)


 c['builders'] = []


 _AddBuilder('Android Builder', 'android', target_bits=32, merge_requests=False)
 _AddBuilder('Android arm64 Builder', 'android', merge_requests=False)
 _AddBuilder('Win Builder', 'win', timeout=3600, target_bits=32)
 _AddBuilder('Win x64 Builder', 'win', timeout=3600)
 _AddBuilder('Mac Builder', 'mac')
 # TODO(krasin): decrease the timeout when https://crbug.com/569732 is fixed.
 _AddBuilder('Linux Builder', 'linux', timeout=4800, merge_requests=False)


 _AddTester('Android Galaxy S5 Perf', 'android', num_shards=3, target_bits=32)
 _AddTester('Android Nexus5 Perf', 'android', num_shards=2, target_bits=32)
 _AddTester('Android Nexus6 Perf', 'android', num_shards=2, target_bits=32)
 _AddTester('Android Nexus7v2 Perf', 'android', num_shards=2, target_bits=32)
 _AddTester('Android Nexus9 Perf', 'android', num_shards=2)
 _AddTester('Android One Perf', 'android', num_shards=2, target_bits=32)


 _AddTester('Win Zenbook Perf', 'win', num_shards=5)
 _AddTester('Win 10 Perf', 'win', num_shards=5)
 _AddTester('Win 8 Perf', 'win', num_shards=5)
 _AddTester('Win 7 Perf', 'win', num_shards=5, target_bits=32)
 _AddTester('Win 7 x64 Perf', 'win', num_shards=5)
 _AddTester('Win 7 ATI GPU Perf', 'win', num_shards=5)
 _AddTester('Win 7 Intel GPU Perf', 'win', num_shards=5)
 _AddTester('Win 7 Nvidia GPU Perf', 'win', num_shards=5)
 _AddTester('Win 7 Low-End Perf', 'win', num_shards=2, target_bits=32)


 _AddTester('Mac 10.11 Perf', 'mac', num_shards=5)
 _AddTester('Mac 10.10 Perf', 'mac', num_shards=5)
 _AddTester('Mac Retina Perf', 'mac', num_shards=5)
 _AddTester('Mac HDD Perf', 'mac', num_shards=5)


 _AddTester('Linux Perf', 'linux', num_shards=5)

 # End Builders.
 # ------------------------------------------------------------------------------


 # ------------------------------------------------------------------------------
 # Schedulers.

 ## configure the Schedulers

 c['schedulers'] = [
     Scheduler(name='chromium',
               branch='master',
               builderNames=_builder_names)
 ]


 for trigger_name, tester_names in _trigger_testers.iteritems():
   c['schedulers'].append(Triggerable(trigger_name, tester_names))

 # End Schedulers.
 # ------------------------------------------------------------------------------


 ####### BUILDSLAVES

 # Associate the slaves to the manual builders. The configuration is in
 # slaves.cfg.
 for builder in c['builders']:
   builder['slavenames'] = slaves.GetSlavesName(builder=builder['name'])

 # The 'slaves' list defines the set of allowable buildslaves. List all the
 # slaves registered to a builder. Remove dupes.
 c['slaves'] = master_utils.AutoSetupSlaves(c['builders'],
                                            config.Master.GetBotPassword())
 master_utils.VerifySetup(c, slaves)


 ####### STATUS TARGETS

 # the 'buildbotURL' string should point to the location where the buildbot's
 # internal web server (usually the html.Waterfall page) is visible. This
 # typically uses the port number set in the Waterfall 'status' entry, but
 # with an externally-visible host name which the buildbot cannot figure out
 # without some help.

 #Must come before AutoSetupMaster().
 c['buildbotURL'] = ActiveMaster.buildbot_url

 # Adds common status and tools to this master.
 master_utils.AutoSetupMaster(c, ActiveMaster,
     public_html='../master.chromium/public_html',
     templates=['./templates', '../master.chromium/templates'],
     tagComparator=master_poller.comparator,
     enable_http_status_push=ActiveMaster.is_production_host)

 # Add more.

 if MAIL_NOTIFIER:
   c['status'].append(chromium_notifier.ChromiumNotifier(
       fromaddr=ActiveMaster.from_address,
       exclusions={},
       relayhost=config.Master.smtp,
       status_header='Perf alert on "%(builder)s":\n%(steps)s',
       subject='%(projectName)s %(builder)s %(result)s',
       extraRecipients=[],
       lookup=master_utils.FilterDomain(),
       sheriffs=['sheriff_perf'],
       public_html='../master.chromium/public_html',
       forgiving_steps=[],
       use_getname=True))

 ####### PROJECT IDENTITY

 # the 'projectName' string will be used to describe the project that this
 # buildbot is working on. For example, it is used as the title of the
 # waterfall HTML page. The 'projectURL' string will be used to provide a link
 # from buildbot HTML pages to your project's home page.

 c['projectName'] = ActiveMaster.project_name
 c['projectURL'] = config.Master.project_url
	# Copyright 2012 The Chromium Authors. All rights reserved.
	# Use of this source code is governed by a BSD-style license that can be
	# found in the LICENSE file.

	# This is the buildmaster config file for the 'chromium.perf' bot. It must
	# be installed as 'master.cfg' in your buildmaster's base directory
	# (although the filename can be changed with the --basedir option to
	# 'mktap buildbot master').

	# It has one job: define a dictionary named BuildmasterConfig. This
	# dictionary has a variety of keys to control different aspects of the
	# buildmaster. They are documented in docs/config.xhtml .

	# This file follows this naming convention:
	# Factories: f_cr_rel_[type]
	# Builders: b_chromium_rel_[os]_[type]
	# BuildDir: chromium-rel-[os]-[type]
	#
	# os = xp/vista/linux/mac
	# type = perf

	from buildbot.scheduler import Scheduler
	from buildbot.scheduler import Triggerable

	from common import chromium_utils

	from master import chromium_notifier
	from master import gitiles_poller
	from master import master_config
	from master import master_utils
	from master import slaves_list
	from master.factory import annotator_factory

	import config
	import master_site_config

	ActiveMaster = master_site_config.ChromiumPerf


	# Enable MAIL_NOTIFIER in production to send perf sheriff alerts.
	MAIL_NOTIFIER = ActiveMaster.is_production_host

	# This is the dictionary that the buildmaster pays attention to. We also use
	# a shorter alias to save typing.
	c = BuildmasterConfig = {}

	# 'slavePortnum' defines the TCP port to listen on. This must match the value
	# configured into the buildslaves (with their --master option)
	c['slavePortnum'] = ActiveMaster.slave_port

	# Disable compression for the stdio files.
	c['logCompressionLimit'] = False

	# Load the list of slaves.
	slaves = slaves_list.SlavesList('slaves.cfg', 'ChromiumPerf')

	config.DatabaseSetup(c, require_dbconfig=ActiveMaster.is_production_host)

	def recipe_factory(recipe):
	f_annotations = annotator_factory.AnnotatorFactory(ActiveMaster)
	return f_annotations.BaseFactory(recipe=recipe)


	# ------------------------------------------------------------------------------
	# Change Sources.

	# Polls config.Master.trunk_url for changes
	master_poller = gitiles_poller.GitilesPoller(
	'https://chromium.googlesource.com/chromium/src')

	c['change_source'] = [master_poller]

	# End Change Sources.
	# ------------------------------------------------------------------------------


	# ----------------------------------------------------------------------------
	# Builders.

	# The 'builders' list defines the Builders. Each one is configured with a
	# dictionary, using the following keys:
	# name (required): the name used to describe this bilder
	# builddir (required): which subdirectory to run the builder in
	# factory (required): a BuildFactory to define how the build is run
	# periodicBuildTime (optional): if set, force a build every N seconds
	# category (optional): it is not used in the normal 'buildbot' meaning. It is
	# used by gatekeeper to determine which steps it should
	# look for to close the tree.
	#

	def _TriggerName(platform, target_bits):
	return '%s_%d' % (platform, target_bits)


	_category_index = enumerate(('builders', 'android', 'win', 'mac', 'linux'))
	_category_index = {category: index + 1 for index, category in _category_index}
	def _CategoryIndex(category):
	return _category_index[category]


	_builder_names = []
	def _AddBuilder(name, platform, merge_requests=True, timeout=1200,
	target_bits=64):
	if platform == 'android':
	recipe = 'android/builder'
	else:
	recipe = 'chromium'

	c['builders'].append({
	'name': name,
	'factory': annotator_factory.AnnotatorFactory().BaseFactory(
	recipe=recipe,
	triggers=[_TriggerName(platform, target_bits)],
	timeout=timeout),
	'category':
	'%d%s\|%s' % (_CategoryIndex('builders'), 'builders', platform),
	'mergeRequests': merge_requests,
	'auto_reboot': False,
	})

	_builder_names.append(name)


	_trigger_testers = {}
	def _AddTesterShard(name, recipe, platform, target_bits):
	c['builders'].append({
	'name': name,
	'factory': recipe_factory(recipe),
	'category': '%d%s\|builder_testers' % (_CategoryIndex(platform), platform),
	})

	trigger_name = _TriggerName(platform, target_bits)
	if trigger_name not in _trigger_testers:
	_trigger_testers[trigger_name] = []
	_trigger_testers[trigger_name].append(name)


	def _AddTester(name, platform, num_shards=1, target_bits=64):
	if platform == 'android':
	recipe = 'android/perf'
	else:
	recipe = 'chromium'

	if num_shards == 1:
	_AddTesterShard(name, recipe, platform, target_bits)
	else:
	for index in xrange(num_shards):
	indexed_name = '%s (%d)' % (name, index + 1)
	_AddTesterShard(indexed_name, recipe, platform, target_bits)


	c['builders'] = []


	_AddBuilder('Android Builder', 'android', target_bits=32, merge_requests=False)
	_AddBuilder('Android arm64 Builder', 'android', merge_requests=False)
	_AddBuilder('Win Builder', 'win', timeout=3600, target_bits=32)
	_AddBuilder('Win x64 Builder', 'win', timeout=3600)
	_AddBuilder('Mac Builder', 'mac')
	# TODO(krasin): decrease the timeout when https://crbug.com/569732 is fixed.
	_AddBuilder('Linux Builder', 'linux', timeout=4800, merge_requests=False)


	_AddTester('Android Galaxy S5 Perf', 'android', num_shards=3, target_bits=32)
	_AddTester('Android Nexus5 Perf', 'android', num_shards=2, target_bits=32)
	_AddTester('Android Nexus6 Perf', 'android', num_shards=2, target_bits=32)
	_AddTester('Android Nexus7v2 Perf', 'android', num_shards=2, target_bits=32)
	_AddTester('Android Nexus9 Perf', 'android', num_shards=2)
	_AddTester('Android One Perf', 'android', num_shards=2, target_bits=32)


	_AddTester('Win Zenbook Perf', 'win', num_shards=5)
	_AddTester('Win 10 Perf', 'win', num_shards=5)
	_AddTester('Win 8 Perf', 'win', num_shards=5)
	_AddTester('Win 7 Perf', 'win', num_shards=5, target_bits=32)
	_AddTester('Win 7 x64 Perf', 'win', num_shards=5)
	_AddTester('Win 7 ATI GPU Perf', 'win', num_shards=5)
	_AddTester('Win 7 Intel GPU Perf', 'win', num_shards=5)
	_AddTester('Win 7 Nvidia GPU Perf', 'win', num_shards=5)
	_AddTester('Win 7 Low-End Perf', 'win', num_shards=2, target_bits=32)


	_AddTester('Mac 10.11 Perf', 'mac', num_shards=5)
	_AddTester('Mac 10.10 Perf', 'mac', num_shards=5)
	_AddTester('Mac Retina Perf', 'mac', num_shards=5)
	_AddTester('Mac HDD Perf', 'mac', num_shards=5)


	_AddTester('Linux Perf', 'linux', num_shards=5)

	# End Builders.
	# ------------------------------------------------------------------------------


	# ------------------------------------------------------------------------------
	# Schedulers.

	## configure the Schedulers

	c['schedulers'] = [
	Scheduler(name='chromium',
	branch='master',
	builderNames=_builder_names)
	]


	for trigger_name, tester_names in _trigger_testers.iteritems():
	c['schedulers'].append(Triggerable(trigger_name, tester_names))

	# End Schedulers.
	# ------------------------------------------------------------------------------


	####### BUILDSLAVES

	# Associate the slaves to the manual builders. The configuration is in
	# slaves.cfg.
	for builder in c['builders']:
	builder['slavenames'] = slaves.GetSlavesName(builder=builder['name'])

	# The 'slaves' list defines the set of allowable buildslaves. List all the
	# slaves registered to a builder. Remove dupes.
	c['slaves'] = master_utils.AutoSetupSlaves(c['builders'],
	config.Master.GetBotPassword())
	master_utils.VerifySetup(c, slaves)


	####### STATUS TARGETS

	# the 'buildbotURL' string should point to the location where the buildbot's
	# internal web server (usually the html.Waterfall page) is visible. This
	# typically uses the port number set in the Waterfall 'status' entry, but
	# with an externally-visible host name which the buildbot cannot figure out
	# without some help.

	#Must come before AutoSetupMaster().
	c['buildbotURL'] = ActiveMaster.buildbot_url

	# Adds common status and tools to this master.
	master_utils.AutoSetupMaster(c, ActiveMaster,
	public_html='../master.chromium/public_html',
	templates=['./templates', '../master.chromium/templates'],
	tagComparator=master_poller.comparator,
	enable_http_status_push=ActiveMaster.is_production_host)

	# Add more.

	if MAIL_NOTIFIER:
	c['status'].append(chromium_notifier.ChromiumNotifier(
	fromaddr=ActiveMaster.from_address,
	exclusions={},
	relayhost=config.Master.smtp,
	status_header='Perf alert on "%(builder)s":\n%(steps)s',
	subject='%(projectName)s %(builder)s %(result)s',
	extraRecipients=[],
	lookup=master_utils.FilterDomain(),
	sheriffs=['sheriff_perf'],
	public_html='../master.chromium/public_html',
	forgiving_steps=[],
	use_getname=True))

	####### PROJECT IDENTITY

	# the 'projectName' string will be used to describe the project that this
	# buildbot is working on. For example, it is used as the title of the
	# waterfall HTML page. The 'projectURL' string will be used to provide a link
	# from buildbot HTML pages to your project's home page.

	c['projectName'] = ActiveMaster.project_name
	c['projectURL'] = config.Master.project_url