| # -*- python -*- |
| # ex: set syntax=python: |
| # Copyright (c) 2012 The Chromium Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| # READ THIS: |
| # See http://dev.chromium.org/developers/testing/chromium-build-infrastructure |
| |
| import os |
| import socket |
| |
| from buildbot.scheduler import Triggerable |
| |
| # These modules come from scripts, which must be in the PYTHONPATH. |
| from master import master_utils |
| from master import slaves_list |
| from master.builders_pools import BuildersPools |
| from master.factory import annotator_factory |
| from master.factory import remote_run_factory |
| from master.try_job_http import TryJobHTTP |
| |
| import config |
| import master_site_config |
| |
| ActiveMaster = master_site_config.TryServerChromiumLinux |
| |
| |
| MAIL_NOTIFIER = ActiveMaster.is_production_host |
| |
| m_annotator = annotator_factory.AnnotatorFactory(ActiveMaster) |
| |
| # This is the dictionary that the buildmaster pays attention to. We also use |
| # a shorter alias to save typing. |
| c = BuildmasterConfig = {} |
| |
| config.DatabaseSetup(c) |
| |
| ####### CHANGESOURCES |
| |
| c['change_source'] = [] |
| |
| # Avoid merging requests. |
| c['mergeRequests'] = False |
| |
| ####### BUILDERS |
| |
| # Try queues. |
| pools = BuildersPools('chrome') |
| |
| chromium_builders = [] |
| chromeos_boards = ('x86-generic', 'amd64-generic', 'daisy') |
| |
| def base_factory(recipe, factory_properties=None, **kwargs): |
| fp = factory_properties or {} |
| fp.update({ |
| '$recipe_engine': {'mode_flags': {'use_subprocess42': True}} |
| }) |
| |
| # Disable "max time without output" timeout if total build timeout |
| # is in effect. |
| if 'max_time' in kwargs and 'timeout' not in kwargs: |
| kwargs['timeout'] = None |
| |
| return m_annotator.BaseFactory(recipe, factory_properties=fp, **kwargs) |
| |
| |
| def m_remote_run(recipe, **kwargs): |
| # Disable "max time without output" timeout if total build timeout |
| # is in effect. |
| if 'max_time' in kwargs and 'timeout' not in kwargs: |
| kwargs['timeout'] = None |
| return remote_run_factory.RemoteRunFactory( |
| active_master=ActiveMaster, |
| repository='https://chromium.googlesource.com/chromium/tools/build.git', |
| recipe=recipe, |
| factory_properties={'path_config': 'kitchen'}, |
| **kwargs) |
| |
| |
| def m_remote_run_chromium_src(recipe, **kwargs): |
| # Disable "max time without output" timeout if total build timeout |
| # is in effect. |
| if 'max_time' in kwargs and 'timeout' not in kwargs: |
| kwargs['timeout'] = None |
| return remote_run_factory.RemoteRunFactory( |
| active_master=ActiveMaster, |
| repository='https://chromium.googlesource.com/chromium/src.git', |
| recipe=recipe, |
| factory_properties={'path_config': 'kitchen'}, |
| use_gitiles=True, |
| **kwargs) |
| |
| |
| for targ in ('_rel', '_dbg'): |
| chromium_builders.extend([{ |
| 'name': 'linux_chromium_compile%s_ng' % targ, |
| 'factory': m_remote_run_chromium_src('chromium_trybot', timeout=3600), |
| 'slavebuilddir': 'remote_run' |
| }, { |
| 'name': 'linux_chromium%s_ng' % targ, |
| 'factory': m_remote_run_chromium_src( |
| 'chromium_trybot', timeout=3600, max_time=master_utils.CQ_MAX_TIME), |
| 'slavebuilddir': 'remote_run' |
| }, { |
| 'name': 'linux_chromium_chromeos%s_ng' % targ, |
| 'factory': m_remote_run_chromium_src( |
| 'chromium_trybot', max_time=master_utils.CQ_MAX_TIME), |
| 'slavebuilddir': 'remote_run' |
| }, { |
| 'name': 'linux_chromium_chromeos_compile%s_ng' % targ, |
| 'factory': m_remote_run_chromium_src( |
| 'chromium_trybot', max_time=master_utils.CQ_MAX_TIME), |
| 'slavebuilddir': 'remote_run' |
| }, { |
| 'name': 'linux_chromium_gn_chromeos%s' % targ, |
| 'factory': m_remote_run_chromium_src( |
| 'chromium_trybot', max_time=master_utils.CQ_MAX_TIME), |
| 'slavebuilddir': 'remote_run' |
| }]) |
| |
| chromium_builders.append({ |
| 'name': 'linux_chromium_chromeos_asan_rel_ng', |
| 'factory': m_remote_run_chromium_src('chromium_trybot'), |
| 'slavebuilddir': 'remote_run', |
| }) |
| |
| chromium_builders.append({ |
| 'name': 'linux_chromium_chromeos_ozone_rel_ng', |
| 'factory': m_remote_run_chromium_src( |
| 'chromium_trybot', max_time=master_utils.CQ_MAX_TIME), |
| 'slavebuilddir': 'remote_run' |
| }) |
| |
| for board in chromeos_boards: |
| chromium_builders.append({ |
| 'name': 'chromeos_%s_chromium_compile_only_ng' % (board,), |
| 'factory': m_remote_run_chromium_src( |
| 'chromium_trybot', max_time=master_utils.CQ_MAX_TIME), |
| 'slavebuilddir': 'remote_run', |
| }) |
| |
| chromium_builders.append({ |
| 'name': 'linux_chromium_msan_rel_ng', |
| 'factory': m_remote_run_chromium_src('chromium_trybot'), |
| 'slavebuilddir': 'remote_run', |
| }) |
| |
| chromium_builders.append({ |
| 'name': 'linux_chromium_chromeos_msan_rel_ng', |
| 'factory': m_remote_run_chromium_src('chromium_trybot'), |
| 'slavebuilddir': 'remote_run', |
| }) |
| |
| chromium_builders.append({ |
| 'name': 'linux_chromium_tsan_rel_ng', |
| 'factory': m_remote_run_chromium_src('chromium_trybot'), |
| 'slavebuilddir': 'remote_run', |
| }) |
| |
| chromium_builders.append({ |
| 'name': 'linux_chromium_cfi_rel_ng', |
| 'factory': m_remote_run_chromium_src('chromium_trybot', timeout=12000), |
| 'slavebuilddir': 'remote_run', |
| }) |
| |
| chromium_builders.extend([{ |
| 'name': 'linux_arm', |
| 'factory': m_remote_run_chromium_src('chromium_trybot'), |
| 'slavebuilddir': 'remote_run', |
| }, |
| ]) |
| |
| chromium_builders.append({ |
| 'name': 'linux_chromium_compile_dbg_32_ng', |
| 'factory': m_remote_run_chromium_src( |
| 'chromium_trybot', max_time=master_utils.CQ_MAX_TIME), |
| 'slavebuilddir': 'remote_run' |
| }) |
| |
| chromium_builders.append({ |
| 'name': 'linux_chromium_dbg_32_ng', |
| 'factory': m_remote_run_chromium_src('chromium_trybot'), |
| 'slavebuilddir': 'remote_run' |
| }) |
| |
| chromium_builders.append({ |
| 'name': 'linux_chromium_archive_rel_ng', |
| 'factory': m_remote_run_chromium_src( |
| 'chromium_trybot', max_time=master_utils.CQ_MAX_TIME), |
| 'slavebuilddir': 'remote_run' |
| }) |
| |
| chromium_builders.append({ |
| 'name': 'linux_chromium_clobber_rel_ng', |
| 'factory': m_remote_run_chromium_src( |
| 'chromium_trybot', max_time=master_utils.CQ_MAX_TIME), |
| 'slavebuilddir': 'remote_run' |
| }) |
| |
| # This bot builds and upload GN binaries to cloud storage. |
| # We do not reuse the 'linux_chromium_gn' slavebuilddir because the upload |
| # bots need the linux sysroots and hence require src-internal. |
| chromium_builders.append({ |
| 'name': 'linux_chromium_gn_upload', |
| 'factory': base_factory('chromium_gn_upload'), |
| 'slavebuilddir': 'build_and_upload_gn', |
| }) |
| |
| chromium_builders.append({ |
| 'name': 'linux_upload_clang', |
| 'factory': base_factory('chromium_upload_clang'), |
| 'slavebuilddir': 'linux_upload_clang', |
| }) |
| |
| # Chromecast builders using chromium_trybot |
| chromium_builders.extend([{ |
| 'name': 'cast_shell_linux', |
| 'factory': m_remote_run_chromium_src( |
| 'chromium_trybot', max_time=master_utils.CQ_MAX_TIME), |
| }, |
| ]) |
| |
| chromium_builders.extend([{ |
| 'name': 'blimp_linux_dbg', |
| 'factory': m_remote_run_chromium_src( |
| 'chromium_trybot', max_time=master_utils.CQ_MAX_TIME), |
| 'slavebuilddir': 'remote_run', |
| }, |
| ]) |
| |
| chromium_builders.extend([{ |
| 'name': 'linux_chromium_headless_rel', |
| 'factory': m_remote_run_chromium_src( |
| 'chromium_trybot', max_time=master_utils.CQ_MAX_TIME), |
| 'slavebuilddir': 'remote_run', |
| }, |
| ]) |
| |
| b_linux_ozone = { |
| 'name': 'linux_chromium_ozone_compile_only_ng', |
| 'factory': m_remote_run('chromium_trybot'), |
| 'slavebuilddir': 'remote_run', |
| } |
| |
| b_linux_site_isolation = { |
| 'name': 'linux_site_isolation', |
| 'factory': m_remote_run_chromium_src('chromium_trybot'), |
| 'slavebuilddir': 'remote_run', |
| } |
| |
| b_linux_chromium_asan_rel_ng = { |
| 'name': 'linux_chromium_asan_rel_ng', |
| 'factory': m_remote_run_chromium_src( |
| 'chromium_trybot', max_time=master_utils.CQ_MAX_TIME), |
| 'slavebuilddir': 'remote_run', |
| } |
| |
| # Optional GPU tryserver |
| chromium_builders.append({ |
| 'name': 'linux_optional_gpu_tests_rel', |
| 'factory': m_remote_run_chromium_src('chromium_trybot', timeout=3600), |
| 'slavebuilddir': 'remote_run' |
| }) |
| |
| # Optional browser-side navigation trybot |
| b_linux_chromium_browser_side_navigation_rel = { |
| 'name': 'linux_chromium_browser_side_navigation_rel', |
| 'factory': m_remote_run_chromium_src('chromium_trybot', timeout=3600), |
| 'slavebuilddir': 'remote_run' |
| } |
| |
| # NaCl SDK try bots |
| b_linux_nacl_sdk = { |
| 'name': 'linux_nacl_sdk', |
| 'factory': base_factory('nacl/sdk'), |
| } |
| |
| # NaCl SDK build-only try bots |
| b_linux_nacl_sdk_build = { |
| 'name': 'linux_nacl_sdk_build', |
| 'factory': base_factory('nacl/sdk_build'), |
| } |
| |
| # Codesearch builders |
| b_codesearch_linux_builder = { |
| 'name': 'Chromium Linux Codesearch Builder', |
| 'factory': base_factory('chromium_codesearch'), |
| } |
| |
| b_codesearch_chromeos_builder = { |
| 'name': 'ChromiumOS Codesearch Builder', |
| 'factory': base_factory('chromium_codesearch'), |
| } |
| |
| # Optional Closure Compilation builder |
| chromium_builders.append({ |
| 'name': 'closure_compilation', |
| 'factory': base_factory('closure_compilation'), |
| # Share build directory with compatible existing builders to save space. |
| 'slavebuilddir': 'linux', |
| }) |
| |
| # Builder specifically to run layout tests with a particular flag (see crbug.com/622865). |
| chromium_builders.append({ |
| 'name': 'linux_layout_tests_slimming_paint_v2', |
| 'factory': m_remote_run_chromium_src('chromium_trybot'), |
| }) |
| |
| chromium_builders.append({ |
| 'name': 'chromium_devtools', |
| 'factory': base_factory('devtools', max_time=master_utils.CQ_MAX_TIME), |
| # Share build directory with compatible existing builders to save space. |
| 'slavebuilddir': 'linux', |
| }) |
| |
| # Presubmit bots |
| def presubmit(name, slavebuilddir=None, extra_props=None): |
| properties = {'repo_name': name} |
| properties.update(extra_props or {}) |
| return { |
| 'name': '%s_presubmit' % name, |
| 'slavebuilddir': (slavebuilddir or ('presubmit_%s' % name)), |
| 'factory': base_factory( |
| 'run_presubmit', properties, |
| max_time=master_utils.CQ_MAX_TIME), |
| } |
| |
| b_chromium_presubmit = presubmit('chromium', slavebuilddir='linux', |
| extra_props={'runhooks': True}) |
| |
| # These variable builders are used by Findit to identify culprit commits for |
| # compile or test failures on the main waterfall. They run on optional slaves |
| # and NOT CQ slaves. |
| variable_builders = [ |
| { |
| 'name': 'linux_chromium_variable', |
| 'factory': m_remote_run('findit/chromium/compile'), |
| 'slavebuilddir': 'linux', |
| }, |
| { |
| 'name': 'linux_chromium_variable_32', |
| 'factory': m_remote_run('findit/chromium/compile'), |
| 'slavebuilddir': 'linux_32', |
| }, |
| { |
| 'name': 'linux_chromium_variable_chrome', |
| 'factory': m_remote_run('findit/chromium/compile'), |
| 'slavebuilddir': 'linux_chrome', |
| }, |
| { |
| 'name': 'linux_chromium_variable_clobber', |
| 'factory': m_remote_run('findit/chromium/compile'), |
| 'slavebuilddir': 'linux_clobber', |
| }, |
| { |
| 'name': 'linux_chromium_variable_32_chrome', |
| 'factory': m_remote_run('findit/chromium/compile'), |
| 'slavebuilddir': 'linux_chrome_32', |
| }, |
| { |
| 'name': 'linux_chromium_variable_32_clobber', |
| 'factory': m_remote_run('findit/chromium/compile'), |
| 'slavebuilddir': 'linux_clobber_32', |
| }, |
| { |
| 'name': 'linux_chromium_asan_variable', |
| 'factory': m_remote_run('findit/chromium/compile'), |
| 'slavebuilddir': 'linux_asan', |
| }, |
| { |
| 'name': 'linux_chromium_chromeos_variable', |
| 'factory': m_remote_run('findit/chromium/compile'), |
| 'slavebuilddir': 'linux_chromeos', |
| }, |
| { |
| 'name': 'linux_chromium_chromeos_variable_chrome', |
| 'factory': m_remote_run('findit/chromium/compile'), |
| 'slavebuilddir': 'linux_chromeos_chrome', |
| }, |
| { |
| 'name': 'linux_chromium_chromeos_asan_variable', |
| 'factory': m_remote_run('findit/chromium/compile'), |
| 'slavebuilddir': 'linux_chromeos_asan', |
| }, |
| { |
| 'name': 'linux_chromium_gn_chromeos_variable', |
| 'factory': m_remote_run('findit/chromium/compile'), |
| 'slavebuilddir': 'linux_chromium_gn_chromeos', |
| }, |
| { |
| 'name': 'linux_chromium_blimp_variable', |
| 'factory': m_remote_run('findit/chromium/compile'), |
| 'slavebuilddir': 'linux_blimp', |
| }, |
| { |
| 'name': 'linux_chromium_cast_variable', |
| 'factory': m_remote_run('findit/chromium/compile'), |
| 'slavebuilddir': 'linux_cast', |
| }, |
| { |
| 'name': 'linux_chromium_webkit_variable', |
| 'factory': m_remote_run('findit/chromium/compile'), |
| 'slavebuilddir': 'linux_layout', |
| }, |
| { |
| 'name': 'linux_chromium_webkit_asan_variable', |
| 'factory': m_remote_run('findit/chromium/compile'), |
| 'slavebuilddir': 'linux_layout_asan', |
| }, |
| { |
| 'name': 'linux_chromium_webkit_leak_variable', |
| 'factory': m_remote_run('findit/chromium/compile'), |
| 'slavebuilddir': 'linux_layout_leak', |
| }, |
| { |
| 'name': 'linux_chromium_webkit_msan_variable', |
| 'factory': m_remote_run('findit/chromium/compile'), |
| 'slavebuilddir': 'linux_layout_msan', |
| }, |
| { |
| 'name': 'linux_chromiumos_full_variable', |
| 'factory': m_remote_run('findit/chromium/compile'), |
| 'slavebuilddir': 'linux_chromeos', |
| }, |
| ] |
| |
| for board in chromeos_boards: |
| variable_builders.append({ |
| 'name': 'chromeos_%s_variable' % (board,), |
| 'factory': m_remote_run('findit/chromium/compile'), |
| 'slavebuilddir': 'chromeos_%s' % (board,), |
| }) |
| |
| c['builders'] = [ |
| b_chromium_presubmit, |
| b_codesearch_chromeos_builder, |
| b_codesearch_linux_builder, |
| b_linux_chromium_asan_rel_ng, |
| b_linux_chromium_browser_side_navigation_rel, |
| b_linux_nacl_sdk, |
| b_linux_nacl_sdk_build, |
| b_linux_ozone, |
| b_linux_site_isolation, |
| ] + chromium_builders + variable_builders |
| |
| |
| # Slaves are loaded from slaves.cfg. |
| slaves = slaves_list.SlavesList('slaves.cfg', 'TryServerChromiumLinux') |
| |
| for builder in c['builders']: |
| # Associate the slaves to the builders. The configuration is in slaves.cfg. |
| builder['slavenames'] = slaves.GetSlavesName(builder=builder['name']) |
| # Sets nextSlave function for slave affinity. |
| builder['nextSlave'] = master_utils.PreferredBuilderNextSlaveFunc() |
| # Don't enable auto_reboot for people testing locally. |
| builder.setdefault('auto_reboot', ActiveMaster.is_production_host) |
| |
| |
| ####### BUILDSLAVES |
| |
| # The 'slaves' list defines the set of allowable buildslaves. List all the |
| # slaves registered to a builder. Remove dupes. |
| c['slaves'] = master_utils.AutoSetupSlaves( |
| c['builders'], |
| config.Master.GetBotPassword(), |
| preferred_builder_dict=slaves.GetPreferredBuildersDict()) |
| |
| # Make sure everything works together. |
| master_utils.VerifySetup(c, slaves) |
| |
| |
| ####### SCHEDULERS |
| |
| # Configure the Schedulers; |
| # Main Tryscheduler for the try queue. groups is defined in the loop above. |
| c['schedulers'] = [] |
| |
| last_good_urls = {'chrome': ActiveMaster.last_good_url, |
| 'blink': ActiveMaster.last_good_blink_url} |
| code_review_sites = {'chrome': ActiveMaster.code_review_site, |
| 'blink': ActiveMaster.code_review_site} |
| |
| c['schedulers'].append(TryJobHTTP( |
| name='try_job_http', |
| port=ActiveMaster.try_job_port, |
| last_good_urls=last_good_urls, |
| code_review_sites=code_review_sites, |
| pools=pools)) |
| |
| ####### STATUS TARGETS |
| |
| # Buildbot master url: |
| # Must come before AutoSetupMaster(). |
| if ActiveMaster.is_production_host: |
| c['buildbotURL'] = ActiveMaster.buildbot_url |
| else: |
| c['buildbotURL'] = 'http://%s:%d/' % ( |
| socket.getfqdn(), ActiveMaster.master_port) |
| |
| # Adds common status and tools to this master. |
| master_utils.AutoSetupMaster(c, ActiveMaster) |
| |
| if MAIL_NOTIFIER: |
| # Add a dumb MailNotifier first so it will be used for BuildSlave with |
| # notify_on_missing set when they go missing. |
| from buildbot.status import mail |
| c['status'].append(mail.MailNotifier( |
| fromaddr=ActiveMaster.from_address, |
| builders=[], |
| relayhost=config.Master.smtp, |
| lookup=master_utils.UsersAreEmails())) |
| |
| # Try job result emails. |
| from master.try_mail_notifier import TryMailNotifier |
| c['status'].append(TryMailNotifier( |
| fromaddr=ActiveMaster.from_address, |
| reply_to=ActiveMaster.reply_to, |
| subject="try %(result)s for %(reason)s @ r%(revision)s", |
| mode='all', |
| relayhost=config.Master.smtp, |
| lookup=master_utils.UsersAreEmails())) |
| |
| |
| # The followings are what is kept on disk. |
| # Keep last try jobs, the default is too low. Must keep at least a few days |
| # worth of try jobs. 3000 is not even a full day but the server is full. Keep |
| # more build objects than log since they are much smaller. |
| c['buildHorizon'] = 6000 |
| c['logHorizon'] = 3000 |
| # Must be at least 2x the number of slaves. |
| c['eventHorizon'] = 200 |
| |
| # Adjust the buildCaches to be 3x the number of slaves per builder. |
| c['autoBuildCacheRatio'] = 3 |
| |
| |
| # Hack buildbot so the Stop build button doesn't work anymore. Otherwise it's |
| # just a pain, user misuse this button all the time. |
| def hack_stop(function): |
| def hook(*args, **kwargs): |
| result = function(*args, **kwargs) |
| result = result.replace('<input type="submit" value="Stop', |
| '<input type="button" onclick="alert(\'' |
| 'For more information, visit ' |
| 'http://dev.chromium.org/developers/try-server-usage' |
| '\');"' |
| ' value="Stop') |
| return result |
| return hook |
| |
| # Only do the hack_stop if we are the production master. This allows |
| # us to keep the STOP button live for local/test instances. |
| #if ActiveMaster.is_production_host: |
| # from buildbot.status.web.builder import StatusResourceBuilder |
| # StatusResourceBuilder.build_line = hack_stop(StatusResourceBuilder.build_line) |
| # from buildbot.status.web import base |
| # base.make_stop_form = hack_stop(base.make_stop_form) |
| |
| |
| ####### PROJECT IDENTITY |
| |
| # The 'projectURL' string will be used to provide a link |
| # from buildbot HTML pages to your project's home page. |
| c['projectURL'] = 'http://dev.chromium.org/developers/testing/try-server-usage' |
| |
| # vi: set ts=4 sts=2 sw=2 et: |