import re
import base64
import subprocess
-import ConfigParser
+import configparser
from buildbot import locks
+from buildbot.changes import filter
+from buildbot.changes.gitpoller import GitPoller
+from buildbot.config import BuilderConfig
+from buildbot.plugins import reporters
+from buildbot.plugins import schedulers
+from buildbot.plugins import steps
+from buildbot.plugins import util
+from buildbot.process import properties
+from buildbot.process.factory import BuildFactory
+from buildbot.process.properties import Interpolate
+from buildbot.process.properties import Property
+from buildbot.schedulers.basic import SingleBranchScheduler
+from buildbot.schedulers.forcesched import BaseParameter
+from buildbot.schedulers.forcesched import ForceScheduler
+from buildbot.schedulers.forcesched import ValidationError
+from buildbot.steps.master import MasterShellCommand
+from buildbot.steps.shell import SetPropertyFromCommand
+from buildbot.steps.shell import ShellCommand
+from buildbot.steps.source.git import Git
+from buildbot.steps.transfer import FileDownload
+from buildbot.steps.transfer import FileUpload
+from buildbot.steps.transfer import StringDownload
+from buildbot.worker import Worker
+
# This is a sample buildmaster config file. It must be installed as
# 'master.cfg' in your buildmaster's base directory.
-ini = ConfigParser.ConfigParser()
+ini = configparser.ConfigParser()
ini.read(os.getenv("BUILDMASTER_CONFIG", "./config.ini"))
# This is the dictionary that the buildmaster pays attention to. We also use
####### BUILDSLAVES
-# The 'slaves' list defines the set of recognized buildslaves. Each element is
-# a BuildSlave object, specifying a unique slave name and password. The same
+# The 'workers' list defines the set of recognized buildslaves. Each element is
+# a Worker object, specifying a unique slave name and password. The same
# slave name and password must be configured on the slave.
-from buildbot.buildslave import BuildSlave
slave_port = 9989
if ini.has_option("phase1", "port"):
- slave_port = ini.getint("phase1", "port")
+ slave_port = ini.get("phase1", "port")
-c['slaves'] = []
+c['workers'] = []
NetLocks = dict()
for section in ini.sections():
if ini.has_option(section, "builds"):
max_builds = ini.getint(section, "builds")
sl_props['max_builds'] = max_builds
+ if max_builds == 1:
+ sl_props['shared_wd'] = True
if ini.has_option(section, "cleanup"):
sl_props['do_cleanup'] = ini.getboolean(section, "cleanup")
if ini.has_option(section, "dl_lock"):
sl_props['shared_wd'] = shared_wd
if shared_wd and (max_builds != 1):
raise ValueError('max_builds must be 1 with shared workdir!')
- c['slaves'].append(BuildSlave(name, password, max_builds = max_builds, properties = sl_props))
+ c['workers'].append(Worker(name, password, max_builds = max_builds, properties = sl_props))
-# 'slavePortnum' defines the TCP port to listen on for connections from slaves.
+# 'slavePortnum' defines the TCP port to listen on for connections from workers.
# This must match the value configured into the buildslaves (with their
# --master option)
-c['slavePortnum'] = slave_port
+c['protocols'] = {'pb': {'port': slave_port}}
# coalesce builds
-c['mergeRequests'] = True
+c['collapseRequests'] = True
# Reduce amount of backlog data
c['buildHorizon'] = 30
line = findtargets.stdout.readline()
if not line:
break
- ta = line.strip().split(' ')
+ ta = line.decode().strip().split(' ')
targets.append(ta[0])
# the 'change_source' setting tells the buildmaster how it should find out
# about source code changes. Here we point to the buildbot clone of pyflakes.
-from buildbot.changes.gitpoller import GitPoller
c['change_source'] = []
c['change_source'].append(GitPoller(
repo_url,
# Configure the Schedulers, which decide how to react to incoming changes. In this
# case, just kick off a 'basebuild' build
-from buildbot.schedulers.basic import SingleBranchScheduler
-from buildbot.schedulers.forcesched import ForceScheduler
-from buildbot.changes import filter
+class TagChoiceParameter(BaseParameter):
+ spec_attributes = ["strict", "choices"]
+ type = "list"
+ strict = True
+
+ def __init__(self, name, label=None, **kw):
+ super().__init__(name, label, **kw)
+ self._choice_list = []
+
+ @property
+ def choices(self):
+ taglist = []
+ basever = re.search(r'-([0-9]+\.[0-9]+)$', repo_branch)
+
+ if basever:
+ findtags = subprocess.Popen(
+ ['git', 'ls-remote', '--tags', repo_url],
+ stdout = subprocess.PIPE)
+
+ while True:
+ line = findtags.stdout.readline()
+
+ if not line:
+ break
+
+ tagver = re.search(r'\brefs/tags/v([0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?)$', line.decode().strip())
+
+ if tagver and tagver[1].find(basever[1]) == 0:
+ taglist.append(tagver[1])
+
+ taglist.sort(reverse=True, key=lambda tag: tag if re.search(r'-rc[0-9]+$', tag) else tag + '-z')
+ taglist.insert(0, '')
+
+ self._choice_list = taglist
+
+ return self._choice_list
+
+ def parse_from_arg(self, s):
+ if self.strict and s not in self._choice_list:
+ raise ValidationError("'%s' does not belong to list of available choices '%s'" % (s, self._choice_list))
+ return s
+
c['schedulers'] = []
c['schedulers'].append(SingleBranchScheduler(
- name="all",
- change_filter=filter.ChangeFilter(branch=repo_branch),
- treeStableTimer=60,
- builderNames=targets))
+ name = "all",
+ change_filter = filter.ChangeFilter(branch=repo_branch),
+ treeStableTimer = 60,
+ builderNames = targets))
c['schedulers'].append(ForceScheduler(
- name="force",
- builderNames=targets))
+ name = "force",
+ buttonName = "Force builds",
+ label = "Force build details",
+ builderNames = [ "00_force_build" ],
+
+ codebases = [
+ util.CodebaseParameter(
+ "",
+ label = "Repository",
+ branch = util.FixedParameter(name = "branch", default = ""),
+ revision = util.FixedParameter(name = "revision", default = ""),
+ repository = util.FixedParameter(name = "repository", default = ""),
+ project = util.FixedParameter(name = "project", default = "")
+ )
+ ],
+
+ reason = util.StringParameter(
+ name = "reason",
+ label = "Reason",
+ default = "Trigger build",
+ required = True,
+ size = 80
+ ),
+
+ properties = [
+ util.NestedParameter(
+ name="options",
+ label="Build Options",
+ layout="vertical",
+ fields=[
+ util.ChoiceStringParameter(
+ name = "target",
+ label = "Build target",
+ default = "all",
+ choices = [ "all" ] + targets
+ ),
+ TagChoiceParameter(
+ name = "tag",
+ label = "Build tag",
+ default = ""
+ )
+ ]
+ )
+ ]
+))
####### BUILDERS
# The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
-# what steps, and which slaves can execute them. Note that any particular build will
+# what steps, and which workers can execute them. Note that any particular build will
# only take place on one slave.
-from buildbot.process.factory import BuildFactory
-from buildbot.steps.source.git import Git
-from buildbot.steps.shell import ShellCommand
-from buildbot.steps.shell import SetPropertyFromCommand
-from buildbot.steps.transfer import FileUpload
-from buildbot.steps.transfer import FileDownload
-from buildbot.steps.transfer import StringDownload
-from buildbot.steps.master import MasterShellCommand
-from buildbot.process.properties import Interpolate
-from buildbot.process import properties
-
-
CleanTargetMap = [
[ "tools", "tools/clean" ],
[ "chain", "toolchain/clean" ],
def IsTaggingRequested(step):
val = step.getProperty("tag")
- if val and re.match("^[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", val):
+ if val and re.match(r"^[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", val):
return True
else:
return False
return repo_branch != "master"
def GetBaseVersion():
- if re.match("^[^-]+-[0-9]+\.[0-9]+$", repo_branch):
+ if re.match(r"^[^-]+-[0-9]+\.[0-9]+$", repo_branch):
return repo_branch.split('-')[1]
else:
return "master"
@properties.renderer
def GetVersionPrefix(props):
basever = GetBaseVersion()
- if props.hasProperty("tag") and re.match("^[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", props["tag"]):
+ if props.hasProperty("tag") and re.match(r"^[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", props["tag"]):
return "%s/" % props["tag"]
elif basever != "master":
return "%s-SNAPSHOT/" % basever
@properties.renderer
def GetNumJobs(props):
if props.hasProperty("max_builds") and props.hasProperty("nproc"):
- return str(int(props["nproc"]) / (props["max_builds"] + other_builds))
+ return str(int(int(props["nproc"]) / (props["max_builds"] + other_builds)))
else:
return "1"
else:
return []
+@util.renderer
+def TagPropertyValue(props):
+ if props.hasProperty("options"):
+ options = props.getProperty("options")
+ if type(options) is dict:
+ return options.get("tag")
+ return None
+
+def IsTargetSelected(target):
+ def CheckTargetProperty(step):
+ try:
+ options = step.getProperty("options")
+ if type(options) is dict:
+ selected_target = options.get("target", "all")
+ if selected_target != "all" and selected_target != target:
+ return False
+ except KeyError:
+ pass
+
+ return True
+
+ return CheckTargetProperty
+
def UsignSec2Pub(seckey, comment="untrusted comment: secret key"):
try:
seckey = base64.b64decode(seckey)
c['builders'] = []
-dlLock = locks.SlaveLock("slave_dl")
+dlLock = locks.WorkerLock("slave_dl")
checkBuiltin = re.sub('[\t\n ]+', ' ', """
checkBuiltin() {
slaveNames = [ ]
-for slave in c['slaves']:
- slaveNames.append(slave.slavename)
+for slave in c['workers']:
+ slaveNames.append(slave.workername)
+
+force_factory = BuildFactory()
+
+c['builders'].append(BuilderConfig(
+ name = "00_force_build",
+ workernames = slaveNames,
+ factory = force_factory))
for target in targets:
ts = target.split('/')
factory.addStep(ShellCommand(
name = "sharedwd",
description = "Setting up shared work directory",
- command = 'test -L "$PWD" || (mkdir -p "../shared-workdir" && rm -rf "$PWD" && ln -s shared-workdir "$PWD")',
+ command = 'test -L "$PWD" || (mkdir -p ../shared-workdir && rm -rf "$PWD" && ln -s shared-workdir "$PWD")',
workdir = ".",
haltOnFailure = True,
doStepIf = IsSharedWorkdir))
factory.addStep(FileDownload(
name = "dlfindbinpl",
mastersrc = scripts_dir + '/findbin.pl',
- slavedest = "../findbin.pl",
- mode = 0755))
+ workerdest = "../findbin.pl",
+ mode = 0o755))
factory.addStep(SetPropertyFromCommand(
name = "gcc",
name = "dlexpiresh",
doStepIf = IsExpireRequested,
mastersrc = scripts_dir + '/expire.sh',
- slavedest = "../expire.sh",
- mode = 0755))
+ workerdest = "../expire.sh",
+ mode = 0o755))
factory.addStep(ShellCommand(
name = "expire",
# cleanup.sh if needed
factory.addStep(FileDownload(
name = "dlcleanupsh",
- mastersrc = scripts_dir + '/cleanup-phase1.sh',
- slavedest = "../cleanup.sh",
- mode = 0755,
+ mastersrc = scripts_dir + '/cleanup.sh',
+ workerdest = "../cleanup.sh",
+ mode = 0o755,
doStepIf = IsCleanupRequested))
factory.addStep(ShellCommand(
name = "cleanold",
description = "Cleaning previous builds",
- command = ["./cleanup.sh", c['buildbotURL'], Interpolate("%(prop:slavename)s"), Interpolate("%(prop:buildername)s"), "full"],
+ command = ["./cleanup.sh", c['buildbotURL'], Interpolate("%(prop:workername)s"), Interpolate("%(prop:buildername)s"), "full"],
workdir = ".",
haltOnFailure = True,
doStepIf = IsCleanupRequested,
factory.addStep(ShellCommand(
name = "cleanup",
description = "Cleaning work area",
- command = ["./cleanup.sh", c['buildbotURL'], Interpolate("%(prop:slavename)s"), Interpolate("%(prop:buildername)s"), "single"],
+ command = ["./cleanup.sh", c['buildbotURL'], Interpolate("%(prop:workername)s"), Interpolate("%(prop:buildername)s"), "single"],
workdir = ".",
haltOnFailure = True,
doStepIf = IsCleanupRequested,
factory.addStep(ShellCommand(
name = "gitcheckout",
description = "Ensure that Git HEAD is sane",
- command = "if [ -d .git ]; then git checkout %s; git branch --set-upstream-to origin/%s; else exit 0; fi" %(repo_branch, repo_branch),
+ command = "if [ -d .git ]; then git checkout -f %s; git branch --set-upstream-to origin/%s; else exit 0; fi" %(repo_branch, repo_branch),
haltOnFailure = True))
# check out the source
factory.addStep(StringDownload(
name = "ccachecc",
s = '#!/bin/sh\nexec ${CCACHE} ${CCC} "$@"\n',
- slavedest = "../ccache_cc.sh",
- mode = 0755,
+ workerdest = "../ccache_cc.sh",
+ mode = 0o755,
))
factory.addStep(StringDownload(
name = "ccachecxx",
s = '#!/bin/sh\nexec ${CCACHE} ${CCXX} "$@"\n',
- slavedest = "../ccache_cxx.sh",
- mode = 0755,
+ workerdest = "../ccache_cxx.sh",
+ mode = 0o755,
))
# Git SSH
factory.addStep(StringDownload(
name = "dlgitclonekey",
s = git_ssh_key,
- slavedest = "../git-clone.key",
- mode = 0600,
+ workerdest = "../git-clone.key",
+ mode = 0o600,
))
factory.addStep(ShellCommand(
factory.addStep(StringDownload(
name = "dlconfigseed",
s = config_seed + '\n',
- slavedest = ".config",
- mode = 0644
+ workerdest = ".config",
+ mode = 0o644
))
# configure
factory.addStep(StringDownload(
name = "dlkeybuildpub",
s = UsignSec2Pub(usign_key, usign_comment),
- slavedest = "key-build.pub",
- mode = 0600,
+ workerdest = "key-build.pub",
+ mode = 0o600,
))
factory.addStep(StringDownload(
name = "dlkeybuild",
s = "# fake private key",
- slavedest = "key-build",
- mode = 0600,
+ workerdest = "key-build",
+ mode = 0o600,
))
factory.addStep(StringDownload(
name = "dlkeybuilducert",
s = "# fake certificate",
- slavedest = "key-build.ucert",
- mode = 0600,
+ workerdest = "key-build.ucert",
+ mode = 0o600,
))
# prepare dl
))
factory.addStep(ShellCommand(
- name = "diffconfig",
- description = "Generating config.seed",
- command=["make", "-j1", "diffconfig", "V=s"],
+ name = "buildinfo",
+ description = "Generating config.buildinfo, version.buildinfo and feeds.buildinfo",
+ command = "make -j1 buildinfo V=s || true",
+ env = MakeEnv(),
+ haltOnFailure = True
+ ))
+
+ factory.addStep(ShellCommand(
+ name = "json_overview_image_info",
+ description = "Generate profiles.json in target folder",
+ command = "make -j1 json_overview_image_info V=s || true",
env = MakeEnv(),
haltOnFailure = True
))
))
factory.addStep(FileUpload(
- slavesrc = "sign.tar.gz",
+ workersrc = "sign.tar.gz",
masterdest = "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1]),
haltOnFailure = True
))
factory.addStep(FileDownload(
name = "dlsigntargz",
mastersrc = "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1]),
- slavedest = "sign.tar.gz",
+ workerdest = "sign.tar.gz",
haltOnFailure = True
))
factory.addStep(FileDownload(
name = "dlsha2rsyncpl",
mastersrc = scripts_dir + '/sha2rsync.pl',
- slavedest = "../sha2rsync.pl",
- mode = 0755,
+ workerdest = "../sha2rsync.pl",
+ mode = 0o755,
))
factory.addStep(ShellCommand(
factory.addStep(FileDownload(
name = "dlrsync.sh",
mastersrc = scripts_dir + '/rsync.sh',
- slavedest = "../rsync.sh",
- mode = 0755
+ workerdest = "../rsync.sh",
+ mode = 0o755
))
# upload new files and update existing ones
name = "sourceupload",
description = "Uploading source archives",
command=["../rsync.sh", "--files-from=sourcelist", "--size-only", "--delay-updates"] + rsync_src_defopts +
- [Interpolate("--partial-dir=.~tmp~%(kw:target)s~%(kw:subtarget)s~%(prop:slavename)s", target=ts[0], subtarget=ts[1]), "-a", "dl/", "%s/" %(rsync_src_url)],
+ [Interpolate("--partial-dir=.~tmp~%(kw:target)s~%(kw:subtarget)s~%(prop:workername)s", target=ts[0], subtarget=ts[1]), "-a", "dl/", "%s/" %(rsync_src_url)],
env={'RSYNC_PASSWORD': rsync_src_key},
haltOnFailure = True,
logEnviron = False,
alwaysRun = True,
))
- from buildbot.config import BuilderConfig
+ c['builders'].append(BuilderConfig(name=target, workernames=slaveNames, factory=factory, nextBuild=GetNextBuild))
- c['builders'].append(BuilderConfig(name=target, slavenames=slaveNames, factory=factory, nextBuild=GetNextBuild))
+ c['schedulers'].append(schedulers.Triggerable(name="trigger_%s" % target, builderNames=[ target ]))
+ force_factory.addStep(steps.Trigger(
+ name = "trigger_%s" % target,
+ description = "Triggering %s build" % target,
+ schedulerNames = [ "trigger_%s" % target ],
+ set_properties = { "reason": Property("reason"), "tag": TagPropertyValue },
+ doStepIf = IsTargetSelected(target)
+ ))
####### STATUS TARGETS
# pushed to these targets. buildbot/status/*.py has a variety to choose from,
# including web pages, email senders, and IRC bots.
-c['status'] = []
-
-from buildbot.status import html
-from buildbot.status.web import authz, auth
-
if ini.has_option("phase1", "status_bind"):
+ c['www'] = {
+ 'port': ini.get("phase1", "status_bind"),
+ 'plugins': {
+ 'waterfall_view': True,
+ 'console_view': True,
+ 'grid_view': True
+ }
+ }
+
if ini.has_option("phase1", "status_user") and ini.has_option("phase1", "status_password"):
- authz_cfg=authz.Authz(
- # change any of these to True to enable; see the manual for more
- # options
- auth=auth.BasicAuth([(ini.get("phase1", "status_user"), ini.get("phase1", "status_password"))]),
- gracefulShutdown = 'auth',
- forceBuild = 'auth', # use this to test your slave once it is set up
- forceAllBuilds = 'auth',
- pingBuilder = False,
- stopBuild = 'auth',
- stopAllBuilds = 'auth',
- cancelPendingBuild = 'auth',
+ c['www']['auth'] = util.UserPasswordAuth([
+ (ini.get("phase1", "status_user"), ini.get("phase1", "status_password"))
+ ])
+ c['www']['authz'] = util.Authz(
+ allowRules=[ util.AnyControlEndpointMatcher(role="admins") ],
+ roleMatchers=[ util.RolesFromUsername(roles=["admins"], usernames=[ini.get("phase1", "status_user")]) ]
)
- c['status'].append(html.WebStatus(http_port=ini.get("phase1", "status_bind"), authz=authz_cfg))
- else:
- c['status'].append(html.WebStatus(http_port=ini.get("phase1", "status_bind")))
-from buildbot.status import words
-
if ini.has_option("irc", "host") and ini.has_option("irc", "nickname") and ini.has_option("irc", "channel"):
irc_host = ini.get("irc", "host")
irc_port = 6667
if ini.has_option("irc", "password"):
irc_pass = ini.get("irc", "password")
- irc = words.IRC(irc_host, irc_nick, port = irc_port, password = irc_pass,
- channels = [{ "channel": irc_chan }],
- notify_events = {
- 'exception': 1,
- 'successToFailure': 1,
- 'failureToSuccess': 1
- }
+ irc = reporters.IRC(irc_host, irc_nick,
+ port = irc_port,
+ password = irc_pass,
+ channels = [ irc_chan ],
+ notify_events = [ 'exception', 'problem', 'recovery' ]
)
- c['status'].append(irc)
+ c['services'].append(irc)
####### DB URL
# this at its default for all but the largest installations.
'db_url' : "sqlite:///state.sqlite",
}
+
+c['buildbotNetUsageData'] = None
projects / buildbot.git / blobdiff