2 # ex: set syntax=python:
9 from buildbot import locks
11 # This is a sample buildmaster config file. It must be installed as
12 # 'master.cfg' in your buildmaster's base directory.
14 ini = ConfigParser.ConfigParser()
15 ini.read("./config.ini")
17 # This is the dictionary that the buildmaster pays attention to. We also use
18 # a shorter alias to save typing.
19 c = BuildmasterConfig = {}
23 # The 'slaves' list defines the set of recognized buildslaves. Each element is
24 # a BuildSlave object, specifying a unique slave name and password. The same
25 # slave name and password must be configured on the slave.
26 from buildbot.buildslave import BuildSlave
30 if ini.has_option("general", "port"):
31 slave_port = ini.getint("general", "port")
36 for section in ini.sections():
37 if section.startswith("slave "):
38 if ini.has_option(section, "name") and ini.has_option(section, "password"):
39 name = ini.get(section, "name")
40 password = ini.get(section, "password")
42 if ini.has_option(section, "builds"):
43 max_builds[name] = ini.getint(section, "builds")
44 c['slaves'].append(BuildSlave(name, password, max_builds = max_builds[name]))
46 # 'slavePortnum' defines the TCP port to listen on for connections from slaves.
47 # This must match the value configured into the buildslaves (with their
49 c['slavePortnum'] = slave_port
52 c['mergeRequests'] = True
54 # Reduce amount of backlog data
55 c['buildHorizon'] = 30
60 home_dir = os.path.abspath(ini.get("general", "homedir"))
68 if ini.has_option("general", "expire"):
69 tree_expire = ini.getint("general", "expire")
71 if ini.has_option("general", "other_builds"):
72 other_builds = ini.getint("general", "other_builds")
74 if ini.has_option("general", "cc_version"):
75 cc_version = ini.get("general", "cc_version").split()
76 if len(cc_version) == 1:
77 cc_version = ["eq", cc_version[0]]
79 repo_url = ini.get("repo", "url")
80 repo_branch = "master"
82 if ini.has_option("repo", "branch"):
83 repo_branch = ini.get("repo", "branch")
85 rsync_bin_url = ini.get("rsync", "binary_url")
86 rsync_bin_key = ini.get("rsync", "binary_password")
91 if ini.has_option("rsync", "source_url"):
92 rsync_src_url = ini.get("rsync", "source_url")
93 rsync_src_key = ini.get("rsync", "source_password")
97 gpg_comment = "Unattended build signature"
98 gpg_passfile = "/dev/null"
100 if ini.has_option("gpg", "home"):
101 gpg_home = ini.get("gpg", "home")
103 if ini.has_option("gpg", "keyid"):
104 gpg_keyid = ini.get("gpg", "keyid")
106 if ini.has_option("gpg", "comment"):
107 gpg_comment = ini.get("gpg", "comment")
109 if ini.has_option("gpg", "passfile"):
110 gpg_passfile = ini.get("gpg", "passfile")
116 if not os.path.isdir(home_dir+'/source.git'):
117 subprocess.call(["git", "clone", "--depth=1", "--branch="+repo_branch, repo_url, home_dir+'/source.git'])
119 subprocess.call(["git", "pull"], cwd = home_dir+'/source.git')
121 findtargets = subprocess.Popen([home_dir+'/dumpinfo.pl', 'targets'],
122 stdout = subprocess.PIPE, cwd = home_dir+'/source.git')
125 line = findtargets.stdout.readline()
128 ta = line.strip().split(' ')
129 targets.append(ta[0])
132 # the 'change_source' setting tells the buildmaster how it should find out
133 # about source code changes. Here we point to the buildbot clone of pyflakes.
135 from buildbot.changes.gitpoller import GitPoller
136 c['change_source'] = []
137 c['change_source'].append(GitPoller(
139 workdir=home_dir+'/work.git', branch=repo_branch,
144 # Configure the Schedulers, which decide how to react to incoming changes. In this
145 # case, just kick off a 'basebuild' build
147 from buildbot.schedulers.basic import SingleBranchScheduler
148 from buildbot.schedulers.forcesched import ForceScheduler
149 from buildbot.changes import filter
151 c['schedulers'].append(SingleBranchScheduler(
153 change_filter=filter.ChangeFilter(branch=repo_branch),
155 builderNames=targets))
157 c['schedulers'].append(ForceScheduler(
159 builderNames=targets))
163 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
164 # what steps, and which slaves can execute them. Note that any particular build will
165 # only take place on one slave.
167 from buildbot.process.factory import BuildFactory
168 from buildbot.steps.source.git import Git
169 from buildbot.steps.shell import ShellCommand
170 from buildbot.steps.shell import SetProperty
171 from buildbot.steps.transfer import FileUpload
172 from buildbot.steps.transfer import FileDownload
173 from buildbot.steps.master import MasterShellCommand
174 from buildbot.process.properties import WithProperties
178 [ "tools", "tools/clean" ],
179 [ "chain", "toolchain/clean" ],
180 [ "linux", "target/linux/clean" ],
181 [ "dir", "dirclean" ],
182 [ "dist", "distclean" ]
185 def IsCleanRequested(pattern):
186 def CheckCleanProperty(step):
187 val = step.getProperty("clean")
188 if val and re.match(pattern, val):
193 return CheckCleanProperty
195 def IsTaggingRequested(step):
196 val = step.getProperty("tag")
197 if val and re.match("^[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", val):
202 def IsNoTaggingRequested(step):
203 return not IsTaggingRequested(step)
205 def IsNoMasterBuild(step):
206 return repo_branch != "master"
208 def GetBaseVersion(props):
209 if re.match("^[^-]+-[0-9]+\.[0-9]+$", repo_branch):
210 return repo_branch.split('-')[1]
214 def GetVersionPrefix(props):
215 basever = GetBaseVersion(props)
216 if props.hasProperty("tag") and re.match("^[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", props["tag"]):
217 return "%s/" % props["tag"]
218 elif basever != "master":
219 return "%s-SNAPSHOT/" % basever
223 def GetNumJobs(props):
224 if props.hasProperty("slavename") and props.hasProperty("nproc"):
225 return ((int(props["nproc"]) / (max_builds[props["slavename"]] + other_builds)) + 1)
230 if props.hasProperty("cc_command"):
231 return props["cc_command"]
236 if props.hasProperty("cxx_command"):
237 return props["cxx_command"]
242 if props.hasProperty("builddir"):
243 return props["builddir"]
244 elif props.hasProperty("workdir"):
245 return props["workdir"]
249 def GetNextBuild(builder, requests):
251 if r.properties and r.properties.hasProperty("tag"):
255 def MakeEnv(overrides=None):
257 'CC': WithProperties("%(cc)s", cc=GetCC),
258 'CXX': WithProperties("%(cxx)s", cxx=GetCXX),
259 'CCACHE_BASEDIR': WithProperties("%(cwd)s", cwd=GetCwd)
261 if overrides is not None:
262 env.update(overrides)
268 dlLock = locks.SlaveLock("slave_dl")
269 tagLock = locks.MasterLock("make_tag")
271 checkBuiltin = re.sub('[\t\n ]+', ' ', """
273 local symbol op path file;
274 for file in $CHANGED_FILES; do
280 while read symbol op path; do
281 case "$symbol" in package-*)
282 symbol="${symbol##*(}";
283 symbol="${symbol%)}";
284 for file in $CHANGED_FILES; do
285 case "$file" in "package/$path/"*)
286 grep -qsx "$symbol=y" .config && return 0
290 done < tmp/.packagedeps;
296 class IfBuiltinShellCommand(ShellCommand):
297 def _quote(self, str):
298 if re.search("[^a-zA-Z0-9/_.-]", str):
299 return "'%s'" %(re.sub("'", "'\"'\"'", str))
302 def setCommand(self, command):
303 if not isinstance(command, (str, unicode)):
304 command = ' '.join(map(self._quote, command))
307 '%s; if checkBuiltin; then %s; else exit 0; fi' %(checkBuiltin, command)
310 def setupEnvironment(self, cmd):
311 slaveEnv = self.slaveEnvironment
315 for request in self.build.requests:
316 for source in request.sources:
317 for change in source.changes:
318 for file in change.files:
319 changedFiles[file] = True
320 fullSlaveEnv = slaveEnv.copy()
321 fullSlaveEnv['CHANGED_FILES'] = ' '.join(changedFiles.keys())
322 cmd.args['env'] = fullSlaveEnv
326 for slave in c['slaves']:
327 slaveNames.append(slave.slavename)
329 for target in targets:
330 ts = target.split('/')
332 factory = BuildFactory()
334 # find number of cores
335 factory.addStep(SetProperty(
338 description = "Finding number of CPUs",
339 command = ["nproc"]))
341 # find gcc and g++ compilers
342 if cc_version is not None:
343 factory.addStep(FileDownload(
344 mastersrc = "findbin.pl",
345 slavedest = "../findbin.pl",
348 factory.addStep(SetProperty(
350 property = "cc_command",
351 description = "Finding gcc command",
352 command = ["../findbin.pl", "gcc", cc_version[0], cc_version[1]],
353 haltOnFailure = True))
355 factory.addStep(SetProperty(
357 property = "cxx_command",
358 description = "Finding g++ command",
359 command = ["../findbin.pl", "g++", cc_version[0], cc_version[1]],
360 haltOnFailure = True))
362 # expire tree if needed
364 factory.addStep(FileDownload(
365 mastersrc = "expire.sh",
366 slavedest = "../expire.sh",
369 factory.addStep(ShellCommand(
371 description = "Checking for build tree expiry",
372 command = ["./expire.sh", str(tree_expire)],
374 haltOnFailure = True,
377 # user-requested clean targets
378 for tuple in CleanTargetMap:
379 factory.addStep(ShellCommand(
381 description = 'User-requested "make %s"' % tuple[1],
382 command = ["make", tuple[1], "V=s"],
384 doStepIf = IsCleanRequested(tuple[0])
387 factory.addStep(MasterShellCommand(
389 description = "Tagging Git repository",
390 command = [home_dir+'/maketag.sh', '-i', '-k', str(gpg_keyid or ''),
391 '-p', str(gpg_passfile or ''), '-v', WithProperties("%(tag:-)s")],
392 path = home_dir+'/source.git',
393 env = {'GNUPGHOME': gpg_home},
394 haltOnFailure = True,
395 doStepIf = IsTaggingRequested,
396 locks = [tagLock.access('exclusive')]
400 factory.addStep(ShellCommand(
401 name = "switchbranch",
402 description = "Checking out Git branch",
403 command = "if [ -d .git ]; then git fetch --depth=1 && git checkout '%s'; else exit 0; fi" % repo_branch,
404 haltOnFailure = True,
405 doStepIf = IsNoTaggingRequested
408 # check out the source
411 branch = repo_branch,
412 mode = 'incremental',
416 factory.addStep(ShellCommand(
418 description = "Fetching Git remote refs",
419 command = ["git", "fetch", "origin", "+refs/heads/%s:refs/remotes/origin/%s" %(repo_branch, repo_branch)],
424 factory.addStep(ShellCommand(
426 description = "Fetching Git tags",
427 command = ["git", "fetch", "--tags", "--", repo_url],
428 haltOnFailure = True,
429 doStepIf = IsTaggingRequested
433 factory.addStep(ShellCommand(
435 description = "Checking out Git tag",
436 command = ["git", "checkout", WithProperties("tags/v%(tag:-)s")],
437 haltOnFailure = True,
438 doStepIf = IsTaggingRequested
441 factory.addStep(ShellCommand(
443 description = "Remove tmp folder",
444 command=["rm", "-rf", "tmp/"]))
447 # factory.addStep(ShellCommand(
448 # name = "feedsconf",
449 # description = "Copy the feeds.conf",
450 # command='''cp ~/feeds.conf ./feeds.conf''' ))
453 factory.addStep(ShellCommand(
454 name = "rmfeedlinks",
455 description = "Remove feed symlinks",
456 command=["rm", "-rf", "package/feeds/"]))
459 factory.addStep(ShellCommand(
460 name = "updatefeeds",
461 description = "Updating feeds",
462 command=["./scripts/feeds", "update"],
466 factory.addStep(ShellCommand(
467 name = "installfeeds",
468 description = "Installing feeds",
469 command=["./scripts/feeds", "install", "-a"],
473 factory.addStep(FileDownload(
474 mastersrc = "config.seed",
475 slavedest = ".config",
480 factory.addStep(ShellCommand(
482 description = "Seeding .config",
483 command = "printf 'CONFIG_TARGET_%s=y\\nCONFIG_TARGET_%s_%s=y\\n' >> .config" %(ts[0], ts[0], ts[1])
486 factory.addStep(ShellCommand(
488 description = "Removing output directory",
489 command = ["rm", "-rf", "bin/"]
492 factory.addStep(ShellCommand(
494 description = "Populating .config",
495 command = ["make", "defconfig"],
500 factory.addStep(ShellCommand(
502 description = "Checking architecture",
503 command = ["grep", "-sq", "CONFIG_TARGET_%s=y" %(ts[0]), ".config"],
511 factory.addStep(SetProperty(
514 description = "Finding libc suffix",
515 command = ["sed", "-ne", '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }', ".config"]))
518 factory.addStep(FileDownload(
519 mastersrc = "ccache.sh",
520 slavedest = "ccache.sh",
525 factory.addStep(ShellCommand(
527 description = "Preparing ccache",
528 command = ["./ccache.sh"]
532 factory.addStep(FileDownload(mastersrc=home_dir+'/key-build', slavedest="key-build", mode=0600))
533 factory.addStep(FileDownload(mastersrc=home_dir+'/key-build.pub', slavedest="key-build.pub", mode=0600))
536 factory.addStep(ShellCommand(
538 description = "Preparing dl/",
539 command = "mkdir -p $HOME/dl && rm -rf ./dl && ln -sf $HOME/dl ./dl",
545 factory.addStep(ShellCommand(
547 description = "Building GNU tar",
548 command = ["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "tools/tar/compile", "V=s"],
554 factory.addStep(ShellCommand(
556 description = "Populating dl/",
557 command = ["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "download", "V=s"],
560 locks = [dlLock.access('exclusive')]
563 factory.addStep(ShellCommand(
565 description = "Cleaning base-files",
566 command=["make", "package/base-files/clean", "V=s"]
570 factory.addStep(ShellCommand(
572 description = "Building tools",
573 command = ["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "tools/install", "V=s"],
578 factory.addStep(ShellCommand(
580 description = "Building toolchain",
581 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "toolchain/install", "V=s"],
586 factory.addStep(ShellCommand(
588 description = "Building kmods",
589 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
591 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
595 factory.addStep(ShellCommand(
597 description = "Cleaning up package build",
598 command=["make", "package/cleanup", "V=s"]
601 factory.addStep(ShellCommand(
603 description = "Building packages",
604 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
606 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
610 # factory.addStep(IfBuiltinShellCommand(
611 factory.addStep(ShellCommand(
613 description = "Installing packages",
614 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "package/install", "V=s"],
619 factory.addStep(ShellCommand(
621 description = "Indexing packages",
622 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "package/index", "V=s"],
627 #factory.addStep(IfBuiltinShellCommand(
628 factory.addStep(ShellCommand(
630 description = "Building images",
631 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "target/install", "V=s"],
636 factory.addStep(ShellCommand(
638 description = "Generating config.seed",
639 command=["make", "-j1", "diffconfig", "V=s"],
644 factory.addStep(ShellCommand(
646 description = "Calculating checksums",
647 command=["make", "-j1", "checksum", "V=s"],
653 if gpg_keyid is not None:
654 factory.addStep(MasterShellCommand(
655 name = "signprepare",
656 description = "Preparing temporary signing directory",
657 command = ["mkdir", "-p", "%s/signing" %(home_dir)],
661 factory.addStep(ShellCommand(
663 description = "Packing files to sign",
664 command = WithProperties("find bin/targets/%s/%s%%(libc)s/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz" %(ts[0], ts[1])),
668 factory.addStep(FileUpload(
669 slavesrc = "sign.tar.gz",
670 masterdest = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
674 factory.addStep(MasterShellCommand(
676 description = "Signing files",
677 command = ["%s/signall.sh" %(home_dir), "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]), gpg_keyid, gpg_comment],
678 env = {'GNUPGHOME': gpg_home, 'PASSFILE': gpg_passfile},
682 factory.addStep(FileDownload(
683 mastersrc = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
684 slavedest = "sign.tar.gz",
688 factory.addStep(ShellCommand(
690 description = "Unpacking signed files",
691 command = ["tar", "-xzf", "sign.tar.gz"],
696 factory.addStep(ShellCommand(
698 description = "Preparing upload directory structure",
699 command = ["mkdir", "-p", WithProperties("tmp/upload/%%(prefix)stargets/%s/%s" %(ts[0], ts[1]), prefix=GetVersionPrefix)],
703 factory.addStep(ShellCommand(
704 name = "linkprepare",
705 description = "Preparing repository symlink",
706 command = ["ln", "-s", "-f", WithProperties("../packages-%(basever)s", basever=GetBaseVersion), WithProperties("tmp/upload/%(prefix)spackages", prefix=GetVersionPrefix)],
707 doStepIf = IsNoMasterBuild,
711 factory.addStep(ShellCommand(
713 description = "Uploading directory structure",
714 command = ["rsync", "-avz", "tmp/upload/", "%s/" %(rsync_bin_url)],
715 env={'RSYNC_PASSWORD': rsync_bin_key},
716 haltOnFailure = True,
720 factory.addStep(ShellCommand(
721 name = "targetupload",
722 description = "Uploading target files",
723 command=["rsync", "--progress", "--delete", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]),
724 "-avz", WithProperties("bin/targets/%s/%s%%(libc)s/" %(ts[0], ts[1])),
725 WithProperties("%s/%%(prefix)stargets/%s/%s/" %(rsync_bin_url, ts[0], ts[1]), prefix=GetVersionPrefix)],
726 env={'RSYNC_PASSWORD': rsync_bin_key},
727 haltOnFailure = True,
731 if rsync_src_url is not None:
732 factory.addStep(ShellCommand(
733 name = "sourceupload",
734 description = "Uploading source archives",
735 command=["rsync", "--progress", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "dl/", "%s/" %(rsync_src_url)],
736 env={'RSYNC_PASSWORD': rsync_src_key},
737 haltOnFailure = True,
742 factory.addStep(ShellCommand(
743 name = "packageupload",
744 description = "Uploading package files",
745 command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "bin/packages/", "%s/packages/" %(rsync_bin_url)],
746 env={'RSYNC_PASSWORD': rsync_bin_key},
747 haltOnFailure = False,
753 factory.addStep(ShellCommand(
755 description = "Uploading logs",
756 command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "logs/", "%s/logs/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
757 env={'RSYNC_PASSWORD': rsync_bin_key},
758 haltOnFailure = False,
763 from buildbot.config import BuilderConfig
765 c['builders'].append(BuilderConfig(name=target, slavenames=slaveNames, factory=factory, nextBuild=GetNextBuild))
768 ####### STATUS TARGETS
770 # 'status' is a list of Status Targets. The results of each build will be
771 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
772 # including web pages, email senders, and IRC bots.
776 from buildbot.status import html
777 from buildbot.status.web import authz, auth
779 if ini.has_option("status", "bind"):
780 if ini.has_option("status", "user") and ini.has_option("status", "password"):
781 authz_cfg=authz.Authz(
782 # change any of these to True to enable; see the manual for more
784 auth=auth.BasicAuth([(ini.get("status", "user"), ini.get("status", "password"))]),
785 gracefulShutdown = 'auth',
786 forceBuild = 'auth', # use this to test your slave once it is set up
787 forceAllBuilds = 'auth',
790 stopAllBuilds = 'auth',
791 cancelPendingBuild = 'auth',
793 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind"), authz=authz_cfg))
795 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind")))
798 from buildbot.status import words
800 if ini.has_option("irc", "host") and ini.has_option("irc", "nickname") and ini.has_option("irc", "channel"):
801 irc_host = ini.get("irc", "host")
803 irc_chan = ini.get("irc", "channel")
804 irc_nick = ini.get("irc", "nickname")
807 if ini.has_option("irc", "port"):
808 irc_port = ini.getint("irc", "port")
810 if ini.has_option("irc", "password"):
811 irc_pass = ini.get("irc", "password")
813 irc = words.IRC(irc_host, irc_nick, port = irc_port, password = irc_pass,
814 channels = [{ "channel": irc_chan }],
817 'successToFailure': 1,
818 'failureToSuccess': 1
822 c['status'].append(irc)
825 ####### PROJECT IDENTITY
827 # the 'title' string will appear at the top of this buildbot
828 # installation's html.WebStatus home page (linked to the
829 # 'titleURL') and is embedded in the title of the waterfall HTML page.
831 c['title'] = ini.get("general", "title")
832 c['titleURL'] = ini.get("general", "title_url")
834 # the 'buildbotURL' string should point to the location where the buildbot's
835 # internal web server (usually the html.WebStatus page) is visible. This
836 # typically uses the port number set in the Waterfall 'status' entry, but
837 # with an externally-visible host name which the buildbot cannot figure out
840 c['buildbotURL'] = ini.get("general", "buildbot_url")
845 # This specifies what database buildbot uses to store its state. You can leave
846 # this at its default for all but the largest installations.
847 'db_url' : "sqlite:///state.sqlite",