2 # ex: set syntax=python:
9 from buildbot import locks
11 # This is a sample buildmaster config file. It must be installed as
12 # 'master.cfg' in your buildmaster's base directory.
14 ini = ConfigParser.ConfigParser()
15 ini.read("./config.ini")
17 # This is the dictionary that the buildmaster pays attention to. We also use
18 # a shorter alias to save typing.
19 c = BuildmasterConfig = {}
23 # The 'slaves' list defines the set of recognized buildslaves. Each element is
24 # a BuildSlave object, specifying a unique slave name and password. The same
25 # slave name and password must be configured on the slave.
26 from buildbot.buildslave import BuildSlave
30 if ini.has_option("general", "port"):
31 slave_port = ini.getint("general", "port")
35 for section in ini.sections():
36 if section.startswith("slave "):
37 if ini.has_option(section, "name") and ini.has_option(section, "password"):
38 name = ini.get(section, "name")
39 password = ini.get(section, "password")
41 if ini.has_option(section, "builds"):
42 max_builds = ini.getint(section, "builds")
43 c['slaves'].append(BuildSlave(name, password, max_builds = max_builds))
45 # 'slavePortnum' defines the TCP port to listen on for connections from slaves.
46 # This must match the value configured into the buildslaves (with their
48 c['slavePortnum'] = slave_port
51 c['mergeRequests'] = True
55 home_dir = os.path.abspath(ini.get("general", "homedir"))
58 if ini.has_option("general", "expire"):
59 tree_expire = ini.getint("general", "expire")
61 repo_url = ini.get("repo", "url")
62 repo_branch = "master"
64 if ini.has_option("repo", "branch"):
65 repo_branch = ini.get("repo", "branch")
67 rsync_bin_url = ini.get("rsync", "binary_url")
68 rsync_bin_key = ini.get("rsync", "binary_password")
73 if ini.has_option("rsync", "source_url"):
74 rsync_src_url = ini.get("rsync", "source_url")
75 rsync_src_key = ini.get("rsync", "source_password")
79 gpg_comment = "Unattended build signature"
80 gpg_passfile = "/dev/null"
82 if ini.has_option("gpg", "home"):
83 gpg_home = ini.get("gpg", "home")
85 if ini.has_option("gpg", "keyid"):
86 gpg_keyid = ini.get("gpg", "keyid")
88 if ini.has_option("gpg", "comment"):
89 gpg_comment = ini.get("gpg", "comment")
91 if ini.has_option("gpg", "passfile"):
92 gpg_passfile = ini.get("gpg", "passfile")
98 if not os.path.isdir(home_dir+'/source.git'):
99 subprocess.call(["git", "clone", "--depth=1", "--branch="+repo_branch, repo_url, home_dir+'/source.git'])
101 subprocess.call(["git", "pull"], cwd = home_dir+'/source.git')
103 findtargets = subprocess.Popen([home_dir+'/dumpinfo.pl', 'targets'],
104 stdout = subprocess.PIPE, cwd = home_dir+'/source.git')
107 line = findtargets.stdout.readline()
110 ta = line.strip().split(' ')
111 targets.append(ta[0])
114 # the 'change_source' setting tells the buildmaster how it should find out
115 # about source code changes. Here we point to the buildbot clone of pyflakes.
117 from buildbot.changes.gitpoller import GitPoller
118 c['change_source'] = []
119 c['change_source'].append(GitPoller(
121 workdir=home_dir+'/work.git', branch='master',
126 # Configure the Schedulers, which decide how to react to incoming changes. In this
127 # case, just kick off a 'basebuild' build
129 from buildbot.schedulers.basic import SingleBranchScheduler
130 from buildbot.schedulers.forcesched import ForceScheduler
131 from buildbot.changes import filter
133 c['schedulers'].append(SingleBranchScheduler(
135 change_filter=filter.ChangeFilter(branch='master'),
137 builderNames=targets))
139 c['schedulers'].append(ForceScheduler(
141 builderNames=targets))
145 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
146 # what steps, and which slaves can execute them. Note that any particular build will
147 # only take place on one slave.
149 from buildbot.process.factory import BuildFactory
150 from buildbot.steps.source.git import Git
151 from buildbot.steps.shell import ShellCommand
152 from buildbot.steps.shell import SetProperty
153 from buildbot.steps.transfer import FileUpload
154 from buildbot.steps.transfer import FileDownload
155 from buildbot.steps.master import MasterShellCommand
156 from buildbot.process.properties import WithProperties
160 [ "tools", "tools/clean" ],
161 [ "chain", "toolchain/clean" ],
162 [ "linux", "target/linux/clean" ],
163 [ "dir", "dirclean" ],
164 [ "dist", "distclean" ]
167 def IsCleanRequested(pattern):
168 def CheckCleanProperty(step):
169 val = step.getProperty("clean")
170 if val and re.match(pattern, val):
175 return CheckCleanProperty
177 def IsTaggingRequested(step):
178 val = step.getProperty("tag")
179 if val and re.match("^[0-9]+\.[0-9]+\.[0-9]+$", val):
184 def IsNoTaggingRequested(step):
185 return not IsTaggingRequested(step)
190 dlLock = locks.SlaveLock("slave_dl")
192 checkBuiltin = re.sub('[\t\n ]+', ' ', """
194 local symbol op path file;
195 for file in $CHANGED_FILES; do
201 while read symbol op path; do
202 case "$symbol" in package-*)
203 symbol="${symbol##*(}";
204 symbol="${symbol%)}";
205 for file in $CHANGED_FILES; do
206 case "$file" in "package/$path/"*)
207 grep -qsx "$symbol=y" .config && return 0
211 done < tmp/.packagedeps;
217 class IfBuiltinShellCommand(ShellCommand):
218 def _quote(self, str):
219 if re.search("[^a-zA-Z0-9/_.-]", str):
220 return "'%s'" %(re.sub("'", "'\"'\"'", str))
223 def setCommand(self, command):
224 if not isinstance(command, (str, unicode)):
225 command = ' '.join(map(self._quote, command))
228 '%s; if checkBuiltin; then %s; else exit 0; fi' %(checkBuiltin, command)
231 def setupEnvironment(self, cmd):
232 slaveEnv = self.slaveEnvironment
236 for request in self.build.requests:
237 for source in request.sources:
238 for change in source.changes:
239 for file in change.files:
240 changedFiles[file] = True
241 fullSlaveEnv = slaveEnv.copy()
242 fullSlaveEnv['CHANGED_FILES'] = ' '.join(changedFiles.keys())
243 cmd.args['env'] = fullSlaveEnv
247 for slave in c['slaves']:
248 slaveNames.append(slave.slavename)
250 for target in targets:
251 ts = target.split('/')
253 factory = BuildFactory()
255 # find number of cores
256 factory.addStep(SetProperty(
259 description = "Finding number of CPUs",
260 command = ["nproc"]))
262 # expire tree if needed
264 factory.addStep(FileDownload(
265 mastersrc = "expire.sh",
266 slavedest = "../expire.sh",
269 factory.addStep(ShellCommand(
271 description = "Checking for build tree expiry",
272 command = ["./expire.sh", str(tree_expire)],
274 haltOnFailure = True,
277 # user-requested clean targets
278 for tuple in CleanTargetMap:
279 factory.addStep(ShellCommand(
281 description = 'User-requested "make %s"' % tuple[1],
282 command = ["make", tuple[1], "V=s"],
283 doStepIf = IsCleanRequested(tuple[0])
286 factory.addStep(MasterShellCommand(
288 description = "Tagging Git repository",
289 command = [home_dir+'/maketag.sh', '-i', '-k', str(gpg_keyid or ''),
290 '-p', str(gpg_passfile or ''), '-v', WithProperties("%(tag:-)s")],
291 path = home_dir+'/source.git',
292 env = {'GNUPGHOME': gpg_home},
293 haltOnFailure = True,
294 doStepIf = IsTaggingRequested
298 factory.addStep(ShellCommand(
299 name = "switchbranch",
300 description = "Checking out Git branch",
301 command = ["git", "checkout", repo_branch],
302 haltOnFailure = True,
303 doStepIf = IsNoTaggingRequested
306 # check out the source
309 branch = repo_branch,
310 mode = 'incremental',
314 factory.addStep(ShellCommand(
316 description = "Fetching Git tags",
317 command = ["git", "fetch", "--tags", "--", repo_url],
318 haltOnFailure = True,
319 doStepIf = IsTaggingRequested
323 factory.addStep(ShellCommand(
325 description = "Checking out Git tag",
326 command = ["git", "checkout", WithProperties("tags/v%(tag:-)s")],
327 haltOnFailure = True,
328 doStepIf = IsTaggingRequested
331 factory.addStep(ShellCommand(
333 description = "Remove tmp folder",
334 command=["rm", "-rf", "tmp/"]))
337 # factory.addStep(ShellCommand(
338 # name = "feedsconf",
339 # description = "Copy the feeds.conf",
340 # command='''cp ~/feeds.conf ./feeds.conf''' ))
343 factory.addStep(ShellCommand(
344 name = "rmfeedlinks",
345 description = "Remove feed symlinks",
346 command=["rm", "-rf", "package/feeds/"]))
349 factory.addStep(ShellCommand(
350 name = "updatefeeds",
351 description = "Updating feeds",
352 command=["./scripts/feeds", "update"]))
355 factory.addStep(ShellCommand(
356 name = "installfeeds",
357 description = "Installing feeds",
358 command=["./scripts/feeds", "install", "-a"]))
361 factory.addStep(FileDownload(
362 mastersrc = "config.seed",
363 slavedest = ".config",
368 factory.addStep(ShellCommand(
370 description = "Seeding .config",
371 command = "printf 'CONFIG_TARGET_%s=y\\nCONFIG_TARGET_%s_%s=y\\n' >> .config" %(ts[0], ts[0], ts[1])
374 factory.addStep(ShellCommand(
376 description = "Removing output directory",
377 command = ["rm", "-rf", "bin/"]
380 factory.addStep(ShellCommand(
382 description = "Populating .config",
383 command = ["make", "defconfig"]
387 factory.addStep(ShellCommand(
389 description = "Checking architecture",
390 command = ["grep", "-sq", "CONFIG_TARGET_%s=y" %(ts[0]), ".config"],
398 factory.addStep(SetProperty(
401 description = "Finding libc suffix",
402 command = ["sed", "-ne", '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }', ".config"]))
405 factory.addStep(FileDownload(mastersrc=home_dir+'/key-build', slavedest="key-build", mode=0600))
406 factory.addStep(FileDownload(mastersrc=home_dir+'/key-build.pub', slavedest="key-build.pub", mode=0600))
409 factory.addStep(ShellCommand(
411 description = "Preparing dl/",
412 command = "mkdir -p $HOME/dl && rm -rf ./dl && ln -sf $HOME/dl ./dl",
418 factory.addStep(ShellCommand(
420 description = "Building GNU tar",
421 command = ["make", WithProperties("-j%(nproc:~4)s"), "tools/tar/install", "V=s"],
426 factory.addStep(ShellCommand(
428 description = "Populating dl/",
429 command = ["make", WithProperties("-j%(nproc:~4)s"), "download", "V=s"],
431 locks = [dlLock.access('exclusive')]
434 factory.addStep(ShellCommand(
436 description = "Cleaning base-files",
437 command=["make", "package/base-files/clean", "V=s"]
441 factory.addStep(ShellCommand(
443 description = "Building tools",
444 command = ["make", WithProperties("-j%(nproc:~4)s"), "tools/install", "V=s"],
448 factory.addStep(ShellCommand(
450 description = "Building toolchain",
451 command=["make", WithProperties("-j%(nproc:~4)s"), "toolchain/install", "V=s"],
455 factory.addStep(ShellCommand(
457 description = "Building kmods",
458 command=["make", WithProperties("-j%(nproc:~4)s"), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
459 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
463 factory.addStep(ShellCommand(
465 description = "Building packages",
466 command=["make", WithProperties("-j%(nproc:~4)s"), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
467 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
471 # factory.addStep(IfBuiltinShellCommand(
472 factory.addStep(ShellCommand(
474 description = "Installing packages",
475 command=["make", WithProperties("-j%(nproc:~4)s"), "package/install", "V=s"],
479 factory.addStep(ShellCommand(
481 description = "Indexing packages",
482 command=["make", WithProperties("-j%(nproc:~4)s"), "package/index", "V=s"],
486 #factory.addStep(IfBuiltinShellCommand(
487 factory.addStep(ShellCommand(
489 description = "Building images",
490 command=["make", WithProperties("-j%(nproc:~4)s"), "target/install", "V=s"],
494 factory.addStep(ShellCommand(
496 description = "Calculating checksums",
497 command=["make", "-j1", "checksum", "V=s"],
502 if gpg_keyid is not None:
503 factory.addStep(MasterShellCommand(
504 name = "signprepare",
505 description = "Preparing temporary signing directory",
506 command = ["mkdir", "-p", "%s/signing" %(home_dir)],
510 factory.addStep(ShellCommand(
512 description = "Packing files to sign",
513 command = WithProperties("find bin/targets/%s/%s%%(libc)s/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz" %(ts[0], ts[1])),
517 factory.addStep(FileUpload(
518 slavesrc = "sign.tar.gz",
519 masterdest = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
523 factory.addStep(MasterShellCommand(
525 description = "Signing files",
526 command = ["%s/signall.sh" %(home_dir), "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]), gpg_keyid, gpg_comment],
527 env = {'GNUPGHOME': gpg_home, 'PASSFILE': gpg_passfile},
531 factory.addStep(FileDownload(
532 mastersrc = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
533 slavedest = "sign.tar.gz",
537 factory.addStep(ShellCommand(
539 description = "Unpacking signed files",
540 command = ["tar", "-xzf", "sign.tar.gz"],
545 factory.addStep(ShellCommand(
546 name = "uploadprepare",
547 description = "Preparing target directory",
548 command=["rsync", "-av", "--include", "/%s/" %(ts[0]), "--include", "/%s/%s/" %(ts[0], ts[1]), "--exclude", "/*", "--exclude", "/*/*", "--exclude", "/%s/%s/*" %(ts[0], ts[1]), "bin/targets/", "%s/targets/" %(rsync_bin_url)],
549 env={'RSYNC_PASSWORD': rsync_bin_key},
550 haltOnFailure = True,
554 factory.addStep(ShellCommand(
555 name = "targetupload",
556 description = "Uploading target files",
557 command=["rsync", "--delete", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", WithProperties("bin/targets/%s/%s%%(libc)s/" %(ts[0], ts[1])), "%s/targets/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
558 env={'RSYNC_PASSWORD': rsync_bin_key},
559 haltOnFailure = True,
563 if rsync_src_url is not None:
564 factory.addStep(ShellCommand(
565 name = "sourceupload",
566 description = "Uploading source archives",
567 command=["rsync", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "dl/", "%s/" %(rsync_src_url)],
568 env={'RSYNC_PASSWORD': rsync_src_key},
569 haltOnFailure = True,
574 factory.addStep(ShellCommand(
575 name = "packageupload",
576 description = "Uploading package files",
577 command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "bin/packages/", "%s/packages/" %(rsync_bin_url)],
578 env={'RSYNC_PASSWORD': rsync_bin_key},
579 haltOnFailure = False,
585 factory.addStep(ShellCommand(
587 description = "Uploading logs",
588 command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "logs/", "%s/logs/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
589 env={'RSYNC_PASSWORD': rsync_bin_key},
590 haltOnFailure = False,
595 from buildbot.config import BuilderConfig
597 c['builders'].append(BuilderConfig(name=target, slavenames=slaveNames, factory=factory))
600 ####### STATUS TARGETS
602 # 'status' is a list of Status Targets. The results of each build will be
603 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
604 # including web pages, email senders, and IRC bots.
608 from buildbot.status import html
609 from buildbot.status.web import authz, auth
611 if ini.has_option("status", "bind"):
612 if ini.has_option("status", "user") and ini.has_option("status", "password"):
613 authz_cfg=authz.Authz(
614 # change any of these to True to enable; see the manual for more
616 auth=auth.BasicAuth([(ini.get("status", "user"), ini.get("status", "password"))]),
617 gracefulShutdown = 'auth',
618 forceBuild = 'auth', # use this to test your slave once it is set up
619 forceAllBuilds = 'auth',
622 stopAllBuilds = 'auth',
623 cancelPendingBuild = 'auth',
625 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind"), authz=authz_cfg))
627 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind")))
630 from buildbot.status import words
632 if ini.has_option("irc", "host") and ini.has_option("irc", "nickname") and ini.has_option("irc", "channel"):
633 irc_host = ini.get("irc", "host")
635 irc_chan = ini.get("irc", "channel")
636 irc_nick = ini.get("irc", "nickname")
639 if ini.has_option("irc", "port"):
640 irc_port = ini.getint("irc", "port")
642 if ini.has_option("irc", "password"):
643 irc_pass = ini.get("irc", "password")
645 irc = words.IRC(irc_host, irc_nick, port = irc_port, password = irc_pass,
646 channels = [{ "channel": irc_chan }],
649 'successToFailure': 1,
650 'failureToSuccess': 1
654 c['status'].append(irc)
657 ####### PROJECT IDENTITY
659 # the 'title' string will appear at the top of this buildbot
660 # installation's html.WebStatus home page (linked to the
661 # 'titleURL') and is embedded in the title of the waterfall HTML page.
663 c['title'] = ini.get("general", "title")
664 c['titleURL'] = ini.get("general", "title_url")
666 # the 'buildbotURL' string should point to the location where the buildbot's
667 # internal web server (usually the html.WebStatus page) is visible. This
668 # typically uses the port number set in the Waterfall 'status' entry, but
669 # with an externally-visible host name which the buildbot cannot figure out
672 c['buildbotURL'] = ini.get("general", "buildbot_url")
677 # This specifies what database buildbot uses to store its state. You can leave
678 # this at its default for all but the largest installations.
679 'db_url' : "sqlite:///state.sqlite",