2 # ex: set syntax=python:
9 from buildbot import locks
11 # This is a sample buildmaster config file. It must be installed as
12 # 'master.cfg' in your buildmaster's base directory.
14 ini = ConfigParser.ConfigParser()
15 ini.read("./config.ini")
17 # This is the dictionary that the buildmaster pays attention to. We also use
18 # a shorter alias to save typing.
19 c = BuildmasterConfig = {}
23 # The 'slaves' list defines the set of recognized buildslaves. Each element is
24 # a BuildSlave object, specifying a unique slave name and password. The same
25 # slave name and password must be configured on the slave.
26 from buildbot.buildslave import BuildSlave
30 if ini.has_option("general", "port"):
31 slave_port = ini.getint("general", "port")
35 for section in ini.sections():
36 if section.startswith("slave "):
37 if ini.has_option(section, "name") and ini.has_option(section, "password"):
38 name = ini.get(section, "name")
39 password = ini.get(section, "password")
41 if ini.has_option(section, "builds"):
42 max_builds = ini.getint(section, "builds")
43 c['slaves'].append(BuildSlave(name, password, max_builds = max_builds))
45 # 'slavePortnum' defines the TCP port to listen on for connections from slaves.
46 # This must match the value configured into the buildslaves (with their
48 c['slavePortnum'] = slave_port
51 c['mergeRequests'] = True
55 home_dir = os.path.abspath(ini.get("general", "homedir"))
58 if ini.has_option("general", "expire"):
59 tree_expire = ini.getint("general", "expire")
61 repo_url = ini.get("repo", "url")
62 repo_branch = "master"
64 if ini.has_option("repo", "branch"):
65 repo_branch = ini.get("repo", "branch")
67 rsync_bin_url = ini.get("rsync", "binary_url")
68 rsync_bin_key = ini.get("rsync", "binary_password")
73 if ini.has_option("rsync", "source_url"):
74 rsync_src_url = ini.get("rsync", "source_url")
75 rsync_src_key = ini.get("rsync", "source_password")
79 gpg_comment = "Unattended build signature"
80 gpg_passfile = "/dev/null"
82 if ini.has_option("gpg", "home"):
83 gpg_home = ini.get("gpg", "home")
85 if ini.has_option("gpg", "keyid"):
86 gpg_keyid = ini.get("gpg", "keyid")
88 if ini.has_option("gpg", "comment"):
89 gpg_comment = ini.get("gpg", "comment")
91 if ini.has_option("gpg", "passfile"):
92 gpg_passfile = ini.get("gpg", "passfile")
98 if not os.path.isdir(home_dir+'/source.git'):
99 subprocess.call(["git", "clone", "--depth=1", repo_url, home_dir+'/source.git'])
100 subprocess.call(["git", "checkout", repo_branch], cwd = home_dir+'/source.git')
102 subprocess.call(["git", "checkout", repo_branch], cwd = home_dir+'/source.git')
103 subprocess.call(["git", "pull"], cwd = home_dir+'/source.git')
105 findtargets = subprocess.Popen([home_dir+'/dumpinfo.pl', 'targets'],
106 stdout = subprocess.PIPE, cwd = home_dir+'/source.git')
109 line = findtargets.stdout.readline()
112 ta = line.strip().split(' ')
113 targets.append(ta[0])
116 # the 'change_source' setting tells the buildmaster how it should find out
117 # about source code changes. Here we point to the buildbot clone of pyflakes.
119 from buildbot.changes.gitpoller import GitPoller
120 c['change_source'] = []
121 c['change_source'].append(GitPoller(
123 workdir=home_dir+'/work.git', branch='master',
128 # Configure the Schedulers, which decide how to react to incoming changes. In this
129 # case, just kick off a 'basebuild' build
131 from buildbot.schedulers.basic import SingleBranchScheduler
132 from buildbot.schedulers.forcesched import ForceScheduler
133 from buildbot.changes import filter
135 c['schedulers'].append(SingleBranchScheduler(
137 change_filter=filter.ChangeFilter(branch='master'),
139 builderNames=targets))
141 c['schedulers'].append(ForceScheduler(
143 builderNames=targets))
147 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
148 # what steps, and which slaves can execute them. Note that any particular build will
149 # only take place on one slave.
151 from buildbot.process.factory import BuildFactory
152 from buildbot.steps.source import Git
153 from buildbot.steps.shell import ShellCommand
154 from buildbot.steps.shell import SetProperty
155 from buildbot.steps.transfer import FileUpload
156 from buildbot.steps.transfer import FileDownload
157 from buildbot.steps.master import MasterShellCommand
158 from buildbot.process.properties import WithProperties
162 [ "tools", "tools/clean" ],
163 [ "chain", "toolchain/clean" ],
164 [ "linux", "target/linux/clean" ],
165 [ "dir", "dirclean" ],
166 [ "dist", "distclean" ]
169 def IsCleanRequested(pattern):
170 def CheckCleanProperty(step):
171 val = step.getProperty("clean")
172 if val and re.match(pattern, val):
177 return CheckCleanProperty
182 dlLock = locks.SlaveLock("slave_dl")
184 checkBuiltin = re.sub('[\t\n ]+', ' ', """
186 local symbol op path file;
187 for file in $CHANGED_FILES; do
193 while read symbol op path; do
194 case "$symbol" in package-*)
195 symbol="${symbol##*(}";
196 symbol="${symbol%)}";
197 for file in $CHANGED_FILES; do
198 case "$file" in "package/$path/"*)
199 grep -qsx "$symbol=y" .config && return 0
203 done < tmp/.packagedeps;
209 class IfBuiltinShellCommand(ShellCommand):
210 def _quote(self, str):
211 if re.search("[^a-zA-Z0-9/_.-]", str):
212 return "'%s'" %(re.sub("'", "'\"'\"'", str))
215 def setCommand(self, command):
216 if not isinstance(command, (str, unicode)):
217 command = ' '.join(map(self._quote, command))
220 '%s; if checkBuiltin; then %s; else exit 0; fi' %(checkBuiltin, command)
223 def setupEnvironment(self, cmd):
224 slaveEnv = self.slaveEnvironment
228 for request in self.build.requests:
229 for source in request.sources:
230 for change in source.changes:
231 for file in change.files:
232 changedFiles[file] = True
233 fullSlaveEnv = slaveEnv.copy()
234 fullSlaveEnv['CHANGED_FILES'] = ' '.join(changedFiles.keys())
235 cmd.args['env'] = fullSlaveEnv
239 for slave in c['slaves']:
240 slaveNames.append(slave.slavename)
242 for target in targets:
243 ts = target.split('/')
245 factory = BuildFactory()
247 # find number of cores
248 factory.addStep(SetProperty(
251 description = "Finding number of CPUs",
252 command = ["nproc"]))
254 # expire tree if needed
256 factory.addStep(FileDownload(
257 mastersrc = "expire.sh",
258 slavedest = "../expire.sh",
261 factory.addStep(ShellCommand(
263 description = "Checking for build tree expiry",
264 command = ["./expire.sh", str(tree_expire)],
266 haltOnFailure = True,
269 # user-requested clean targets
270 for tuple in CleanTargetMap:
271 factory.addStep(ShellCommand(
273 description = 'User-requested "make %s"' % tuple[1],
274 command = ["make", tuple[1], "V=s"],
275 doStepIf = IsCleanRequested(tuple[0])
278 # check out the source
279 factory.addStep(Git(repourl=repo_url, branch=repo_branch, mode='update'))
281 factory.addStep(ShellCommand(
283 description = "Remove tmp folder",
284 command=["rm", "-rf", "tmp/"]))
287 # factory.addStep(ShellCommand(
288 # name = "feedsconf",
289 # description = "Copy the feeds.conf",
290 # command='''cp ~/feeds.conf ./feeds.conf''' ))
293 factory.addStep(ShellCommand(
294 name = "rmfeedlinks",
295 description = "Remove feed symlinks",
296 command=["rm", "-rf", "package/feeds/"]))
299 factory.addStep(ShellCommand(
300 name = "updatefeeds",
301 description = "Updating feeds",
302 command=["./scripts/feeds", "update"]))
305 factory.addStep(ShellCommand(
306 name = "installfeeds",
307 description = "Installing feeds",
308 command=["./scripts/feeds", "install", "-a"]))
311 factory.addStep(FileDownload(
312 mastersrc = "config.seed",
313 slavedest = ".config",
318 factory.addStep(ShellCommand(
320 description = "Seeding .config",
321 command = "printf 'CONFIG_TARGET_%s=y\\nCONFIG_TARGET_%s_%s=y\\n' >> .config" %(ts[0], ts[0], ts[1])
324 factory.addStep(ShellCommand(
326 description = "Removing output directory",
327 command = ["rm", "-rf", "bin/"]
330 factory.addStep(ShellCommand(
332 description = "Populating .config",
333 command = ["make", "defconfig"]
337 factory.addStep(ShellCommand(
339 description = "Checking architecture",
340 command = ["grep", "-sq", "CONFIG_TARGET_%s=y" %(ts[0]), ".config"],
348 factory.addStep(SetProperty(
351 description = "Finding libc suffix",
352 command = ["sed", "-ne", '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }', ".config"]))
355 factory.addStep(FileDownload(mastersrc=home_dir+'/key-build', slavedest="key-build", mode=0600))
356 factory.addStep(FileDownload(mastersrc=home_dir+'/key-build.pub', slavedest="key-build.pub", mode=0600))
359 factory.addStep(ShellCommand(
361 description = "Preparing dl/",
362 command = "mkdir -p $HOME/dl && rm -rf ./dl && ln -sf $HOME/dl ./dl",
368 factory.addStep(ShellCommand(
370 description = "Building GNU tar",
371 command = ["make", WithProperties("-j%(nproc:~4)s"), "tools/tar/install", "V=s"],
376 factory.addStep(ShellCommand(
378 description = "Populating dl/",
379 command = ["make", WithProperties("-j%(nproc:~4)s"), "download", "V=s"],
381 locks = [dlLock.access('exclusive')]
384 factory.addStep(ShellCommand(
386 description = "Cleaning base-files",
387 command=["make", "package/base-files/clean", "V=s"]
391 factory.addStep(ShellCommand(
393 description = "Building tools",
394 command = ["make", WithProperties("-j%(nproc:~4)s"), "tools/install", "V=s"],
398 factory.addStep(ShellCommand(
400 description = "Building toolchain",
401 command=["make", WithProperties("-j%(nproc:~4)s"), "toolchain/install", "V=s"],
405 factory.addStep(ShellCommand(
407 description = "Building kmods",
408 command=["make", WithProperties("-j%(nproc:~4)s"), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
409 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
413 factory.addStep(ShellCommand(
415 description = "Building packages",
416 command=["make", WithProperties("-j%(nproc:~4)s"), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
417 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
421 # factory.addStep(IfBuiltinShellCommand(
422 factory.addStep(ShellCommand(
424 description = "Installing packages",
425 command=["make", WithProperties("-j%(nproc:~4)s"), "package/install", "V=s"],
429 factory.addStep(ShellCommand(
431 description = "Indexing packages",
432 command=["make", WithProperties("-j%(nproc:~4)s"), "package/index", "V=s"],
436 #factory.addStep(IfBuiltinShellCommand(
437 factory.addStep(ShellCommand(
439 description = "Building images",
440 command=["make", WithProperties("-j%(nproc:~4)s"), "target/install", "V=s"],
444 factory.addStep(ShellCommand(
446 description = "Calculating checksums",
447 command=["make", "-j1", "checksum", "V=s"],
452 if gpg_keyid is not None:
453 factory.addStep(MasterShellCommand(
454 name = "signprepare",
455 description = "Preparing temporary signing directory",
456 command = ["mkdir", "-p", "%s/signing" %(home_dir)],
460 factory.addStep(ShellCommand(
462 description = "Packing files to sign",
463 command = WithProperties("find bin/targets/%s/%s%%(libc)s/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz" %(ts[0], ts[1])),
467 factory.addStep(FileUpload(
468 slavesrc = "sign.tar.gz",
469 masterdest = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
473 factory.addStep(MasterShellCommand(
475 description = "Signing files",
476 command = ["%s/signall.sh" %(home_dir), "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]), gpg_keyid, gpg_comment],
477 env = {'GNUPGHOME': gpg_home, 'PASSFILE': gpg_passfile},
481 factory.addStep(FileDownload(
482 mastersrc = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
483 slavedest = "sign.tar.gz",
487 factory.addStep(ShellCommand(
489 description = "Unpacking signed files",
490 command = ["tar", "-xzf", "sign.tar.gz"],
495 factory.addStep(ShellCommand(
496 name = "uploadprepare",
497 description = "Preparing target directory",
498 command=["rsync", "-av", "--include", "/%s/" %(ts[0]), "--include", "/%s/%s/" %(ts[0], ts[1]), "--exclude", "/*", "--exclude", "/*/*", "--exclude", "/%s/%s/*" %(ts[0], ts[1]), "bin/targets/", "%s/targets/" %(rsync_bin_url)],
499 env={'RSYNC_PASSWORD': rsync_bin_key},
500 haltOnFailure = True,
504 factory.addStep(ShellCommand(
505 name = "targetupload",
506 description = "Uploading target files",
507 command=["rsync", "--delete", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", WithProperties("bin/targets/%s/%s%%(libc)s/" %(ts[0], ts[1])), "%s/targets/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
508 env={'RSYNC_PASSWORD': rsync_bin_key},
509 haltOnFailure = True,
513 if rsync_src_url is not None:
514 factory.addStep(ShellCommand(
515 name = "sourceupload",
516 description = "Uploading source archives",
517 command=["rsync", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "dl/", "%s/" %(rsync_src_url)],
518 env={'RSYNC_PASSWORD': rsync_src_key},
519 haltOnFailure = True,
524 factory.addStep(ShellCommand(
525 name = "packageupload",
526 description = "Uploading package files",
527 command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "bin/packages/", "%s/packages/" %(rsync_bin_url)],
528 env={'RSYNC_PASSWORD': rsync_bin_key},
529 haltOnFailure = False,
535 factory.addStep(ShellCommand(
537 description = "Uploading logs",
538 command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "logs/", "%s/logs/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
539 env={'RSYNC_PASSWORD': rsync_bin_key},
540 haltOnFailure = False,
545 from buildbot.config import BuilderConfig
547 c['builders'].append(BuilderConfig(name=target, slavenames=slaveNames, factory=factory))
550 ####### STATUS TARGETS
552 # 'status' is a list of Status Targets. The results of each build will be
553 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
554 # including web pages, email senders, and IRC bots.
558 from buildbot.status import html
559 from buildbot.status.web import authz, auth
561 if ini.has_option("status", "bind"):
562 if ini.has_option("status", "user") and ini.has_option("status", "password"):
563 authz_cfg=authz.Authz(
564 # change any of these to True to enable; see the manual for more
566 auth=auth.BasicAuth([(ini.get("status", "user"), ini.get("status", "password"))]),
567 gracefulShutdown = 'auth',
568 forceBuild = 'auth', # use this to test your slave once it is set up
569 forceAllBuilds = 'auth',
572 stopAllBuilds = 'auth',
573 cancelPendingBuild = 'auth',
575 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind"), authz=authz_cfg))
577 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind")))
580 from buildbot.status import words
582 if ini.has_option("irc", "host") and ini.has_option("irc", "nickname") and ini.has_option("irc", "channel"):
583 irc_host = ini.get("irc", "host")
585 irc_chan = ini.get("irc", "channel")
586 irc_nick = ini.get("irc", "nickname")
589 if ini.has_option("irc", "port"):
590 irc_port = ini.getint("irc", "port")
592 if ini.has_option("irc", "password"):
593 irc_pass = ini.get("irc", "password")
595 irc = words.IRC(irc_host, irc_nick, port = irc_port, password = irc_pass,
596 channels = [{ "channel": irc_chan }],
599 'successToFailure': 1,
600 'failureToSuccess': 1
604 c['status'].append(irc)
607 ####### PROJECT IDENTITY
609 # the 'title' string will appear at the top of this buildbot
610 # installation's html.WebStatus home page (linked to the
611 # 'titleURL') and is embedded in the title of the waterfall HTML page.
613 c['title'] = ini.get("general", "title")
614 c['titleURL'] = ini.get("general", "title_url")
616 # the 'buildbotURL' string should point to the location where the buildbot's
617 # internal web server (usually the html.WebStatus page) is visible. This
618 # typically uses the port number set in the Waterfall 'status' entry, but
619 # with an externally-visible host name which the buildbot cannot figure out
622 c['buildbotURL'] = ini.get("general", "buildbot_url")
627 # This specifies what database buildbot uses to store its state. You can leave
628 # this at its default for all but the largest installations.
629 'db_url' : "sqlite:///state.sqlite",