2 # ex: set syntax=python:
9 from buildbot import locks
11 # This is a sample buildmaster config file. It must be installed as
12 # 'master.cfg' in your buildmaster's base directory.
14 ini = ConfigParser.ConfigParser()
15 ini.read("./config.ini")
17 # This is the dictionary that the buildmaster pays attention to. We also use
18 # a shorter alias to save typing.
19 c = BuildmasterConfig = {}
23 # The 'slaves' list defines the set of recognized buildslaves. Each element is
24 # a BuildSlave object, specifying a unique slave name and password. The same
25 # slave name and password must be configured on the slave.
26 from buildbot.buildslave import BuildSlave
30 if ini.has_option("general", "port"):
31 slave_port = ini.getint("general", "port")
35 for section in ini.sections():
36 if section.startswith("slave "):
37 if ini.has_option(section, "name") and ini.has_option(section, "password"):
38 name = ini.get(section, "name")
39 password = ini.get(section, "password")
41 if ini.has_option(section, "builds"):
42 max_builds = ini.getint(section, "builds")
43 c['slaves'].append(BuildSlave(name, password, max_builds = max_builds))
45 # 'slavePortnum' defines the TCP port to listen on for connections from slaves.
46 # This must match the value configured into the buildslaves (with their
48 c['slavePortnum'] = slave_port
51 c['mergeRequests'] = True
55 home_dir = os.path.abspath(ini.get("general", "homedir"))
58 if ini.has_option("general", "expire"):
59 tree_expire = ini.getint("general", "expire")
61 repo_url = ini.get("repo", "url")
62 repo_branch = "master"
64 if ini.has_option("repo", "branch"):
65 repo_branch = ini.get("repo", "branch")
67 rsync_bin_url = ini.get("rsync", "binary_url")
68 rsync_bin_key = ini.get("rsync", "binary_password")
73 if ini.has_option("rsync", "source_url"):
74 rsync_src_url = ini.get("rsync", "source_url")
75 rsync_src_key = ini.get("rsync", "source_password")
79 gpg_comment = "Unattended build signature"
80 gpg_passfile = "/dev/null"
82 if ini.has_option("gpg", "home"):
83 gpg_home = ini.get("gpg", "home")
85 if ini.has_option("gpg", "keyid"):
86 gpg_keyid = ini.get("gpg", "keyid")
88 if ini.has_option("gpg", "comment"):
89 gpg_comment = ini.get("gpg", "comment")
91 if ini.has_option("gpg", "passfile"):
92 gpg_passfile = ini.get("gpg", "passfile")
98 if not os.path.isdir(home_dir+'/source.git'):
99 subprocess.call(["git", "clone", "--depth=1", "--branch="+repo_branch, repo_url, home_dir+'/source.git'])
101 subprocess.call(["git", "pull"], cwd = home_dir+'/source.git')
103 findtargets = subprocess.Popen([home_dir+'/dumpinfo.pl', 'targets'],
104 stdout = subprocess.PIPE, cwd = home_dir+'/source.git')
107 line = findtargets.stdout.readline()
110 ta = line.strip().split(' ')
111 targets.append(ta[0])
114 # the 'change_source' setting tells the buildmaster how it should find out
115 # about source code changes. Here we point to the buildbot clone of pyflakes.
117 from buildbot.changes.gitpoller import GitPoller
118 c['change_source'] = []
119 c['change_source'].append(GitPoller(
121 workdir=home_dir+'/work.git', branch='master',
126 # Configure the Schedulers, which decide how to react to incoming changes. In this
127 # case, just kick off a 'basebuild' build
129 from buildbot.schedulers.basic import SingleBranchScheduler
130 from buildbot.schedulers.forcesched import ForceScheduler
131 from buildbot.changes import filter
133 c['schedulers'].append(SingleBranchScheduler(
135 change_filter=filter.ChangeFilter(branch='master'),
137 builderNames=targets))
139 c['schedulers'].append(ForceScheduler(
141 builderNames=targets))
145 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
146 # what steps, and which slaves can execute them. Note that any particular build will
147 # only take place on one slave.
149 from buildbot.process.factory import BuildFactory
150 from buildbot.steps.source import Git
151 from buildbot.steps.shell import ShellCommand
152 from buildbot.steps.shell import SetProperty
153 from buildbot.steps.transfer import FileUpload
154 from buildbot.steps.transfer import FileDownload
155 from buildbot.steps.master import MasterShellCommand
156 from buildbot.process.properties import WithProperties
160 [ "tools", "tools/clean" ],
161 [ "chain", "toolchain/clean" ],
162 [ "linux", "target/linux/clean" ],
163 [ "dir", "dirclean" ],
164 [ "dist", "distclean" ]
167 def IsCleanRequested(pattern):
168 def CheckCleanProperty(step):
169 val = step.getProperty("clean")
170 if val and re.match(pattern, val):
175 return CheckCleanProperty
180 dlLock = locks.SlaveLock("slave_dl")
182 checkBuiltin = re.sub('[\t\n ]+', ' ', """
184 local symbol op path file;
185 for file in $CHANGED_FILES; do
191 while read symbol op path; do
192 case "$symbol" in package-*)
193 symbol="${symbol##*(}";
194 symbol="${symbol%)}";
195 for file in $CHANGED_FILES; do
196 case "$file" in "package/$path/"*)
197 grep -qsx "$symbol=y" .config && return 0
201 done < tmp/.packagedeps;
207 class IfBuiltinShellCommand(ShellCommand):
208 def _quote(self, str):
209 if re.search("[^a-zA-Z0-9/_.-]", str):
210 return "'%s'" %(re.sub("'", "'\"'\"'", str))
213 def setCommand(self, command):
214 if not isinstance(command, (str, unicode)):
215 command = ' '.join(map(self._quote, command))
218 '%s; if checkBuiltin; then %s; else exit 0; fi' %(checkBuiltin, command)
221 def setupEnvironment(self, cmd):
222 slaveEnv = self.slaveEnvironment
226 for request in self.build.requests:
227 for source in request.sources:
228 for change in source.changes:
229 for file in change.files:
230 changedFiles[file] = True
231 fullSlaveEnv = slaveEnv.copy()
232 fullSlaveEnv['CHANGED_FILES'] = ' '.join(changedFiles.keys())
233 cmd.args['env'] = fullSlaveEnv
237 for slave in c['slaves']:
238 slaveNames.append(slave.slavename)
240 for target in targets:
241 ts = target.split('/')
243 factory = BuildFactory()
245 # find number of cores
246 factory.addStep(SetProperty(
249 description = "Finding number of CPUs",
250 command = ["nproc"]))
252 # expire tree if needed
254 factory.addStep(FileDownload(
255 mastersrc = "expire.sh",
256 slavedest = "../expire.sh",
259 factory.addStep(ShellCommand(
261 description = "Checking for build tree expiry",
262 command = ["./expire.sh", str(tree_expire)],
264 haltOnFailure = True,
267 # user-requested clean targets
268 for tuple in CleanTargetMap:
269 factory.addStep(ShellCommand(
271 description = 'User-requested "make %s"' % tuple[1],
272 command = ["make", tuple[1], "V=s"],
273 doStepIf = IsCleanRequested(tuple[0])
276 # check out the source
277 factory.addStep(Git(repourl=repo_url, branch=repo_branch, mode='update'))
279 factory.addStep(ShellCommand(
281 description = "Remove tmp folder",
282 command=["rm", "-rf", "tmp/"]))
285 # factory.addStep(ShellCommand(
286 # name = "feedsconf",
287 # description = "Copy the feeds.conf",
288 # command='''cp ~/feeds.conf ./feeds.conf''' ))
291 factory.addStep(ShellCommand(
292 name = "rmfeedlinks",
293 description = "Remove feed symlinks",
294 command=["rm", "-rf", "package/feeds/"]))
297 factory.addStep(ShellCommand(
298 name = "updatefeeds",
299 description = "Updating feeds",
300 command=["./scripts/feeds", "update"]))
303 factory.addStep(ShellCommand(
304 name = "installfeeds",
305 description = "Installing feeds",
306 command=["./scripts/feeds", "install", "-a"]))
309 factory.addStep(FileDownload(
310 mastersrc = "config.seed",
311 slavedest = ".config",
316 factory.addStep(ShellCommand(
318 description = "Seeding .config",
319 command = "printf 'CONFIG_TARGET_%s=y\\nCONFIG_TARGET_%s_%s=y\\n' >> .config" %(ts[0], ts[0], ts[1])
322 factory.addStep(ShellCommand(
324 description = "Removing output directory",
325 command = ["rm", "-rf", "bin/"]
328 factory.addStep(ShellCommand(
330 description = "Populating .config",
331 command = ["make", "defconfig"]
335 factory.addStep(ShellCommand(
337 description = "Checking architecture",
338 command = ["grep", "-sq", "CONFIG_TARGET_%s=y" %(ts[0]), ".config"],
346 factory.addStep(SetProperty(
349 description = "Finding libc suffix",
350 command = ["sed", "-ne", '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }', ".config"]))
353 factory.addStep(FileDownload(mastersrc=home_dir+'/key-build', slavedest="key-build", mode=0600))
354 factory.addStep(FileDownload(mastersrc=home_dir+'/key-build.pub', slavedest="key-build.pub", mode=0600))
357 factory.addStep(ShellCommand(
359 description = "Preparing dl/",
360 command = "mkdir -p $HOME/dl && rm -rf ./dl && ln -sf $HOME/dl ./dl",
366 factory.addStep(ShellCommand(
368 description = "Building GNU tar",
369 command = ["make", WithProperties("-j%(nproc:~4)s"), "tools/tar/install", "V=s"],
374 factory.addStep(ShellCommand(
376 description = "Populating dl/",
377 command = ["make", WithProperties("-j%(nproc:~4)s"), "download", "V=s"],
379 locks = [dlLock.access('exclusive')]
382 factory.addStep(ShellCommand(
384 description = "Cleaning base-files",
385 command=["make", "package/base-files/clean", "V=s"]
389 factory.addStep(ShellCommand(
391 description = "Building tools",
392 command = ["make", WithProperties("-j%(nproc:~4)s"), "tools/install", "V=s"],
396 factory.addStep(ShellCommand(
398 description = "Building toolchain",
399 command=["make", WithProperties("-j%(nproc:~4)s"), "toolchain/install", "V=s"],
403 factory.addStep(ShellCommand(
405 description = "Building kmods",
406 command=["make", WithProperties("-j%(nproc:~4)s"), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
407 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
411 factory.addStep(ShellCommand(
413 description = "Building packages",
414 command=["make", WithProperties("-j%(nproc:~4)s"), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
415 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
419 # factory.addStep(IfBuiltinShellCommand(
420 factory.addStep(ShellCommand(
422 description = "Installing packages",
423 command=["make", WithProperties("-j%(nproc:~4)s"), "package/install", "V=s"],
427 factory.addStep(ShellCommand(
429 description = "Indexing packages",
430 command=["make", WithProperties("-j%(nproc:~4)s"), "package/index", "V=s"],
434 #factory.addStep(IfBuiltinShellCommand(
435 factory.addStep(ShellCommand(
437 description = "Building images",
438 command=["make", WithProperties("-j%(nproc:~4)s"), "target/install", "V=s"],
442 factory.addStep(ShellCommand(
444 description = "Calculating checksums",
445 command=["make", "-j1", "checksum", "V=s"],
450 if gpg_keyid is not None:
451 factory.addStep(MasterShellCommand(
452 name = "signprepare",
453 description = "Preparing temporary signing directory",
454 command = ["mkdir", "-p", "%s/signing" %(home_dir)],
458 factory.addStep(ShellCommand(
460 description = "Packing files to sign",
461 command = WithProperties("find bin/targets/%s/%s%%(libc)s/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz" %(ts[0], ts[1])),
465 factory.addStep(FileUpload(
466 slavesrc = "sign.tar.gz",
467 masterdest = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
471 factory.addStep(MasterShellCommand(
473 description = "Signing files",
474 command = ["%s/signall.sh" %(home_dir), "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]), gpg_keyid, gpg_comment],
475 env = {'GNUPGHOME': gpg_home, 'PASSFILE': gpg_passfile},
479 factory.addStep(FileDownload(
480 mastersrc = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
481 slavedest = "sign.tar.gz",
485 factory.addStep(ShellCommand(
487 description = "Unpacking signed files",
488 command = ["tar", "-xzf", "sign.tar.gz"],
493 factory.addStep(ShellCommand(
494 name = "uploadprepare",
495 description = "Preparing target directory",
496 command=["rsync", "-av", "--include", "/%s/" %(ts[0]), "--include", "/%s/%s/" %(ts[0], ts[1]), "--exclude", "/*", "--exclude", "/*/*", "--exclude", "/%s/%s/*" %(ts[0], ts[1]), "bin/targets/", "%s/targets/" %(rsync_bin_url)],
497 env={'RSYNC_PASSWORD': rsync_bin_key},
498 haltOnFailure = True,
502 factory.addStep(ShellCommand(
503 name = "targetupload",
504 description = "Uploading target files",
505 command=["rsync", "--delete", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", WithProperties("bin/targets/%s/%s%%(libc)s/" %(ts[0], ts[1])), "%s/targets/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
506 env={'RSYNC_PASSWORD': rsync_bin_key},
507 haltOnFailure = True,
511 if rsync_src_url is not None:
512 factory.addStep(ShellCommand(
513 name = "sourceupload",
514 description = "Uploading source archives",
515 command=["rsync", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "dl/", "%s/" %(rsync_src_url)],
516 env={'RSYNC_PASSWORD': rsync_src_key},
517 haltOnFailure = True,
522 factory.addStep(ShellCommand(
523 name = "packageupload",
524 description = "Uploading package files",
525 command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "bin/packages/", "%s/packages/" %(rsync_bin_url)],
526 env={'RSYNC_PASSWORD': rsync_bin_key},
527 haltOnFailure = False,
533 factory.addStep(ShellCommand(
535 description = "Uploading logs",
536 command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "logs/", "%s/logs/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
537 env={'RSYNC_PASSWORD': rsync_bin_key},
538 haltOnFailure = False,
543 from buildbot.config import BuilderConfig
545 c['builders'].append(BuilderConfig(name=target, slavenames=slaveNames, factory=factory))
548 ####### STATUS TARGETS
550 # 'status' is a list of Status Targets. The results of each build will be
551 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
552 # including web pages, email senders, and IRC bots.
556 from buildbot.status import html
557 from buildbot.status.web import authz, auth
559 if ini.has_option("status", "bind"):
560 if ini.has_option("status", "user") and ini.has_option("status", "password"):
561 authz_cfg=authz.Authz(
562 # change any of these to True to enable; see the manual for more
564 auth=auth.BasicAuth([(ini.get("status", "user"), ini.get("status", "password"))]),
565 gracefulShutdown = 'auth',
566 forceBuild = 'auth', # use this to test your slave once it is set up
567 forceAllBuilds = 'auth',
570 stopAllBuilds = 'auth',
571 cancelPendingBuild = 'auth',
573 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind"), authz=authz_cfg))
575 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind")))
578 from buildbot.status import words
580 if ini.has_option("irc", "host") and ini.has_option("irc", "nickname") and ini.has_option("irc", "channel"):
581 irc_host = ini.get("irc", "host")
583 irc_chan = ini.get("irc", "channel")
584 irc_nick = ini.get("irc", "nickname")
587 if ini.has_option("irc", "port"):
588 irc_port = ini.getint("irc", "port")
590 if ini.has_option("irc", "password"):
591 irc_pass = ini.get("irc", "password")
593 irc = words.IRC(irc_host, irc_nick, port = irc_port, password = irc_pass,
594 channels = [{ "channel": irc_chan }],
597 'successToFailure': 1,
598 'failureToSuccess': 1
602 c['status'].append(irc)
605 ####### PROJECT IDENTITY
607 # the 'title' string will appear at the top of this buildbot
608 # installation's html.WebStatus home page (linked to the
609 # 'titleURL') and is embedded in the title of the waterfall HTML page.
611 c['title'] = ini.get("general", "title")
612 c['titleURL'] = ini.get("general", "title_url")
614 # the 'buildbotURL' string should point to the location where the buildbot's
615 # internal web server (usually the html.WebStatus page) is visible. This
616 # typically uses the port number set in the Waterfall 'status' entry, but
617 # with an externally-visible host name which the buildbot cannot figure out
620 c['buildbotURL'] = ini.get("general", "buildbot_url")
625 # This specifies what database buildbot uses to store its state. You can leave
626 # this at its default for all but the largest installations.
627 'db_url' : "sqlite:///state.sqlite",