2 # ex: set syntax=python:
9 from buildbot import locks
11 # This is a sample buildmaster config file. It must be installed as
12 # 'master.cfg' in your buildmaster's base directory.
14 ini = ConfigParser.ConfigParser()
15 ini.read("./config.ini")
17 # This is the dictionary that the buildmaster pays attention to. We also use
18 # a shorter alias to save typing.
19 c = BuildmasterConfig = {}
23 # The 'slaves' list defines the set of recognized buildslaves. Each element is
24 # a BuildSlave object, specifying a unique slave name and password. The same
25 # slave name and password must be configured on the slave.
26 from buildbot.buildslave import BuildSlave
30 if ini.has_option("general", "port"):
31 slave_port = ini.getint("general", "port")
35 for section in ini.sections():
36 if section.startswith("slave "):
37 if ini.has_option(section, "name") and ini.has_option(section, "password"):
38 name = ini.get(section, "name")
39 password = ini.get(section, "password")
41 if ini.has_option(section, "builds"):
42 max_builds = ini.getint(section, "builds")
43 c['slaves'].append(BuildSlave(name, password, max_builds = max_builds))
45 # 'slavePortnum' defines the TCP port to listen on for connections from slaves.
46 # This must match the value configured into the buildslaves (with their
48 c['slavePortnum'] = slave_port
51 c['mergeRequests'] = True
55 home_dir = os.path.abspath(ini.get("general", "homedir"))
58 if ini.has_option("general", "expire"):
59 tree_expire = ini.getint("general", "expire")
61 repo_url = ini.get("repo", "url")
62 repo_branch = "master"
64 if ini.has_option("repo", "branch"):
65 repo_branch = ini.get("repo", "branch")
67 rsync_bin_url = ini.get("rsync", "binary_url")
68 rsync_bin_key = ini.get("rsync", "binary_password")
73 if ini.has_option("rsync", "source_url"):
74 rsync_src_url = ini.get("rsync", "source_url")
75 rsync_src_key = ini.get("rsync", "source_password")
79 gpg_comment = "Unattended build signature"
80 gpg_passfile = "/dev/null"
82 if ini.has_option("gpg", "home"):
83 gpg_home = ini.get("gpg", "home")
85 if ini.has_option("gpg", "keyid"):
86 gpg_keyid = ini.get("gpg", "keyid")
88 if ini.has_option("gpg", "comment"):
89 gpg_comment = ini.get("gpg", "comment")
91 if ini.has_option("gpg", "passfile"):
92 gpg_passfile = ini.get("gpg", "passfile")
98 if not os.path.isdir(home_dir+'/source.git'):
99 subprocess.call(["git", "clone", "--depth=1", "--branch="+repo_branch, repo_url, home_dir+'/source.git'])
101 subprocess.call(["git", "pull"], cwd = home_dir+'/source.git')
103 findtargets = subprocess.Popen([home_dir+'/dumpinfo.pl', 'targets'],
104 stdout = subprocess.PIPE, cwd = home_dir+'/source.git')
107 line = findtargets.stdout.readline()
110 ta = line.strip().split(' ')
111 targets.append(ta[0])
114 # the 'change_source' setting tells the buildmaster how it should find out
115 # about source code changes. Here we point to the buildbot clone of pyflakes.
117 from buildbot.changes.gitpoller import GitPoller
118 c['change_source'] = []
119 c['change_source'].append(GitPoller(
121 workdir=home_dir+'/work.git', branch='master',
126 # Configure the Schedulers, which decide how to react to incoming changes. In this
127 # case, just kick off a 'basebuild' build
129 from buildbot.schedulers.basic import SingleBranchScheduler
130 from buildbot.schedulers.forcesched import ForceScheduler
131 from buildbot.changes import filter
133 c['schedulers'].append(SingleBranchScheduler(
135 change_filter=filter.ChangeFilter(branch='master'),
137 builderNames=targets))
139 c['schedulers'].append(ForceScheduler(
141 builderNames=targets))
145 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
146 # what steps, and which slaves can execute them. Note that any particular build will
147 # only take place on one slave.
149 from buildbot.process.factory import BuildFactory
150 from buildbot.steps.source.git import Git
151 from buildbot.steps.shell import ShellCommand
152 from buildbot.steps.shell import SetProperty
153 from buildbot.steps.transfer import FileUpload
154 from buildbot.steps.transfer import FileDownload
155 from buildbot.steps.master import MasterShellCommand
156 from buildbot.process.properties import WithProperties
160 [ "tools", "tools/clean" ],
161 [ "chain", "toolchain/clean" ],
162 [ "linux", "target/linux/clean" ],
163 [ "dir", "dirclean" ],
164 [ "dist", "distclean" ]
167 def IsCleanRequested(pattern):
168 def CheckCleanProperty(step):
169 val = step.getProperty("clean")
170 if val and re.match(pattern, val):
175 return CheckCleanProperty
180 dlLock = locks.SlaveLock("slave_dl")
182 checkBuiltin = re.sub('[\t\n ]+', ' ', """
184 local symbol op path file;
185 for file in $CHANGED_FILES; do
191 while read symbol op path; do
192 case "$symbol" in package-*)
193 symbol="${symbol##*(}";
194 symbol="${symbol%)}";
195 for file in $CHANGED_FILES; do
196 case "$file" in "package/$path/"*)
197 grep -qsx "$symbol=y" .config && return 0
201 done < tmp/.packagedeps;
207 class IfBuiltinShellCommand(ShellCommand):
208 def _quote(self, str):
209 if re.search("[^a-zA-Z0-9/_.-]", str):
210 return "'%s'" %(re.sub("'", "'\"'\"'", str))
213 def setCommand(self, command):
214 if not isinstance(command, (str, unicode)):
215 command = ' '.join(map(self._quote, command))
218 '%s; if checkBuiltin; then %s; else exit 0; fi' %(checkBuiltin, command)
221 def setupEnvironment(self, cmd):
222 slaveEnv = self.slaveEnvironment
226 for request in self.build.requests:
227 for source in request.sources:
228 for change in source.changes:
229 for file in change.files:
230 changedFiles[file] = True
231 fullSlaveEnv = slaveEnv.copy()
232 fullSlaveEnv['CHANGED_FILES'] = ' '.join(changedFiles.keys())
233 cmd.args['env'] = fullSlaveEnv
237 for slave in c['slaves']:
238 slaveNames.append(slave.slavename)
240 for target in targets:
241 ts = target.split('/')
243 factory = BuildFactory()
245 # find number of cores
246 factory.addStep(SetProperty(
249 description = "Finding number of CPUs",
250 command = ["nproc"]))
252 # expire tree if needed
254 factory.addStep(FileDownload(
255 mastersrc = "expire.sh",
256 slavedest = "../expire.sh",
259 factory.addStep(ShellCommand(
261 description = "Checking for build tree expiry",
262 command = ["./expire.sh", str(tree_expire)],
264 haltOnFailure = True,
267 # user-requested clean targets
268 for tuple in CleanTargetMap:
269 factory.addStep(ShellCommand(
271 description = 'User-requested "make %s"' % tuple[1],
272 command = ["make", tuple[1], "V=s"],
273 doStepIf = IsCleanRequested(tuple[0])
276 # check out the source
279 branch = repo_branch,
280 mode = 'incremental',
283 factory.addStep(ShellCommand(
285 description = "Remove tmp folder",
286 command=["rm", "-rf", "tmp/"]))
289 # factory.addStep(ShellCommand(
290 # name = "feedsconf",
291 # description = "Copy the feeds.conf",
292 # command='''cp ~/feeds.conf ./feeds.conf''' ))
295 factory.addStep(ShellCommand(
296 name = "rmfeedlinks",
297 description = "Remove feed symlinks",
298 command=["rm", "-rf", "package/feeds/"]))
301 factory.addStep(ShellCommand(
302 name = "updatefeeds",
303 description = "Updating feeds",
304 command=["./scripts/feeds", "update"]))
307 factory.addStep(ShellCommand(
308 name = "installfeeds",
309 description = "Installing feeds",
310 command=["./scripts/feeds", "install", "-a"]))
313 factory.addStep(FileDownload(
314 mastersrc = "config.seed",
315 slavedest = ".config",
320 factory.addStep(ShellCommand(
322 description = "Seeding .config",
323 command = "printf 'CONFIG_TARGET_%s=y\\nCONFIG_TARGET_%s_%s=y\\n' >> .config" %(ts[0], ts[0], ts[1])
326 factory.addStep(ShellCommand(
328 description = "Removing output directory",
329 command = ["rm", "-rf", "bin/"]
332 factory.addStep(ShellCommand(
334 description = "Populating .config",
335 command = ["make", "defconfig"]
339 factory.addStep(ShellCommand(
341 description = "Checking architecture",
342 command = ["grep", "-sq", "CONFIG_TARGET_%s=y" %(ts[0]), ".config"],
350 factory.addStep(SetProperty(
353 description = "Finding libc suffix",
354 command = ["sed", "-ne", '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }', ".config"]))
357 factory.addStep(FileDownload(mastersrc=home_dir+'/key-build', slavedest="key-build", mode=0600))
358 factory.addStep(FileDownload(mastersrc=home_dir+'/key-build.pub', slavedest="key-build.pub", mode=0600))
361 factory.addStep(ShellCommand(
363 description = "Preparing dl/",
364 command = "mkdir -p $HOME/dl && rm -rf ./dl && ln -sf $HOME/dl ./dl",
370 factory.addStep(ShellCommand(
372 description = "Building GNU tar",
373 command = ["make", WithProperties("-j%(nproc:~4)s"), "tools/tar/install", "V=s"],
378 factory.addStep(ShellCommand(
380 description = "Populating dl/",
381 command = ["make", WithProperties("-j%(nproc:~4)s"), "download", "V=s"],
383 locks = [dlLock.access('exclusive')]
386 factory.addStep(ShellCommand(
388 description = "Cleaning base-files",
389 command=["make", "package/base-files/clean", "V=s"]
393 factory.addStep(ShellCommand(
395 description = "Building tools",
396 command = ["make", WithProperties("-j%(nproc:~4)s"), "tools/install", "V=s"],
400 factory.addStep(ShellCommand(
402 description = "Building toolchain",
403 command=["make", WithProperties("-j%(nproc:~4)s"), "toolchain/install", "V=s"],
407 factory.addStep(ShellCommand(
409 description = "Building kmods",
410 command=["make", WithProperties("-j%(nproc:~4)s"), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
411 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
415 factory.addStep(ShellCommand(
417 description = "Building packages",
418 command=["make", WithProperties("-j%(nproc:~4)s"), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
419 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
423 # factory.addStep(IfBuiltinShellCommand(
424 factory.addStep(ShellCommand(
426 description = "Installing packages",
427 command=["make", WithProperties("-j%(nproc:~4)s"), "package/install", "V=s"],
431 factory.addStep(ShellCommand(
433 description = "Indexing packages",
434 command=["make", WithProperties("-j%(nproc:~4)s"), "package/index", "V=s"],
438 #factory.addStep(IfBuiltinShellCommand(
439 factory.addStep(ShellCommand(
441 description = "Building images",
442 command=["make", WithProperties("-j%(nproc:~4)s"), "target/install", "V=s"],
446 factory.addStep(ShellCommand(
448 description = "Calculating checksums",
449 command=["make", "-j1", "checksum", "V=s"],
454 if gpg_keyid is not None:
455 factory.addStep(MasterShellCommand(
456 name = "signprepare",
457 description = "Preparing temporary signing directory",
458 command = ["mkdir", "-p", "%s/signing" %(home_dir)],
462 factory.addStep(ShellCommand(
464 description = "Packing files to sign",
465 command = WithProperties("find bin/targets/%s/%s%%(libc)s/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz" %(ts[0], ts[1])),
469 factory.addStep(FileUpload(
470 slavesrc = "sign.tar.gz",
471 masterdest = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
475 factory.addStep(MasterShellCommand(
477 description = "Signing files",
478 command = ["%s/signall.sh" %(home_dir), "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]), gpg_keyid, gpg_comment],
479 env = {'GNUPGHOME': gpg_home, 'PASSFILE': gpg_passfile},
483 factory.addStep(FileDownload(
484 mastersrc = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
485 slavedest = "sign.tar.gz",
489 factory.addStep(ShellCommand(
491 description = "Unpacking signed files",
492 command = ["tar", "-xzf", "sign.tar.gz"],
497 factory.addStep(ShellCommand(
498 name = "uploadprepare",
499 description = "Preparing target directory",
500 command=["rsync", "-av", "--include", "/%s/" %(ts[0]), "--include", "/%s/%s/" %(ts[0], ts[1]), "--exclude", "/*", "--exclude", "/*/*", "--exclude", "/%s/%s/*" %(ts[0], ts[1]), "bin/targets/", "%s/targets/" %(rsync_bin_url)],
501 env={'RSYNC_PASSWORD': rsync_bin_key},
502 haltOnFailure = True,
506 factory.addStep(ShellCommand(
507 name = "targetupload",
508 description = "Uploading target files",
509 command=["rsync", "--delete", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", WithProperties("bin/targets/%s/%s%%(libc)s/" %(ts[0], ts[1])), "%s/targets/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
510 env={'RSYNC_PASSWORD': rsync_bin_key},
511 haltOnFailure = True,
515 if rsync_src_url is not None:
516 factory.addStep(ShellCommand(
517 name = "sourceupload",
518 description = "Uploading source archives",
519 command=["rsync", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "dl/", "%s/" %(rsync_src_url)],
520 env={'RSYNC_PASSWORD': rsync_src_key},
521 haltOnFailure = True,
526 factory.addStep(ShellCommand(
527 name = "packageupload",
528 description = "Uploading package files",
529 command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "bin/packages/", "%s/packages/" %(rsync_bin_url)],
530 env={'RSYNC_PASSWORD': rsync_bin_key},
531 haltOnFailure = False,
537 factory.addStep(ShellCommand(
539 description = "Uploading logs",
540 command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "logs/", "%s/logs/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
541 env={'RSYNC_PASSWORD': rsync_bin_key},
542 haltOnFailure = False,
547 from buildbot.config import BuilderConfig
549 c['builders'].append(BuilderConfig(name=target, slavenames=slaveNames, factory=factory))
552 ####### STATUS TARGETS
554 # 'status' is a list of Status Targets. The results of each build will be
555 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
556 # including web pages, email senders, and IRC bots.
560 from buildbot.status import html
561 from buildbot.status.web import authz, auth
563 if ini.has_option("status", "bind"):
564 if ini.has_option("status", "user") and ini.has_option("status", "password"):
565 authz_cfg=authz.Authz(
566 # change any of these to True to enable; see the manual for more
568 auth=auth.BasicAuth([(ini.get("status", "user"), ini.get("status", "password"))]),
569 gracefulShutdown = 'auth',
570 forceBuild = 'auth', # use this to test your slave once it is set up
571 forceAllBuilds = 'auth',
574 stopAllBuilds = 'auth',
575 cancelPendingBuild = 'auth',
577 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind"), authz=authz_cfg))
579 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind")))
582 from buildbot.status import words
584 if ini.has_option("irc", "host") and ini.has_option("irc", "nickname") and ini.has_option("irc", "channel"):
585 irc_host = ini.get("irc", "host")
587 irc_chan = ini.get("irc", "channel")
588 irc_nick = ini.get("irc", "nickname")
591 if ini.has_option("irc", "port"):
592 irc_port = ini.getint("irc", "port")
594 if ini.has_option("irc", "password"):
595 irc_pass = ini.get("irc", "password")
597 irc = words.IRC(irc_host, irc_nick, port = irc_port, password = irc_pass,
598 channels = [{ "channel": irc_chan }],
601 'successToFailure': 1,
602 'failureToSuccess': 1
606 c['status'].append(irc)
609 ####### PROJECT IDENTITY
611 # the 'title' string will appear at the top of this buildbot
612 # installation's html.WebStatus home page (linked to the
613 # 'titleURL') and is embedded in the title of the waterfall HTML page.
615 c['title'] = ini.get("general", "title")
616 c['titleURL'] = ini.get("general", "title_url")
618 # the 'buildbotURL' string should point to the location where the buildbot's
619 # internal web server (usually the html.WebStatus page) is visible. This
620 # typically uses the port number set in the Waterfall 'status' entry, but
621 # with an externally-visible host name which the buildbot cannot figure out
624 c['buildbotURL'] = ini.get("general", "buildbot_url")
629 # This specifies what database buildbot uses to store its state. You can leave
630 # this at its default for all but the largest installations.
631 'db_url' : "sqlite:///state.sqlite",