2 # ex: set syntax=python:
9 from buildbot import locks
11 # This is a sample buildmaster config file. It must be installed as
12 # 'master.cfg' in your buildmaster's base directory.
14 ini = ConfigParser.ConfigParser()
15 ini.read("./config.ini")
17 # This is the dictionary that the buildmaster pays attention to. We also use
18 # a shorter alias to save typing.
19 c = BuildmasterConfig = {}
23 # The 'slaves' list defines the set of recognized buildslaves. Each element is
24 # a BuildSlave object, specifying a unique slave name and password. The same
25 # slave name and password must be configured on the slave.
26 from buildbot.buildslave import BuildSlave
30 for section in ini.sections():
31 if section.startswith("slave "):
32 if ini.has_option(section, "name") and ini.has_option(section, "password"):
33 name = ini.get(section, "name")
34 password = ini.get(section, "password")
36 if ini.has_option(section, "builds"):
37 max_builds = ini.getint(section, "builds")
38 c['slaves'].append(BuildSlave(name, password, max_builds = max_builds))
40 # 'slavePortnum' defines the TCP port to listen on for connections from slaves.
41 # This must match the value configured into the buildslaves (with their
43 c['slavePortnum'] = 9989
46 c['mergeRequests'] = True
50 home_dir = os.path.abspath(ini.get("general", "homedir"))
53 if ini.has_option("general", "expire"):
54 tree_expire = ini.getint("general", "expire")
56 repo_url = ini.get("repo", "url")
58 rsync_bin_url = ini.get("rsync", "binary_url")
59 rsync_bin_key = ini.get("rsync", "binary_password")
64 if ini.has_option("rsync", "source_url"):
65 rsync_src_url = ini.get("rsync", "source_url")
66 rsync_src_key = ini.get("rsync", "source_password")
69 gpg_comment = "Unattended build signature"
70 gpg_passfile = "/dev/null"
72 if ini.has_option("gpg", "keyid"):
73 gpg_keyid = ini.get("gpg", "keyid")
75 if ini.has_option("gpg", "comment"):
76 gpg_comment = ini.get("gpg", "comment")
78 if ini.has_option("gpg", "passfile"):
79 gpg_passfile = ini.get("gpg", "passfile")
85 if not os.path.isdir(home_dir+'/source.git'):
86 subprocess.call(["git", "clone", "--depth=1", repo_url, home_dir+'/source.git'])
88 findtargets = subprocess.Popen([home_dir+'/dumpinfo.pl', 'targets'],
89 stdout = subprocess.PIPE, cwd = home_dir+'/source.git')
92 line = findtargets.stdout.readline()
95 ta = line.strip().split(' ')
99 # the 'change_source' setting tells the buildmaster how it should find out
100 # about source code changes. Here we point to the buildbot clone of pyflakes.
102 from buildbot.changes.gitpoller import GitPoller
103 c['change_source'] = []
104 c['change_source'].append(GitPoller(
106 workdir=home_dir+'/source.git', branch='master',
111 # Configure the Schedulers, which decide how to react to incoming changes. In this
112 # case, just kick off a 'basebuild' build
114 from buildbot.schedulers.basic import SingleBranchScheduler
115 from buildbot.schedulers.forcesched import ForceScheduler
116 from buildbot.changes import filter
118 c['schedulers'].append(SingleBranchScheduler(
120 change_filter=filter.ChangeFilter(branch='master'),
122 builderNames=targets))
124 c['schedulers'].append(ForceScheduler(
126 builderNames=targets))
130 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
131 # what steps, and which slaves can execute them. Note that any particular build will
132 # only take place on one slave.
134 from buildbot.process.factory import BuildFactory
135 from buildbot.steps.source import Git
136 from buildbot.steps.shell import ShellCommand
137 from buildbot.steps.shell import SetProperty
138 from buildbot.steps.transfer import FileUpload
139 from buildbot.steps.transfer import FileDownload
140 from buildbot.steps.master import MasterShellCommand
141 from buildbot.process.properties import WithProperties
145 "^tools/": "tools/clean",
146 "^toolchain/": "toolchain/clean",
147 "^target/linux/": "target/linux/clean",
148 "^(config|include)/": "dirclean"
151 def IsAffected(pattern):
152 def CheckAffected(change):
153 for request in change.build.requests:
154 for source in request.sources:
155 for change in source.changes:
156 for file in change.files:
157 if re.match(pattern, file):
162 def isPathBuiltin(path):
165 conf = open(".config", "r")
168 line = conf.readline()
171 m = re.match("^(CONFIG_PACKAGE_.+?)=y", line)
173 incl[m.group(1)] = True
177 deps = open("tmp/.packagedeps", "r")
180 line = deps.readline()
183 m = re.match("^package-\$\((CONFIG_PACKAGE_.+?)\) \+= (\S+)", line)
184 if m and incl.get(m.group(1)) == True:
185 pkgs["package/%s" % m.group(2)] = True
190 if pkgs.get(path) == True:
192 path = os.path.dirname(path)
196 def isChangeBuiltin(change):
198 # for request in change.build.requests:
199 # for source in request.sources:
200 # for change in source.changes:
201 # for file in change.files:
202 # if isPathBuiltin(file):
209 dlLock = locks.SlaveLock("slave_dl")
211 checkBuiltin = re.sub('[\t\n ]+', ' ', """
213 local symbol op path file;
214 for file in $CHANGED_FILES; do
220 while read symbol op path; do
221 case "$symbol" in package-*)
222 symbol="${symbol##*(}";
223 symbol="${symbol%)}";
224 for file in $CHANGED_FILES; do
225 case "$file" in "package/$path/"*)
226 grep -qsx "$symbol=y" .config && return 0
230 done < tmp/.packagedeps;
236 class IfBuiltinShellCommand(ShellCommand):
237 def _quote(self, str):
238 if re.search("[^a-zA-Z0-9/_.-]", str):
239 return "'%s'" %(re.sub("'", "'\"'\"'", str))
242 def setCommand(self, command):
243 if not isinstance(command, (str, unicode)):
244 command = ' '.join(map(self._quote, command))
247 '%s; if checkBuiltin; then %s; else exit 0; fi' %(checkBuiltin, command)
250 def setupEnvironment(self, cmd):
251 slaveEnv = self.slaveEnvironment
255 for request in self.build.requests:
256 for source in request.sources:
257 for change in source.changes:
258 for file in change.files:
259 changedFiles[file] = True
260 fullSlaveEnv = slaveEnv.copy()
261 fullSlaveEnv['CHANGED_FILES'] = ' '.join(changedFiles.keys())
262 cmd.args['env'] = fullSlaveEnv
266 for slave in c['slaves']:
267 slaveNames.append(slave.slavename)
269 for target in targets:
270 ts = target.split('/')
272 factory = BuildFactory()
274 # find number of cores
275 factory.addStep(SetProperty(
278 description = "Finding number of CPUs",
279 command = ["nproc"]))
281 # expire tree if needed
283 factory.addStep(FileDownload(
284 mastersrc = "expire.sh",
285 slavedest = "../expire.sh",
288 factory.addStep(ShellCommand(
290 description = "Checking for build tree expiry",
291 command = ["./expire.sh", str(tree_expire)],
293 haltOnFailure = True,
296 # check out the source
297 factory.addStep(Git(repourl=repo_url, mode='update'))
299 factory.addStep(ShellCommand(
301 description = "Remove tmp folder",
302 command=["rm", "-rf", "tmp/"]))
305 # factory.addStep(ShellCommand(
306 # name = "feedsconf",
307 # description = "Copy the feeds.conf",
308 # command='''cp ~/feeds.conf ./feeds.conf''' ))
311 factory.addStep(ShellCommand(
312 name = "rmfeedlinks",
313 description = "Remove feed symlinks",
314 command=["rm", "-rf", "package/feeds/"]))
317 factory.addStep(ShellCommand(
318 name = "updatefeeds",
319 description = "Updating feeds",
320 command=["./scripts/feeds", "update"]))
323 factory.addStep(ShellCommand(
324 name = "installfeeds",
325 description = "Installing feeds",
326 command=["./scripts/feeds", "install", "-a"]))
329 factory.addStep(ShellCommand(
331 description = "Seeding .config",
332 command='''cat <<EOT > .config
334 CONFIG_TARGET_%s_%s=y
335 CONFIG_ALL_NONSHARED=y
338 # CONFIG_IB_STANDALONE is not set
341 CONFIG_SIGNED_PACKAGES=y
342 # CONFIG_PER_FEED_REPO_ADD_COMMENTED is not set
343 CONFIG_KERNEL_KALLSYMS=y
344 CONFIG_COLLECT_KERNEL_DEBUG=y
345 CONFIG_TARGET_ALL_PROFILES=y
346 CONFIG_TARGET_MULTI_PROFILE=y
347 CONFIG_TARGET_PER_DEVICE_ROOTFS=y
348 EOT''' %(ts[0], ts[0], ts[1]) ))
350 factory.addStep(ShellCommand(
352 description = "Removing output directory",
353 command = ["rm", "-rf", "bin/"]
356 factory.addStep(ShellCommand(
358 description = "Populating .config",
359 command = ["make", "defconfig"]
363 factory.addStep(ShellCommand(
365 description = "Checking architecture",
366 command = ["grep", "-sq", "CONFIG_TARGET_%s=y" %(ts[0]), ".config"],
374 factory.addStep(SetProperty(
377 description = "Finding libc suffix",
378 command = ["sed", "-ne", '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }', ".config"]))
381 factory.addStep(FileDownload(mastersrc=home_dir+'/key-build', slavedest="key-build", mode=0600))
382 factory.addStep(FileDownload(mastersrc=home_dir+'/key-build.pub', slavedest="key-build.pub", mode=0600))
385 factory.addStep(ShellCommand(
387 description = "Preparing dl/",
388 command = "mkdir -p $HOME/dl && ln -sf $HOME/dl ./dl",
394 factory.addStep(ShellCommand(
396 description = "Populating dl/",
397 command = ["make", WithProperties("-j%(nproc:~4)s"), "download", "V=s"],
399 locks = [dlLock.access('exclusive')]
402 factory.addStep(ShellCommand(
404 description = "Cleaning base-files",
405 command=["make", "package/base-files/clean", "V=s"]
408 # optional clean steps
409 for pattern, maketarget in MakeTargetMap.items():
410 factory.addStep(ShellCommand(
412 description = maketarget,
413 command=["make", maketarget, "V=s"], doStepIf=IsAffected(pattern)
417 factory.addStep(ShellCommand(
419 description = "Building tools",
420 command = ["make", WithProperties("-j%(nproc:~4)s"), "tools/install", "V=s"],
424 factory.addStep(ShellCommand(
426 description = "Building toolchain",
427 command=["make", WithProperties("-j%(nproc:~4)s"), "toolchain/install", "V=s"],
431 factory.addStep(ShellCommand(
433 description = "Building kmods",
434 command=["make", WithProperties("-j%(nproc:~4)s"), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
435 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
439 factory.addStep(ShellCommand(
441 description = "Building packages",
442 command=["make", WithProperties("-j%(nproc:~4)s"), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
443 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
447 # factory.addStep(IfBuiltinShellCommand(
448 factory.addStep(ShellCommand(
450 description = "Installing packages",
451 command=["make", WithProperties("-j%(nproc:~4)s"), "package/install", "V=s"],
452 doStepIf = isChangeBuiltin,
456 factory.addStep(ShellCommand(
458 description = "Indexing packages",
459 command=["make", WithProperties("-j%(nproc:~4)s"), "package/index", "V=s"],
463 #factory.addStep(IfBuiltinShellCommand(
464 factory.addStep(ShellCommand(
466 description = "Building images",
467 command=["make", WithProperties("-j%(nproc:~4)s"), "target/install", "V=s"],
468 doStepIf = isChangeBuiltin,
472 factory.addStep(ShellCommand(
474 description = "Calculating checksums",
475 command=["make", "-j1", "checksum", "V=s"],
476 doStepIf = isChangeBuiltin,
481 if gpg_keyid is not None:
482 factory.addStep(MasterShellCommand(
483 name = "signprepare",
484 description = "Preparing temporary signing directory",
485 command = ["mkdir", "-p", "%s/signing" %(home_dir)],
489 factory.addStep(ShellCommand(
491 description = "Packing files to sign",
492 command = WithProperties("find bin/targets/%s/%s%%(libc)s/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz" %(ts[0], ts[1])),
496 factory.addStep(FileUpload(
497 slavesrc = "sign.tar.gz",
498 masterdest = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
502 factory.addStep(MasterShellCommand(
504 description = "Signing files",
505 command = ["%s/signall.sh" %(home_dir), "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]), gpg_keyid, gpg_passfile, gpg_comment],
509 factory.addStep(FileDownload(
510 mastersrc = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
511 slavedest = "sign.tar.gz",
515 factory.addStep(ShellCommand(
517 description = "Unpacking signed files",
518 command = ["tar", "-xzf", "sign.tar.gz"],
523 factory.addStep(ShellCommand(
524 name = "uploadprepare",
525 description = "Preparing target directory",
526 command=["rsync", "-av", "--include", "/%s/" %(ts[0]), "--include", "/%s/%s/" %(ts[0], ts[1]), "--exclude", "/*", "--exclude", "/*/*", "--exclude", "/%s/%s/*" %(ts[0], ts[1]), "bin/targets/", "%s/targets/" %(rsync_bin_url)],
527 env={'RSYNC_PASSWORD': rsync_bin_key},
528 haltOnFailure = True,
532 factory.addStep(ShellCommand(
533 name = "targetupload",
534 description = "Uploading target files",
535 command=["rsync", "--delete", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", WithProperties("bin/targets/%s/%s%%(libc)s/" %(ts[0], ts[1])), "%s/targets/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
536 env={'RSYNC_PASSWORD': rsync_bin_key},
537 haltOnFailure = True,
541 if rsync_src_url is not None:
542 factory.addStep(ShellCommand(
543 name = "sourceupload",
544 description = "Uploading source archives",
545 command=["rsync", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "dl/", "%s/" %(rsync_src_url)],
546 env={'RSYNC_PASSWORD': rsync_src_key},
547 haltOnFailure = True,
552 factory.addStep(ShellCommand(
553 name = "packageupload",
554 description = "Uploading package files",
555 command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "bin/packages/", "%s/packages/" %(rsync_bin_url)],
556 env={'RSYNC_PASSWORD': rsync_bin_key},
557 haltOnFailure = False,
563 factory.addStep(ShellCommand(
565 description = "Uploading logs",
566 command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "logs/", "%s/logs/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
567 env={'RSYNC_PASSWORD': rsync_bin_key},
568 haltOnFailure = False,
573 from buildbot.config import BuilderConfig
575 c['builders'].append(BuilderConfig(name=target, slavenames=slaveNames, factory=factory))
578 ####### STATUS TARGETS
580 # 'status' is a list of Status Targets. The results of each build will be
581 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
582 # including web pages, email senders, and IRC bots.
586 from buildbot.status import html
587 from buildbot.status.web import authz, auth
589 if ini.has_option("status", "bind"):
590 if ini.has_option("status", "user") and ini.has_option("status", "password"):
591 authz_cfg=authz.Authz(
592 # change any of these to True to enable; see the manual for more
594 auth=auth.BasicAuth([(ini.get("status", "user"), ini.get("status", "password"))]),
595 gracefulShutdown = 'auth',
596 forceBuild = 'auth', # use this to test your slave once it is set up
597 forceAllBuilds = 'auth',
600 stopAllBuilds = 'auth',
601 cancelPendingBuild = 'auth',
603 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind"), authz=authz_cfg))
605 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind")))
608 from buildbot.status import words
610 if ini.has_option("irc", "host") and ini.has_option("irc", "nickname") and ini.has_option("irc", "channel"):
611 irc_host = ini.get("irc", "host")
613 irc_chan = ini.get("irc", "channel")
614 irc_nick = ini.get("irc", "nickname")
617 if ini.has_option("irc", "port"):
618 irc_port = ini.getint("irc", "port")
620 if ini.has_option("irc", "password"):
621 irc_pass = ini.get("irc", "password")
623 irc = words.IRC(irc_host, irc_nick, port = irc_port, password = irc_pass,
624 channels = [{ "channel": irc_chan }],
627 'successToFailure': 1,
628 'failureToSuccess': 1
632 c['status'].append(irc)
635 ####### PROJECT IDENTITY
637 # the 'title' string will appear at the top of this buildbot
638 # installation's html.WebStatus home page (linked to the
639 # 'titleURL') and is embedded in the title of the waterfall HTML page.
641 c['title'] = ini.get("general", "title")
642 c['titleURL'] = ini.get("general", "title_url")
644 # the 'buildbotURL' string should point to the location where the buildbot's
645 # internal web server (usually the html.WebStatus page) is visible. This
646 # typically uses the port number set in the Waterfall 'status' entry, but
647 # with an externally-visible host name which the buildbot cannot figure out
650 c['buildbotURL'] = ini.get("general", "buildbot_url")
655 # This specifies what database buildbot uses to store its state. You can leave
656 # this at its default for all but the largest installations.
657 'db_url' : "sqlite:///state.sqlite",