2 # ex: set syntax=python:
9 from buildbot import locks
11 # This is a sample buildmaster config file. It must be installed as
12 # 'master.cfg' in your buildmaster's base directory.
14 ini = ConfigParser.ConfigParser()
15 ini.read("./config.ini")
17 # This is the dictionary that the buildmaster pays attention to. We also use
18 # a shorter alias to save typing.
19 c = BuildmasterConfig = {}
23 # The 'slaves' list defines the set of recognized buildslaves. Each element is
24 # a BuildSlave object, specifying a unique slave name and password. The same
25 # slave name and password must be configured on the slave.
26 from buildbot.buildslave import BuildSlave
30 if ini.has_option("general", "port"):
31 slave_port = ini.getint("general", "port")
35 for section in ini.sections():
36 if section.startswith("slave "):
37 if ini.has_option(section, "name") and ini.has_option(section, "password"):
38 name = ini.get(section, "name")
39 password = ini.get(section, "password")
41 if ini.has_option(section, "builds"):
42 max_builds = ini.getint(section, "builds")
43 c['slaves'].append(BuildSlave(name, password, max_builds = max_builds))
45 # 'slavePortnum' defines the TCP port to listen on for connections from slaves.
46 # This must match the value configured into the buildslaves (with their
48 c['slavePortnum'] = slave_port
51 c['mergeRequests'] = True
55 home_dir = os.path.abspath(ini.get("general", "homedir"))
58 if ini.has_option("general", "expire"):
59 tree_expire = ini.getint("general", "expire")
61 repo_url = ini.get("repo", "url")
63 rsync_bin_url = ini.get("rsync", "binary_url")
64 rsync_bin_key = ini.get("rsync", "binary_password")
69 if ini.has_option("rsync", "source_url"):
70 rsync_src_url = ini.get("rsync", "source_url")
71 rsync_src_key = ini.get("rsync", "source_password")
74 gpg_comment = "Unattended build signature"
75 gpg_passfile = "/dev/null"
77 if ini.has_option("gpg", "keyid"):
78 gpg_keyid = ini.get("gpg", "keyid")
80 if ini.has_option("gpg", "comment"):
81 gpg_comment = ini.get("gpg", "comment")
83 if ini.has_option("gpg", "passfile"):
84 gpg_passfile = ini.get("gpg", "passfile")
90 if not os.path.isdir(home_dir+'/source.git'):
91 subprocess.call(["git", "clone", "--depth=1", repo_url, home_dir+'/source.git'])
93 findtargets = subprocess.Popen([home_dir+'/dumpinfo.pl', 'targets'],
94 stdout = subprocess.PIPE, cwd = home_dir+'/source.git')
97 line = findtargets.stdout.readline()
100 ta = line.strip().split(' ')
101 targets.append(ta[0])
104 # the 'change_source' setting tells the buildmaster how it should find out
105 # about source code changes. Here we point to the buildbot clone of pyflakes.
107 from buildbot.changes.gitpoller import GitPoller
108 c['change_source'] = []
109 c['change_source'].append(GitPoller(
111 workdir=home_dir+'/source.git', branch='master',
116 # Configure the Schedulers, which decide how to react to incoming changes. In this
117 # case, just kick off a 'basebuild' build
119 from buildbot.schedulers.basic import SingleBranchScheduler
120 from buildbot.schedulers.forcesched import ForceScheduler
121 from buildbot.changes import filter
123 c['schedulers'].append(SingleBranchScheduler(
125 change_filter=filter.ChangeFilter(branch='master'),
127 builderNames=targets))
129 c['schedulers'].append(ForceScheduler(
131 builderNames=targets))
135 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
136 # what steps, and which slaves can execute them. Note that any particular build will
137 # only take place on one slave.
139 from buildbot.process.factory import BuildFactory
140 from buildbot.steps.source import Git
141 from buildbot.steps.shell import ShellCommand
142 from buildbot.steps.shell import SetProperty
143 from buildbot.steps.transfer import FileUpload
144 from buildbot.steps.transfer import FileDownload
145 from buildbot.steps.master import MasterShellCommand
146 from buildbot.process.properties import WithProperties
150 [ "tools", "tools/clean" ],
151 [ "chain", "toolchain/clean" ],
152 [ "linux", "target/linux/clean" ],
153 [ "dir", "dirclean" ],
154 [ "dist", "distclean" ]
157 def IsCleanRequested(pattern):
158 def CheckCleanProperty(step):
159 val = step.getProperty("clean")
160 if val and re.match(pattern, val):
165 return CheckCleanProperty
170 dlLock = locks.SlaveLock("slave_dl")
172 checkBuiltin = re.sub('[\t\n ]+', ' ', """
174 local symbol op path file;
175 for file in $CHANGED_FILES; do
181 while read symbol op path; do
182 case "$symbol" in package-*)
183 symbol="${symbol##*(}";
184 symbol="${symbol%)}";
185 for file in $CHANGED_FILES; do
186 case "$file" in "package/$path/"*)
187 grep -qsx "$symbol=y" .config && return 0
191 done < tmp/.packagedeps;
197 class IfBuiltinShellCommand(ShellCommand):
198 def _quote(self, str):
199 if re.search("[^a-zA-Z0-9/_.-]", str):
200 return "'%s'" %(re.sub("'", "'\"'\"'", str))
203 def setCommand(self, command):
204 if not isinstance(command, (str, unicode)):
205 command = ' '.join(map(self._quote, command))
208 '%s; if checkBuiltin; then %s; else exit 0; fi' %(checkBuiltin, command)
211 def setupEnvironment(self, cmd):
212 slaveEnv = self.slaveEnvironment
216 for request in self.build.requests:
217 for source in request.sources:
218 for change in source.changes:
219 for file in change.files:
220 changedFiles[file] = True
221 fullSlaveEnv = slaveEnv.copy()
222 fullSlaveEnv['CHANGED_FILES'] = ' '.join(changedFiles.keys())
223 cmd.args['env'] = fullSlaveEnv
227 for slave in c['slaves']:
228 slaveNames.append(slave.slavename)
230 for target in targets:
231 ts = target.split('/')
233 factory = BuildFactory()
235 # find number of cores
236 factory.addStep(SetProperty(
239 description = "Finding number of CPUs",
240 command = ["nproc"]))
242 # expire tree if needed
244 factory.addStep(FileDownload(
245 mastersrc = "expire.sh",
246 slavedest = "../expire.sh",
249 factory.addStep(ShellCommand(
251 description = "Checking for build tree expiry",
252 command = ["./expire.sh", str(tree_expire)],
254 haltOnFailure = True,
257 # user-requested clean targets
258 for tuple in CleanTargetMap:
259 factory.addStep(ShellCommand(
261 description = 'User-requested "make %s"' % tuple[1],
262 command = ["make", tuple[1], "V=s"],
263 doStepIf = IsCleanRequested(tuple[0])
266 # check out the source
267 factory.addStep(Git(repourl=repo_url, mode='update'))
269 factory.addStep(ShellCommand(
271 description = "Remove tmp folder",
272 command=["rm", "-rf", "tmp/"]))
275 # factory.addStep(ShellCommand(
276 # name = "feedsconf",
277 # description = "Copy the feeds.conf",
278 # command='''cp ~/feeds.conf ./feeds.conf''' ))
281 factory.addStep(ShellCommand(
282 name = "rmfeedlinks",
283 description = "Remove feed symlinks",
284 command=["rm", "-rf", "package/feeds/"]))
287 factory.addStep(ShellCommand(
288 name = "updatefeeds",
289 description = "Updating feeds",
290 command=["./scripts/feeds", "update"]))
293 factory.addStep(ShellCommand(
294 name = "installfeeds",
295 description = "Installing feeds",
296 command=["./scripts/feeds", "install", "-a"]))
299 factory.addStep(ShellCommand(
301 description = "Seeding .config",
302 command='''cat <<EOT > .config
304 CONFIG_TARGET_%s_%s=y
305 CONFIG_ALL_NONSHARED=y
308 # CONFIG_IB_STANDALONE is not set
311 CONFIG_SIGNED_PACKAGES=y
312 # CONFIG_PER_FEED_REPO_ADD_COMMENTED is not set
313 CONFIG_KERNEL_KALLSYMS=y
314 CONFIG_COLLECT_KERNEL_DEBUG=y
315 CONFIG_TARGET_ALL_PROFILES=y
316 CONFIG_TARGET_MULTI_PROFILE=y
317 CONFIG_TARGET_PER_DEVICE_ROOTFS=y
318 EOT''' %(ts[0], ts[0], ts[1]) ))
320 factory.addStep(ShellCommand(
322 description = "Removing output directory",
323 command = ["rm", "-rf", "bin/"]
326 factory.addStep(ShellCommand(
328 description = "Populating .config",
329 command = ["make", "defconfig"]
333 factory.addStep(ShellCommand(
335 description = "Checking architecture",
336 command = ["grep", "-sq", "CONFIG_TARGET_%s=y" %(ts[0]), ".config"],
344 factory.addStep(SetProperty(
347 description = "Finding libc suffix",
348 command = ["sed", "-ne", '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }', ".config"]))
351 factory.addStep(FileDownload(mastersrc=home_dir+'/key-build', slavedest="key-build", mode=0600))
352 factory.addStep(FileDownload(mastersrc=home_dir+'/key-build.pub', slavedest="key-build.pub", mode=0600))
355 factory.addStep(ShellCommand(
357 description = "Preparing dl/",
358 command = "mkdir -p $HOME/dl && ln -sf $HOME/dl ./dl",
364 factory.addStep(ShellCommand(
366 description = "Building GNU tar",
367 command = ["make", WithProperties("-j%(nproc:~4)s"), "tools/tar/install", "V=s"],
372 factory.addStep(ShellCommand(
374 description = "Populating dl/",
375 command = ["make", WithProperties("-j%(nproc:~4)s"), "download", "V=s"],
377 locks = [dlLock.access('exclusive')]
380 factory.addStep(ShellCommand(
382 description = "Cleaning base-files",
383 command=["make", "package/base-files/clean", "V=s"]
387 factory.addStep(ShellCommand(
389 description = "Building tools",
390 command = ["make", WithProperties("-j%(nproc:~4)s"), "tools/install", "V=s"],
394 factory.addStep(ShellCommand(
396 description = "Building toolchain",
397 command=["make", WithProperties("-j%(nproc:~4)s"), "toolchain/install", "V=s"],
401 factory.addStep(ShellCommand(
403 description = "Building kmods",
404 command=["make", WithProperties("-j%(nproc:~4)s"), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
405 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
409 factory.addStep(ShellCommand(
411 description = "Building packages",
412 command=["make", WithProperties("-j%(nproc:~4)s"), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
413 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
417 # factory.addStep(IfBuiltinShellCommand(
418 factory.addStep(ShellCommand(
420 description = "Installing packages",
421 command=["make", WithProperties("-j%(nproc:~4)s"), "package/install", "V=s"],
425 factory.addStep(ShellCommand(
427 description = "Indexing packages",
428 command=["make", WithProperties("-j%(nproc:~4)s"), "package/index", "V=s"],
432 #factory.addStep(IfBuiltinShellCommand(
433 factory.addStep(ShellCommand(
435 description = "Building images",
436 command=["make", WithProperties("-j%(nproc:~4)s"), "target/install", "V=s"],
440 factory.addStep(ShellCommand(
442 description = "Calculating checksums",
443 command=["make", "-j1", "checksum", "V=s"],
448 if gpg_keyid is not None:
449 factory.addStep(MasterShellCommand(
450 name = "signprepare",
451 description = "Preparing temporary signing directory",
452 command = ["mkdir", "-p", "%s/signing" %(home_dir)],
456 factory.addStep(ShellCommand(
458 description = "Packing files to sign",
459 command = WithProperties("find bin/targets/%s/%s%%(libc)s/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz" %(ts[0], ts[1])),
463 factory.addStep(FileUpload(
464 slavesrc = "sign.tar.gz",
465 masterdest = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
469 factory.addStep(MasterShellCommand(
471 description = "Signing files",
472 command = ["%s/signall.sh" %(home_dir), "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]), gpg_keyid, gpg_passfile, gpg_comment],
476 factory.addStep(FileDownload(
477 mastersrc = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
478 slavedest = "sign.tar.gz",
482 factory.addStep(ShellCommand(
484 description = "Unpacking signed files",
485 command = ["tar", "-xzf", "sign.tar.gz"],
490 factory.addStep(ShellCommand(
491 name = "uploadprepare",
492 description = "Preparing target directory",
493 command=["rsync", "-av", "--include", "/%s/" %(ts[0]), "--include", "/%s/%s/" %(ts[0], ts[1]), "--exclude", "/*", "--exclude", "/*/*", "--exclude", "/%s/%s/*" %(ts[0], ts[1]), "bin/targets/", "%s/targets/" %(rsync_bin_url)],
494 env={'RSYNC_PASSWORD': rsync_bin_key},
495 haltOnFailure = True,
499 factory.addStep(ShellCommand(
500 name = "targetupload",
501 description = "Uploading target files",
502 command=["rsync", "--delete", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", WithProperties("bin/targets/%s/%s%%(libc)s/" %(ts[0], ts[1])), "%s/targets/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
503 env={'RSYNC_PASSWORD': rsync_bin_key},
504 haltOnFailure = True,
508 if rsync_src_url is not None:
509 factory.addStep(ShellCommand(
510 name = "sourceupload",
511 description = "Uploading source archives",
512 command=["rsync", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "dl/", "%s/" %(rsync_src_url)],
513 env={'RSYNC_PASSWORD': rsync_src_key},
514 haltOnFailure = True,
519 factory.addStep(ShellCommand(
520 name = "packageupload",
521 description = "Uploading package files",
522 command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "bin/packages/", "%s/packages/" %(rsync_bin_url)],
523 env={'RSYNC_PASSWORD': rsync_bin_key},
524 haltOnFailure = False,
530 factory.addStep(ShellCommand(
532 description = "Uploading logs",
533 command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "logs/", "%s/logs/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
534 env={'RSYNC_PASSWORD': rsync_bin_key},
535 haltOnFailure = False,
540 from buildbot.config import BuilderConfig
542 c['builders'].append(BuilderConfig(name=target, slavenames=slaveNames, factory=factory))
545 ####### STATUS TARGETS
547 # 'status' is a list of Status Targets. The results of each build will be
548 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
549 # including web pages, email senders, and IRC bots.
553 from buildbot.status import html
554 from buildbot.status.web import authz, auth
556 if ini.has_option("status", "bind"):
557 if ini.has_option("status", "user") and ini.has_option("status", "password"):
558 authz_cfg=authz.Authz(
559 # change any of these to True to enable; see the manual for more
561 auth=auth.BasicAuth([(ini.get("status", "user"), ini.get("status", "password"))]),
562 gracefulShutdown = 'auth',
563 forceBuild = 'auth', # use this to test your slave once it is set up
564 forceAllBuilds = 'auth',
567 stopAllBuilds = 'auth',
568 cancelPendingBuild = 'auth',
570 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind"), authz=authz_cfg))
572 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind")))
575 from buildbot.status import words
577 if ini.has_option("irc", "host") and ini.has_option("irc", "nickname") and ini.has_option("irc", "channel"):
578 irc_host = ini.get("irc", "host")
580 irc_chan = ini.get("irc", "channel")
581 irc_nick = ini.get("irc", "nickname")
584 if ini.has_option("irc", "port"):
585 irc_port = ini.getint("irc", "port")
587 if ini.has_option("irc", "password"):
588 irc_pass = ini.get("irc", "password")
590 irc = words.IRC(irc_host, irc_nick, port = irc_port, password = irc_pass,
591 channels = [{ "channel": irc_chan }],
594 'successToFailure': 1,
595 'failureToSuccess': 1
599 c['status'].append(irc)
602 ####### PROJECT IDENTITY
604 # the 'title' string will appear at the top of this buildbot
605 # installation's html.WebStatus home page (linked to the
606 # 'titleURL') and is embedded in the title of the waterfall HTML page.
608 c['title'] = ini.get("general", "title")
609 c['titleURL'] = ini.get("general", "title_url")
611 # the 'buildbotURL' string should point to the location where the buildbot's
612 # internal web server (usually the html.WebStatus page) is visible. This
613 # typically uses the port number set in the Waterfall 'status' entry, but
614 # with an externally-visible host name which the buildbot cannot figure out
617 c['buildbotURL'] = ini.get("general", "buildbot_url")
622 # This specifies what database buildbot uses to store its state. You can leave
623 # this at its default for all but the largest installations.
624 'db_url' : "sqlite:///state.sqlite",