2 # ex: set syntax=python:
9 from buildbot import locks
11 # This is a sample buildmaster config file. It must be installed as
12 # 'master.cfg' in your buildmaster's base directory.
14 ini = ConfigParser.ConfigParser()
15 ini.read("./config.ini")
17 # This is the dictionary that the buildmaster pays attention to. We also use
18 # a shorter alias to save typing.
19 c = BuildmasterConfig = {}
23 # The 'slaves' list defines the set of recognized buildslaves. Each element is
24 # a BuildSlave object, specifying a unique slave name and password. The same
25 # slave name and password must be configured on the slave.
26 from buildbot.buildslave import BuildSlave
30 if ini.has_option("general", "port"):
31 slave_port = ini.getint("general", "port")
35 for section in ini.sections():
36 if section.startswith("slave "):
37 if ini.has_option(section, "name") and ini.has_option(section, "password"):
38 name = ini.get(section, "name")
39 password = ini.get(section, "password")
41 if ini.has_option(section, "builds"):
42 max_builds = ini.getint(section, "builds")
43 c['slaves'].append(BuildSlave(name, password, max_builds = max_builds))
45 # 'slavePortnum' defines the TCP port to listen on for connections from slaves.
46 # This must match the value configured into the buildslaves (with their
48 c['slavePortnum'] = slave_port
51 c['mergeRequests'] = True
55 home_dir = os.path.abspath(ini.get("general", "homedir"))
58 if ini.has_option("general", "expire"):
59 tree_expire = ini.getint("general", "expire")
61 repo_url = ini.get("repo", "url")
62 repo_branch = "master"
64 if ini.has_option("repo", "branch"):
65 repo_branch = ini.get("repo", "branch")
67 rsync_bin_url = ini.get("rsync", "binary_url")
68 rsync_bin_key = ini.get("rsync", "binary_password")
73 if ini.has_option("rsync", "source_url"):
74 rsync_src_url = ini.get("rsync", "source_url")
75 rsync_src_key = ini.get("rsync", "source_password")
78 gpg_comment = "Unattended build signature"
79 gpg_passfile = "/dev/null"
81 if ini.has_option("gpg", "keyid"):
82 gpg_keyid = ini.get("gpg", "keyid")
84 if ini.has_option("gpg", "comment"):
85 gpg_comment = ini.get("gpg", "comment")
87 if ini.has_option("gpg", "passfile"):
88 gpg_passfile = ini.get("gpg", "passfile")
94 if not os.path.isdir(home_dir+'/source.git'):
95 subprocess.call(["git", "clone", "--depth=1", repo_url, home_dir+'/source.git'])
96 subprocess.call(["git", "checkout", repo_branch], cwd = home_dir+'/source.git')
98 subprocess.call(["git", "checkout", repo_branch], cwd = home_dir+'/source.git')
99 subprocess.call(["git", "pull"], cwd = home_dir+'/source.git')
101 findtargets = subprocess.Popen([home_dir+'/dumpinfo.pl', 'targets'],
102 stdout = subprocess.PIPE, cwd = home_dir+'/source.git')
105 line = findtargets.stdout.readline()
108 ta = line.strip().split(' ')
109 targets.append(ta[0])
112 # the 'change_source' setting tells the buildmaster how it should find out
113 # about source code changes. Here we point to the buildbot clone of pyflakes.
115 from buildbot.changes.gitpoller import GitPoller
116 c['change_source'] = []
117 c['change_source'].append(GitPoller(
119 workdir=home_dir+'/work.git', branch='master',
124 # Configure the Schedulers, which decide how to react to incoming changes. In this
125 # case, just kick off a 'basebuild' build
127 from buildbot.schedulers.basic import SingleBranchScheduler
128 from buildbot.schedulers.forcesched import ForceScheduler
129 from buildbot.changes import filter
131 c['schedulers'].append(SingleBranchScheduler(
133 change_filter=filter.ChangeFilter(branch='master'),
135 builderNames=targets))
137 c['schedulers'].append(ForceScheduler(
139 builderNames=targets))
143 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
144 # what steps, and which slaves can execute them. Note that any particular build will
145 # only take place on one slave.
147 from buildbot.process.factory import BuildFactory
148 from buildbot.steps.source import Git
149 from buildbot.steps.shell import ShellCommand
150 from buildbot.steps.shell import SetProperty
151 from buildbot.steps.transfer import FileUpload
152 from buildbot.steps.transfer import FileDownload
153 from buildbot.steps.master import MasterShellCommand
154 from buildbot.process.properties import WithProperties
158 [ "tools", "tools/clean" ],
159 [ "chain", "toolchain/clean" ],
160 [ "linux", "target/linux/clean" ],
161 [ "dir", "dirclean" ],
162 [ "dist", "distclean" ]
165 def IsCleanRequested(pattern):
166 def CheckCleanProperty(step):
167 val = step.getProperty("clean")
168 if val and re.match(pattern, val):
173 return CheckCleanProperty
178 dlLock = locks.SlaveLock("slave_dl")
180 checkBuiltin = re.sub('[\t\n ]+', ' ', """
182 local symbol op path file;
183 for file in $CHANGED_FILES; do
189 while read symbol op path; do
190 case "$symbol" in package-*)
191 symbol="${symbol##*(}";
192 symbol="${symbol%)}";
193 for file in $CHANGED_FILES; do
194 case "$file" in "package/$path/"*)
195 grep -qsx "$symbol=y" .config && return 0
199 done < tmp/.packagedeps;
205 class IfBuiltinShellCommand(ShellCommand):
206 def _quote(self, str):
207 if re.search("[^a-zA-Z0-9/_.-]", str):
208 return "'%s'" %(re.sub("'", "'\"'\"'", str))
211 def setCommand(self, command):
212 if not isinstance(command, (str, unicode)):
213 command = ' '.join(map(self._quote, command))
216 '%s; if checkBuiltin; then %s; else exit 0; fi' %(checkBuiltin, command)
219 def setupEnvironment(self, cmd):
220 slaveEnv = self.slaveEnvironment
224 for request in self.build.requests:
225 for source in request.sources:
226 for change in source.changes:
227 for file in change.files:
228 changedFiles[file] = True
229 fullSlaveEnv = slaveEnv.copy()
230 fullSlaveEnv['CHANGED_FILES'] = ' '.join(changedFiles.keys())
231 cmd.args['env'] = fullSlaveEnv
235 for slave in c['slaves']:
236 slaveNames.append(slave.slavename)
238 for target in targets:
239 ts = target.split('/')
241 factory = BuildFactory()
243 # find number of cores
244 factory.addStep(SetProperty(
247 description = "Finding number of CPUs",
248 command = ["nproc"]))
250 # expire tree if needed
252 factory.addStep(FileDownload(
253 mastersrc = "expire.sh",
254 slavedest = "../expire.sh",
257 factory.addStep(ShellCommand(
259 description = "Checking for build tree expiry",
260 command = ["./expire.sh", str(tree_expire)],
262 haltOnFailure = True,
265 # user-requested clean targets
266 for tuple in CleanTargetMap:
267 factory.addStep(ShellCommand(
269 description = 'User-requested "make %s"' % tuple[1],
270 command = ["make", tuple[1], "V=s"],
271 doStepIf = IsCleanRequested(tuple[0])
274 # check out the source
275 factory.addStep(Git(repourl=repo_url, branch=repo_branch, mode='update'))
277 factory.addStep(ShellCommand(
279 description = "Remove tmp folder",
280 command=["rm", "-rf", "tmp/"]))
283 # factory.addStep(ShellCommand(
284 # name = "feedsconf",
285 # description = "Copy the feeds.conf",
286 # command='''cp ~/feeds.conf ./feeds.conf''' ))
289 factory.addStep(ShellCommand(
290 name = "rmfeedlinks",
291 description = "Remove feed symlinks",
292 command=["rm", "-rf", "package/feeds/"]))
295 factory.addStep(ShellCommand(
296 name = "updatefeeds",
297 description = "Updating feeds",
298 command=["./scripts/feeds", "update"]))
301 factory.addStep(ShellCommand(
302 name = "installfeeds",
303 description = "Installing feeds",
304 command=["./scripts/feeds", "install", "-a"]))
307 factory.addStep(FileDownload(
308 mastersrc = "config.seed",
309 slavedest = ".config",
314 factory.addStep(ShellCommand(
316 description = "Seeding .config",
317 command = "printf 'CONFIG_TARGET_%s=y\\nCONFIG_TARGET_%s_%s=y\\n' >> .config" %(ts[0], ts[0], ts[1])
320 factory.addStep(ShellCommand(
322 description = "Removing output directory",
323 command = ["rm", "-rf", "bin/"]
326 factory.addStep(ShellCommand(
328 description = "Populating .config",
329 command = ["make", "defconfig"]
333 factory.addStep(ShellCommand(
335 description = "Checking architecture",
336 command = ["grep", "-sq", "CONFIG_TARGET_%s=y" %(ts[0]), ".config"],
344 factory.addStep(SetProperty(
347 description = "Finding libc suffix",
348 command = ["sed", "-ne", '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }', ".config"]))
351 factory.addStep(FileDownload(mastersrc=home_dir+'/key-build', slavedest="key-build", mode=0600))
352 factory.addStep(FileDownload(mastersrc=home_dir+'/key-build.pub', slavedest="key-build.pub", mode=0600))
355 factory.addStep(ShellCommand(
357 description = "Preparing dl/",
358 command = "mkdir -p $HOME/dl && rm -rf ./dl && ln -sf $HOME/dl ./dl",
364 factory.addStep(ShellCommand(
366 description = "Building GNU tar",
367 command = ["make", WithProperties("-j%(nproc:~4)s"), "tools/tar/install", "V=s"],
372 factory.addStep(ShellCommand(
374 description = "Populating dl/",
375 command = ["make", WithProperties("-j%(nproc:~4)s"), "download", "V=s"],
377 locks = [dlLock.access('exclusive')]
380 factory.addStep(ShellCommand(
382 description = "Cleaning base-files",
383 command=["make", "package/base-files/clean", "V=s"]
387 factory.addStep(ShellCommand(
389 description = "Building tools",
390 command = ["make", WithProperties("-j%(nproc:~4)s"), "tools/install", "V=s"],
394 factory.addStep(ShellCommand(
396 description = "Building toolchain",
397 command=["make", WithProperties("-j%(nproc:~4)s"), "toolchain/install", "V=s"],
401 factory.addStep(ShellCommand(
403 description = "Building kmods",
404 command=["make", WithProperties("-j%(nproc:~4)s"), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
405 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
409 factory.addStep(ShellCommand(
411 description = "Building packages",
412 command=["make", WithProperties("-j%(nproc:~4)s"), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
413 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
417 # factory.addStep(IfBuiltinShellCommand(
418 factory.addStep(ShellCommand(
420 description = "Installing packages",
421 command=["make", WithProperties("-j%(nproc:~4)s"), "package/install", "V=s"],
425 factory.addStep(ShellCommand(
427 description = "Indexing packages",
428 command=["make", WithProperties("-j%(nproc:~4)s"), "package/index", "V=s"],
432 #factory.addStep(IfBuiltinShellCommand(
433 factory.addStep(ShellCommand(
435 description = "Building images",
436 command=["make", WithProperties("-j%(nproc:~4)s"), "target/install", "V=s"],
440 factory.addStep(ShellCommand(
442 description = "Calculating checksums",
443 command=["make", "-j1", "checksum", "V=s"],
448 if gpg_keyid is not None:
449 factory.addStep(MasterShellCommand(
450 name = "signprepare",
451 description = "Preparing temporary signing directory",
452 command = ["mkdir", "-p", "%s/signing" %(home_dir)],
456 factory.addStep(ShellCommand(
458 description = "Packing files to sign",
459 command = WithProperties("find bin/targets/%s/%s%%(libc)s/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz" %(ts[0], ts[1])),
463 factory.addStep(FileUpload(
464 slavesrc = "sign.tar.gz",
465 masterdest = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
469 factory.addStep(MasterShellCommand(
471 description = "Signing files",
472 command = ["%s/signall.sh" %(home_dir), "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]), gpg_keyid, gpg_passfile, gpg_comment],
476 factory.addStep(FileDownload(
477 mastersrc = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
478 slavedest = "sign.tar.gz",
482 factory.addStep(ShellCommand(
484 description = "Unpacking signed files",
485 command = ["tar", "-xzf", "sign.tar.gz"],
490 factory.addStep(ShellCommand(
491 name = "uploadprepare",
492 description = "Preparing target directory",
493 command=["rsync", "-av", "--include", "/%s/" %(ts[0]), "--include", "/%s/%s/" %(ts[0], ts[1]), "--exclude", "/*", "--exclude", "/*/*", "--exclude", "/%s/%s/*" %(ts[0], ts[1]), "bin/targets/", "%s/targets/" %(rsync_bin_url)],
494 env={'RSYNC_PASSWORD': rsync_bin_key},
495 haltOnFailure = True,
499 factory.addStep(ShellCommand(
500 name = "targetupload",
501 description = "Uploading target files",
502 command=["rsync", "--delete", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", WithProperties("bin/targets/%s/%s%%(libc)s/" %(ts[0], ts[1])), "%s/targets/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
503 env={'RSYNC_PASSWORD': rsync_bin_key},
504 haltOnFailure = True,
508 if rsync_src_url is not None:
509 factory.addStep(ShellCommand(
510 name = "sourceupload",
511 description = "Uploading source archives",
512 command=["rsync", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "dl/", "%s/" %(rsync_src_url)],
513 env={'RSYNC_PASSWORD': rsync_src_key},
514 haltOnFailure = True,
519 factory.addStep(ShellCommand(
520 name = "packageupload",
521 description = "Uploading package files",
522 command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "bin/packages/", "%s/packages/" %(rsync_bin_url)],
523 env={'RSYNC_PASSWORD': rsync_bin_key},
524 haltOnFailure = False,
530 factory.addStep(ShellCommand(
532 description = "Uploading logs",
533 command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "logs/", "%s/logs/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
534 env={'RSYNC_PASSWORD': rsync_bin_key},
535 haltOnFailure = False,
540 from buildbot.config import BuilderConfig
542 c['builders'].append(BuilderConfig(name=target, slavenames=slaveNames, factory=factory))
545 ####### STATUS TARGETS
547 # 'status' is a list of Status Targets. The results of each build will be
548 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
549 # including web pages, email senders, and IRC bots.
553 from buildbot.status import html
554 from buildbot.status.web import authz, auth
556 if ini.has_option("status", "bind"):
557 if ini.has_option("status", "user") and ini.has_option("status", "password"):
558 authz_cfg=authz.Authz(
559 # change any of these to True to enable; see the manual for more
561 auth=auth.BasicAuth([(ini.get("status", "user"), ini.get("status", "password"))]),
562 gracefulShutdown = 'auth',
563 forceBuild = 'auth', # use this to test your slave once it is set up
564 forceAllBuilds = 'auth',
567 stopAllBuilds = 'auth',
568 cancelPendingBuild = 'auth',
570 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind"), authz=authz_cfg))
572 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind")))
575 from buildbot.status import words
577 if ini.has_option("irc", "host") and ini.has_option("irc", "nickname") and ini.has_option("irc", "channel"):
578 irc_host = ini.get("irc", "host")
580 irc_chan = ini.get("irc", "channel")
581 irc_nick = ini.get("irc", "nickname")
584 if ini.has_option("irc", "port"):
585 irc_port = ini.getint("irc", "port")
587 if ini.has_option("irc", "password"):
588 irc_pass = ini.get("irc", "password")
590 irc = words.IRC(irc_host, irc_nick, port = irc_port, password = irc_pass,
591 channels = [{ "channel": irc_chan }],
594 'successToFailure': 1,
595 'failureToSuccess': 1
599 c['status'].append(irc)
602 ####### PROJECT IDENTITY
604 # the 'title' string will appear at the top of this buildbot
605 # installation's html.WebStatus home page (linked to the
606 # 'titleURL') and is embedded in the title of the waterfall HTML page.
608 c['title'] = ini.get("general", "title")
609 c['titleURL'] = ini.get("general", "title_url")
611 # the 'buildbotURL' string should point to the location where the buildbot's
612 # internal web server (usually the html.WebStatus page) is visible. This
613 # typically uses the port number set in the Waterfall 'status' entry, but
614 # with an externally-visible host name which the buildbot cannot figure out
617 c['buildbotURL'] = ini.get("general", "buildbot_url")
622 # This specifies what database buildbot uses to store its state. You can leave
623 # this at its default for all but the largest installations.
624 'db_url' : "sqlite:///state.sqlite",