2 # ex: set syntax=python:
9 from buildbot import locks
11 # This is a sample buildmaster config file. It must be installed as
12 # 'master.cfg' in your buildmaster's base directory.
14 ini = ConfigParser.ConfigParser()
15 ini.read("./config.ini")
17 # This is the dictionary that the buildmaster pays attention to. We also use
18 # a shorter alias to save typing.
19 c = BuildmasterConfig = {}
23 # The 'slaves' list defines the set of recognized buildslaves. Each element is
24 # a BuildSlave object, specifying a unique slave name and password. The same
25 # slave name and password must be configured on the slave.
26 from buildbot.buildslave import BuildSlave
30 if ini.has_option("general", "port"):
31 slave_port = ini.getint("general", "port")
35 for section in ini.sections():
36 if section.startswith("slave "):
37 if ini.has_option(section, "name") and ini.has_option(section, "password"):
38 name = ini.get(section, "name")
39 password = ini.get(section, "password")
41 if ini.has_option(section, "builds"):
42 max_builds = ini.getint(section, "builds")
43 c['slaves'].append(BuildSlave(name, password, max_builds = max_builds))
45 # 'slavePortnum' defines the TCP port to listen on for connections from slaves.
46 # This must match the value configured into the buildslaves (with their
48 c['slavePortnum'] = slave_port
51 c['mergeRequests'] = True
55 home_dir = os.path.abspath(ini.get("general", "homedir"))
58 if ini.has_option("general", "expire"):
59 tree_expire = ini.getint("general", "expire")
61 repo_url = ini.get("repo", "url")
62 repo_branch = "master"
64 if ini.has_option("repo", "branch"):
65 repo_branch = ini.get("repo", "branch")
67 rsync_bin_url = ini.get("rsync", "binary_url")
68 rsync_bin_key = ini.get("rsync", "binary_password")
73 if ini.has_option("rsync", "source_url"):
74 rsync_src_url = ini.get("rsync", "source_url")
75 rsync_src_key = ini.get("rsync", "source_password")
79 gpg_comment = "Unattended build signature"
80 gpg_passfile = "/dev/null"
82 if ini.has_option("gpg", "home"):
83 gpg_home = ini.get("gpg", "home")
85 if ini.has_option("gpg", "keyid"):
86 gpg_keyid = ini.get("gpg", "keyid")
88 if ini.has_option("gpg", "comment"):
89 gpg_comment = ini.get("gpg", "comment")
91 if ini.has_option("gpg", "passfile"):
92 gpg_passfile = ini.get("gpg", "passfile")
98 if not os.path.isdir(home_dir+'/source.git'):
99 subprocess.call(["git", "clone", "--depth=1", "--branch="+repo_branch, repo_url, home_dir+'/source.git'])
101 subprocess.call(["git", "pull"], cwd = home_dir+'/source.git')
103 findtargets = subprocess.Popen([home_dir+'/dumpinfo.pl', 'targets'],
104 stdout = subprocess.PIPE, cwd = home_dir+'/source.git')
107 line = findtargets.stdout.readline()
110 ta = line.strip().split(' ')
111 targets.append(ta[0])
114 # the 'change_source' setting tells the buildmaster how it should find out
115 # about source code changes. Here we point to the buildbot clone of pyflakes.
117 from buildbot.changes.gitpoller import GitPoller
118 c['change_source'] = []
119 c['change_source'].append(GitPoller(
121 workdir=home_dir+'/work.git', branch='master',
126 # Configure the Schedulers, which decide how to react to incoming changes. In this
127 # case, just kick off a 'basebuild' build
129 from buildbot.schedulers.basic import SingleBranchScheduler
130 from buildbot.schedulers.forcesched import ForceScheduler
131 from buildbot.changes import filter
133 c['schedulers'].append(SingleBranchScheduler(
135 change_filter=filter.ChangeFilter(branch='master'),
137 builderNames=targets))
139 c['schedulers'].append(ForceScheduler(
141 builderNames=targets))
145 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
146 # what steps, and which slaves can execute them. Note that any particular build will
147 # only take place on one slave.
149 from buildbot.process.factory import BuildFactory
150 from buildbot.steps.source.git import Git
151 from buildbot.steps.shell import ShellCommand
152 from buildbot.steps.shell import SetProperty
153 from buildbot.steps.transfer import FileUpload
154 from buildbot.steps.transfer import FileDownload
155 from buildbot.steps.master import MasterShellCommand
156 from buildbot.process.properties import WithProperties
160 [ "tools", "tools/clean" ],
161 [ "chain", "toolchain/clean" ],
162 [ "linux", "target/linux/clean" ],
163 [ "dir", "dirclean" ],
164 [ "dist", "distclean" ]
167 def IsCleanRequested(pattern):
168 def CheckCleanProperty(step):
169 val = step.getProperty("clean")
170 if val and re.match(pattern, val):
175 return CheckCleanProperty
177 def IsTaggingRequested(step):
178 val = step.getProperty("tag")
179 if val and re.match("^[0-9]+\.[0-9]+\.[0-9]+$", val):
184 def IsNoTaggingRequested(step):
185 return not IsTaggingRequested(step)
187 def IsNoMasterBuild(step):
188 return repo_branch != "master"
190 def GetBaseVersion(props):
191 if re.match("^[^-]+-[0-9]+\.[0-9]+$", repo_branch):
192 return repo_branch.split('-')[1]
196 def GetVersionPrefix(props):
197 basever = GetBaseVersion(props)
198 if props.hasProperty("tag") and re.match("^[0-9]+\.[0-9]+\.[0-9]+$", props["tag"]):
199 return "%s/" % props["tag"]
200 elif basever != "master":
201 return "%s-HEAD/" % basever
208 dlLock = locks.SlaveLock("slave_dl")
210 checkBuiltin = re.sub('[\t\n ]+', ' ', """
212 local symbol op path file;
213 for file in $CHANGED_FILES; do
219 while read symbol op path; do
220 case "$symbol" in package-*)
221 symbol="${symbol##*(}";
222 symbol="${symbol%)}";
223 for file in $CHANGED_FILES; do
224 case "$file" in "package/$path/"*)
225 grep -qsx "$symbol=y" .config && return 0
229 done < tmp/.packagedeps;
235 class IfBuiltinShellCommand(ShellCommand):
236 def _quote(self, str):
237 if re.search("[^a-zA-Z0-9/_.-]", str):
238 return "'%s'" %(re.sub("'", "'\"'\"'", str))
241 def setCommand(self, command):
242 if not isinstance(command, (str, unicode)):
243 command = ' '.join(map(self._quote, command))
246 '%s; if checkBuiltin; then %s; else exit 0; fi' %(checkBuiltin, command)
249 def setupEnvironment(self, cmd):
250 slaveEnv = self.slaveEnvironment
254 for request in self.build.requests:
255 for source in request.sources:
256 for change in source.changes:
257 for file in change.files:
258 changedFiles[file] = True
259 fullSlaveEnv = slaveEnv.copy()
260 fullSlaveEnv['CHANGED_FILES'] = ' '.join(changedFiles.keys())
261 cmd.args['env'] = fullSlaveEnv
265 for slave in c['slaves']:
266 slaveNames.append(slave.slavename)
268 for target in targets:
269 ts = target.split('/')
271 factory = BuildFactory()
273 # find number of cores
274 factory.addStep(SetProperty(
277 description = "Finding number of CPUs",
278 command = ["nproc"]))
280 # expire tree if needed
282 factory.addStep(FileDownload(
283 mastersrc = "expire.sh",
284 slavedest = "../expire.sh",
287 factory.addStep(ShellCommand(
289 description = "Checking for build tree expiry",
290 command = ["./expire.sh", str(tree_expire)],
292 haltOnFailure = True,
295 # user-requested clean targets
296 for tuple in CleanTargetMap:
297 factory.addStep(ShellCommand(
299 description = 'User-requested "make %s"' % tuple[1],
300 command = ["make", tuple[1], "V=s"],
301 doStepIf = IsCleanRequested(tuple[0])
304 factory.addStep(MasterShellCommand(
306 description = "Tagging Git repository",
307 command = [home_dir+'/maketag.sh', '-i', '-k', str(gpg_keyid or ''),
308 '-p', str(gpg_passfile or ''), '-v', WithProperties("%(tag:-)s")],
309 path = home_dir+'/source.git',
310 env = {'GNUPGHOME': gpg_home},
311 haltOnFailure = True,
312 doStepIf = IsTaggingRequested
316 factory.addStep(ShellCommand(
317 name = "switchbranch",
318 description = "Checking out Git branch",
319 command = "if [ -d .git ]; then git checkout '%s'; else exit 0; fi" % repo_branch,
320 haltOnFailure = True,
321 doStepIf = IsNoTaggingRequested
324 # check out the source
327 branch = repo_branch,
328 mode = 'incremental',
332 factory.addStep(ShellCommand(
334 description = "Fetching Git tags",
335 command = ["git", "fetch", "--tags", "--", repo_url],
336 haltOnFailure = True,
337 doStepIf = IsTaggingRequested
341 factory.addStep(ShellCommand(
343 description = "Checking out Git tag",
344 command = ["git", "checkout", WithProperties("tags/v%(tag:-)s")],
345 haltOnFailure = True,
346 doStepIf = IsTaggingRequested
349 factory.addStep(ShellCommand(
351 description = "Remove tmp folder",
352 command=["rm", "-rf", "tmp/"]))
355 # factory.addStep(ShellCommand(
356 # name = "feedsconf",
357 # description = "Copy the feeds.conf",
358 # command='''cp ~/feeds.conf ./feeds.conf''' ))
361 factory.addStep(ShellCommand(
362 name = "rmfeedlinks",
363 description = "Remove feed symlinks",
364 command=["rm", "-rf", "package/feeds/"]))
367 factory.addStep(ShellCommand(
368 name = "updatefeeds",
369 description = "Updating feeds",
370 command=["./scripts/feeds", "update"]))
373 factory.addStep(ShellCommand(
374 name = "installfeeds",
375 description = "Installing feeds",
376 command=["./scripts/feeds", "install", "-a"]))
379 factory.addStep(FileDownload(
380 mastersrc = "config.seed",
381 slavedest = ".config",
386 factory.addStep(ShellCommand(
388 description = "Seeding .config",
389 command = "printf 'CONFIG_TARGET_%s=y\\nCONFIG_TARGET_%s_%s=y\\n' >> .config" %(ts[0], ts[0], ts[1])
392 factory.addStep(ShellCommand(
394 description = "Removing output directory",
395 command = ["rm", "-rf", "bin/"]
398 factory.addStep(ShellCommand(
400 description = "Populating .config",
401 command = ["make", "defconfig"]
405 factory.addStep(ShellCommand(
407 description = "Checking architecture",
408 command = ["grep", "-sq", "CONFIG_TARGET_%s=y" %(ts[0]), ".config"],
416 factory.addStep(SetProperty(
419 description = "Finding libc suffix",
420 command = ["sed", "-ne", '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }', ".config"]))
423 factory.addStep(FileDownload(mastersrc=home_dir+'/key-build', slavedest="key-build", mode=0600))
424 factory.addStep(FileDownload(mastersrc=home_dir+'/key-build.pub', slavedest="key-build.pub", mode=0600))
427 factory.addStep(ShellCommand(
429 description = "Preparing dl/",
430 command = "mkdir -p $HOME/dl && rm -rf ./dl && ln -sf $HOME/dl ./dl",
436 factory.addStep(ShellCommand(
438 description = "Building GNU tar",
439 command = ["make", WithProperties("-j%(nproc:~4)s"), "tools/tar/install", "V=s"],
444 factory.addStep(ShellCommand(
446 description = "Populating dl/",
447 command = ["make", WithProperties("-j%(nproc:~4)s"), "download", "V=s"],
449 locks = [dlLock.access('exclusive')]
452 factory.addStep(ShellCommand(
454 description = "Cleaning base-files",
455 command=["make", "package/base-files/clean", "V=s"]
459 factory.addStep(ShellCommand(
461 description = "Building tools",
462 command = ["make", WithProperties("-j%(nproc:~4)s"), "tools/install", "V=s"],
466 factory.addStep(ShellCommand(
468 description = "Building toolchain",
469 command=["make", WithProperties("-j%(nproc:~4)s"), "toolchain/install", "V=s"],
473 factory.addStep(ShellCommand(
475 description = "Building kmods",
476 command=["make", WithProperties("-j%(nproc:~4)s"), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
477 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
481 factory.addStep(ShellCommand(
483 description = "Building packages",
484 command=["make", WithProperties("-j%(nproc:~4)s"), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
485 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
489 # factory.addStep(IfBuiltinShellCommand(
490 factory.addStep(ShellCommand(
492 description = "Installing packages",
493 command=["make", WithProperties("-j%(nproc:~4)s"), "package/install", "V=s"],
497 factory.addStep(ShellCommand(
499 description = "Indexing packages",
500 command=["make", WithProperties("-j%(nproc:~4)s"), "package/index", "V=s"],
504 #factory.addStep(IfBuiltinShellCommand(
505 factory.addStep(ShellCommand(
507 description = "Building images",
508 command=["make", WithProperties("-j%(nproc:~4)s"), "target/install", "V=s"],
512 factory.addStep(ShellCommand(
514 description = "Calculating checksums",
515 command=["make", "-j1", "checksum", "V=s"],
520 if gpg_keyid is not None:
521 factory.addStep(MasterShellCommand(
522 name = "signprepare",
523 description = "Preparing temporary signing directory",
524 command = ["mkdir", "-p", "%s/signing" %(home_dir)],
528 factory.addStep(ShellCommand(
530 description = "Packing files to sign",
531 command = WithProperties("find bin/targets/%s/%s%%(libc)s/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz" %(ts[0], ts[1])),
535 factory.addStep(FileUpload(
536 slavesrc = "sign.tar.gz",
537 masterdest = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
541 factory.addStep(MasterShellCommand(
543 description = "Signing files",
544 command = ["%s/signall.sh" %(home_dir), "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]), gpg_keyid, gpg_comment],
545 env = {'GNUPGHOME': gpg_home, 'PASSFILE': gpg_passfile},
549 factory.addStep(FileDownload(
550 mastersrc = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
551 slavedest = "sign.tar.gz",
555 factory.addStep(ShellCommand(
557 description = "Unpacking signed files",
558 command = ["tar", "-xzf", "sign.tar.gz"],
563 factory.addStep(ShellCommand(
565 description = "Preparing upload directory structure",
566 command = ["mkdir", "-p", WithProperties("tmp/upload/%%(prefix)stargets/%s/%s" %(ts[0], ts[1]), prefix=GetVersionPrefix)],
570 factory.addStep(ShellCommand(
571 name = "linkprepare",
572 description = "Preparing repository symlink",
573 command = ["ln", "-s", "-f", WithProperties("../packages-%(basever)s", basever=GetBaseVersion), WithProperties("tmp/upload/%(prefix)spackages", prefix=GetVersionPrefix)],
574 doStepIf = IsNoMasterBuild,
578 factory.addStep(ShellCommand(
580 description = "Uploading directory structure",
581 command = ["rsync", "-avz", "tmp/upload/", "%s/" %(rsync_bin_url)],
582 env={'RSYNC_PASSWORD': rsync_bin_key},
583 haltOnFailure = True,
587 factory.addStep(ShellCommand(
588 name = "targetupload",
589 description = "Uploading target files",
590 command=["rsync", "--delete", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]),
591 "-avz", WithProperties("bin/targets/%s/%s%%(libc)s/" %(ts[0], ts[1])),
592 WithProperties("%s/%%(prefix)stargets/%s/%s/" %(rsync_bin_url, ts[0], ts[1]), prefix=GetVersionPrefix)],
593 env={'RSYNC_PASSWORD': rsync_bin_key},
594 haltOnFailure = True,
598 if rsync_src_url is not None:
599 factory.addStep(ShellCommand(
600 name = "sourceupload",
601 description = "Uploading source archives",
602 command=["rsync", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "dl/", "%s/" %(rsync_src_url)],
603 env={'RSYNC_PASSWORD': rsync_src_key},
604 haltOnFailure = True,
609 factory.addStep(ShellCommand(
610 name = "packageupload",
611 description = "Uploading package files",
612 command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "bin/packages/", "%s/packages/" %(rsync_bin_url)],
613 env={'RSYNC_PASSWORD': rsync_bin_key},
614 haltOnFailure = False,
620 factory.addStep(ShellCommand(
622 description = "Uploading logs",
623 command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "logs/", "%s/logs/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
624 env={'RSYNC_PASSWORD': rsync_bin_key},
625 haltOnFailure = False,
630 from buildbot.config import BuilderConfig
632 c['builders'].append(BuilderConfig(name=target, slavenames=slaveNames, factory=factory))
635 ####### STATUS TARGETS
637 # 'status' is a list of Status Targets. The results of each build will be
638 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
639 # including web pages, email senders, and IRC bots.
643 from buildbot.status import html
644 from buildbot.status.web import authz, auth
646 if ini.has_option("status", "bind"):
647 if ini.has_option("status", "user") and ini.has_option("status", "password"):
648 authz_cfg=authz.Authz(
649 # change any of these to True to enable; see the manual for more
651 auth=auth.BasicAuth([(ini.get("status", "user"), ini.get("status", "password"))]),
652 gracefulShutdown = 'auth',
653 forceBuild = 'auth', # use this to test your slave once it is set up
654 forceAllBuilds = 'auth',
657 stopAllBuilds = 'auth',
658 cancelPendingBuild = 'auth',
660 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind"), authz=authz_cfg))
662 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind")))
665 from buildbot.status import words
667 if ini.has_option("irc", "host") and ini.has_option("irc", "nickname") and ini.has_option("irc", "channel"):
668 irc_host = ini.get("irc", "host")
670 irc_chan = ini.get("irc", "channel")
671 irc_nick = ini.get("irc", "nickname")
674 if ini.has_option("irc", "port"):
675 irc_port = ini.getint("irc", "port")
677 if ini.has_option("irc", "password"):
678 irc_pass = ini.get("irc", "password")
680 irc = words.IRC(irc_host, irc_nick, port = irc_port, password = irc_pass,
681 channels = [{ "channel": irc_chan }],
684 'successToFailure': 1,
685 'failureToSuccess': 1
689 c['status'].append(irc)
692 ####### PROJECT IDENTITY
694 # the 'title' string will appear at the top of this buildbot
695 # installation's html.WebStatus home page (linked to the
696 # 'titleURL') and is embedded in the title of the waterfall HTML page.
698 c['title'] = ini.get("general", "title")
699 c['titleURL'] = ini.get("general", "title_url")
701 # the 'buildbotURL' string should point to the location where the buildbot's
702 # internal web server (usually the html.WebStatus page) is visible. This
703 # typically uses the port number set in the Waterfall 'status' entry, but
704 # with an externally-visible host name which the buildbot cannot figure out
707 c['buildbotURL'] = ini.get("general", "buildbot_url")
712 # This specifies what database buildbot uses to store its state. You can leave
713 # this at its default for all but the largest installations.
714 'db_url' : "sqlite:///state.sqlite",