2 # ex: set syntax=python:
9 from buildbot import locks
11 # This is a sample buildmaster config file. It must be installed as
12 # 'master.cfg' in your buildmaster's base directory.
14 ini = ConfigParser.ConfigParser()
15 ini.read("./config.ini")
17 # This is the dictionary that the buildmaster pays attention to. We also use
18 # a shorter alias to save typing.
19 c = BuildmasterConfig = {}
23 # The 'slaves' list defines the set of recognized buildslaves. Each element is
24 # a BuildSlave object, specifying a unique slave name and password. The same
25 # slave name and password must be configured on the slave.
26 from buildbot.buildslave import BuildSlave
30 for section in ini.sections():
31 if section.startswith("slave "):
32 if ini.has_option(section, "name") and ini.has_option(section, "password"):
33 name = ini.get(section, "name")
34 password = ini.get(section, "password")
36 if ini.has_option(section, "builds"):
37 max_builds = ini.getint(section, "builds")
38 c['slaves'].append(BuildSlave(name, password, max_builds = max_builds))
40 # 'slavePortnum' defines the TCP port to listen on for connections from slaves.
41 # This must match the value configured into the buildslaves (with their
43 c['slavePortnum'] = 9989
46 c['mergeRequests'] = True
50 home_dir = os.path.abspath(ini.get("general", "homedir"))
52 repo_url = ini.get("repo", "url")
54 rsync_bin_url = ini.get("rsync", "binary_url")
55 rsync_bin_key = ini.get("rsync", "binary_password")
60 if ini.has_option("rsync", "source_url"):
61 rsync_src_url = ini.get("rsync", "source_url")
62 rsync_src_key = ini.get("rsync", "source_password")
65 gpg_comment = "Unattended build signature"
66 gpg_passfile = "/dev/null"
68 if ini.has_option("gpg", "keyid"):
69 gpg_keyid = ini.get("gpg", "keyid")
71 if ini.has_option("gpg", "comment"):
72 gpg_comment = ini.get("gpg", "comment")
74 if ini.has_option("gpg", "passfile"):
75 gpg_passfile = ini.get("gpg", "passfile")
81 findtargets = subprocess.Popen([home_dir+'/dumpinfo.pl', 'targets'],
82 stdout = subprocess.PIPE, cwd = home_dir+'/source.git')
85 line = findtargets.stdout.readline()
88 ta = line.strip().split(' ')
92 # the 'change_source' setting tells the buildmaster how it should find out
93 # about source code changes. Here we point to the buildbot clone of pyflakes.
95 from buildbot.changes.gitpoller import GitPoller
96 c['change_source'] = []
97 c['change_source'].append(GitPoller(
99 workdir=home_dir+'/source.git', branch='master',
104 # Configure the Schedulers, which decide how to react to incoming changes. In this
105 # case, just kick off a 'basebuild' build
107 from buildbot.schedulers.basic import SingleBranchScheduler
108 from buildbot.schedulers.forcesched import ForceScheduler
109 from buildbot.changes import filter
111 c['schedulers'].append(SingleBranchScheduler(
113 change_filter=filter.ChangeFilter(branch='master'),
115 builderNames=targets))
117 c['schedulers'].append(ForceScheduler(
119 builderNames=targets))
123 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
124 # what steps, and which slaves can execute them. Note that any particular build will
125 # only take place on one slave.
127 from buildbot.process.factory import BuildFactory
128 from buildbot.steps.source import Git
129 from buildbot.steps.shell import ShellCommand
130 from buildbot.steps.shell import SetProperty
131 from buildbot.steps.transfer import FileUpload
132 from buildbot.steps.transfer import FileDownload
133 from buildbot.steps.master import MasterShellCommand
134 from buildbot.process.properties import WithProperties
138 "^tools/": "tools/clean",
139 "^toolchain/": "toolchain/clean",
140 "^target/linux/": "target/linux/clean",
141 "^(config|include)/": "dirclean"
144 def IsAffected(pattern):
145 def CheckAffected(change):
146 for request in change.build.requests:
147 for source in request.sources:
148 for change in source.changes:
149 for file in change.files:
150 if re.match(pattern, file):
155 def isPathBuiltin(path):
158 conf = open(".config", "r")
161 line = conf.readline()
164 m = re.match("^(CONFIG_PACKAGE_.+?)=y", line)
166 incl[m.group(1)] = True
170 deps = open("tmp/.packagedeps", "r")
173 line = deps.readline()
176 m = re.match("^package-\$\((CONFIG_PACKAGE_.+?)\) \+= (\S+)", line)
177 if m and incl.get(m.group(1)) == True:
178 pkgs["package/%s" % m.group(2)] = True
183 if pkgs.get(path) == True:
185 path = os.path.dirname(path)
189 def isChangeBuiltin(change):
191 # for request in change.build.requests:
192 # for source in request.sources:
193 # for change in source.changes:
194 # for file in change.files:
195 # if isPathBuiltin(file):
202 dlLock = locks.SlaveLock("slave_dl")
204 checkBuiltin = re.sub('[\t\n ]+', ' ', """
206 local symbol op path file;
207 for file in $CHANGED_FILES; do
213 while read symbol op path; do
214 case "$symbol" in package-*)
215 symbol="${symbol##*(}";
216 symbol="${symbol%)}";
217 for file in $CHANGED_FILES; do
218 case "$file" in "package/$path/"*)
219 grep -qsx "$symbol=y" .config && return 0
223 done < tmp/.packagedeps;
229 class IfBuiltinShellCommand(ShellCommand):
230 def _quote(self, str):
231 if re.search("[^a-zA-Z0-9/_.-]", str):
232 return "'%s'" %(re.sub("'", "'\"'\"'", str))
235 def setCommand(self, command):
236 if not isinstance(command, (str, unicode)):
237 command = ' '.join(map(self._quote, command))
240 '%s; if checkBuiltin; then %s; else exit 0; fi' %(checkBuiltin, command)
243 def setupEnvironment(self, cmd):
244 slaveEnv = self.slaveEnvironment
248 for request in self.build.requests:
249 for source in request.sources:
250 for change in source.changes:
251 for file in change.files:
252 changedFiles[file] = True
253 fullSlaveEnv = slaveEnv.copy()
254 fullSlaveEnv['CHANGED_FILES'] = ' '.join(changedFiles.keys())
255 cmd.args['env'] = fullSlaveEnv
259 for slave in c['slaves']:
260 slaveNames.append(slave.slavename)
262 for target in targets:
263 ts = target.split('/')
265 factory = BuildFactory()
267 # find number of cores
268 factory.addStep(SetProperty(
271 description = "Finding number of CPUs",
272 command = ["nproc"]))
274 # check out the source
275 factory.addStep(Git(repourl=repo_url, mode='update'))
277 factory.addStep(ShellCommand(
279 description = "Remove tmp folder",
280 command=["rm", "-rf", "tmp/"]))
283 # factory.addStep(ShellCommand(
284 # name = "feedsconf",
285 # description = "Copy the feeds.conf",
286 # command='''cp ~/feeds.conf ./feeds.conf''' ))
289 factory.addStep(ShellCommand(
290 name = "rmfeedlinks",
291 description = "Remove feed symlinks",
292 command=["rm", "-rf", "package/feeds/"]))
295 factory.addStep(ShellCommand(
296 name = "updatefeeds",
297 description = "Updating feeds",
298 command=["./scripts/feeds", "update"]))
301 factory.addStep(ShellCommand(
302 name = "installfeeds",
303 description = "Installing feeds",
304 command=["./scripts/feeds", "install", "-a"]))
307 factory.addStep(ShellCommand(
309 description = "Seeding .config",
310 command='''cat <<EOT > .config
312 CONFIG_TARGET_%s_%s=y
313 CONFIG_ALL_NONSHARED=y
316 # CONFIG_IB_STANDALONE is not set
319 CONFIG_SIGNED_PACKAGES=y
320 # CONFIG_PER_FEED_REPO_ADD_COMMENTED is not set
321 CONFIG_KERNEL_KALLSYMS=y
322 CONFIG_COLLECT_KERNEL_DEBUG=y
323 CONFIG_TARGET_ALL_PROFILES=y
324 CONFIG_TARGET_MULTI_PROFILE=y
325 CONFIG_TARGET_PER_DEVICE_ROOTFS=y
326 EOT''' %(ts[0], ts[0], ts[1]) ))
328 factory.addStep(ShellCommand(
330 description = "Removing output directory",
331 command = ["rm", "-rf", "bin/"]
334 factory.addStep(ShellCommand(
336 description = "Populating .config",
337 command = ["make", "defconfig"]
341 factory.addStep(ShellCommand(
343 description = "Checking architecture",
344 command = ["grep", "-sq", "CONFIG_TARGET_%s=y" %(ts[0]), ".config"],
352 factory.addStep(SetProperty(
355 description = "Finding libc suffix",
356 command = ["sed", "-ne", '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }', ".config"]))
359 factory.addStep(FileDownload(mastersrc=home_dir+'/key-build', slavedest="key-build", mode=0600))
360 factory.addStep(FileDownload(mastersrc=home_dir+'/key-build.pub', slavedest="key-build.pub", mode=0600))
363 factory.addStep(ShellCommand(
365 description = "Preparing dl/",
366 command = "mkdir -p $HOME/dl && ln -sf $HOME/dl ./dl",
372 factory.addStep(ShellCommand(
374 description = "Populating dl/",
375 command = ["make", WithProperties("-j%(nproc:~4)s"), "download", "V=s"],
377 locks = [dlLock.access('exclusive')]
380 factory.addStep(ShellCommand(
382 description = "Cleaning base-files",
383 command=["make", "package/base-files/clean", "V=s"]
386 # optional clean steps
387 for pattern, maketarget in MakeTargetMap.items():
388 factory.addStep(ShellCommand(
390 description = maketarget,
391 command=["make", maketarget, "V=s"], doStepIf=IsAffected(pattern)
395 factory.addStep(ShellCommand(
397 description = "Building tools",
398 command = ["make", WithProperties("-j%(nproc:~4)s"), "tools/install", "V=s"],
402 factory.addStep(ShellCommand(
404 description = "Building toolchain",
405 command=["make", WithProperties("-j%(nproc:~4)s"), "toolchain/install", "V=s"],
409 factory.addStep(ShellCommand(
411 description = "Building kmods",
412 command=["make", WithProperties("-j%(nproc:~4)s"), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
413 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
417 factory.addStep(ShellCommand(
419 description = "Building packages",
420 command=["make", WithProperties("-j%(nproc:~4)s"), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
421 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
425 # factory.addStep(IfBuiltinShellCommand(
426 factory.addStep(ShellCommand(
428 description = "Installing packages",
429 command=["make", WithProperties("-j%(nproc:~4)s"), "package/install", "V=s"],
430 doStepIf = isChangeBuiltin,
434 factory.addStep(ShellCommand(
436 description = "Indexing packages",
437 command=["make", WithProperties("-j%(nproc:~4)s"), "package/index", "V=s"],
441 #factory.addStep(IfBuiltinShellCommand(
442 factory.addStep(ShellCommand(
444 description = "Building images",
445 command=["make", "-j1", "target/install", "V=s"],
446 doStepIf = isChangeBuiltin,
450 factory.addStep(ShellCommand(
452 description = "Calculating checksums",
453 command=["make", "-j1", "checksum", "V=s"],
454 doStepIf = isChangeBuiltin,
459 if gpg_keyid is not None:
460 factory.addStep(MasterShellCommand(
461 name = "signprepare",
462 description = "Preparing temporary signing directory",
463 command = ["mkdir", "-p", "%s/signing" %(home_dir)],
467 factory.addStep(ShellCommand(
469 description = "Packing files to sign",
470 command = WithProperties("find bin/targets/%s/%s%%(libc)s/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz" %(ts[0], ts[1])),
474 factory.addStep(FileUpload(
475 slavesrc = "sign.tar.gz",
476 masterdest = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
480 factory.addStep(MasterShellCommand(
482 description = "Signing files",
483 command = ["%s/signall.sh" %(home_dir), "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]), gpg_keyid, gpg_passfile, gpg_comment],
487 factory.addStep(FileDownload(
488 mastersrc = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
489 slavedest = "sign.tar.gz",
493 factory.addStep(ShellCommand(
495 description = "Unpacking signed files",
496 command = ["tar", "-xzf", "sign.tar.gz"],
501 factory.addStep(ShellCommand(
502 name = "uploadprepare",
503 description = "Preparing target directory",
504 command=["rsync", "-av", "--include", "/%s/" %(ts[0]), "--include", "/%s/%s/" %(ts[0], ts[1]), "--exclude", "/*", "--exclude", "/*/*", "--exclude", "/%s/%s/*" %(ts[0], ts[1]), "bin/targets/", "%s/targets/" %(rsync_bin_url)],
505 env={'RSYNC_PASSWORD': rsync_bin_key},
506 haltOnFailure = True,
510 factory.addStep(ShellCommand(
511 name = "targetupload",
512 description = "Uploading target files",
513 command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", WithProperties("bin/targets/%s/%s%%(libc)s/" %(ts[0], ts[1])), "%s/targets/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
514 env={'RSYNC_PASSWORD': rsync_bin_key},
515 haltOnFailure = True,
519 if rsync_src_url is not None:
520 factory.addStep(ShellCommand(
521 name = "sourceupload",
522 description = "Uploading source archives",
523 command=["rsync", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "dl/", "%s/" %(rsync_src_url)],
524 env={'RSYNC_PASSWORD': rsync_src_key},
525 haltOnFailure = True,
530 factory.addStep(ShellCommand(
531 name = "packageupload",
532 description = "Uploading package files",
533 command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "bin/packages/", "%s/packages/" %(rsync_bin_url)],
534 env={'RSYNC_PASSWORD': rsync_bin_key},
535 haltOnFailure = False,
541 factory.addStep(ShellCommand(
543 description = "Uploading logs",
544 command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "logs/", "%s/logs/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
545 env={'RSYNC_PASSWORD': rsync_bin_key},
546 haltOnFailure = False,
551 from buildbot.config import BuilderConfig
553 c['builders'].append(BuilderConfig(name=target, slavenames=slaveNames, factory=factory))
556 ####### STATUS TARGETS
558 # 'status' is a list of Status Targets. The results of each build will be
559 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
560 # including web pages, email senders, and IRC bots.
564 from buildbot.status import html
565 from buildbot.status.web import authz, auth
567 if ini.has_option("status", "bind"):
568 if ini.has_option("status", "user") and ini.has_option("status", "password"):
569 authz_cfg=authz.Authz(
570 # change any of these to True to enable; see the manual for more
572 auth=auth.BasicAuth([(ini.get("status", "user"), ini.get("status", "password"))]),
573 gracefulShutdown = 'auth',
574 forceBuild = 'auth', # use this to test your slave once it is set up
575 forceAllBuilds = 'auth',
578 stopAllBuilds = 'auth',
579 cancelPendingBuild = 'auth',
581 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind"), authz=authz_cfg))
583 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind")))
586 from buildbot.status import words
588 if ini.has_option("irc", "host") and ini.has_option("irc", "nickname") and ini.has_option("irc", "channel"):
589 irc_host = ini.get("irc", "host")
591 irc_chan = ini.get("irc", "channel")
592 irc_nick = ini.get("irc", "nickname")
595 if ini.has_option("irc", "port"):
596 irc_port = ini.getint("irc", "port")
598 if ini.has_option("irc", "password"):
599 irc_pass = ini.get("irc", "password")
601 irc = words.IRC(irc_host, irc_nick, port = irc_port, password = irc_pass,
602 channels = [{ "channel": irc_chan }],
605 'successToFailure': 1,
606 'failureToSuccess': 1
610 c['status'].append(irc)
613 ####### PROJECT IDENTITY
615 # the 'title' string will appear at the top of this buildbot
616 # installation's html.WebStatus home page (linked to the
617 # 'titleURL') and is embedded in the title of the waterfall HTML page.
619 c['title'] = ini.get("general", "title")
620 c['titleURL'] = ini.get("general", "title_url")
622 # the 'buildbotURL' string should point to the location where the buildbot's
623 # internal web server (usually the html.WebStatus page) is visible. This
624 # typically uses the port number set in the Waterfall 'status' entry, but
625 # with an externally-visible host name which the buildbot cannot figure out
628 c['buildbotURL'] = ini.get("general", "buildbot_url")
633 # This specifies what database buildbot uses to store its state. You can leave
634 # this at its default for all but the largest installations.
635 'db_url' : "sqlite:///state.sqlite",