2e7c195ff8bf0add79dc3d8d0f67fe9c53364c20
[buildbot.git] / phase1 / master.cfg
1 # -*- python -*-
2 # ex: set syntax=python:
3
4 import os
5 import re
6 import subprocess
7 import ConfigParser
8
9 from buildbot import locks
10
11 # This is a sample buildmaster config file. It must be installed as
12 # 'master.cfg' in your buildmaster's base directory.
13
14 ini = ConfigParser.ConfigParser()
15 ini.read("./config.ini")
16
17 # This is the dictionary that the buildmaster pays attention to. We also use
18 # a shorter alias to save typing.
19 c = BuildmasterConfig = {}
20
21 ####### BUILDSLAVES
22
23 # The 'slaves' list defines the set of recognized buildslaves. Each element is
24 # a BuildSlave object, specifying a unique slave name and password. The same
25 # slave name and password must be configured on the slave.
26 from buildbot.buildslave import BuildSlave
27
28 slave_port = 9989
29
30 if ini.has_option("general", "port"):
31 slave_port = ini.getint("general", "port")
32
33 c['slaves'] = []
34
35 for section in ini.sections():
36 if section.startswith("slave "):
37 if ini.has_option(section, "name") and ini.has_option(section, "password"):
38 name = ini.get(section, "name")
39 password = ini.get(section, "password")
40 max_builds = 1
41 if ini.has_option(section, "builds"):
42 max_builds = ini.getint(section, "builds")
43 c['slaves'].append(BuildSlave(name, password, max_builds = max_builds))
44
45 # 'slavePortnum' defines the TCP port to listen on for connections from slaves.
46 # This must match the value configured into the buildslaves (with their
47 # --master option)
48 c['slavePortnum'] = slave_port
49
50 # coalesce builds
51 c['mergeRequests'] = True
52
53 ####### CHANGESOURCES
54
55 home_dir = os.path.abspath(ini.get("general", "homedir"))
56 tree_expire = 0
57
58 if ini.has_option("general", "expire"):
59 tree_expire = ini.getint("general", "expire")
60
61 repo_url = ini.get("repo", "url")
62 repo_branch = "master"
63
64 if ini.has_option("repo", "branch"):
65 repo_branch = ini.get("repo", "branch")
66
67 rsync_bin_url = ini.get("rsync", "binary_url")
68 rsync_bin_key = ini.get("rsync", "binary_password")
69
70 rsync_src_url = None
71 rsync_src_key = None
72
73 if ini.has_option("rsync", "source_url"):
74 rsync_src_url = ini.get("rsync", "source_url")
75 rsync_src_key = ini.get("rsync", "source_password")
76
77 gpg_home = "~/.gnupg"
78 gpg_keyid = None
79 gpg_comment = "Unattended build signature"
80 gpg_passfile = "/dev/null"
81
82 if ini.has_option("gpg", "home"):
83 gpg_home = ini.get("gpg", "home")
84
85 if ini.has_option("gpg", "keyid"):
86 gpg_keyid = ini.get("gpg", "keyid")
87
88 if ini.has_option("gpg", "comment"):
89 gpg_comment = ini.get("gpg", "comment")
90
91 if ini.has_option("gpg", "passfile"):
92 gpg_passfile = ini.get("gpg", "passfile")
93
94
95 # find targets
96 targets = [ ]
97
98 if not os.path.isdir(home_dir+'/source.git'):
99 subprocess.call(["git", "clone", "--depth=1", repo_url, home_dir+'/source.git'])
100 subprocess.call(["git", "checkout", repo_branch], cwd = home_dir+'/source.git')
101 else:
102 subprocess.call(["git", "checkout", repo_branch], cwd = home_dir+'/source.git')
103 subprocess.call(["git", "pull"], cwd = home_dir+'/source.git')
104
105 findtargets = subprocess.Popen([home_dir+'/dumpinfo.pl', 'targets'],
106 stdout = subprocess.PIPE, cwd = home_dir+'/source.git')
107
108 while True:
109 line = findtargets.stdout.readline()
110 if not line:
111 break
112 ta = line.strip().split(' ')
113 targets.append(ta[0])
114
115
116 # the 'change_source' setting tells the buildmaster how it should find out
117 # about source code changes. Here we point to the buildbot clone of pyflakes.
118
119 from buildbot.changes.gitpoller import GitPoller
120 c['change_source'] = []
121 c['change_source'].append(GitPoller(
122 repo_url,
123 workdir=home_dir+'/work.git', branch='master',
124 pollinterval=300))
125
126 ####### SCHEDULERS
127
128 # Configure the Schedulers, which decide how to react to incoming changes. In this
129 # case, just kick off a 'basebuild' build
130
131 from buildbot.schedulers.basic import SingleBranchScheduler
132 from buildbot.schedulers.forcesched import ForceScheduler
133 from buildbot.changes import filter
134 c['schedulers'] = []
135 c['schedulers'].append(SingleBranchScheduler(
136 name="all",
137 change_filter=filter.ChangeFilter(branch='master'),
138 treeStableTimer=60,
139 builderNames=targets))
140
141 c['schedulers'].append(ForceScheduler(
142 name="force",
143 builderNames=targets))
144
145 ####### BUILDERS
146
147 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
148 # what steps, and which slaves can execute them. Note that any particular build will
149 # only take place on one slave.
150
151 from buildbot.process.factory import BuildFactory
152 from buildbot.steps.source import Git
153 from buildbot.steps.shell import ShellCommand
154 from buildbot.steps.shell import SetProperty
155 from buildbot.steps.transfer import FileUpload
156 from buildbot.steps.transfer import FileDownload
157 from buildbot.steps.master import MasterShellCommand
158 from buildbot.process.properties import WithProperties
159
160
161 CleanTargetMap = [
162 [ "tools", "tools/clean" ],
163 [ "chain", "toolchain/clean" ],
164 [ "linux", "target/linux/clean" ],
165 [ "dir", "dirclean" ],
166 [ "dist", "distclean" ]
167 ]
168
169 def IsCleanRequested(pattern):
170 def CheckCleanProperty(step):
171 val = step.getProperty("clean")
172 if val and re.match(pattern, val):
173 return True
174 else:
175 return False
176
177 return CheckCleanProperty
178
179
180 c['builders'] = []
181
182 dlLock = locks.SlaveLock("slave_dl")
183
184 checkBuiltin = re.sub('[\t\n ]+', ' ', """
185 checkBuiltin() {
186 local symbol op path file;
187 for file in $CHANGED_FILES; do
188 case "$file" in
189 package/*/*) : ;;
190 *) return 0 ;;
191 esac;
192 done;
193 while read symbol op path; do
194 case "$symbol" in package-*)
195 symbol="${symbol##*(}";
196 symbol="${symbol%)}";
197 for file in $CHANGED_FILES; do
198 case "$file" in "package/$path/"*)
199 grep -qsx "$symbol=y" .config && return 0
200 ;; esac;
201 done;
202 esac;
203 done < tmp/.packagedeps;
204 return 1;
205 }
206 """).strip()
207
208
209 class IfBuiltinShellCommand(ShellCommand):
210 def _quote(self, str):
211 if re.search("[^a-zA-Z0-9/_.-]", str):
212 return "'%s'" %(re.sub("'", "'\"'\"'", str))
213 return str
214
215 def setCommand(self, command):
216 if not isinstance(command, (str, unicode)):
217 command = ' '.join(map(self._quote, command))
218 self.command = [
219 '/bin/sh', '-c',
220 '%s; if checkBuiltin; then %s; else exit 0; fi' %(checkBuiltin, command)
221 ]
222
223 def setupEnvironment(self, cmd):
224 slaveEnv = self.slaveEnvironment
225 if slaveEnv is None:
226 slaveEnv = { }
227 changedFiles = { }
228 for request in self.build.requests:
229 for source in request.sources:
230 for change in source.changes:
231 for file in change.files:
232 changedFiles[file] = True
233 fullSlaveEnv = slaveEnv.copy()
234 fullSlaveEnv['CHANGED_FILES'] = ' '.join(changedFiles.keys())
235 cmd.args['env'] = fullSlaveEnv
236
237 slaveNames = [ ]
238
239 for slave in c['slaves']:
240 slaveNames.append(slave.slavename)
241
242 for target in targets:
243 ts = target.split('/')
244
245 factory = BuildFactory()
246
247 # find number of cores
248 factory.addStep(SetProperty(
249 name = "nproc",
250 property = "nproc",
251 description = "Finding number of CPUs",
252 command = ["nproc"]))
253
254 # expire tree if needed
255 if tree_expire > 0:
256 factory.addStep(FileDownload(
257 mastersrc = "expire.sh",
258 slavedest = "../expire.sh",
259 mode = 0755))
260
261 factory.addStep(ShellCommand(
262 name = "expire",
263 description = "Checking for build tree expiry",
264 command = ["./expire.sh", str(tree_expire)],
265 workdir = ".",
266 haltOnFailure = True,
267 timeout = 2400))
268
269 # user-requested clean targets
270 for tuple in CleanTargetMap:
271 factory.addStep(ShellCommand(
272 name = tuple[1],
273 description = 'User-requested "make %s"' % tuple[1],
274 command = ["make", tuple[1], "V=s"],
275 doStepIf = IsCleanRequested(tuple[0])
276 ))
277
278 # check out the source
279 factory.addStep(Git(repourl=repo_url, branch=repo_branch, mode='update'))
280
281 factory.addStep(ShellCommand(
282 name = "rmtmp",
283 description = "Remove tmp folder",
284 command=["rm", "-rf", "tmp/"]))
285
286 # feed
287 # factory.addStep(ShellCommand(
288 # name = "feedsconf",
289 # description = "Copy the feeds.conf",
290 # command='''cp ~/feeds.conf ./feeds.conf''' ))
291
292 # feed
293 factory.addStep(ShellCommand(
294 name = "rmfeedlinks",
295 description = "Remove feed symlinks",
296 command=["rm", "-rf", "package/feeds/"]))
297
298 # feed
299 factory.addStep(ShellCommand(
300 name = "updatefeeds",
301 description = "Updating feeds",
302 command=["./scripts/feeds", "update"]))
303
304 # feed
305 factory.addStep(ShellCommand(
306 name = "installfeeds",
307 description = "Installing feeds",
308 command=["./scripts/feeds", "install", "-a"]))
309
310 # seed config
311 factory.addStep(FileDownload(
312 mastersrc = "config.seed",
313 slavedest = ".config",
314 mode = 0644
315 ))
316
317 # configure
318 factory.addStep(ShellCommand(
319 name = "newconfig",
320 description = "Seeding .config",
321 command = "printf 'CONFIG_TARGET_%s=y\\nCONFIG_TARGET_%s_%s=y\\n' >> .config" %(ts[0], ts[0], ts[1])
322 ))
323
324 factory.addStep(ShellCommand(
325 name = "delbin",
326 description = "Removing output directory",
327 command = ["rm", "-rf", "bin/"]
328 ))
329
330 factory.addStep(ShellCommand(
331 name = "defconfig",
332 description = "Populating .config",
333 command = ["make", "defconfig"]
334 ))
335
336 # check arch
337 factory.addStep(ShellCommand(
338 name = "checkarch",
339 description = "Checking architecture",
340 command = ["grep", "-sq", "CONFIG_TARGET_%s=y" %(ts[0]), ".config"],
341 logEnviron = False,
342 want_stdout = False,
343 want_stderr = False,
344 haltOnFailure = True
345 ))
346
347 # find libc suffix
348 factory.addStep(SetProperty(
349 name = "libc",
350 property = "libc",
351 description = "Finding libc suffix",
352 command = ["sed", "-ne", '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }', ".config"]))
353
354 # install build key
355 factory.addStep(FileDownload(mastersrc=home_dir+'/key-build', slavedest="key-build", mode=0600))
356 factory.addStep(FileDownload(mastersrc=home_dir+'/key-build.pub', slavedest="key-build.pub", mode=0600))
357
358 # prepare dl
359 factory.addStep(ShellCommand(
360 name = "dldir",
361 description = "Preparing dl/",
362 command = "mkdir -p $HOME/dl && rm -rf ./dl && ln -sf $HOME/dl ./dl",
363 logEnviron = False,
364 want_stdout = False
365 ))
366
367 # prepare tar
368 factory.addStep(ShellCommand(
369 name = "dltar",
370 description = "Building GNU tar",
371 command = ["make", WithProperties("-j%(nproc:~4)s"), "tools/tar/install", "V=s"],
372 haltOnFailure = True
373 ))
374
375 # populate dl
376 factory.addStep(ShellCommand(
377 name = "dlrun",
378 description = "Populating dl/",
379 command = ["make", WithProperties("-j%(nproc:~4)s"), "download", "V=s"],
380 logEnviron = False,
381 locks = [dlLock.access('exclusive')]
382 ))
383
384 factory.addStep(ShellCommand(
385 name = "cleanbase",
386 description = "Cleaning base-files",
387 command=["make", "package/base-files/clean", "V=s"]
388 ))
389
390 # build
391 factory.addStep(ShellCommand(
392 name = "tools",
393 description = "Building tools",
394 command = ["make", WithProperties("-j%(nproc:~4)s"), "tools/install", "V=s"],
395 haltOnFailure = True
396 ))
397
398 factory.addStep(ShellCommand(
399 name = "toolchain",
400 description = "Building toolchain",
401 command=["make", WithProperties("-j%(nproc:~4)s"), "toolchain/install", "V=s"],
402 haltOnFailure = True
403 ))
404
405 factory.addStep(ShellCommand(
406 name = "kmods",
407 description = "Building kmods",
408 command=["make", WithProperties("-j%(nproc:~4)s"), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
409 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
410 haltOnFailure = True
411 ))
412
413 factory.addStep(ShellCommand(
414 name = "pkgbuild",
415 description = "Building packages",
416 command=["make", WithProperties("-j%(nproc:~4)s"), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
417 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
418 haltOnFailure = True
419 ))
420
421 # factory.addStep(IfBuiltinShellCommand(
422 factory.addStep(ShellCommand(
423 name = "pkginstall",
424 description = "Installing packages",
425 command=["make", WithProperties("-j%(nproc:~4)s"), "package/install", "V=s"],
426 haltOnFailure = True
427 ))
428
429 factory.addStep(ShellCommand(
430 name = "pkgindex",
431 description = "Indexing packages",
432 command=["make", WithProperties("-j%(nproc:~4)s"), "package/index", "V=s"],
433 haltOnFailure = True
434 ))
435
436 #factory.addStep(IfBuiltinShellCommand(
437 factory.addStep(ShellCommand(
438 name = "images",
439 description = "Building images",
440 command=["make", WithProperties("-j%(nproc:~4)s"), "target/install", "V=s"],
441 haltOnFailure = True
442 ))
443
444 factory.addStep(ShellCommand(
445 name = "checksums",
446 description = "Calculating checksums",
447 command=["make", "-j1", "checksum", "V=s"],
448 haltOnFailure = True
449 ))
450
451 # sign
452 if gpg_keyid is not None:
453 factory.addStep(MasterShellCommand(
454 name = "signprepare",
455 description = "Preparing temporary signing directory",
456 command = ["mkdir", "-p", "%s/signing" %(home_dir)],
457 haltOnFailure = True
458 ))
459
460 factory.addStep(ShellCommand(
461 name = "signpack",
462 description = "Packing files to sign",
463 command = WithProperties("find bin/targets/%s/%s%%(libc)s/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz" %(ts[0], ts[1])),
464 haltOnFailure = True
465 ))
466
467 factory.addStep(FileUpload(
468 slavesrc = "sign.tar.gz",
469 masterdest = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
470 haltOnFailure = True
471 ))
472
473 factory.addStep(MasterShellCommand(
474 name = "signfiles",
475 description = "Signing files",
476 command = ["%s/signall.sh" %(home_dir), "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]), gpg_keyid, gpg_comment],
477 env = {'GNUPGHOME': gpg_home, 'PASSFILE': gpg_passfile},
478 haltOnFailure = True
479 ))
480
481 factory.addStep(FileDownload(
482 mastersrc = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
483 slavedest = "sign.tar.gz",
484 haltOnFailure = True
485 ))
486
487 factory.addStep(ShellCommand(
488 name = "signunpack",
489 description = "Unpacking signed files",
490 command = ["tar", "-xzf", "sign.tar.gz"],
491 haltOnFailure = True
492 ))
493
494 # upload
495 factory.addStep(ShellCommand(
496 name = "uploadprepare",
497 description = "Preparing target directory",
498 command=["rsync", "-av", "--include", "/%s/" %(ts[0]), "--include", "/%s/%s/" %(ts[0], ts[1]), "--exclude", "/*", "--exclude", "/*/*", "--exclude", "/%s/%s/*" %(ts[0], ts[1]), "bin/targets/", "%s/targets/" %(rsync_bin_url)],
499 env={'RSYNC_PASSWORD': rsync_bin_key},
500 haltOnFailure = True,
501 logEnviron = False
502 ))
503
504 factory.addStep(ShellCommand(
505 name = "targetupload",
506 description = "Uploading target files",
507 command=["rsync", "--delete", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", WithProperties("bin/targets/%s/%s%%(libc)s/" %(ts[0], ts[1])), "%s/targets/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
508 env={'RSYNC_PASSWORD': rsync_bin_key},
509 haltOnFailure = True,
510 logEnviron = False
511 ))
512
513 if rsync_src_url is not None:
514 factory.addStep(ShellCommand(
515 name = "sourceupload",
516 description = "Uploading source archives",
517 command=["rsync", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "dl/", "%s/" %(rsync_src_url)],
518 env={'RSYNC_PASSWORD': rsync_src_key},
519 haltOnFailure = True,
520 logEnviron = False
521 ))
522
523 if False:
524 factory.addStep(ShellCommand(
525 name = "packageupload",
526 description = "Uploading package files",
527 command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "bin/packages/", "%s/packages/" %(rsync_bin_url)],
528 env={'RSYNC_PASSWORD': rsync_bin_key},
529 haltOnFailure = False,
530 logEnviron = False
531 ))
532
533 # logs
534 if False:
535 factory.addStep(ShellCommand(
536 name = "upload",
537 description = "Uploading logs",
538 command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "logs/", "%s/logs/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
539 env={'RSYNC_PASSWORD': rsync_bin_key},
540 haltOnFailure = False,
541 alwaysRun = True,
542 logEnviron = False
543 ))
544
545 from buildbot.config import BuilderConfig
546
547 c['builders'].append(BuilderConfig(name=target, slavenames=slaveNames, factory=factory))
548
549
550 ####### STATUS TARGETS
551
552 # 'status' is a list of Status Targets. The results of each build will be
553 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
554 # including web pages, email senders, and IRC bots.
555
556 c['status'] = []
557
558 from buildbot.status import html
559 from buildbot.status.web import authz, auth
560
561 if ini.has_option("status", "bind"):
562 if ini.has_option("status", "user") and ini.has_option("status", "password"):
563 authz_cfg=authz.Authz(
564 # change any of these to True to enable; see the manual for more
565 # options
566 auth=auth.BasicAuth([(ini.get("status", "user"), ini.get("status", "password"))]),
567 gracefulShutdown = 'auth',
568 forceBuild = 'auth', # use this to test your slave once it is set up
569 forceAllBuilds = 'auth',
570 pingBuilder = False,
571 stopBuild = 'auth',
572 stopAllBuilds = 'auth',
573 cancelPendingBuild = 'auth',
574 )
575 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind"), authz=authz_cfg))
576 else:
577 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind")))
578
579
580 from buildbot.status import words
581
582 if ini.has_option("irc", "host") and ini.has_option("irc", "nickname") and ini.has_option("irc", "channel"):
583 irc_host = ini.get("irc", "host")
584 irc_port = 6667
585 irc_chan = ini.get("irc", "channel")
586 irc_nick = ini.get("irc", "nickname")
587 irc_pass = None
588
589 if ini.has_option("irc", "port"):
590 irc_port = ini.getint("irc", "port")
591
592 if ini.has_option("irc", "password"):
593 irc_pass = ini.get("irc", "password")
594
595 irc = words.IRC(irc_host, irc_nick, port = irc_port, password = irc_pass,
596 channels = [{ "channel": irc_chan }],
597 notify_events = {
598 'exception': 1,
599 'successToFailure': 1,
600 'failureToSuccess': 1
601 }
602 )
603
604 c['status'].append(irc)
605
606
607 ####### PROJECT IDENTITY
608
609 # the 'title' string will appear at the top of this buildbot
610 # installation's html.WebStatus home page (linked to the
611 # 'titleURL') and is embedded in the title of the waterfall HTML page.
612
613 c['title'] = ini.get("general", "title")
614 c['titleURL'] = ini.get("general", "title_url")
615
616 # the 'buildbotURL' string should point to the location where the buildbot's
617 # internal web server (usually the html.WebStatus page) is visible. This
618 # typically uses the port number set in the Waterfall 'status' entry, but
619 # with an externally-visible host name which the buildbot cannot figure out
620 # without some help.
621
622 c['buildbotURL'] = ini.get("general", "buildbot_url")
623
624 ####### DB URL
625
626 c['db'] = {
627 # This specifies what database buildbot uses to store its state. You can leave
628 # this at its default for all but the largest installations.
629 'db_url' : "sqlite:///state.sqlite",
630 }