2103a59cfc74c781ad39cebdc2060556456aadad
[buildbot.git] / phase1 / master.cfg
1 # -*- python -*-
2 # ex: set syntax=python:
3
4 import os
5 import re
6 import subprocess
7 import ConfigParser
8
9 from buildbot import locks
10
11 # This is a sample buildmaster config file. It must be installed as
12 # 'master.cfg' in your buildmaster's base directory.
13
14 ini = ConfigParser.ConfigParser()
15 ini.read("./config.ini")
16
17 # This is the dictionary that the buildmaster pays attention to. We also use
18 # a shorter alias to save typing.
19 c = BuildmasterConfig = {}
20
21 ####### BUILDSLAVES
22
23 # The 'slaves' list defines the set of recognized buildslaves. Each element is
24 # a BuildSlave object, specifying a unique slave name and password. The same
25 # slave name and password must be configured on the slave.
26 from buildbot.buildslave import BuildSlave
27
28 slave_port = 9989
29
30 if ini.has_option("general", "port"):
31 slave_port = ini.getint("general", "port")
32
33 c['slaves'] = []
34
35 for section in ini.sections():
36 if section.startswith("slave "):
37 if ini.has_option(section, "name") and ini.has_option(section, "password"):
38 name = ini.get(section, "name")
39 password = ini.get(section, "password")
40 max_builds = 1
41 if ini.has_option(section, "builds"):
42 max_builds = ini.getint(section, "builds")
43 c['slaves'].append(BuildSlave(name, password, max_builds = max_builds))
44
45 # 'slavePortnum' defines the TCP port to listen on for connections from slaves.
46 # This must match the value configured into the buildslaves (with their
47 # --master option)
48 c['slavePortnum'] = slave_port
49
50 # coalesce builds
51 c['mergeRequests'] = True
52
53 ####### CHANGESOURCES
54
55 home_dir = os.path.abspath(ini.get("general", "homedir"))
56 tree_expire = 0
57
58 if ini.has_option("general", "expire"):
59 tree_expire = ini.getint("general", "expire")
60
61 repo_url = ini.get("repo", "url")
62 repo_branch = "master"
63
64 if ini.has_option("repo", "branch"):
65 repo_branch = ini.get("repo", "branch")
66
67 rsync_bin_url = ini.get("rsync", "binary_url")
68 rsync_bin_key = ini.get("rsync", "binary_password")
69
70 rsync_src_url = None
71 rsync_src_key = None
72
73 if ini.has_option("rsync", "source_url"):
74 rsync_src_url = ini.get("rsync", "source_url")
75 rsync_src_key = ini.get("rsync", "source_password")
76
77 gpg_keyid = None
78 gpg_comment = "Unattended build signature"
79 gpg_passfile = "/dev/null"
80
81 if ini.has_option("gpg", "keyid"):
82 gpg_keyid = ini.get("gpg", "keyid")
83
84 if ini.has_option("gpg", "comment"):
85 gpg_comment = ini.get("gpg", "comment")
86
87 if ini.has_option("gpg", "passfile"):
88 gpg_passfile = ini.get("gpg", "passfile")
89
90
91 # find targets
92 targets = [ ]
93
94 if not os.path.isdir(home_dir+'/source.git'):
95 subprocess.call(["git", "clone", "--depth=1", repo_url, home_dir+'/source.git'])
96 subprocess.call(["git", "checkout", repo_branch], cwd = home_dir+'/source.git')
97
98 findtargets = subprocess.Popen([home_dir+'/dumpinfo.pl', 'targets'],
99 stdout = subprocess.PIPE, cwd = home_dir+'/source.git')
100
101 while True:
102 line = findtargets.stdout.readline()
103 if not line:
104 break
105 ta = line.strip().split(' ')
106 targets.append(ta[0])
107
108
109 # the 'change_source' setting tells the buildmaster how it should find out
110 # about source code changes. Here we point to the buildbot clone of pyflakes.
111
112 from buildbot.changes.gitpoller import GitPoller
113 c['change_source'] = []
114 c['change_source'].append(GitPoller(
115 repo_url,
116 workdir=home_dir+'/source.git', branch='master',
117 pollinterval=300))
118
119 ####### SCHEDULERS
120
121 # Configure the Schedulers, which decide how to react to incoming changes. In this
122 # case, just kick off a 'basebuild' build
123
124 from buildbot.schedulers.basic import SingleBranchScheduler
125 from buildbot.schedulers.forcesched import ForceScheduler
126 from buildbot.changes import filter
127 c['schedulers'] = []
128 c['schedulers'].append(SingleBranchScheduler(
129 name="all",
130 change_filter=filter.ChangeFilter(branch='master'),
131 treeStableTimer=60,
132 builderNames=targets))
133
134 c['schedulers'].append(ForceScheduler(
135 name="force",
136 builderNames=targets))
137
138 ####### BUILDERS
139
140 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
141 # what steps, and which slaves can execute them. Note that any particular build will
142 # only take place on one slave.
143
144 from buildbot.process.factory import BuildFactory
145 from buildbot.steps.source import Git
146 from buildbot.steps.shell import ShellCommand
147 from buildbot.steps.shell import SetProperty
148 from buildbot.steps.transfer import FileUpload
149 from buildbot.steps.transfer import FileDownload
150 from buildbot.steps.master import MasterShellCommand
151 from buildbot.process.properties import WithProperties
152
153
154 CleanTargetMap = [
155 [ "tools", "tools/clean" ],
156 [ "chain", "toolchain/clean" ],
157 [ "linux", "target/linux/clean" ],
158 [ "dir", "dirclean" ],
159 [ "dist", "distclean" ]
160 ]
161
162 def IsCleanRequested(pattern):
163 def CheckCleanProperty(step):
164 val = step.getProperty("clean")
165 if val and re.match(pattern, val):
166 return True
167 else:
168 return False
169
170 return CheckCleanProperty
171
172
173 c['builders'] = []
174
175 dlLock = locks.SlaveLock("slave_dl")
176
177 checkBuiltin = re.sub('[\t\n ]+', ' ', """
178 checkBuiltin() {
179 local symbol op path file;
180 for file in $CHANGED_FILES; do
181 case "$file" in
182 package/*/*) : ;;
183 *) return 0 ;;
184 esac;
185 done;
186 while read symbol op path; do
187 case "$symbol" in package-*)
188 symbol="${symbol##*(}";
189 symbol="${symbol%)}";
190 for file in $CHANGED_FILES; do
191 case "$file" in "package/$path/"*)
192 grep -qsx "$symbol=y" .config && return 0
193 ;; esac;
194 done;
195 esac;
196 done < tmp/.packagedeps;
197 return 1;
198 }
199 """).strip()
200
201
202 class IfBuiltinShellCommand(ShellCommand):
203 def _quote(self, str):
204 if re.search("[^a-zA-Z0-9/_.-]", str):
205 return "'%s'" %(re.sub("'", "'\"'\"'", str))
206 return str
207
208 def setCommand(self, command):
209 if not isinstance(command, (str, unicode)):
210 command = ' '.join(map(self._quote, command))
211 self.command = [
212 '/bin/sh', '-c',
213 '%s; if checkBuiltin; then %s; else exit 0; fi' %(checkBuiltin, command)
214 ]
215
216 def setupEnvironment(self, cmd):
217 slaveEnv = self.slaveEnvironment
218 if slaveEnv is None:
219 slaveEnv = { }
220 changedFiles = { }
221 for request in self.build.requests:
222 for source in request.sources:
223 for change in source.changes:
224 for file in change.files:
225 changedFiles[file] = True
226 fullSlaveEnv = slaveEnv.copy()
227 fullSlaveEnv['CHANGED_FILES'] = ' '.join(changedFiles.keys())
228 cmd.args['env'] = fullSlaveEnv
229
230 slaveNames = [ ]
231
232 for slave in c['slaves']:
233 slaveNames.append(slave.slavename)
234
235 for target in targets:
236 ts = target.split('/')
237
238 factory = BuildFactory()
239
240 # find number of cores
241 factory.addStep(SetProperty(
242 name = "nproc",
243 property = "nproc",
244 description = "Finding number of CPUs",
245 command = ["nproc"]))
246
247 # expire tree if needed
248 if tree_expire > 0:
249 factory.addStep(FileDownload(
250 mastersrc = "expire.sh",
251 slavedest = "../expire.sh",
252 mode = 0755))
253
254 factory.addStep(ShellCommand(
255 name = "expire",
256 description = "Checking for build tree expiry",
257 command = ["./expire.sh", str(tree_expire)],
258 workdir = ".",
259 haltOnFailure = True,
260 timeout = 2400))
261
262 # user-requested clean targets
263 for tuple in CleanTargetMap:
264 factory.addStep(ShellCommand(
265 name = tuple[1],
266 description = 'User-requested "make %s"' % tuple[1],
267 command = ["make", tuple[1], "V=s"],
268 doStepIf = IsCleanRequested(tuple[0])
269 ))
270
271 # check out the source
272 factory.addStep(Git(repourl=repo_url, branch=repo_branch, mode='update'))
273
274 factory.addStep(ShellCommand(
275 name = "rmtmp",
276 description = "Remove tmp folder",
277 command=["rm", "-rf", "tmp/"]))
278
279 # feed
280 # factory.addStep(ShellCommand(
281 # name = "feedsconf",
282 # description = "Copy the feeds.conf",
283 # command='''cp ~/feeds.conf ./feeds.conf''' ))
284
285 # feed
286 factory.addStep(ShellCommand(
287 name = "rmfeedlinks",
288 description = "Remove feed symlinks",
289 command=["rm", "-rf", "package/feeds/"]))
290
291 # feed
292 factory.addStep(ShellCommand(
293 name = "updatefeeds",
294 description = "Updating feeds",
295 command=["./scripts/feeds", "update"]))
296
297 # feed
298 factory.addStep(ShellCommand(
299 name = "installfeeds",
300 description = "Installing feeds",
301 command=["./scripts/feeds", "install", "-a"]))
302
303 # configure
304 factory.addStep(ShellCommand(
305 name = "newconfig",
306 description = "Seeding .config",
307 command='''cat <<EOT > .config
308 CONFIG_TARGET_%s=y
309 CONFIG_TARGET_%s_%s=y
310 CONFIG_ALL_NONSHARED=y
311 CONFIG_SDK=y
312 CONFIG_IB=y
313 # CONFIG_IB_STANDALONE is not set
314 CONFIG_DEVEL=y
315 CONFIG_CCACHE=y
316 CONFIG_SIGNED_PACKAGES=y
317 # CONFIG_PER_FEED_REPO_ADD_COMMENTED is not set
318 CONFIG_KERNEL_KALLSYMS=y
319 CONFIG_COLLECT_KERNEL_DEBUG=y
320 CONFIG_TARGET_ALL_PROFILES=y
321 CONFIG_TARGET_MULTI_PROFILE=y
322 CONFIG_TARGET_PER_DEVICE_ROOTFS=y
323 EOT''' %(ts[0], ts[0], ts[1]) ))
324
325 factory.addStep(ShellCommand(
326 name = "delbin",
327 description = "Removing output directory",
328 command = ["rm", "-rf", "bin/"]
329 ))
330
331 factory.addStep(ShellCommand(
332 name = "defconfig",
333 description = "Populating .config",
334 command = ["make", "defconfig"]
335 ))
336
337 # check arch
338 factory.addStep(ShellCommand(
339 name = "checkarch",
340 description = "Checking architecture",
341 command = ["grep", "-sq", "CONFIG_TARGET_%s=y" %(ts[0]), ".config"],
342 logEnviron = False,
343 want_stdout = False,
344 want_stderr = False,
345 haltOnFailure = True
346 ))
347
348 # find libc suffix
349 factory.addStep(SetProperty(
350 name = "libc",
351 property = "libc",
352 description = "Finding libc suffix",
353 command = ["sed", "-ne", '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }', ".config"]))
354
355 # install build key
356 factory.addStep(FileDownload(mastersrc=home_dir+'/key-build', slavedest="key-build", mode=0600))
357 factory.addStep(FileDownload(mastersrc=home_dir+'/key-build.pub', slavedest="key-build.pub", mode=0600))
358
359 # prepare dl
360 factory.addStep(ShellCommand(
361 name = "dldir",
362 description = "Preparing dl/",
363 command = "mkdir -p $HOME/dl && rm -rf ./dl && ln -sf $HOME/dl ./dl",
364 logEnviron = False,
365 want_stdout = False
366 ))
367
368 # prepare tar
369 factory.addStep(ShellCommand(
370 name = "dltar",
371 description = "Building GNU tar",
372 command = ["make", WithProperties("-j%(nproc:~4)s"), "tools/tar/install", "V=s"],
373 haltOnFailure = True
374 ))
375
376 # populate dl
377 factory.addStep(ShellCommand(
378 name = "dlrun",
379 description = "Populating dl/",
380 command = ["make", WithProperties("-j%(nproc:~4)s"), "download", "V=s"],
381 logEnviron = False,
382 locks = [dlLock.access('exclusive')]
383 ))
384
385 factory.addStep(ShellCommand(
386 name = "cleanbase",
387 description = "Cleaning base-files",
388 command=["make", "package/base-files/clean", "V=s"]
389 ))
390
391 # build
392 factory.addStep(ShellCommand(
393 name = "tools",
394 description = "Building tools",
395 command = ["make", WithProperties("-j%(nproc:~4)s"), "tools/install", "V=s"],
396 haltOnFailure = True
397 ))
398
399 factory.addStep(ShellCommand(
400 name = "toolchain",
401 description = "Building toolchain",
402 command=["make", WithProperties("-j%(nproc:~4)s"), "toolchain/install", "V=s"],
403 haltOnFailure = True
404 ))
405
406 factory.addStep(ShellCommand(
407 name = "kmods",
408 description = "Building kmods",
409 command=["make", WithProperties("-j%(nproc:~4)s"), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
410 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
411 haltOnFailure = True
412 ))
413
414 factory.addStep(ShellCommand(
415 name = "pkgbuild",
416 description = "Building packages",
417 command=["make", WithProperties("-j%(nproc:~4)s"), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
418 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
419 haltOnFailure = True
420 ))
421
422 # factory.addStep(IfBuiltinShellCommand(
423 factory.addStep(ShellCommand(
424 name = "pkginstall",
425 description = "Installing packages",
426 command=["make", WithProperties("-j%(nproc:~4)s"), "package/install", "V=s"],
427 haltOnFailure = True
428 ))
429
430 factory.addStep(ShellCommand(
431 name = "pkgindex",
432 description = "Indexing packages",
433 command=["make", WithProperties("-j%(nproc:~4)s"), "package/index", "V=s"],
434 haltOnFailure = True
435 ))
436
437 #factory.addStep(IfBuiltinShellCommand(
438 factory.addStep(ShellCommand(
439 name = "images",
440 description = "Building images",
441 command=["make", WithProperties("-j%(nproc:~4)s"), "target/install", "V=s"],
442 haltOnFailure = True
443 ))
444
445 factory.addStep(ShellCommand(
446 name = "checksums",
447 description = "Calculating checksums",
448 command=["make", "-j1", "checksum", "V=s"],
449 haltOnFailure = True
450 ))
451
452 # sign
453 if gpg_keyid is not None:
454 factory.addStep(MasterShellCommand(
455 name = "signprepare",
456 description = "Preparing temporary signing directory",
457 command = ["mkdir", "-p", "%s/signing" %(home_dir)],
458 haltOnFailure = True
459 ))
460
461 factory.addStep(ShellCommand(
462 name = "signpack",
463 description = "Packing files to sign",
464 command = WithProperties("find bin/targets/%s/%s%%(libc)s/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz" %(ts[0], ts[1])),
465 haltOnFailure = True
466 ))
467
468 factory.addStep(FileUpload(
469 slavesrc = "sign.tar.gz",
470 masterdest = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
471 haltOnFailure = True
472 ))
473
474 factory.addStep(MasterShellCommand(
475 name = "signfiles",
476 description = "Signing files",
477 command = ["%s/signall.sh" %(home_dir), "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]), gpg_keyid, gpg_passfile, gpg_comment],
478 haltOnFailure = True
479 ))
480
481 factory.addStep(FileDownload(
482 mastersrc = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
483 slavedest = "sign.tar.gz",
484 haltOnFailure = True
485 ))
486
487 factory.addStep(ShellCommand(
488 name = "signunpack",
489 description = "Unpacking signed files",
490 command = ["tar", "-xzf", "sign.tar.gz"],
491 haltOnFailure = True
492 ))
493
494 # upload
495 factory.addStep(ShellCommand(
496 name = "uploadprepare",
497 description = "Preparing target directory",
498 command=["rsync", "-av", "--include", "/%s/" %(ts[0]), "--include", "/%s/%s/" %(ts[0], ts[1]), "--exclude", "/*", "--exclude", "/*/*", "--exclude", "/%s/%s/*" %(ts[0], ts[1]), "bin/targets/", "%s/targets/" %(rsync_bin_url)],
499 env={'RSYNC_PASSWORD': rsync_bin_key},
500 haltOnFailure = True,
501 logEnviron = False
502 ))
503
504 factory.addStep(ShellCommand(
505 name = "targetupload",
506 description = "Uploading target files",
507 command=["rsync", "--delete", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", WithProperties("bin/targets/%s/%s%%(libc)s/" %(ts[0], ts[1])), "%s/targets/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
508 env={'RSYNC_PASSWORD': rsync_bin_key},
509 haltOnFailure = True,
510 logEnviron = False
511 ))
512
513 if rsync_src_url is not None:
514 factory.addStep(ShellCommand(
515 name = "sourceupload",
516 description = "Uploading source archives",
517 command=["rsync", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "dl/", "%s/" %(rsync_src_url)],
518 env={'RSYNC_PASSWORD': rsync_src_key},
519 haltOnFailure = True,
520 logEnviron = False
521 ))
522
523 if False:
524 factory.addStep(ShellCommand(
525 name = "packageupload",
526 description = "Uploading package files",
527 command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "bin/packages/", "%s/packages/" %(rsync_bin_url)],
528 env={'RSYNC_PASSWORD': rsync_bin_key},
529 haltOnFailure = False,
530 logEnviron = False
531 ))
532
533 # logs
534 if False:
535 factory.addStep(ShellCommand(
536 name = "upload",
537 description = "Uploading logs",
538 command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "logs/", "%s/logs/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
539 env={'RSYNC_PASSWORD': rsync_bin_key},
540 haltOnFailure = False,
541 alwaysRun = True,
542 logEnviron = False
543 ))
544
545 from buildbot.config import BuilderConfig
546
547 c['builders'].append(BuilderConfig(name=target, slavenames=slaveNames, factory=factory))
548
549
550 ####### STATUS TARGETS
551
552 # 'status' is a list of Status Targets. The results of each build will be
553 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
554 # including web pages, email senders, and IRC bots.
555
556 c['status'] = []
557
558 from buildbot.status import html
559 from buildbot.status.web import authz, auth
560
561 if ini.has_option("status", "bind"):
562 if ini.has_option("status", "user") and ini.has_option("status", "password"):
563 authz_cfg=authz.Authz(
564 # change any of these to True to enable; see the manual for more
565 # options
566 auth=auth.BasicAuth([(ini.get("status", "user"), ini.get("status", "password"))]),
567 gracefulShutdown = 'auth',
568 forceBuild = 'auth', # use this to test your slave once it is set up
569 forceAllBuilds = 'auth',
570 pingBuilder = False,
571 stopBuild = 'auth',
572 stopAllBuilds = 'auth',
573 cancelPendingBuild = 'auth',
574 )
575 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind"), authz=authz_cfg))
576 else:
577 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind")))
578
579
580 from buildbot.status import words
581
582 if ini.has_option("irc", "host") and ini.has_option("irc", "nickname") and ini.has_option("irc", "channel"):
583 irc_host = ini.get("irc", "host")
584 irc_port = 6667
585 irc_chan = ini.get("irc", "channel")
586 irc_nick = ini.get("irc", "nickname")
587 irc_pass = None
588
589 if ini.has_option("irc", "port"):
590 irc_port = ini.getint("irc", "port")
591
592 if ini.has_option("irc", "password"):
593 irc_pass = ini.get("irc", "password")
594
595 irc = words.IRC(irc_host, irc_nick, port = irc_port, password = irc_pass,
596 channels = [{ "channel": irc_chan }],
597 notify_events = {
598 'exception': 1,
599 'successToFailure': 1,
600 'failureToSuccess': 1
601 }
602 )
603
604 c['status'].append(irc)
605
606
607 ####### PROJECT IDENTITY
608
609 # the 'title' string will appear at the top of this buildbot
610 # installation's html.WebStatus home page (linked to the
611 # 'titleURL') and is embedded in the title of the waterfall HTML page.
612
613 c['title'] = ini.get("general", "title")
614 c['titleURL'] = ini.get("general", "title_url")
615
616 # the 'buildbotURL' string should point to the location where the buildbot's
617 # internal web server (usually the html.WebStatus page) is visible. This
618 # typically uses the port number set in the Waterfall 'status' entry, but
619 # with an externally-visible host name which the buildbot cannot figure out
620 # without some help.
621
622 c['buildbotURL'] = ini.get("general", "buildbot_url")
623
624 ####### DB URL
625
626 c['db'] = {
627 # This specifies what database buildbot uses to store its state. You can leave
628 # this at its default for all but the largest installations.
629 'db_url' : "sqlite:///state.sqlite",
630 }