5eb70f4411a400c544e2ac112f476b35178796ef
[buildbot.git] / phase1 / master.cfg
1 # -*- python -*-
2 # ex: set syntax=python:
3
4 import os
5 import re
6 import subprocess
7 import ConfigParser
8
9 from buildbot import locks
10
11 # This is a sample buildmaster config file. It must be installed as
12 # 'master.cfg' in your buildmaster's base directory.
13
14 ini = ConfigParser.ConfigParser()
15 ini.read("./config.ini")
16
17 # This is the dictionary that the buildmaster pays attention to. We also use
18 # a shorter alias to save typing.
19 c = BuildmasterConfig = {}
20
21 ####### BUILDSLAVES
22
23 # The 'slaves' list defines the set of recognized buildslaves. Each element is
24 # a BuildSlave object, specifying a unique slave name and password. The same
25 # slave name and password must be configured on the slave.
26 from buildbot.buildslave import BuildSlave
27
28 c['slaves'] = []
29
30 for section in ini.sections():
31 if section.startswith("slave "):
32 if ini.has_option(section, "name") and ini.has_option(section, "password"):
33 name = ini.get(section, "name")
34 password = ini.get(section, "password")
35 max_builds = 1
36 if ini.has_option(section, "builds"):
37 max_builds = ini.getint(section, "builds")
38 c['slaves'].append(BuildSlave(name, password, max_builds = max_builds))
39
40 # 'slavePortnum' defines the TCP port to listen on for connections from slaves.
41 # This must match the value configured into the buildslaves (with their
42 # --master option)
43 c['slavePortnum'] = 9989
44
45 # coalesce builds
46 c['mergeRequests'] = True
47
48 ####### CHANGESOURCES
49
50 home_dir = os.path.abspath(ini.get("general", "homedir"))
51
52 repo_url = ini.get("repo", "url")
53
54 rsync_bin_url = ini.get("rsync", "binary_url")
55 rsync_bin_key = ini.get("rsync", "binary_password")
56
57 rsync_src_url = None
58 rsync_src_key = None
59
60 if ini.has_option("rsync", "source_url"):
61 rsync_src_url = ini.get("rsync", "source_url")
62 rsync_src_key = ini.get("rsync", "source_password")
63
64 gpg_keyid = None
65 gpg_comment = "Unattended build signature"
66 gpg_passfile = "/dev/null"
67
68 if ini.has_option("gpg", "keyid"):
69 gpg_keyid = ini.get("gpg", "keyid")
70
71 if ini.has_option("gpg", "comment"):
72 gpg_comment = ini.get("gpg", "comment")
73
74 if ini.has_option("gpg", "passfile"):
75 gpg_passfile = ini.get("gpg", "passfile")
76
77
78 # find targets
79 targets = [ ]
80
81 findtargets = subprocess.Popen([home_dir+'/dumpinfo.pl', 'targets'],
82 stdout = subprocess.PIPE, cwd = home_dir+'/source.git')
83
84 while True:
85 line = findtargets.stdout.readline()
86 if not line:
87 break
88 ta = line.strip().split(' ')
89 targets.append(ta[0])
90
91
92 # the 'change_source' setting tells the buildmaster how it should find out
93 # about source code changes. Here we point to the buildbot clone of pyflakes.
94
95 from buildbot.changes.gitpoller import GitPoller
96 c['change_source'] = []
97 c['change_source'].append(GitPoller(
98 repo_url,
99 workdir=home_dir+'/source.git', branch='master',
100 pollinterval=300))
101
102 ####### SCHEDULERS
103
104 # Configure the Schedulers, which decide how to react to incoming changes. In this
105 # case, just kick off a 'basebuild' build
106
107 from buildbot.schedulers.basic import SingleBranchScheduler
108 from buildbot.schedulers.forcesched import ForceScheduler
109 from buildbot.changes import filter
110 c['schedulers'] = []
111 c['schedulers'].append(SingleBranchScheduler(
112 name="all",
113 change_filter=filter.ChangeFilter(branch='master'),
114 treeStableTimer=60,
115 builderNames=targets))
116
117 c['schedulers'].append(ForceScheduler(
118 name="force",
119 builderNames=targets))
120
121 ####### BUILDERS
122
123 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
124 # what steps, and which slaves can execute them. Note that any particular build will
125 # only take place on one slave.
126
127 from buildbot.process.factory import BuildFactory
128 from buildbot.steps.source import Git
129 from buildbot.steps.shell import ShellCommand
130 from buildbot.steps.shell import SetProperty
131 from buildbot.steps.transfer import FileUpload
132 from buildbot.steps.transfer import FileDownload
133 from buildbot.steps.master import MasterShellCommand
134 from buildbot.process.properties import WithProperties
135
136
137 MakeTargetMap = {
138 "^tools/": "tools/clean",
139 "^toolchain/": "toolchain/clean",
140 "^target/linux/": "target/linux/clean",
141 "^(config|include)/": "dirclean"
142 }
143
144 def IsAffected(pattern):
145 def CheckAffected(change):
146 for request in change.build.requests:
147 for source in request.sources:
148 for change in source.changes:
149 for file in change.files:
150 if re.match(pattern, file):
151 return True
152 return False
153 return CheckAffected
154
155 def isPathBuiltin(path):
156 incl = {}
157 pkgs = {}
158 conf = open(".config", "r")
159
160 while True:
161 line = conf.readline()
162 if line == '':
163 break
164 m = re.match("^(CONFIG_PACKAGE_.+?)=y", line)
165 if m:
166 incl[m.group(1)] = True
167
168 conf.close()
169
170 deps = open("tmp/.packagedeps", "r")
171
172 while True:
173 line = deps.readline()
174 if line == '':
175 break
176 m = re.match("^package-\$\((CONFIG_PACKAGE_.+?)\) \+= (\S+)", line)
177 if m and incl.get(m.group(1)) == True:
178 pkgs["package/%s" % m.group(2)] = True
179
180 deps.close()
181
182 while path != '':
183 if pkgs.get(path) == True:
184 return True
185 path = os.path.dirname(path)
186
187 return False
188
189 def isChangeBuiltin(change):
190 return True
191 # for request in change.build.requests:
192 # for source in request.sources:
193 # for change in source.changes:
194 # for file in change.files:
195 # if isPathBuiltin(file):
196 # return True
197 # return False
198
199
200 c['builders'] = []
201
202 dlLock = locks.SlaveLock("slave_dl")
203
204 checkBuiltin = re.sub('[\t\n ]+', ' ', """
205 checkBuiltin() {
206 local symbol op path file;
207 for file in $CHANGED_FILES; do
208 case "$file" in
209 package/*/*) : ;;
210 *) return 0 ;;
211 esac;
212 done;
213 while read symbol op path; do
214 case "$symbol" in package-*)
215 symbol="${symbol##*(}";
216 symbol="${symbol%)}";
217 for file in $CHANGED_FILES; do
218 case "$file" in "package/$path/"*)
219 grep -qsx "$symbol=y" .config && return 0
220 ;; esac;
221 done;
222 esac;
223 done < tmp/.packagedeps;
224 return 1;
225 }
226 """).strip()
227
228
229 class IfBuiltinShellCommand(ShellCommand):
230 def _quote(self, str):
231 if re.search("[^a-zA-Z0-9/_.-]", str):
232 return "'%s'" %(re.sub("'", "'\"'\"'", str))
233 return str
234
235 def setCommand(self, command):
236 if not isinstance(command, (str, unicode)):
237 command = ' '.join(map(self._quote, command))
238 self.command = [
239 '/bin/sh', '-c',
240 '%s; if checkBuiltin; then %s; else exit 0; fi' %(checkBuiltin, command)
241 ]
242
243 def setupEnvironment(self, cmd):
244 slaveEnv = self.slaveEnvironment
245 if slaveEnv is None:
246 slaveEnv = { }
247 changedFiles = { }
248 for request in self.build.requests:
249 for source in request.sources:
250 for change in source.changes:
251 for file in change.files:
252 changedFiles[file] = True
253 fullSlaveEnv = slaveEnv.copy()
254 fullSlaveEnv['CHANGED_FILES'] = ' '.join(changedFiles.keys())
255 cmd.args['env'] = fullSlaveEnv
256
257 slaveNames = [ ]
258
259 for slave in c['slaves']:
260 slaveNames.append(slave.slavename)
261
262 for target in targets:
263 ts = target.split('/')
264
265 factory = BuildFactory()
266
267 # find number of cores
268 factory.addStep(SetProperty(
269 name = "nproc",
270 property = "nproc",
271 description = "Finding number of CPUs",
272 command = ["nproc"]))
273
274 # check out the source
275 factory.addStep(Git(repourl=repo_url, mode='update'))
276
277 factory.addStep(ShellCommand(
278 name = "rmtmp",
279 description = "Remove tmp folder",
280 command=["rm", "-rf", "tmp/"]))
281
282 # feed
283 # factory.addStep(ShellCommand(
284 # name = "feedsconf",
285 # description = "Copy the feeds.conf",
286 # command='''cp ~/feeds.conf ./feeds.conf''' ))
287
288 # feed
289 factory.addStep(ShellCommand(
290 name = "rmfeedlinks",
291 description = "Remove feed symlinks",
292 command=["rm", "-rf", "package/feeds/"]))
293
294 # feed
295 factory.addStep(ShellCommand(
296 name = "updatefeeds",
297 description = "Updating feeds",
298 command=["./scripts/feeds", "update"]))
299
300 # feed
301 factory.addStep(ShellCommand(
302 name = "installfeeds",
303 description = "Installing feeds",
304 command=["./scripts/feeds", "install", "-a"]))
305
306 # configure
307 factory.addStep(ShellCommand(
308 name = "newconfig",
309 description = "Seeding .config",
310 command='''cat <<EOT > .config
311 CONFIG_TARGET_%s=y
312 CONFIG_TARGET_%s_%s=y
313 CONFIG_ALL_NONSHARED=y
314 CONFIG_SDK=y
315 CONFIG_IB=y
316 # CONFIG_IB_STANDALONE is not set
317 CONFIG_DEVEL=y
318 CONFIG_CCACHE=y
319 CONFIG_SIGNED_PACKAGES=y
320 # CONFIG_PER_FEED_REPO_ADD_COMMENTED is not set
321 CONFIG_KERNEL_KALLSYMS=y
322 CONFIG_COLLECT_KERNEL_DEBUG=y
323 CONFIG_TARGET_ALL_PROFILES=y
324 CONFIG_TARGET_MULTI_PROFILE=y
325 CONFIG_TARGET_PER_DEVICE_ROOTFS=y
326 EOT''' %(ts[0], ts[0], ts[1]) ))
327
328 factory.addStep(ShellCommand(
329 name = "delbin",
330 description = "Removing output directory",
331 command = ["rm", "-rf", "bin/"]
332 ))
333
334 factory.addStep(ShellCommand(
335 name = "defconfig",
336 description = "Populating .config",
337 command = ["make", "defconfig"]
338 ))
339
340 # check arch
341 factory.addStep(ShellCommand(
342 name = "checkarch",
343 description = "Checking architecture",
344 command = ["grep", "-sq", "CONFIG_TARGET_%s=y" %(ts[0]), ".config"],
345 logEnviron = False,
346 want_stdout = False,
347 want_stderr = False,
348 haltOnFailure = True
349 ))
350
351 # find libc suffix
352 factory.addStep(SetProperty(
353 name = "libc",
354 property = "libc",
355 description = "Finding libc suffix",
356 command = ["sed", "-ne", '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }', ".config"]))
357
358 # install build key
359 factory.addStep(FileDownload(mastersrc=home_dir+'/key-build', slavedest="key-build", mode=0600))
360 factory.addStep(FileDownload(mastersrc=home_dir+'/key-build.pub', slavedest="key-build.pub", mode=0600))
361
362 # prepare dl
363 factory.addStep(ShellCommand(
364 name = "dldir",
365 description = "Preparing dl/",
366 command = "mkdir -p $HOME/dl && ln -sf $HOME/dl ./dl",
367 logEnviron = False,
368 want_stdout = False
369 ))
370
371 # populate dl
372 factory.addStep(ShellCommand(
373 name = "dlrun",
374 description = "Populating dl/",
375 command = ["make", WithProperties("-j%(nproc:~4)s"), "download", "V=s"],
376 logEnviron = False,
377 locks = [dlLock.access('exclusive')]
378 ))
379
380 factory.addStep(ShellCommand(
381 name = "cleanbase",
382 description = "Cleaning base-files",
383 command=["make", "package/base-files/clean", "V=s"]
384 ))
385
386 # optional clean steps
387 for pattern, maketarget in MakeTargetMap.items():
388 factory.addStep(ShellCommand(
389 name = maketarget,
390 description = maketarget,
391 command=["make", maketarget, "V=s"], doStepIf=IsAffected(pattern)
392 ))
393
394 # build
395 factory.addStep(ShellCommand(
396 name = "tools",
397 description = "Building tools",
398 command = ["make", WithProperties("-j%(nproc:~4)s"), "tools/install", "V=s"],
399 haltOnFailure = True
400 ))
401
402 factory.addStep(ShellCommand(
403 name = "toolchain",
404 description = "Building toolchain",
405 command=["make", WithProperties("-j%(nproc:~4)s"), "toolchain/install", "V=s"],
406 haltOnFailure = True
407 ))
408
409 factory.addStep(ShellCommand(
410 name = "kmods",
411 description = "Building kmods",
412 command=["make", WithProperties("-j%(nproc:~4)s"), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
413 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
414 haltOnFailure = True
415 ))
416
417 factory.addStep(ShellCommand(
418 name = "pkgbuild",
419 description = "Building packages",
420 command=["make", WithProperties("-j%(nproc:~4)s"), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
421 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
422 haltOnFailure = True
423 ))
424
425 # factory.addStep(IfBuiltinShellCommand(
426 factory.addStep(ShellCommand(
427 name = "pkginstall",
428 description = "Installing packages",
429 command=["make", WithProperties("-j%(nproc:~4)s"), "package/install", "V=s"],
430 doStepIf = isChangeBuiltin,
431 haltOnFailure = True
432 ))
433
434 factory.addStep(ShellCommand(
435 name = "pkgindex",
436 description = "Indexing packages",
437 command=["make", WithProperties("-j%(nproc:~4)s"), "package/index", "V=s"],
438 haltOnFailure = True
439 ))
440
441 #factory.addStep(IfBuiltinShellCommand(
442 factory.addStep(ShellCommand(
443 name = "images",
444 description = "Building images",
445 command=["make", "-j1", "target/install", "V=s"],
446 doStepIf = isChangeBuiltin,
447 haltOnFailure = True
448 ))
449
450 factory.addStep(ShellCommand(
451 name = "checksums",
452 description = "Calculating checksums",
453 command=["make", "-j1", "checksum", "V=s"],
454 doStepIf = isChangeBuiltin,
455 haltOnFailure = True
456 ))
457
458 # sign
459 if gpg_keyid is not None:
460 factory.addStep(MasterShellCommand(
461 name = "signprepare",
462 description = "Preparing temporary signing directory",
463 command = ["mkdir", "-p", "%s/signing" %(home_dir)],
464 haltOnFailure = True
465 ))
466
467 factory.addStep(ShellCommand(
468 name = "signpack",
469 description = "Packing files to sign",
470 command = WithProperties("find bin/targets/%s/%s%%(libc)s/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz" %(ts[0], ts[1])),
471 haltOnFailure = True
472 ))
473
474 factory.addStep(FileUpload(
475 slavesrc = "sign.tar.gz",
476 masterdest = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
477 haltOnFailure = True
478 ))
479
480 factory.addStep(MasterShellCommand(
481 name = "signfiles",
482 description = "Signing files",
483 command = ["%s/signall.sh" %(home_dir), "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]), gpg_keyid, gpg_passfile, gpg_comment],
484 haltOnFailure = True
485 ))
486
487 factory.addStep(FileDownload(
488 mastersrc = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
489 slavedest = "sign.tar.gz",
490 haltOnFailure = True
491 ))
492
493 factory.addStep(ShellCommand(
494 name = "signunpack",
495 description = "Unpacking signed files",
496 command = ["tar", "-xzf", "sign.tar.gz"],
497 haltOnFailure = True
498 ))
499
500 # upload
501 factory.addStep(ShellCommand(
502 name = "uploadprepare",
503 description = "Preparing target directory",
504 command=["rsync", "-av", "--include", "/%s/" %(ts[0]), "--include", "/%s/%s/" %(ts[0], ts[1]), "--exclude", "/*", "--exclude", "/*/*", "--exclude", "/%s/%s/*" %(ts[0], ts[1]), "bin/targets/", "%s/targets/" %(rsync_bin_url)],
505 env={'RSYNC_PASSWORD': rsync_bin_key},
506 haltOnFailure = True,
507 logEnviron = False
508 ))
509
510 factory.addStep(ShellCommand(
511 name = "targetupload",
512 description = "Uploading target files",
513 command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", WithProperties("bin/targets/%s/%s%%(libc)s/" %(ts[0], ts[1])), "%s/targets/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
514 env={'RSYNC_PASSWORD': rsync_bin_key},
515 haltOnFailure = True,
516 logEnviron = False
517 ))
518
519 if rsync_src_url is not None:
520 factory.addStep(ShellCommand(
521 name = "sourceupload",
522 description = "Uploading source archives",
523 command=["rsync", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "dl/", "%s/" %(rsync_src_url)],
524 env={'RSYNC_PASSWORD': rsync_src_key},
525 haltOnFailure = True,
526 logEnviron = False
527 ))
528
529 if False:
530 factory.addStep(ShellCommand(
531 name = "packageupload",
532 description = "Uploading package files",
533 command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "bin/packages/", "%s/packages/" %(rsync_bin_url)],
534 env={'RSYNC_PASSWORD': rsync_bin_key},
535 haltOnFailure = False,
536 logEnviron = False
537 ))
538
539 # logs
540 if False:
541 factory.addStep(ShellCommand(
542 name = "upload",
543 description = "Uploading logs",
544 command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "logs/", "%s/logs/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
545 env={'RSYNC_PASSWORD': rsync_bin_key},
546 haltOnFailure = False,
547 alwaysRun = True,
548 logEnviron = False
549 ))
550
551 from buildbot.config import BuilderConfig
552
553 c['builders'].append(BuilderConfig(name=target, slavenames=slaveNames, factory=factory))
554
555
556 ####### STATUS TARGETS
557
558 # 'status' is a list of Status Targets. The results of each build will be
559 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
560 # including web pages, email senders, and IRC bots.
561
562 c['status'] = []
563
564 from buildbot.status import html
565 from buildbot.status.web import authz, auth
566
567 if ini.has_option("status", "bind"):
568 if ini.has_option("status", "user") and ini.has_option("status", "password"):
569 authz_cfg=authz.Authz(
570 # change any of these to True to enable; see the manual for more
571 # options
572 auth=auth.BasicAuth([(ini.get("status", "user"), ini.get("status", "password"))]),
573 gracefulShutdown = 'auth',
574 forceBuild = 'auth', # use this to test your slave once it is set up
575 forceAllBuilds = 'auth',
576 pingBuilder = False,
577 stopBuild = 'auth',
578 stopAllBuilds = 'auth',
579 cancelPendingBuild = 'auth',
580 )
581 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind"), authz=authz_cfg))
582 else:
583 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind")))
584
585
586 from buildbot.status import words
587
588 if ini.has_option("irc", "host") and ini.has_option("irc", "nickname") and ini.has_option("irc", "channel"):
589 irc_host = ini.get("irc", "host")
590 irc_port = 6667
591 irc_chan = ini.get("irc", "channel")
592 irc_nick = ini.get("irc", "nickname")
593 irc_pass = None
594
595 if ini.has_option("irc", "port"):
596 irc_port = ini.getint("irc", "port")
597
598 if ini.has_option("irc", "password"):
599 irc_pass = ini.get("irc", "password")
600
601 irc = words.IRC(irc_host, irc_nick, port = irc_port, password = irc_pass,
602 channels = [{ "channel": irc_chan }],
603 notify_events = {
604 'exception': 1,
605 'successToFailure': 1,
606 'failureToSuccess': 1
607 }
608 )
609
610 c['status'].append(irc)
611
612
613 ####### PROJECT IDENTITY
614
615 # the 'title' string will appear at the top of this buildbot
616 # installation's html.WebStatus home page (linked to the
617 # 'titleURL') and is embedded in the title of the waterfall HTML page.
618
619 c['title'] = ini.get("general", "title")
620 c['titleURL'] = ini.get("general", "title_url")
621
622 # the 'buildbotURL' string should point to the location where the buildbot's
623 # internal web server (usually the html.WebStatus page) is visible. This
624 # typically uses the port number set in the Waterfall 'status' entry, but
625 # with an externally-visible host name which the buildbot cannot figure out
626 # without some help.
627
628 c['buildbotURL'] = ini.get("general", "buildbot_url")
629
630 ####### DB URL
631
632 c['db'] = {
633 # This specifies what database buildbot uses to store its state. You can leave
634 # this at its default for all but the largest installations.
635 'db_url' : "sqlite:///state.sqlite",
636 }