phase1: parallelize target/install phase
[buildbot.git] / phase1 / master.cfg
1 # -*- python -*-
2 # ex: set syntax=python:
3
4 import os
5 import re
6 import subprocess
7 import ConfigParser
8
9 from buildbot import locks
10
11 # This is a sample buildmaster config file. It must be installed as
12 # 'master.cfg' in your buildmaster's base directory.
13
14 ini = ConfigParser.ConfigParser()
15 ini.read("./config.ini")
16
17 # This is the dictionary that the buildmaster pays attention to. We also use
18 # a shorter alias to save typing.
19 c = BuildmasterConfig = {}
20
21 ####### BUILDSLAVES
22
23 # The 'slaves' list defines the set of recognized buildslaves. Each element is
24 # a BuildSlave object, specifying a unique slave name and password. The same
25 # slave name and password must be configured on the slave.
26 from buildbot.buildslave import BuildSlave
27
28 c['slaves'] = []
29
30 for section in ini.sections():
31 if section.startswith("slave "):
32 if ini.has_option(section, "name") and ini.has_option(section, "password"):
33 name = ini.get(section, "name")
34 password = ini.get(section, "password")
35 max_builds = 1
36 if ini.has_option(section, "builds"):
37 max_builds = ini.getint(section, "builds")
38 c['slaves'].append(BuildSlave(name, password, max_builds = max_builds))
39
40 # 'slavePortnum' defines the TCP port to listen on for connections from slaves.
41 # This must match the value configured into the buildslaves (with their
42 # --master option)
43 c['slavePortnum'] = 9989
44
45 # coalesce builds
46 c['mergeRequests'] = True
47
48 ####### CHANGESOURCES
49
50 home_dir = os.path.abspath(ini.get("general", "homedir"))
51 tree_expire = 0
52
53 if ini.has_option("general", "expire"):
54 tree_expire = ini.getint("general", "expire")
55
56 repo_url = ini.get("repo", "url")
57
58 rsync_bin_url = ini.get("rsync", "binary_url")
59 rsync_bin_key = ini.get("rsync", "binary_password")
60
61 rsync_src_url = None
62 rsync_src_key = None
63
64 if ini.has_option("rsync", "source_url"):
65 rsync_src_url = ini.get("rsync", "source_url")
66 rsync_src_key = ini.get("rsync", "source_password")
67
68 gpg_keyid = None
69 gpg_comment = "Unattended build signature"
70 gpg_passfile = "/dev/null"
71
72 if ini.has_option("gpg", "keyid"):
73 gpg_keyid = ini.get("gpg", "keyid")
74
75 if ini.has_option("gpg", "comment"):
76 gpg_comment = ini.get("gpg", "comment")
77
78 if ini.has_option("gpg", "passfile"):
79 gpg_passfile = ini.get("gpg", "passfile")
80
81
82 # find targets
83 targets = [ ]
84
85 findtargets = subprocess.Popen([home_dir+'/dumpinfo.pl', 'targets'],
86 stdout = subprocess.PIPE, cwd = home_dir+'/source.git')
87
88 while True:
89 line = findtargets.stdout.readline()
90 if not line:
91 break
92 ta = line.strip().split(' ')
93 targets.append(ta[0])
94
95
96 # the 'change_source' setting tells the buildmaster how it should find out
97 # about source code changes. Here we point to the buildbot clone of pyflakes.
98
99 from buildbot.changes.gitpoller import GitPoller
100 c['change_source'] = []
101 c['change_source'].append(GitPoller(
102 repo_url,
103 workdir=home_dir+'/source.git', branch='master',
104 pollinterval=300))
105
106 ####### SCHEDULERS
107
108 # Configure the Schedulers, which decide how to react to incoming changes. In this
109 # case, just kick off a 'basebuild' build
110
111 from buildbot.schedulers.basic import SingleBranchScheduler
112 from buildbot.schedulers.forcesched import ForceScheduler
113 from buildbot.changes import filter
114 c['schedulers'] = []
115 c['schedulers'].append(SingleBranchScheduler(
116 name="all",
117 change_filter=filter.ChangeFilter(branch='master'),
118 treeStableTimer=60,
119 builderNames=targets))
120
121 c['schedulers'].append(ForceScheduler(
122 name="force",
123 builderNames=targets))
124
125 ####### BUILDERS
126
127 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
128 # what steps, and which slaves can execute them. Note that any particular build will
129 # only take place on one slave.
130
131 from buildbot.process.factory import BuildFactory
132 from buildbot.steps.source import Git
133 from buildbot.steps.shell import ShellCommand
134 from buildbot.steps.shell import SetProperty
135 from buildbot.steps.transfer import FileUpload
136 from buildbot.steps.transfer import FileDownload
137 from buildbot.steps.master import MasterShellCommand
138 from buildbot.process.properties import WithProperties
139
140
141 MakeTargetMap = {
142 "^tools/": "tools/clean",
143 "^toolchain/": "toolchain/clean",
144 "^target/linux/": "target/linux/clean",
145 "^(config|include)/": "dirclean"
146 }
147
148 def IsAffected(pattern):
149 def CheckAffected(change):
150 for request in change.build.requests:
151 for source in request.sources:
152 for change in source.changes:
153 for file in change.files:
154 if re.match(pattern, file):
155 return True
156 return False
157 return CheckAffected
158
159 def isPathBuiltin(path):
160 incl = {}
161 pkgs = {}
162 conf = open(".config", "r")
163
164 while True:
165 line = conf.readline()
166 if line == '':
167 break
168 m = re.match("^(CONFIG_PACKAGE_.+?)=y", line)
169 if m:
170 incl[m.group(1)] = True
171
172 conf.close()
173
174 deps = open("tmp/.packagedeps", "r")
175
176 while True:
177 line = deps.readline()
178 if line == '':
179 break
180 m = re.match("^package-\$\((CONFIG_PACKAGE_.+?)\) \+= (\S+)", line)
181 if m and incl.get(m.group(1)) == True:
182 pkgs["package/%s" % m.group(2)] = True
183
184 deps.close()
185
186 while path != '':
187 if pkgs.get(path) == True:
188 return True
189 path = os.path.dirname(path)
190
191 return False
192
193 def isChangeBuiltin(change):
194 return True
195 # for request in change.build.requests:
196 # for source in request.sources:
197 # for change in source.changes:
198 # for file in change.files:
199 # if isPathBuiltin(file):
200 # return True
201 # return False
202
203
204 c['builders'] = []
205
206 dlLock = locks.SlaveLock("slave_dl")
207
208 checkBuiltin = re.sub('[\t\n ]+', ' ', """
209 checkBuiltin() {
210 local symbol op path file;
211 for file in $CHANGED_FILES; do
212 case "$file" in
213 package/*/*) : ;;
214 *) return 0 ;;
215 esac;
216 done;
217 while read symbol op path; do
218 case "$symbol" in package-*)
219 symbol="${symbol##*(}";
220 symbol="${symbol%)}";
221 for file in $CHANGED_FILES; do
222 case "$file" in "package/$path/"*)
223 grep -qsx "$symbol=y" .config && return 0
224 ;; esac;
225 done;
226 esac;
227 done < tmp/.packagedeps;
228 return 1;
229 }
230 """).strip()
231
232
233 class IfBuiltinShellCommand(ShellCommand):
234 def _quote(self, str):
235 if re.search("[^a-zA-Z0-9/_.-]", str):
236 return "'%s'" %(re.sub("'", "'\"'\"'", str))
237 return str
238
239 def setCommand(self, command):
240 if not isinstance(command, (str, unicode)):
241 command = ' '.join(map(self._quote, command))
242 self.command = [
243 '/bin/sh', '-c',
244 '%s; if checkBuiltin; then %s; else exit 0; fi' %(checkBuiltin, command)
245 ]
246
247 def setupEnvironment(self, cmd):
248 slaveEnv = self.slaveEnvironment
249 if slaveEnv is None:
250 slaveEnv = { }
251 changedFiles = { }
252 for request in self.build.requests:
253 for source in request.sources:
254 for change in source.changes:
255 for file in change.files:
256 changedFiles[file] = True
257 fullSlaveEnv = slaveEnv.copy()
258 fullSlaveEnv['CHANGED_FILES'] = ' '.join(changedFiles.keys())
259 cmd.args['env'] = fullSlaveEnv
260
261 slaveNames = [ ]
262
263 for slave in c['slaves']:
264 slaveNames.append(slave.slavename)
265
266 for target in targets:
267 ts = target.split('/')
268
269 factory = BuildFactory()
270
271 # find number of cores
272 factory.addStep(SetProperty(
273 name = "nproc",
274 property = "nproc",
275 description = "Finding number of CPUs",
276 command = ["nproc"]))
277
278 # expire tree if needed
279 if tree_expire > 0:
280 factory.addStep(FileDownload(
281 mastersrc = "expire.sh",
282 slavedest = "../expire.sh",
283 mode = 0755))
284
285 factory.addStep(ShellCommand(
286 name = "expire",
287 description = "Checking for build tree expiry",
288 command = ["./expire.sh", str(tree_expire)],
289 workdir = ".",
290 haltOnFailure = True,
291 timeout = 2400))
292
293 # check out the source
294 factory.addStep(Git(repourl=repo_url, mode='update'))
295
296 factory.addStep(ShellCommand(
297 name = "rmtmp",
298 description = "Remove tmp folder",
299 command=["rm", "-rf", "tmp/"]))
300
301 # feed
302 # factory.addStep(ShellCommand(
303 # name = "feedsconf",
304 # description = "Copy the feeds.conf",
305 # command='''cp ~/feeds.conf ./feeds.conf''' ))
306
307 # feed
308 factory.addStep(ShellCommand(
309 name = "rmfeedlinks",
310 description = "Remove feed symlinks",
311 command=["rm", "-rf", "package/feeds/"]))
312
313 # feed
314 factory.addStep(ShellCommand(
315 name = "updatefeeds",
316 description = "Updating feeds",
317 command=["./scripts/feeds", "update"]))
318
319 # feed
320 factory.addStep(ShellCommand(
321 name = "installfeeds",
322 description = "Installing feeds",
323 command=["./scripts/feeds", "install", "-a"]))
324
325 # configure
326 factory.addStep(ShellCommand(
327 name = "newconfig",
328 description = "Seeding .config",
329 command='''cat <<EOT > .config
330 CONFIG_TARGET_%s=y
331 CONFIG_TARGET_%s_%s=y
332 CONFIG_ALL_NONSHARED=y
333 CONFIG_SDK=y
334 CONFIG_IB=y
335 # CONFIG_IB_STANDALONE is not set
336 CONFIG_DEVEL=y
337 CONFIG_CCACHE=y
338 CONFIG_SIGNED_PACKAGES=y
339 # CONFIG_PER_FEED_REPO_ADD_COMMENTED is not set
340 CONFIG_KERNEL_KALLSYMS=y
341 CONFIG_COLLECT_KERNEL_DEBUG=y
342 CONFIG_TARGET_ALL_PROFILES=y
343 CONFIG_TARGET_MULTI_PROFILE=y
344 CONFIG_TARGET_PER_DEVICE_ROOTFS=y
345 EOT''' %(ts[0], ts[0], ts[1]) ))
346
347 factory.addStep(ShellCommand(
348 name = "delbin",
349 description = "Removing output directory",
350 command = ["rm", "-rf", "bin/"]
351 ))
352
353 factory.addStep(ShellCommand(
354 name = "defconfig",
355 description = "Populating .config",
356 command = ["make", "defconfig"]
357 ))
358
359 # check arch
360 factory.addStep(ShellCommand(
361 name = "checkarch",
362 description = "Checking architecture",
363 command = ["grep", "-sq", "CONFIG_TARGET_%s=y" %(ts[0]), ".config"],
364 logEnviron = False,
365 want_stdout = False,
366 want_stderr = False,
367 haltOnFailure = True
368 ))
369
370 # find libc suffix
371 factory.addStep(SetProperty(
372 name = "libc",
373 property = "libc",
374 description = "Finding libc suffix",
375 command = ["sed", "-ne", '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }', ".config"]))
376
377 # install build key
378 factory.addStep(FileDownload(mastersrc=home_dir+'/key-build', slavedest="key-build", mode=0600))
379 factory.addStep(FileDownload(mastersrc=home_dir+'/key-build.pub', slavedest="key-build.pub", mode=0600))
380
381 # prepare dl
382 factory.addStep(ShellCommand(
383 name = "dldir",
384 description = "Preparing dl/",
385 command = "mkdir -p $HOME/dl && ln -sf $HOME/dl ./dl",
386 logEnviron = False,
387 want_stdout = False
388 ))
389
390 # populate dl
391 factory.addStep(ShellCommand(
392 name = "dlrun",
393 description = "Populating dl/",
394 command = ["make", WithProperties("-j%(nproc:~4)s"), "download", "V=s"],
395 logEnviron = False,
396 locks = [dlLock.access('exclusive')]
397 ))
398
399 factory.addStep(ShellCommand(
400 name = "cleanbase",
401 description = "Cleaning base-files",
402 command=["make", "package/base-files/clean", "V=s"]
403 ))
404
405 # optional clean steps
406 for pattern, maketarget in MakeTargetMap.items():
407 factory.addStep(ShellCommand(
408 name = maketarget,
409 description = maketarget,
410 command=["make", maketarget, "V=s"], doStepIf=IsAffected(pattern)
411 ))
412
413 # build
414 factory.addStep(ShellCommand(
415 name = "tools",
416 description = "Building tools",
417 command = ["make", WithProperties("-j%(nproc:~4)s"), "tools/install", "V=s"],
418 haltOnFailure = True
419 ))
420
421 factory.addStep(ShellCommand(
422 name = "toolchain",
423 description = "Building toolchain",
424 command=["make", WithProperties("-j%(nproc:~4)s"), "toolchain/install", "V=s"],
425 haltOnFailure = True
426 ))
427
428 factory.addStep(ShellCommand(
429 name = "kmods",
430 description = "Building kmods",
431 command=["make", WithProperties("-j%(nproc:~4)s"), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
432 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
433 haltOnFailure = True
434 ))
435
436 factory.addStep(ShellCommand(
437 name = "pkgbuild",
438 description = "Building packages",
439 command=["make", WithProperties("-j%(nproc:~4)s"), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
440 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
441 haltOnFailure = True
442 ))
443
444 # factory.addStep(IfBuiltinShellCommand(
445 factory.addStep(ShellCommand(
446 name = "pkginstall",
447 description = "Installing packages",
448 command=["make", WithProperties("-j%(nproc:~4)s"), "package/install", "V=s"],
449 doStepIf = isChangeBuiltin,
450 haltOnFailure = True
451 ))
452
453 factory.addStep(ShellCommand(
454 name = "pkgindex",
455 description = "Indexing packages",
456 command=["make", WithProperties("-j%(nproc:~4)s"), "package/index", "V=s"],
457 haltOnFailure = True
458 ))
459
460 #factory.addStep(IfBuiltinShellCommand(
461 factory.addStep(ShellCommand(
462 name = "images",
463 description = "Building images",
464 command=["make", WithProperties("-j%(nproc:~4)s"), "target/install", "V=s"],
465 doStepIf = isChangeBuiltin,
466 haltOnFailure = True
467 ))
468
469 factory.addStep(ShellCommand(
470 name = "checksums",
471 description = "Calculating checksums",
472 command=["make", "-j1", "checksum", "V=s"],
473 doStepIf = isChangeBuiltin,
474 haltOnFailure = True
475 ))
476
477 # sign
478 if gpg_keyid is not None:
479 factory.addStep(MasterShellCommand(
480 name = "signprepare",
481 description = "Preparing temporary signing directory",
482 command = ["mkdir", "-p", "%s/signing" %(home_dir)],
483 haltOnFailure = True
484 ))
485
486 factory.addStep(ShellCommand(
487 name = "signpack",
488 description = "Packing files to sign",
489 command = WithProperties("find bin/targets/%s/%s%%(libc)s/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz" %(ts[0], ts[1])),
490 haltOnFailure = True
491 ))
492
493 factory.addStep(FileUpload(
494 slavesrc = "sign.tar.gz",
495 masterdest = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
496 haltOnFailure = True
497 ))
498
499 factory.addStep(MasterShellCommand(
500 name = "signfiles",
501 description = "Signing files",
502 command = ["%s/signall.sh" %(home_dir), "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]), gpg_keyid, gpg_passfile, gpg_comment],
503 haltOnFailure = True
504 ))
505
506 factory.addStep(FileDownload(
507 mastersrc = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
508 slavedest = "sign.tar.gz",
509 haltOnFailure = True
510 ))
511
512 factory.addStep(ShellCommand(
513 name = "signunpack",
514 description = "Unpacking signed files",
515 command = ["tar", "-xzf", "sign.tar.gz"],
516 haltOnFailure = True
517 ))
518
519 # upload
520 factory.addStep(ShellCommand(
521 name = "uploadprepare",
522 description = "Preparing target directory",
523 command=["rsync", "-av", "--include", "/%s/" %(ts[0]), "--include", "/%s/%s/" %(ts[0], ts[1]), "--exclude", "/*", "--exclude", "/*/*", "--exclude", "/%s/%s/*" %(ts[0], ts[1]), "bin/targets/", "%s/targets/" %(rsync_bin_url)],
524 env={'RSYNC_PASSWORD': rsync_bin_key},
525 haltOnFailure = True,
526 logEnviron = False
527 ))
528
529 factory.addStep(ShellCommand(
530 name = "targetupload",
531 description = "Uploading target files",
532 command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", WithProperties("bin/targets/%s/%s%%(libc)s/" %(ts[0], ts[1])), "%s/targets/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
533 env={'RSYNC_PASSWORD': rsync_bin_key},
534 haltOnFailure = True,
535 logEnviron = False
536 ))
537
538 if rsync_src_url is not None:
539 factory.addStep(ShellCommand(
540 name = "sourceupload",
541 description = "Uploading source archives",
542 command=["rsync", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "dl/", "%s/" %(rsync_src_url)],
543 env={'RSYNC_PASSWORD': rsync_src_key},
544 haltOnFailure = True,
545 logEnviron = False
546 ))
547
548 if False:
549 factory.addStep(ShellCommand(
550 name = "packageupload",
551 description = "Uploading package files",
552 command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "bin/packages/", "%s/packages/" %(rsync_bin_url)],
553 env={'RSYNC_PASSWORD': rsync_bin_key},
554 haltOnFailure = False,
555 logEnviron = False
556 ))
557
558 # logs
559 if False:
560 factory.addStep(ShellCommand(
561 name = "upload",
562 description = "Uploading logs",
563 command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "logs/", "%s/logs/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
564 env={'RSYNC_PASSWORD': rsync_bin_key},
565 haltOnFailure = False,
566 alwaysRun = True,
567 logEnviron = False
568 ))
569
570 from buildbot.config import BuilderConfig
571
572 c['builders'].append(BuilderConfig(name=target, slavenames=slaveNames, factory=factory))
573
574
575 ####### STATUS TARGETS
576
577 # 'status' is a list of Status Targets. The results of each build will be
578 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
579 # including web pages, email senders, and IRC bots.
580
581 c['status'] = []
582
583 from buildbot.status import html
584 from buildbot.status.web import authz, auth
585
586 if ini.has_option("status", "bind"):
587 if ini.has_option("status", "user") and ini.has_option("status", "password"):
588 authz_cfg=authz.Authz(
589 # change any of these to True to enable; see the manual for more
590 # options
591 auth=auth.BasicAuth([(ini.get("status", "user"), ini.get("status", "password"))]),
592 gracefulShutdown = 'auth',
593 forceBuild = 'auth', # use this to test your slave once it is set up
594 forceAllBuilds = 'auth',
595 pingBuilder = False,
596 stopBuild = 'auth',
597 stopAllBuilds = 'auth',
598 cancelPendingBuild = 'auth',
599 )
600 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind"), authz=authz_cfg))
601 else:
602 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind")))
603
604
605 from buildbot.status import words
606
607 if ini.has_option("irc", "host") and ini.has_option("irc", "nickname") and ini.has_option("irc", "channel"):
608 irc_host = ini.get("irc", "host")
609 irc_port = 6667
610 irc_chan = ini.get("irc", "channel")
611 irc_nick = ini.get("irc", "nickname")
612 irc_pass = None
613
614 if ini.has_option("irc", "port"):
615 irc_port = ini.getint("irc", "port")
616
617 if ini.has_option("irc", "password"):
618 irc_pass = ini.get("irc", "password")
619
620 irc = words.IRC(irc_host, irc_nick, port = irc_port, password = irc_pass,
621 channels = [{ "channel": irc_chan }],
622 notify_events = {
623 'exception': 1,
624 'successToFailure': 1,
625 'failureToSuccess': 1
626 }
627 )
628
629 c['status'].append(irc)
630
631
632 ####### PROJECT IDENTITY
633
634 # the 'title' string will appear at the top of this buildbot
635 # installation's html.WebStatus home page (linked to the
636 # 'titleURL') and is embedded in the title of the waterfall HTML page.
637
638 c['title'] = ini.get("general", "title")
639 c['titleURL'] = ini.get("general", "title_url")
640
641 # the 'buildbotURL' string should point to the location where the buildbot's
642 # internal web server (usually the html.WebStatus page) is visible. This
643 # typically uses the port number set in the Waterfall 'status' entry, but
644 # with an externally-visible host name which the buildbot cannot figure out
645 # without some help.
646
647 c['buildbotURL'] = ini.get("general", "buildbot_url")
648
649 ####### DB URL
650
651 c['db'] = {
652 # This specifies what database buildbot uses to store its state. You can leave
653 # this at its default for all but the largest installations.
654 'db_url' : "sqlite:///state.sqlite",
655 }