phase1: initialize source.git on first start
[buildbot.git] / phase1 / master.cfg
1 # -*- python -*-
2 # ex: set syntax=python:
3
4 import os
5 import re
6 import subprocess
7 import ConfigParser
8
9 from buildbot import locks
10
11 # This is a sample buildmaster config file. It must be installed as
12 # 'master.cfg' in your buildmaster's base directory.
13
14 ini = ConfigParser.ConfigParser()
15 ini.read("./config.ini")
16
17 # This is the dictionary that the buildmaster pays attention to. We also use
18 # a shorter alias to save typing.
19 c = BuildmasterConfig = {}
20
21 ####### BUILDSLAVES
22
23 # The 'slaves' list defines the set of recognized buildslaves. Each element is
24 # a BuildSlave object, specifying a unique slave name and password. The same
25 # slave name and password must be configured on the slave.
26 from buildbot.buildslave import BuildSlave
27
28 c['slaves'] = []
29
30 for section in ini.sections():
31 if section.startswith("slave "):
32 if ini.has_option(section, "name") and ini.has_option(section, "password"):
33 name = ini.get(section, "name")
34 password = ini.get(section, "password")
35 max_builds = 1
36 if ini.has_option(section, "builds"):
37 max_builds = ini.getint(section, "builds")
38 c['slaves'].append(BuildSlave(name, password, max_builds = max_builds))
39
40 # 'slavePortnum' defines the TCP port to listen on for connections from slaves.
41 # This must match the value configured into the buildslaves (with their
42 # --master option)
43 c['slavePortnum'] = 9989
44
45 # coalesce builds
46 c['mergeRequests'] = True
47
48 ####### CHANGESOURCES
49
50 home_dir = os.path.abspath(ini.get("general", "homedir"))
51 tree_expire = 0
52
53 if ini.has_option("general", "expire"):
54 tree_expire = ini.getint("general", "expire")
55
56 repo_url = ini.get("repo", "url")
57
58 rsync_bin_url = ini.get("rsync", "binary_url")
59 rsync_bin_key = ini.get("rsync", "binary_password")
60
61 rsync_src_url = None
62 rsync_src_key = None
63
64 if ini.has_option("rsync", "source_url"):
65 rsync_src_url = ini.get("rsync", "source_url")
66 rsync_src_key = ini.get("rsync", "source_password")
67
68 gpg_keyid = None
69 gpg_comment = "Unattended build signature"
70 gpg_passfile = "/dev/null"
71
72 if ini.has_option("gpg", "keyid"):
73 gpg_keyid = ini.get("gpg", "keyid")
74
75 if ini.has_option("gpg", "comment"):
76 gpg_comment = ini.get("gpg", "comment")
77
78 if ini.has_option("gpg", "passfile"):
79 gpg_passfile = ini.get("gpg", "passfile")
80
81
82 # find targets
83 targets = [ ]
84
85 if not os.path.isdir(home_dir+'/source.git'):
86 subprocess.call(["git", "clone", "--depth=1", repo_url, home_dir+'/source.git'])
87
88 findtargets = subprocess.Popen([home_dir+'/dumpinfo.pl', 'targets'],
89 stdout = subprocess.PIPE, cwd = home_dir+'/source.git')
90
91 while True:
92 line = findtargets.stdout.readline()
93 if not line:
94 break
95 ta = line.strip().split(' ')
96 targets.append(ta[0])
97
98
99 # the 'change_source' setting tells the buildmaster how it should find out
100 # about source code changes. Here we point to the buildbot clone of pyflakes.
101
102 from buildbot.changes.gitpoller import GitPoller
103 c['change_source'] = []
104 c['change_source'].append(GitPoller(
105 repo_url,
106 workdir=home_dir+'/source.git', branch='master',
107 pollinterval=300))
108
109 ####### SCHEDULERS
110
111 # Configure the Schedulers, which decide how to react to incoming changes. In this
112 # case, just kick off a 'basebuild' build
113
114 from buildbot.schedulers.basic import SingleBranchScheduler
115 from buildbot.schedulers.forcesched import ForceScheduler
116 from buildbot.changes import filter
117 c['schedulers'] = []
118 c['schedulers'].append(SingleBranchScheduler(
119 name="all",
120 change_filter=filter.ChangeFilter(branch='master'),
121 treeStableTimer=60,
122 builderNames=targets))
123
124 c['schedulers'].append(ForceScheduler(
125 name="force",
126 builderNames=targets))
127
128 ####### BUILDERS
129
130 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
131 # what steps, and which slaves can execute them. Note that any particular build will
132 # only take place on one slave.
133
134 from buildbot.process.factory import BuildFactory
135 from buildbot.steps.source import Git
136 from buildbot.steps.shell import ShellCommand
137 from buildbot.steps.shell import SetProperty
138 from buildbot.steps.transfer import FileUpload
139 from buildbot.steps.transfer import FileDownload
140 from buildbot.steps.master import MasterShellCommand
141 from buildbot.process.properties import WithProperties
142
143
144 MakeTargetMap = {
145 "^tools/": "tools/clean",
146 "^toolchain/": "toolchain/clean",
147 "^target/linux/": "target/linux/clean",
148 "^(config|include)/": "dirclean"
149 }
150
151 def IsAffected(pattern):
152 def CheckAffected(change):
153 for request in change.build.requests:
154 for source in request.sources:
155 for change in source.changes:
156 for file in change.files:
157 if re.match(pattern, file):
158 return True
159 return False
160 return CheckAffected
161
162 def isPathBuiltin(path):
163 incl = {}
164 pkgs = {}
165 conf = open(".config", "r")
166
167 while True:
168 line = conf.readline()
169 if line == '':
170 break
171 m = re.match("^(CONFIG_PACKAGE_.+?)=y", line)
172 if m:
173 incl[m.group(1)] = True
174
175 conf.close()
176
177 deps = open("tmp/.packagedeps", "r")
178
179 while True:
180 line = deps.readline()
181 if line == '':
182 break
183 m = re.match("^package-\$\((CONFIG_PACKAGE_.+?)\) \+= (\S+)", line)
184 if m and incl.get(m.group(1)) == True:
185 pkgs["package/%s" % m.group(2)] = True
186
187 deps.close()
188
189 while path != '':
190 if pkgs.get(path) == True:
191 return True
192 path = os.path.dirname(path)
193
194 return False
195
196 def isChangeBuiltin(change):
197 return True
198 # for request in change.build.requests:
199 # for source in request.sources:
200 # for change in source.changes:
201 # for file in change.files:
202 # if isPathBuiltin(file):
203 # return True
204 # return False
205
206
207 c['builders'] = []
208
209 dlLock = locks.SlaveLock("slave_dl")
210
211 checkBuiltin = re.sub('[\t\n ]+', ' ', """
212 checkBuiltin() {
213 local symbol op path file;
214 for file in $CHANGED_FILES; do
215 case "$file" in
216 package/*/*) : ;;
217 *) return 0 ;;
218 esac;
219 done;
220 while read symbol op path; do
221 case "$symbol" in package-*)
222 symbol="${symbol##*(}";
223 symbol="${symbol%)}";
224 for file in $CHANGED_FILES; do
225 case "$file" in "package/$path/"*)
226 grep -qsx "$symbol=y" .config && return 0
227 ;; esac;
228 done;
229 esac;
230 done < tmp/.packagedeps;
231 return 1;
232 }
233 """).strip()
234
235
236 class IfBuiltinShellCommand(ShellCommand):
237 def _quote(self, str):
238 if re.search("[^a-zA-Z0-9/_.-]", str):
239 return "'%s'" %(re.sub("'", "'\"'\"'", str))
240 return str
241
242 def setCommand(self, command):
243 if not isinstance(command, (str, unicode)):
244 command = ' '.join(map(self._quote, command))
245 self.command = [
246 '/bin/sh', '-c',
247 '%s; if checkBuiltin; then %s; else exit 0; fi' %(checkBuiltin, command)
248 ]
249
250 def setupEnvironment(self, cmd):
251 slaveEnv = self.slaveEnvironment
252 if slaveEnv is None:
253 slaveEnv = { }
254 changedFiles = { }
255 for request in self.build.requests:
256 for source in request.sources:
257 for change in source.changes:
258 for file in change.files:
259 changedFiles[file] = True
260 fullSlaveEnv = slaveEnv.copy()
261 fullSlaveEnv['CHANGED_FILES'] = ' '.join(changedFiles.keys())
262 cmd.args['env'] = fullSlaveEnv
263
264 slaveNames = [ ]
265
266 for slave in c['slaves']:
267 slaveNames.append(slave.slavename)
268
269 for target in targets:
270 ts = target.split('/')
271
272 factory = BuildFactory()
273
274 # find number of cores
275 factory.addStep(SetProperty(
276 name = "nproc",
277 property = "nproc",
278 description = "Finding number of CPUs",
279 command = ["nproc"]))
280
281 # expire tree if needed
282 if tree_expire > 0:
283 factory.addStep(FileDownload(
284 mastersrc = "expire.sh",
285 slavedest = "../expire.sh",
286 mode = 0755))
287
288 factory.addStep(ShellCommand(
289 name = "expire",
290 description = "Checking for build tree expiry",
291 command = ["./expire.sh", str(tree_expire)],
292 workdir = ".",
293 haltOnFailure = True,
294 timeout = 2400))
295
296 # check out the source
297 factory.addStep(Git(repourl=repo_url, mode='update'))
298
299 factory.addStep(ShellCommand(
300 name = "rmtmp",
301 description = "Remove tmp folder",
302 command=["rm", "-rf", "tmp/"]))
303
304 # feed
305 # factory.addStep(ShellCommand(
306 # name = "feedsconf",
307 # description = "Copy the feeds.conf",
308 # command='''cp ~/feeds.conf ./feeds.conf''' ))
309
310 # feed
311 factory.addStep(ShellCommand(
312 name = "rmfeedlinks",
313 description = "Remove feed symlinks",
314 command=["rm", "-rf", "package/feeds/"]))
315
316 # feed
317 factory.addStep(ShellCommand(
318 name = "updatefeeds",
319 description = "Updating feeds",
320 command=["./scripts/feeds", "update"]))
321
322 # feed
323 factory.addStep(ShellCommand(
324 name = "installfeeds",
325 description = "Installing feeds",
326 command=["./scripts/feeds", "install", "-a"]))
327
328 # configure
329 factory.addStep(ShellCommand(
330 name = "newconfig",
331 description = "Seeding .config",
332 command='''cat <<EOT > .config
333 CONFIG_TARGET_%s=y
334 CONFIG_TARGET_%s_%s=y
335 CONFIG_ALL_NONSHARED=y
336 CONFIG_SDK=y
337 CONFIG_IB=y
338 # CONFIG_IB_STANDALONE is not set
339 CONFIG_DEVEL=y
340 CONFIG_CCACHE=y
341 CONFIG_SIGNED_PACKAGES=y
342 # CONFIG_PER_FEED_REPO_ADD_COMMENTED is not set
343 CONFIG_KERNEL_KALLSYMS=y
344 CONFIG_COLLECT_KERNEL_DEBUG=y
345 CONFIG_TARGET_ALL_PROFILES=y
346 CONFIG_TARGET_MULTI_PROFILE=y
347 CONFIG_TARGET_PER_DEVICE_ROOTFS=y
348 EOT''' %(ts[0], ts[0], ts[1]) ))
349
350 factory.addStep(ShellCommand(
351 name = "delbin",
352 description = "Removing output directory",
353 command = ["rm", "-rf", "bin/"]
354 ))
355
356 factory.addStep(ShellCommand(
357 name = "defconfig",
358 description = "Populating .config",
359 command = ["make", "defconfig"]
360 ))
361
362 # check arch
363 factory.addStep(ShellCommand(
364 name = "checkarch",
365 description = "Checking architecture",
366 command = ["grep", "-sq", "CONFIG_TARGET_%s=y" %(ts[0]), ".config"],
367 logEnviron = False,
368 want_stdout = False,
369 want_stderr = False,
370 haltOnFailure = True
371 ))
372
373 # find libc suffix
374 factory.addStep(SetProperty(
375 name = "libc",
376 property = "libc",
377 description = "Finding libc suffix",
378 command = ["sed", "-ne", '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }', ".config"]))
379
380 # install build key
381 factory.addStep(FileDownload(mastersrc=home_dir+'/key-build', slavedest="key-build", mode=0600))
382 factory.addStep(FileDownload(mastersrc=home_dir+'/key-build.pub', slavedest="key-build.pub", mode=0600))
383
384 # prepare dl
385 factory.addStep(ShellCommand(
386 name = "dldir",
387 description = "Preparing dl/",
388 command = "mkdir -p $HOME/dl && ln -sf $HOME/dl ./dl",
389 logEnviron = False,
390 want_stdout = False
391 ))
392
393 # populate dl
394 factory.addStep(ShellCommand(
395 name = "dlrun",
396 description = "Populating dl/",
397 command = ["make", WithProperties("-j%(nproc:~4)s"), "download", "V=s"],
398 logEnviron = False,
399 locks = [dlLock.access('exclusive')]
400 ))
401
402 factory.addStep(ShellCommand(
403 name = "cleanbase",
404 description = "Cleaning base-files",
405 command=["make", "package/base-files/clean", "V=s"]
406 ))
407
408 # optional clean steps
409 for pattern, maketarget in MakeTargetMap.items():
410 factory.addStep(ShellCommand(
411 name = maketarget,
412 description = maketarget,
413 command=["make", maketarget, "V=s"], doStepIf=IsAffected(pattern)
414 ))
415
416 # build
417 factory.addStep(ShellCommand(
418 name = "tools",
419 description = "Building tools",
420 command = ["make", WithProperties("-j%(nproc:~4)s"), "tools/install", "V=s"],
421 haltOnFailure = True
422 ))
423
424 factory.addStep(ShellCommand(
425 name = "toolchain",
426 description = "Building toolchain",
427 command=["make", WithProperties("-j%(nproc:~4)s"), "toolchain/install", "V=s"],
428 haltOnFailure = True
429 ))
430
431 factory.addStep(ShellCommand(
432 name = "kmods",
433 description = "Building kmods",
434 command=["make", WithProperties("-j%(nproc:~4)s"), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
435 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
436 haltOnFailure = True
437 ))
438
439 factory.addStep(ShellCommand(
440 name = "pkgbuild",
441 description = "Building packages",
442 command=["make", WithProperties("-j%(nproc:~4)s"), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
443 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
444 haltOnFailure = True
445 ))
446
447 # factory.addStep(IfBuiltinShellCommand(
448 factory.addStep(ShellCommand(
449 name = "pkginstall",
450 description = "Installing packages",
451 command=["make", WithProperties("-j%(nproc:~4)s"), "package/install", "V=s"],
452 doStepIf = isChangeBuiltin,
453 haltOnFailure = True
454 ))
455
456 factory.addStep(ShellCommand(
457 name = "pkgindex",
458 description = "Indexing packages",
459 command=["make", WithProperties("-j%(nproc:~4)s"), "package/index", "V=s"],
460 haltOnFailure = True
461 ))
462
463 #factory.addStep(IfBuiltinShellCommand(
464 factory.addStep(ShellCommand(
465 name = "images",
466 description = "Building images",
467 command=["make", WithProperties("-j%(nproc:~4)s"), "target/install", "V=s"],
468 doStepIf = isChangeBuiltin,
469 haltOnFailure = True
470 ))
471
472 factory.addStep(ShellCommand(
473 name = "checksums",
474 description = "Calculating checksums",
475 command=["make", "-j1", "checksum", "V=s"],
476 doStepIf = isChangeBuiltin,
477 haltOnFailure = True
478 ))
479
480 # sign
481 if gpg_keyid is not None:
482 factory.addStep(MasterShellCommand(
483 name = "signprepare",
484 description = "Preparing temporary signing directory",
485 command = ["mkdir", "-p", "%s/signing" %(home_dir)],
486 haltOnFailure = True
487 ))
488
489 factory.addStep(ShellCommand(
490 name = "signpack",
491 description = "Packing files to sign",
492 command = WithProperties("find bin/targets/%s/%s%%(libc)s/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz" %(ts[0], ts[1])),
493 haltOnFailure = True
494 ))
495
496 factory.addStep(FileUpload(
497 slavesrc = "sign.tar.gz",
498 masterdest = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
499 haltOnFailure = True
500 ))
501
502 factory.addStep(MasterShellCommand(
503 name = "signfiles",
504 description = "Signing files",
505 command = ["%s/signall.sh" %(home_dir), "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]), gpg_keyid, gpg_passfile, gpg_comment],
506 haltOnFailure = True
507 ))
508
509 factory.addStep(FileDownload(
510 mastersrc = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
511 slavedest = "sign.tar.gz",
512 haltOnFailure = True
513 ))
514
515 factory.addStep(ShellCommand(
516 name = "signunpack",
517 description = "Unpacking signed files",
518 command = ["tar", "-xzf", "sign.tar.gz"],
519 haltOnFailure = True
520 ))
521
522 # upload
523 factory.addStep(ShellCommand(
524 name = "uploadprepare",
525 description = "Preparing target directory",
526 command=["rsync", "-av", "--include", "/%s/" %(ts[0]), "--include", "/%s/%s/" %(ts[0], ts[1]), "--exclude", "/*", "--exclude", "/*/*", "--exclude", "/%s/%s/*" %(ts[0], ts[1]), "bin/targets/", "%s/targets/" %(rsync_bin_url)],
527 env={'RSYNC_PASSWORD': rsync_bin_key},
528 haltOnFailure = True,
529 logEnviron = False
530 ))
531
532 factory.addStep(ShellCommand(
533 name = "targetupload",
534 description = "Uploading target files",
535 command=["rsync", "--delete", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", WithProperties("bin/targets/%s/%s%%(libc)s/" %(ts[0], ts[1])), "%s/targets/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
536 env={'RSYNC_PASSWORD': rsync_bin_key},
537 haltOnFailure = True,
538 logEnviron = False
539 ))
540
541 if rsync_src_url is not None:
542 factory.addStep(ShellCommand(
543 name = "sourceupload",
544 description = "Uploading source archives",
545 command=["rsync", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "dl/", "%s/" %(rsync_src_url)],
546 env={'RSYNC_PASSWORD': rsync_src_key},
547 haltOnFailure = True,
548 logEnviron = False
549 ))
550
551 if False:
552 factory.addStep(ShellCommand(
553 name = "packageupload",
554 description = "Uploading package files",
555 command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "bin/packages/", "%s/packages/" %(rsync_bin_url)],
556 env={'RSYNC_PASSWORD': rsync_bin_key},
557 haltOnFailure = False,
558 logEnviron = False
559 ))
560
561 # logs
562 if False:
563 factory.addStep(ShellCommand(
564 name = "upload",
565 description = "Uploading logs",
566 command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "logs/", "%s/logs/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
567 env={'RSYNC_PASSWORD': rsync_bin_key},
568 haltOnFailure = False,
569 alwaysRun = True,
570 logEnviron = False
571 ))
572
573 from buildbot.config import BuilderConfig
574
575 c['builders'].append(BuilderConfig(name=target, slavenames=slaveNames, factory=factory))
576
577
578 ####### STATUS TARGETS
579
580 # 'status' is a list of Status Targets. The results of each build will be
581 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
582 # including web pages, email senders, and IRC bots.
583
584 c['status'] = []
585
586 from buildbot.status import html
587 from buildbot.status.web import authz, auth
588
589 if ini.has_option("status", "bind"):
590 if ini.has_option("status", "user") and ini.has_option("status", "password"):
591 authz_cfg=authz.Authz(
592 # change any of these to True to enable; see the manual for more
593 # options
594 auth=auth.BasicAuth([(ini.get("status", "user"), ini.get("status", "password"))]),
595 gracefulShutdown = 'auth',
596 forceBuild = 'auth', # use this to test your slave once it is set up
597 forceAllBuilds = 'auth',
598 pingBuilder = False,
599 stopBuild = 'auth',
600 stopAllBuilds = 'auth',
601 cancelPendingBuild = 'auth',
602 )
603 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind"), authz=authz_cfg))
604 else:
605 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind")))
606
607
608 from buildbot.status import words
609
610 if ini.has_option("irc", "host") and ini.has_option("irc", "nickname") and ini.has_option("irc", "channel"):
611 irc_host = ini.get("irc", "host")
612 irc_port = 6667
613 irc_chan = ini.get("irc", "channel")
614 irc_nick = ini.get("irc", "nickname")
615 irc_pass = None
616
617 if ini.has_option("irc", "port"):
618 irc_port = ini.getint("irc", "port")
619
620 if ini.has_option("irc", "password"):
621 irc_pass = ini.get("irc", "password")
622
623 irc = words.IRC(irc_host, irc_nick, port = irc_port, password = irc_pass,
624 channels = [{ "channel": irc_chan }],
625 notify_events = {
626 'exception': 1,
627 'successToFailure': 1,
628 'failureToSuccess': 1
629 }
630 )
631
632 c['status'].append(irc)
633
634
635 ####### PROJECT IDENTITY
636
637 # the 'title' string will appear at the top of this buildbot
638 # installation's html.WebStatus home page (linked to the
639 # 'titleURL') and is embedded in the title of the waterfall HTML page.
640
641 c['title'] = ini.get("general", "title")
642 c['titleURL'] = ini.get("general", "title_url")
643
644 # the 'buildbotURL' string should point to the location where the buildbot's
645 # internal web server (usually the html.WebStatus page) is visible. This
646 # typically uses the port number set in the Waterfall 'status' entry, but
647 # with an externally-visible host name which the buildbot cannot figure out
648 # without some help.
649
650 c['buildbotURL'] = ini.get("general", "buildbot_url")
651
652 ####### DB URL
653
654 c['db'] = {
655 # This specifies what database buildbot uses to store its state. You can leave
656 # this at its default for all but the largest installations.
657 'db_url' : "sqlite:///state.sqlite",
658 }