phase1: use a different name for the Git clone work directory
[buildbot.git] / phase1 / master.cfg
1 # -*- python -*-
2 # ex: set syntax=python:
3
4 import os
5 import re
6 import subprocess
7 import ConfigParser
8
9 from buildbot import locks
10
11 # This is a sample buildmaster config file. It must be installed as
12 # 'master.cfg' in your buildmaster's base directory.
13
14 ini = ConfigParser.ConfigParser()
15 ini.read("./config.ini")
16
17 # This is the dictionary that the buildmaster pays attention to. We also use
18 # a shorter alias to save typing.
19 c = BuildmasterConfig = {}
20
21 ####### BUILDSLAVES
22
23 # The 'slaves' list defines the set of recognized buildslaves. Each element is
24 # a BuildSlave object, specifying a unique slave name and password. The same
25 # slave name and password must be configured on the slave.
26 from buildbot.buildslave import BuildSlave
27
28 slave_port = 9989
29
30 if ini.has_option("general", "port"):
31 slave_port = ini.getint("general", "port")
32
33 c['slaves'] = []
34
35 for section in ini.sections():
36 if section.startswith("slave "):
37 if ini.has_option(section, "name") and ini.has_option(section, "password"):
38 name = ini.get(section, "name")
39 password = ini.get(section, "password")
40 max_builds = 1
41 if ini.has_option(section, "builds"):
42 max_builds = ini.getint(section, "builds")
43 c['slaves'].append(BuildSlave(name, password, max_builds = max_builds))
44
45 # 'slavePortnum' defines the TCP port to listen on for connections from slaves.
46 # This must match the value configured into the buildslaves (with their
47 # --master option)
48 c['slavePortnum'] = slave_port
49
50 # coalesce builds
51 c['mergeRequests'] = True
52
53 ####### CHANGESOURCES
54
55 home_dir = os.path.abspath(ini.get("general", "homedir"))
56 tree_expire = 0
57
58 if ini.has_option("general", "expire"):
59 tree_expire = ini.getint("general", "expire")
60
61 repo_url = ini.get("repo", "url")
62 repo_branch = "master"
63
64 if ini.has_option("repo", "branch"):
65 repo_branch = ini.get("repo", "branch")
66
67 rsync_bin_url = ini.get("rsync", "binary_url")
68 rsync_bin_key = ini.get("rsync", "binary_password")
69
70 rsync_src_url = None
71 rsync_src_key = None
72
73 if ini.has_option("rsync", "source_url"):
74 rsync_src_url = ini.get("rsync", "source_url")
75 rsync_src_key = ini.get("rsync", "source_password")
76
77 gpg_keyid = None
78 gpg_comment = "Unattended build signature"
79 gpg_passfile = "/dev/null"
80
81 if ini.has_option("gpg", "keyid"):
82 gpg_keyid = ini.get("gpg", "keyid")
83
84 if ini.has_option("gpg", "comment"):
85 gpg_comment = ini.get("gpg", "comment")
86
87 if ini.has_option("gpg", "passfile"):
88 gpg_passfile = ini.get("gpg", "passfile")
89
90
91 # find targets
92 targets = [ ]
93
94 if not os.path.isdir(home_dir+'/source.git'):
95 subprocess.call(["git", "clone", "--depth=1", repo_url, home_dir+'/source.git'])
96 subprocess.call(["git", "checkout", repo_branch], cwd = home_dir+'/source.git')
97 else:
98 subprocess.call(["git", "checkout", repo_branch], cwd = home_dir+'/source.git')
99 subprocess.call(["git", "pull"], cwd = home_dir+'/source.git')
100
101 findtargets = subprocess.Popen([home_dir+'/dumpinfo.pl', 'targets'],
102 stdout = subprocess.PIPE, cwd = home_dir+'/source.git')
103
104 while True:
105 line = findtargets.stdout.readline()
106 if not line:
107 break
108 ta = line.strip().split(' ')
109 targets.append(ta[0])
110
111
112 # the 'change_source' setting tells the buildmaster how it should find out
113 # about source code changes. Here we point to the buildbot clone of pyflakes.
114
115 from buildbot.changes.gitpoller import GitPoller
116 c['change_source'] = []
117 c['change_source'].append(GitPoller(
118 repo_url,
119 workdir=home_dir+'/work.git', branch='master',
120 pollinterval=300))
121
122 ####### SCHEDULERS
123
124 # Configure the Schedulers, which decide how to react to incoming changes. In this
125 # case, just kick off a 'basebuild' build
126
127 from buildbot.schedulers.basic import SingleBranchScheduler
128 from buildbot.schedulers.forcesched import ForceScheduler
129 from buildbot.changes import filter
130 c['schedulers'] = []
131 c['schedulers'].append(SingleBranchScheduler(
132 name="all",
133 change_filter=filter.ChangeFilter(branch='master'),
134 treeStableTimer=60,
135 builderNames=targets))
136
137 c['schedulers'].append(ForceScheduler(
138 name="force",
139 builderNames=targets))
140
141 ####### BUILDERS
142
143 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
144 # what steps, and which slaves can execute them. Note that any particular build will
145 # only take place on one slave.
146
147 from buildbot.process.factory import BuildFactory
148 from buildbot.steps.source import Git
149 from buildbot.steps.shell import ShellCommand
150 from buildbot.steps.shell import SetProperty
151 from buildbot.steps.transfer import FileUpload
152 from buildbot.steps.transfer import FileDownload
153 from buildbot.steps.master import MasterShellCommand
154 from buildbot.process.properties import WithProperties
155
156
157 CleanTargetMap = [
158 [ "tools", "tools/clean" ],
159 [ "chain", "toolchain/clean" ],
160 [ "linux", "target/linux/clean" ],
161 [ "dir", "dirclean" ],
162 [ "dist", "distclean" ]
163 ]
164
165 def IsCleanRequested(pattern):
166 def CheckCleanProperty(step):
167 val = step.getProperty("clean")
168 if val and re.match(pattern, val):
169 return True
170 else:
171 return False
172
173 return CheckCleanProperty
174
175
176 c['builders'] = []
177
178 dlLock = locks.SlaveLock("slave_dl")
179
180 checkBuiltin = re.sub('[\t\n ]+', ' ', """
181 checkBuiltin() {
182 local symbol op path file;
183 for file in $CHANGED_FILES; do
184 case "$file" in
185 package/*/*) : ;;
186 *) return 0 ;;
187 esac;
188 done;
189 while read symbol op path; do
190 case "$symbol" in package-*)
191 symbol="${symbol##*(}";
192 symbol="${symbol%)}";
193 for file in $CHANGED_FILES; do
194 case "$file" in "package/$path/"*)
195 grep -qsx "$symbol=y" .config && return 0
196 ;; esac;
197 done;
198 esac;
199 done < tmp/.packagedeps;
200 return 1;
201 }
202 """).strip()
203
204
205 class IfBuiltinShellCommand(ShellCommand):
206 def _quote(self, str):
207 if re.search("[^a-zA-Z0-9/_.-]", str):
208 return "'%s'" %(re.sub("'", "'\"'\"'", str))
209 return str
210
211 def setCommand(self, command):
212 if not isinstance(command, (str, unicode)):
213 command = ' '.join(map(self._quote, command))
214 self.command = [
215 '/bin/sh', '-c',
216 '%s; if checkBuiltin; then %s; else exit 0; fi' %(checkBuiltin, command)
217 ]
218
219 def setupEnvironment(self, cmd):
220 slaveEnv = self.slaveEnvironment
221 if slaveEnv is None:
222 slaveEnv = { }
223 changedFiles = { }
224 for request in self.build.requests:
225 for source in request.sources:
226 for change in source.changes:
227 for file in change.files:
228 changedFiles[file] = True
229 fullSlaveEnv = slaveEnv.copy()
230 fullSlaveEnv['CHANGED_FILES'] = ' '.join(changedFiles.keys())
231 cmd.args['env'] = fullSlaveEnv
232
233 slaveNames = [ ]
234
235 for slave in c['slaves']:
236 slaveNames.append(slave.slavename)
237
238 for target in targets:
239 ts = target.split('/')
240
241 factory = BuildFactory()
242
243 # find number of cores
244 factory.addStep(SetProperty(
245 name = "nproc",
246 property = "nproc",
247 description = "Finding number of CPUs",
248 command = ["nproc"]))
249
250 # expire tree if needed
251 if tree_expire > 0:
252 factory.addStep(FileDownload(
253 mastersrc = "expire.sh",
254 slavedest = "../expire.sh",
255 mode = 0755))
256
257 factory.addStep(ShellCommand(
258 name = "expire",
259 description = "Checking for build tree expiry",
260 command = ["./expire.sh", str(tree_expire)],
261 workdir = ".",
262 haltOnFailure = True,
263 timeout = 2400))
264
265 # user-requested clean targets
266 for tuple in CleanTargetMap:
267 factory.addStep(ShellCommand(
268 name = tuple[1],
269 description = 'User-requested "make %s"' % tuple[1],
270 command = ["make", tuple[1], "V=s"],
271 doStepIf = IsCleanRequested(tuple[0])
272 ))
273
274 # check out the source
275 factory.addStep(Git(repourl=repo_url, branch=repo_branch, mode='update'))
276
277 factory.addStep(ShellCommand(
278 name = "rmtmp",
279 description = "Remove tmp folder",
280 command=["rm", "-rf", "tmp/"]))
281
282 # feed
283 # factory.addStep(ShellCommand(
284 # name = "feedsconf",
285 # description = "Copy the feeds.conf",
286 # command='''cp ~/feeds.conf ./feeds.conf''' ))
287
288 # feed
289 factory.addStep(ShellCommand(
290 name = "rmfeedlinks",
291 description = "Remove feed symlinks",
292 command=["rm", "-rf", "package/feeds/"]))
293
294 # feed
295 factory.addStep(ShellCommand(
296 name = "updatefeeds",
297 description = "Updating feeds",
298 command=["./scripts/feeds", "update"]))
299
300 # feed
301 factory.addStep(ShellCommand(
302 name = "installfeeds",
303 description = "Installing feeds",
304 command=["./scripts/feeds", "install", "-a"]))
305
306 # seed config
307 factory.addStep(FileDownload(
308 mastersrc = "config.seed",
309 slavedest = ".config",
310 mode = 0644
311 ))
312
313 # configure
314 factory.addStep(ShellCommand(
315 name = "newconfig",
316 description = "Seeding .config",
317 command = "printf 'CONFIG_TARGET_%s=y\\nCONFIG_TARGET_%s_%s=y\\n' >> .config" %(ts[0], ts[0], ts[1])
318 ))
319
320 factory.addStep(ShellCommand(
321 name = "delbin",
322 description = "Removing output directory",
323 command = ["rm", "-rf", "bin/"]
324 ))
325
326 factory.addStep(ShellCommand(
327 name = "defconfig",
328 description = "Populating .config",
329 command = ["make", "defconfig"]
330 ))
331
332 # check arch
333 factory.addStep(ShellCommand(
334 name = "checkarch",
335 description = "Checking architecture",
336 command = ["grep", "-sq", "CONFIG_TARGET_%s=y" %(ts[0]), ".config"],
337 logEnviron = False,
338 want_stdout = False,
339 want_stderr = False,
340 haltOnFailure = True
341 ))
342
343 # find libc suffix
344 factory.addStep(SetProperty(
345 name = "libc",
346 property = "libc",
347 description = "Finding libc suffix",
348 command = ["sed", "-ne", '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }', ".config"]))
349
350 # install build key
351 factory.addStep(FileDownload(mastersrc=home_dir+'/key-build', slavedest="key-build", mode=0600))
352 factory.addStep(FileDownload(mastersrc=home_dir+'/key-build.pub', slavedest="key-build.pub", mode=0600))
353
354 # prepare dl
355 factory.addStep(ShellCommand(
356 name = "dldir",
357 description = "Preparing dl/",
358 command = "mkdir -p $HOME/dl && rm -rf ./dl && ln -sf $HOME/dl ./dl",
359 logEnviron = False,
360 want_stdout = False
361 ))
362
363 # prepare tar
364 factory.addStep(ShellCommand(
365 name = "dltar",
366 description = "Building GNU tar",
367 command = ["make", WithProperties("-j%(nproc:~4)s"), "tools/tar/install", "V=s"],
368 haltOnFailure = True
369 ))
370
371 # populate dl
372 factory.addStep(ShellCommand(
373 name = "dlrun",
374 description = "Populating dl/",
375 command = ["make", WithProperties("-j%(nproc:~4)s"), "download", "V=s"],
376 logEnviron = False,
377 locks = [dlLock.access('exclusive')]
378 ))
379
380 factory.addStep(ShellCommand(
381 name = "cleanbase",
382 description = "Cleaning base-files",
383 command=["make", "package/base-files/clean", "V=s"]
384 ))
385
386 # build
387 factory.addStep(ShellCommand(
388 name = "tools",
389 description = "Building tools",
390 command = ["make", WithProperties("-j%(nproc:~4)s"), "tools/install", "V=s"],
391 haltOnFailure = True
392 ))
393
394 factory.addStep(ShellCommand(
395 name = "toolchain",
396 description = "Building toolchain",
397 command=["make", WithProperties("-j%(nproc:~4)s"), "toolchain/install", "V=s"],
398 haltOnFailure = True
399 ))
400
401 factory.addStep(ShellCommand(
402 name = "kmods",
403 description = "Building kmods",
404 command=["make", WithProperties("-j%(nproc:~4)s"), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
405 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
406 haltOnFailure = True
407 ))
408
409 factory.addStep(ShellCommand(
410 name = "pkgbuild",
411 description = "Building packages",
412 command=["make", WithProperties("-j%(nproc:~4)s"), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
413 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
414 haltOnFailure = True
415 ))
416
417 # factory.addStep(IfBuiltinShellCommand(
418 factory.addStep(ShellCommand(
419 name = "pkginstall",
420 description = "Installing packages",
421 command=["make", WithProperties("-j%(nproc:~4)s"), "package/install", "V=s"],
422 haltOnFailure = True
423 ))
424
425 factory.addStep(ShellCommand(
426 name = "pkgindex",
427 description = "Indexing packages",
428 command=["make", WithProperties("-j%(nproc:~4)s"), "package/index", "V=s"],
429 haltOnFailure = True
430 ))
431
432 #factory.addStep(IfBuiltinShellCommand(
433 factory.addStep(ShellCommand(
434 name = "images",
435 description = "Building images",
436 command=["make", WithProperties("-j%(nproc:~4)s"), "target/install", "V=s"],
437 haltOnFailure = True
438 ))
439
440 factory.addStep(ShellCommand(
441 name = "checksums",
442 description = "Calculating checksums",
443 command=["make", "-j1", "checksum", "V=s"],
444 haltOnFailure = True
445 ))
446
447 # sign
448 if gpg_keyid is not None:
449 factory.addStep(MasterShellCommand(
450 name = "signprepare",
451 description = "Preparing temporary signing directory",
452 command = ["mkdir", "-p", "%s/signing" %(home_dir)],
453 haltOnFailure = True
454 ))
455
456 factory.addStep(ShellCommand(
457 name = "signpack",
458 description = "Packing files to sign",
459 command = WithProperties("find bin/targets/%s/%s%%(libc)s/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz" %(ts[0], ts[1])),
460 haltOnFailure = True
461 ))
462
463 factory.addStep(FileUpload(
464 slavesrc = "sign.tar.gz",
465 masterdest = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
466 haltOnFailure = True
467 ))
468
469 factory.addStep(MasterShellCommand(
470 name = "signfiles",
471 description = "Signing files",
472 command = ["%s/signall.sh" %(home_dir), "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]), gpg_keyid, gpg_passfile, gpg_comment],
473 haltOnFailure = True
474 ))
475
476 factory.addStep(FileDownload(
477 mastersrc = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
478 slavedest = "sign.tar.gz",
479 haltOnFailure = True
480 ))
481
482 factory.addStep(ShellCommand(
483 name = "signunpack",
484 description = "Unpacking signed files",
485 command = ["tar", "-xzf", "sign.tar.gz"],
486 haltOnFailure = True
487 ))
488
489 # upload
490 factory.addStep(ShellCommand(
491 name = "uploadprepare",
492 description = "Preparing target directory",
493 command=["rsync", "-av", "--include", "/%s/" %(ts[0]), "--include", "/%s/%s/" %(ts[0], ts[1]), "--exclude", "/*", "--exclude", "/*/*", "--exclude", "/%s/%s/*" %(ts[0], ts[1]), "bin/targets/", "%s/targets/" %(rsync_bin_url)],
494 env={'RSYNC_PASSWORD': rsync_bin_key},
495 haltOnFailure = True,
496 logEnviron = False
497 ))
498
499 factory.addStep(ShellCommand(
500 name = "targetupload",
501 description = "Uploading target files",
502 command=["rsync", "--delete", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", WithProperties("bin/targets/%s/%s%%(libc)s/" %(ts[0], ts[1])), "%s/targets/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
503 env={'RSYNC_PASSWORD': rsync_bin_key},
504 haltOnFailure = True,
505 logEnviron = False
506 ))
507
508 if rsync_src_url is not None:
509 factory.addStep(ShellCommand(
510 name = "sourceupload",
511 description = "Uploading source archives",
512 command=["rsync", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "dl/", "%s/" %(rsync_src_url)],
513 env={'RSYNC_PASSWORD': rsync_src_key},
514 haltOnFailure = True,
515 logEnviron = False
516 ))
517
518 if False:
519 factory.addStep(ShellCommand(
520 name = "packageupload",
521 description = "Uploading package files",
522 command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "bin/packages/", "%s/packages/" %(rsync_bin_url)],
523 env={'RSYNC_PASSWORD': rsync_bin_key},
524 haltOnFailure = False,
525 logEnviron = False
526 ))
527
528 # logs
529 if False:
530 factory.addStep(ShellCommand(
531 name = "upload",
532 description = "Uploading logs",
533 command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "logs/", "%s/logs/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
534 env={'RSYNC_PASSWORD': rsync_bin_key},
535 haltOnFailure = False,
536 alwaysRun = True,
537 logEnviron = False
538 ))
539
540 from buildbot.config import BuilderConfig
541
542 c['builders'].append(BuilderConfig(name=target, slavenames=slaveNames, factory=factory))
543
544
545 ####### STATUS TARGETS
546
547 # 'status' is a list of Status Targets. The results of each build will be
548 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
549 # including web pages, email senders, and IRC bots.
550
551 c['status'] = []
552
553 from buildbot.status import html
554 from buildbot.status.web import authz, auth
555
556 if ini.has_option("status", "bind"):
557 if ini.has_option("status", "user") and ini.has_option("status", "password"):
558 authz_cfg=authz.Authz(
559 # change any of these to True to enable; see the manual for more
560 # options
561 auth=auth.BasicAuth([(ini.get("status", "user"), ini.get("status", "password"))]),
562 gracefulShutdown = 'auth',
563 forceBuild = 'auth', # use this to test your slave once it is set up
564 forceAllBuilds = 'auth',
565 pingBuilder = False,
566 stopBuild = 'auth',
567 stopAllBuilds = 'auth',
568 cancelPendingBuild = 'auth',
569 )
570 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind"), authz=authz_cfg))
571 else:
572 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind")))
573
574
575 from buildbot.status import words
576
577 if ini.has_option("irc", "host") and ini.has_option("irc", "nickname") and ini.has_option("irc", "channel"):
578 irc_host = ini.get("irc", "host")
579 irc_port = 6667
580 irc_chan = ini.get("irc", "channel")
581 irc_nick = ini.get("irc", "nickname")
582 irc_pass = None
583
584 if ini.has_option("irc", "port"):
585 irc_port = ini.getint("irc", "port")
586
587 if ini.has_option("irc", "password"):
588 irc_pass = ini.get("irc", "password")
589
590 irc = words.IRC(irc_host, irc_nick, port = irc_port, password = irc_pass,
591 channels = [{ "channel": irc_chan }],
592 notify_events = {
593 'exception': 1,
594 'successToFailure': 1,
595 'failureToSuccess': 1
596 }
597 )
598
599 c['status'].append(irc)
600
601
602 ####### PROJECT IDENTITY
603
604 # the 'title' string will appear at the top of this buildbot
605 # installation's html.WebStatus home page (linked to the
606 # 'titleURL') and is embedded in the title of the waterfall HTML page.
607
608 c['title'] = ini.get("general", "title")
609 c['titleURL'] = ini.get("general", "title_url")
610
611 # the 'buildbotURL' string should point to the location where the buildbot's
612 # internal web server (usually the html.WebStatus page) is visible. This
613 # typically uses the port number set in the Waterfall 'status' entry, but
614 # with an externally-visible host name which the buildbot cannot figure out
615 # without some help.
616
617 c['buildbotURL'] = ini.get("general", "buildbot_url")
618
619 ####### DB URL
620
621 c['db'] = {
622 # This specifies what database buildbot uses to store its state. You can leave
623 # this at its default for all but the largest installations.
624 'db_url' : "sqlite:///state.sqlite",
625 }