phase1: support creating tags on demand
[buildbot.git] / phase1 / master.cfg
1 # -*- python -*-
2 # ex: set syntax=python:
3
4 import os
5 import re
6 import subprocess
7 import ConfigParser
8
9 from buildbot import locks
10
11 # This is a sample buildmaster config file. It must be installed as
12 # 'master.cfg' in your buildmaster's base directory.
13
14 ini = ConfigParser.ConfigParser()
15 ini.read("./config.ini")
16
17 # This is the dictionary that the buildmaster pays attention to. We also use
18 # a shorter alias to save typing.
19 c = BuildmasterConfig = {}
20
21 ####### BUILDSLAVES
22
23 # The 'slaves' list defines the set of recognized buildslaves. Each element is
24 # a BuildSlave object, specifying a unique slave name and password. The same
25 # slave name and password must be configured on the slave.
26 from buildbot.buildslave import BuildSlave
27
28 slave_port = 9989
29
30 if ini.has_option("general", "port"):
31 slave_port = ini.getint("general", "port")
32
33 c['slaves'] = []
34
35 for section in ini.sections():
36 if section.startswith("slave "):
37 if ini.has_option(section, "name") and ini.has_option(section, "password"):
38 name = ini.get(section, "name")
39 password = ini.get(section, "password")
40 max_builds = 1
41 if ini.has_option(section, "builds"):
42 max_builds = ini.getint(section, "builds")
43 c['slaves'].append(BuildSlave(name, password, max_builds = max_builds))
44
45 # 'slavePortnum' defines the TCP port to listen on for connections from slaves.
46 # This must match the value configured into the buildslaves (with their
47 # --master option)
48 c['slavePortnum'] = slave_port
49
50 # coalesce builds
51 c['mergeRequests'] = True
52
53 ####### CHANGESOURCES
54
55 home_dir = os.path.abspath(ini.get("general", "homedir"))
56 tree_expire = 0
57
58 if ini.has_option("general", "expire"):
59 tree_expire = ini.getint("general", "expire")
60
61 repo_url = ini.get("repo", "url")
62 repo_branch = "master"
63
64 if ini.has_option("repo", "branch"):
65 repo_branch = ini.get("repo", "branch")
66
67 rsync_bin_url = ini.get("rsync", "binary_url")
68 rsync_bin_key = ini.get("rsync", "binary_password")
69
70 rsync_src_url = None
71 rsync_src_key = None
72
73 if ini.has_option("rsync", "source_url"):
74 rsync_src_url = ini.get("rsync", "source_url")
75 rsync_src_key = ini.get("rsync", "source_password")
76
77 gpg_home = "~/.gnupg"
78 gpg_keyid = None
79 gpg_comment = "Unattended build signature"
80 gpg_passfile = "/dev/null"
81
82 if ini.has_option("gpg", "home"):
83 gpg_home = ini.get("gpg", "home")
84
85 if ini.has_option("gpg", "keyid"):
86 gpg_keyid = ini.get("gpg", "keyid")
87
88 if ini.has_option("gpg", "comment"):
89 gpg_comment = ini.get("gpg", "comment")
90
91 if ini.has_option("gpg", "passfile"):
92 gpg_passfile = ini.get("gpg", "passfile")
93
94
95 # find targets
96 targets = [ ]
97
98 if not os.path.isdir(home_dir+'/source.git'):
99 subprocess.call(["git", "clone", "--depth=1", "--branch="+repo_branch, repo_url, home_dir+'/source.git'])
100 else:
101 subprocess.call(["git", "pull"], cwd = home_dir+'/source.git')
102
103 findtargets = subprocess.Popen([home_dir+'/dumpinfo.pl', 'targets'],
104 stdout = subprocess.PIPE, cwd = home_dir+'/source.git')
105
106 while True:
107 line = findtargets.stdout.readline()
108 if not line:
109 break
110 ta = line.strip().split(' ')
111 targets.append(ta[0])
112
113
114 # the 'change_source' setting tells the buildmaster how it should find out
115 # about source code changes. Here we point to the buildbot clone of pyflakes.
116
117 from buildbot.changes.gitpoller import GitPoller
118 c['change_source'] = []
119 c['change_source'].append(GitPoller(
120 repo_url,
121 workdir=home_dir+'/work.git', branch='master',
122 pollinterval=300))
123
124 ####### SCHEDULERS
125
126 # Configure the Schedulers, which decide how to react to incoming changes. In this
127 # case, just kick off a 'basebuild' build
128
129 from buildbot.schedulers.basic import SingleBranchScheduler
130 from buildbot.schedulers.forcesched import ForceScheduler
131 from buildbot.changes import filter
132 c['schedulers'] = []
133 c['schedulers'].append(SingleBranchScheduler(
134 name="all",
135 change_filter=filter.ChangeFilter(branch='master'),
136 treeStableTimer=60,
137 builderNames=targets))
138
139 c['schedulers'].append(ForceScheduler(
140 name="force",
141 builderNames=targets))
142
143 ####### BUILDERS
144
145 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
146 # what steps, and which slaves can execute them. Note that any particular build will
147 # only take place on one slave.
148
149 from buildbot.process.factory import BuildFactory
150 from buildbot.steps.source.git import Git
151 from buildbot.steps.shell import ShellCommand
152 from buildbot.steps.shell import SetProperty
153 from buildbot.steps.transfer import FileUpload
154 from buildbot.steps.transfer import FileDownload
155 from buildbot.steps.master import MasterShellCommand
156 from buildbot.process.properties import WithProperties
157
158
159 CleanTargetMap = [
160 [ "tools", "tools/clean" ],
161 [ "chain", "toolchain/clean" ],
162 [ "linux", "target/linux/clean" ],
163 [ "dir", "dirclean" ],
164 [ "dist", "distclean" ]
165 ]
166
167 def IsCleanRequested(pattern):
168 def CheckCleanProperty(step):
169 val = step.getProperty("clean")
170 if val and re.match(pattern, val):
171 return True
172 else:
173 return False
174
175 return CheckCleanProperty
176
177 def IsTaggingRequested(step):
178 val = step.getProperty("tag")
179 if val and re.match("^[0-9]+\.[0-9]+\.[0-9]+$", val):
180 return True
181 else:
182 return False
183
184 def IsNoTaggingRequested(step):
185 return not IsTaggingRequested(step)
186
187
188 c['builders'] = []
189
190 dlLock = locks.SlaveLock("slave_dl")
191
192 checkBuiltin = re.sub('[\t\n ]+', ' ', """
193 checkBuiltin() {
194 local symbol op path file;
195 for file in $CHANGED_FILES; do
196 case "$file" in
197 package/*/*) : ;;
198 *) return 0 ;;
199 esac;
200 done;
201 while read symbol op path; do
202 case "$symbol" in package-*)
203 symbol="${symbol##*(}";
204 symbol="${symbol%)}";
205 for file in $CHANGED_FILES; do
206 case "$file" in "package/$path/"*)
207 grep -qsx "$symbol=y" .config && return 0
208 ;; esac;
209 done;
210 esac;
211 done < tmp/.packagedeps;
212 return 1;
213 }
214 """).strip()
215
216
217 class IfBuiltinShellCommand(ShellCommand):
218 def _quote(self, str):
219 if re.search("[^a-zA-Z0-9/_.-]", str):
220 return "'%s'" %(re.sub("'", "'\"'\"'", str))
221 return str
222
223 def setCommand(self, command):
224 if not isinstance(command, (str, unicode)):
225 command = ' '.join(map(self._quote, command))
226 self.command = [
227 '/bin/sh', '-c',
228 '%s; if checkBuiltin; then %s; else exit 0; fi' %(checkBuiltin, command)
229 ]
230
231 def setupEnvironment(self, cmd):
232 slaveEnv = self.slaveEnvironment
233 if slaveEnv is None:
234 slaveEnv = { }
235 changedFiles = { }
236 for request in self.build.requests:
237 for source in request.sources:
238 for change in source.changes:
239 for file in change.files:
240 changedFiles[file] = True
241 fullSlaveEnv = slaveEnv.copy()
242 fullSlaveEnv['CHANGED_FILES'] = ' '.join(changedFiles.keys())
243 cmd.args['env'] = fullSlaveEnv
244
245 slaveNames = [ ]
246
247 for slave in c['slaves']:
248 slaveNames.append(slave.slavename)
249
250 for target in targets:
251 ts = target.split('/')
252
253 factory = BuildFactory()
254
255 # find number of cores
256 factory.addStep(SetProperty(
257 name = "nproc",
258 property = "nproc",
259 description = "Finding number of CPUs",
260 command = ["nproc"]))
261
262 # expire tree if needed
263 if tree_expire > 0:
264 factory.addStep(FileDownload(
265 mastersrc = "expire.sh",
266 slavedest = "../expire.sh",
267 mode = 0755))
268
269 factory.addStep(ShellCommand(
270 name = "expire",
271 description = "Checking for build tree expiry",
272 command = ["./expire.sh", str(tree_expire)],
273 workdir = ".",
274 haltOnFailure = True,
275 timeout = 2400))
276
277 # user-requested clean targets
278 for tuple in CleanTargetMap:
279 factory.addStep(ShellCommand(
280 name = tuple[1],
281 description = 'User-requested "make %s"' % tuple[1],
282 command = ["make", tuple[1], "V=s"],
283 doStepIf = IsCleanRequested(tuple[0])
284 ))
285
286 factory.addStep(MasterShellCommand(
287 name = "maketag",
288 description = "Tagging Git repository",
289 command = [home_dir+'/maketag.sh', '-i', '-k', str(gpg_keyid or ''),
290 '-p', str(gpg_passfile or ''), '-v', WithProperties("%(tag:-)s")],
291 path = home_dir+'/source.git',
292 env = {'GNUPGHOME': gpg_home},
293 haltOnFailure = True,
294 doStepIf = IsTaggingRequested
295 ))
296
297 # switch to branch
298 factory.addStep(ShellCommand(
299 name = "switchbranch",
300 description = "Checking out Git branch",
301 command = ["git", "checkout", repo_branch],
302 haltOnFailure = True,
303 doStepIf = IsNoTaggingRequested
304 ))
305
306 # check out the source
307 factory.addStep(Git(
308 repourl = repo_url,
309 branch = repo_branch,
310 mode = 'incremental',
311 method = 'clean'))
312
313 # fetch tags
314 factory.addStep(ShellCommand(
315 name = "fetchtag",
316 description = "Fetching Git tags",
317 command = ["git", "fetch", "--tags", "--", repo_url],
318 haltOnFailure = True,
319 doStepIf = IsTaggingRequested
320 ))
321
322 # switch to tag
323 factory.addStep(ShellCommand(
324 name = "switchtag",
325 description = "Checking out Git tag",
326 command = ["git", "checkout", WithProperties("tags/v%(tag:-)s")],
327 haltOnFailure = True,
328 doStepIf = IsTaggingRequested
329 ))
330
331 factory.addStep(ShellCommand(
332 name = "rmtmp",
333 description = "Remove tmp folder",
334 command=["rm", "-rf", "tmp/"]))
335
336 # feed
337 # factory.addStep(ShellCommand(
338 # name = "feedsconf",
339 # description = "Copy the feeds.conf",
340 # command='''cp ~/feeds.conf ./feeds.conf''' ))
341
342 # feed
343 factory.addStep(ShellCommand(
344 name = "rmfeedlinks",
345 description = "Remove feed symlinks",
346 command=["rm", "-rf", "package/feeds/"]))
347
348 # feed
349 factory.addStep(ShellCommand(
350 name = "updatefeeds",
351 description = "Updating feeds",
352 command=["./scripts/feeds", "update"]))
353
354 # feed
355 factory.addStep(ShellCommand(
356 name = "installfeeds",
357 description = "Installing feeds",
358 command=["./scripts/feeds", "install", "-a"]))
359
360 # seed config
361 factory.addStep(FileDownload(
362 mastersrc = "config.seed",
363 slavedest = ".config",
364 mode = 0644
365 ))
366
367 # configure
368 factory.addStep(ShellCommand(
369 name = "newconfig",
370 description = "Seeding .config",
371 command = "printf 'CONFIG_TARGET_%s=y\\nCONFIG_TARGET_%s_%s=y\\n' >> .config" %(ts[0], ts[0], ts[1])
372 ))
373
374 factory.addStep(ShellCommand(
375 name = "delbin",
376 description = "Removing output directory",
377 command = ["rm", "-rf", "bin/"]
378 ))
379
380 factory.addStep(ShellCommand(
381 name = "defconfig",
382 description = "Populating .config",
383 command = ["make", "defconfig"]
384 ))
385
386 # check arch
387 factory.addStep(ShellCommand(
388 name = "checkarch",
389 description = "Checking architecture",
390 command = ["grep", "-sq", "CONFIG_TARGET_%s=y" %(ts[0]), ".config"],
391 logEnviron = False,
392 want_stdout = False,
393 want_stderr = False,
394 haltOnFailure = True
395 ))
396
397 # find libc suffix
398 factory.addStep(SetProperty(
399 name = "libc",
400 property = "libc",
401 description = "Finding libc suffix",
402 command = ["sed", "-ne", '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }', ".config"]))
403
404 # install build key
405 factory.addStep(FileDownload(mastersrc=home_dir+'/key-build', slavedest="key-build", mode=0600))
406 factory.addStep(FileDownload(mastersrc=home_dir+'/key-build.pub', slavedest="key-build.pub", mode=0600))
407
408 # prepare dl
409 factory.addStep(ShellCommand(
410 name = "dldir",
411 description = "Preparing dl/",
412 command = "mkdir -p $HOME/dl && rm -rf ./dl && ln -sf $HOME/dl ./dl",
413 logEnviron = False,
414 want_stdout = False
415 ))
416
417 # prepare tar
418 factory.addStep(ShellCommand(
419 name = "dltar",
420 description = "Building GNU tar",
421 command = ["make", WithProperties("-j%(nproc:~4)s"), "tools/tar/install", "V=s"],
422 haltOnFailure = True
423 ))
424
425 # populate dl
426 factory.addStep(ShellCommand(
427 name = "dlrun",
428 description = "Populating dl/",
429 command = ["make", WithProperties("-j%(nproc:~4)s"), "download", "V=s"],
430 logEnviron = False,
431 locks = [dlLock.access('exclusive')]
432 ))
433
434 factory.addStep(ShellCommand(
435 name = "cleanbase",
436 description = "Cleaning base-files",
437 command=["make", "package/base-files/clean", "V=s"]
438 ))
439
440 # build
441 factory.addStep(ShellCommand(
442 name = "tools",
443 description = "Building tools",
444 command = ["make", WithProperties("-j%(nproc:~4)s"), "tools/install", "V=s"],
445 haltOnFailure = True
446 ))
447
448 factory.addStep(ShellCommand(
449 name = "toolchain",
450 description = "Building toolchain",
451 command=["make", WithProperties("-j%(nproc:~4)s"), "toolchain/install", "V=s"],
452 haltOnFailure = True
453 ))
454
455 factory.addStep(ShellCommand(
456 name = "kmods",
457 description = "Building kmods",
458 command=["make", WithProperties("-j%(nproc:~4)s"), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
459 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
460 haltOnFailure = True
461 ))
462
463 factory.addStep(ShellCommand(
464 name = "pkgbuild",
465 description = "Building packages",
466 command=["make", WithProperties("-j%(nproc:~4)s"), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
467 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
468 haltOnFailure = True
469 ))
470
471 # factory.addStep(IfBuiltinShellCommand(
472 factory.addStep(ShellCommand(
473 name = "pkginstall",
474 description = "Installing packages",
475 command=["make", WithProperties("-j%(nproc:~4)s"), "package/install", "V=s"],
476 haltOnFailure = True
477 ))
478
479 factory.addStep(ShellCommand(
480 name = "pkgindex",
481 description = "Indexing packages",
482 command=["make", WithProperties("-j%(nproc:~4)s"), "package/index", "V=s"],
483 haltOnFailure = True
484 ))
485
486 #factory.addStep(IfBuiltinShellCommand(
487 factory.addStep(ShellCommand(
488 name = "images",
489 description = "Building images",
490 command=["make", WithProperties("-j%(nproc:~4)s"), "target/install", "V=s"],
491 haltOnFailure = True
492 ))
493
494 factory.addStep(ShellCommand(
495 name = "checksums",
496 description = "Calculating checksums",
497 command=["make", "-j1", "checksum", "V=s"],
498 haltOnFailure = True
499 ))
500
501 # sign
502 if gpg_keyid is not None:
503 factory.addStep(MasterShellCommand(
504 name = "signprepare",
505 description = "Preparing temporary signing directory",
506 command = ["mkdir", "-p", "%s/signing" %(home_dir)],
507 haltOnFailure = True
508 ))
509
510 factory.addStep(ShellCommand(
511 name = "signpack",
512 description = "Packing files to sign",
513 command = WithProperties("find bin/targets/%s/%s%%(libc)s/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz" %(ts[0], ts[1])),
514 haltOnFailure = True
515 ))
516
517 factory.addStep(FileUpload(
518 slavesrc = "sign.tar.gz",
519 masterdest = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
520 haltOnFailure = True
521 ))
522
523 factory.addStep(MasterShellCommand(
524 name = "signfiles",
525 description = "Signing files",
526 command = ["%s/signall.sh" %(home_dir), "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]), gpg_keyid, gpg_comment],
527 env = {'GNUPGHOME': gpg_home, 'PASSFILE': gpg_passfile},
528 haltOnFailure = True
529 ))
530
531 factory.addStep(FileDownload(
532 mastersrc = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
533 slavedest = "sign.tar.gz",
534 haltOnFailure = True
535 ))
536
537 factory.addStep(ShellCommand(
538 name = "signunpack",
539 description = "Unpacking signed files",
540 command = ["tar", "-xzf", "sign.tar.gz"],
541 haltOnFailure = True
542 ))
543
544 # upload
545 factory.addStep(ShellCommand(
546 name = "uploadprepare",
547 description = "Preparing target directory",
548 command=["rsync", "-av", "--include", "/%s/" %(ts[0]), "--include", "/%s/%s/" %(ts[0], ts[1]), "--exclude", "/*", "--exclude", "/*/*", "--exclude", "/%s/%s/*" %(ts[0], ts[1]), "bin/targets/", "%s/targets/" %(rsync_bin_url)],
549 env={'RSYNC_PASSWORD': rsync_bin_key},
550 haltOnFailure = True,
551 logEnviron = False
552 ))
553
554 factory.addStep(ShellCommand(
555 name = "targetupload",
556 description = "Uploading target files",
557 command=["rsync", "--delete", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", WithProperties("bin/targets/%s/%s%%(libc)s/" %(ts[0], ts[1])), "%s/targets/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
558 env={'RSYNC_PASSWORD': rsync_bin_key},
559 haltOnFailure = True,
560 logEnviron = False
561 ))
562
563 if rsync_src_url is not None:
564 factory.addStep(ShellCommand(
565 name = "sourceupload",
566 description = "Uploading source archives",
567 command=["rsync", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "dl/", "%s/" %(rsync_src_url)],
568 env={'RSYNC_PASSWORD': rsync_src_key},
569 haltOnFailure = True,
570 logEnviron = False
571 ))
572
573 if False:
574 factory.addStep(ShellCommand(
575 name = "packageupload",
576 description = "Uploading package files",
577 command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "bin/packages/", "%s/packages/" %(rsync_bin_url)],
578 env={'RSYNC_PASSWORD': rsync_bin_key},
579 haltOnFailure = False,
580 logEnviron = False
581 ))
582
583 # logs
584 if False:
585 factory.addStep(ShellCommand(
586 name = "upload",
587 description = "Uploading logs",
588 command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "logs/", "%s/logs/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
589 env={'RSYNC_PASSWORD': rsync_bin_key},
590 haltOnFailure = False,
591 alwaysRun = True,
592 logEnviron = False
593 ))
594
595 from buildbot.config import BuilderConfig
596
597 c['builders'].append(BuilderConfig(name=target, slavenames=slaveNames, factory=factory))
598
599
600 ####### STATUS TARGETS
601
602 # 'status' is a list of Status Targets. The results of each build will be
603 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
604 # including web pages, email senders, and IRC bots.
605
606 c['status'] = []
607
608 from buildbot.status import html
609 from buildbot.status.web import authz, auth
610
611 if ini.has_option("status", "bind"):
612 if ini.has_option("status", "user") and ini.has_option("status", "password"):
613 authz_cfg=authz.Authz(
614 # change any of these to True to enable; see the manual for more
615 # options
616 auth=auth.BasicAuth([(ini.get("status", "user"), ini.get("status", "password"))]),
617 gracefulShutdown = 'auth',
618 forceBuild = 'auth', # use this to test your slave once it is set up
619 forceAllBuilds = 'auth',
620 pingBuilder = False,
621 stopBuild = 'auth',
622 stopAllBuilds = 'auth',
623 cancelPendingBuild = 'auth',
624 )
625 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind"), authz=authz_cfg))
626 else:
627 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind")))
628
629
630 from buildbot.status import words
631
632 if ini.has_option("irc", "host") and ini.has_option("irc", "nickname") and ini.has_option("irc", "channel"):
633 irc_host = ini.get("irc", "host")
634 irc_port = 6667
635 irc_chan = ini.get("irc", "channel")
636 irc_nick = ini.get("irc", "nickname")
637 irc_pass = None
638
639 if ini.has_option("irc", "port"):
640 irc_port = ini.getint("irc", "port")
641
642 if ini.has_option("irc", "password"):
643 irc_pass = ini.get("irc", "password")
644
645 irc = words.IRC(irc_host, irc_nick, port = irc_port, password = irc_pass,
646 channels = [{ "channel": irc_chan }],
647 notify_events = {
648 'exception': 1,
649 'successToFailure': 1,
650 'failureToSuccess': 1
651 }
652 )
653
654 c['status'].append(irc)
655
656
657 ####### PROJECT IDENTITY
658
659 # the 'title' string will appear at the top of this buildbot
660 # installation's html.WebStatus home page (linked to the
661 # 'titleURL') and is embedded in the title of the waterfall HTML page.
662
663 c['title'] = ini.get("general", "title")
664 c['titleURL'] = ini.get("general", "title_url")
665
666 # the 'buildbotURL' string should point to the location where the buildbot's
667 # internal web server (usually the html.WebStatus page) is visible. This
668 # typically uses the port number set in the Waterfall 'status' entry, but
669 # with an externally-visible host name which the buildbot cannot figure out
670 # without some help.
671
672 c['buildbotURL'] = ini.get("general", "buildbot_url")
673
674 ####### DB URL
675
676 c['db'] = {
677 # This specifies what database buildbot uses to store its state. You can leave
678 # this at its default for all but the largest installations.
679 'db_url' : "sqlite:///state.sqlite",
680 }