phase1: switch away from deprecated slave side Git step
[buildbot.git] / phase1 / master.cfg
1 # -*- python -*-
2 # ex: set syntax=python:
3
4 import os
5 import re
6 import subprocess
7 import ConfigParser
8
9 from buildbot import locks
10
11 # This is a sample buildmaster config file. It must be installed as
12 # 'master.cfg' in your buildmaster's base directory.
13
14 ini = ConfigParser.ConfigParser()
15 ini.read("./config.ini")
16
17 # This is the dictionary that the buildmaster pays attention to. We also use
18 # a shorter alias to save typing.
19 c = BuildmasterConfig = {}
20
21 ####### BUILDSLAVES
22
23 # The 'slaves' list defines the set of recognized buildslaves. Each element is
24 # a BuildSlave object, specifying a unique slave name and password. The same
25 # slave name and password must be configured on the slave.
26 from buildbot.buildslave import BuildSlave
27
28 slave_port = 9989
29
30 if ini.has_option("general", "port"):
31 slave_port = ini.getint("general", "port")
32
33 c['slaves'] = []
34
35 for section in ini.sections():
36 if section.startswith("slave "):
37 if ini.has_option(section, "name") and ini.has_option(section, "password"):
38 name = ini.get(section, "name")
39 password = ini.get(section, "password")
40 max_builds = 1
41 if ini.has_option(section, "builds"):
42 max_builds = ini.getint(section, "builds")
43 c['slaves'].append(BuildSlave(name, password, max_builds = max_builds))
44
45 # 'slavePortnum' defines the TCP port to listen on for connections from slaves.
46 # This must match the value configured into the buildslaves (with their
47 # --master option)
48 c['slavePortnum'] = slave_port
49
50 # coalesce builds
51 c['mergeRequests'] = True
52
53 ####### CHANGESOURCES
54
55 home_dir = os.path.abspath(ini.get("general", "homedir"))
56 tree_expire = 0
57
58 if ini.has_option("general", "expire"):
59 tree_expire = ini.getint("general", "expire")
60
61 repo_url = ini.get("repo", "url")
62 repo_branch = "master"
63
64 if ini.has_option("repo", "branch"):
65 repo_branch = ini.get("repo", "branch")
66
67 rsync_bin_url = ini.get("rsync", "binary_url")
68 rsync_bin_key = ini.get("rsync", "binary_password")
69
70 rsync_src_url = None
71 rsync_src_key = None
72
73 if ini.has_option("rsync", "source_url"):
74 rsync_src_url = ini.get("rsync", "source_url")
75 rsync_src_key = ini.get("rsync", "source_password")
76
77 gpg_home = "~/.gnupg"
78 gpg_keyid = None
79 gpg_comment = "Unattended build signature"
80 gpg_passfile = "/dev/null"
81
82 if ini.has_option("gpg", "home"):
83 gpg_home = ini.get("gpg", "home")
84
85 if ini.has_option("gpg", "keyid"):
86 gpg_keyid = ini.get("gpg", "keyid")
87
88 if ini.has_option("gpg", "comment"):
89 gpg_comment = ini.get("gpg", "comment")
90
91 if ini.has_option("gpg", "passfile"):
92 gpg_passfile = ini.get("gpg", "passfile")
93
94
95 # find targets
96 targets = [ ]
97
98 if not os.path.isdir(home_dir+'/source.git'):
99 subprocess.call(["git", "clone", "--depth=1", "--branch="+repo_branch, repo_url, home_dir+'/source.git'])
100 else:
101 subprocess.call(["git", "pull"], cwd = home_dir+'/source.git')
102
103 findtargets = subprocess.Popen([home_dir+'/dumpinfo.pl', 'targets'],
104 stdout = subprocess.PIPE, cwd = home_dir+'/source.git')
105
106 while True:
107 line = findtargets.stdout.readline()
108 if not line:
109 break
110 ta = line.strip().split(' ')
111 targets.append(ta[0])
112
113
114 # the 'change_source' setting tells the buildmaster how it should find out
115 # about source code changes. Here we point to the buildbot clone of pyflakes.
116
117 from buildbot.changes.gitpoller import GitPoller
118 c['change_source'] = []
119 c['change_source'].append(GitPoller(
120 repo_url,
121 workdir=home_dir+'/work.git', branch='master',
122 pollinterval=300))
123
124 ####### SCHEDULERS
125
126 # Configure the Schedulers, which decide how to react to incoming changes. In this
127 # case, just kick off a 'basebuild' build
128
129 from buildbot.schedulers.basic import SingleBranchScheduler
130 from buildbot.schedulers.forcesched import ForceScheduler
131 from buildbot.changes import filter
132 c['schedulers'] = []
133 c['schedulers'].append(SingleBranchScheduler(
134 name="all",
135 change_filter=filter.ChangeFilter(branch='master'),
136 treeStableTimer=60,
137 builderNames=targets))
138
139 c['schedulers'].append(ForceScheduler(
140 name="force",
141 builderNames=targets))
142
143 ####### BUILDERS
144
145 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
146 # what steps, and which slaves can execute them. Note that any particular build will
147 # only take place on one slave.
148
149 from buildbot.process.factory import BuildFactory
150 from buildbot.steps.source.git import Git
151 from buildbot.steps.shell import ShellCommand
152 from buildbot.steps.shell import SetProperty
153 from buildbot.steps.transfer import FileUpload
154 from buildbot.steps.transfer import FileDownload
155 from buildbot.steps.master import MasterShellCommand
156 from buildbot.process.properties import WithProperties
157
158
159 CleanTargetMap = [
160 [ "tools", "tools/clean" ],
161 [ "chain", "toolchain/clean" ],
162 [ "linux", "target/linux/clean" ],
163 [ "dir", "dirclean" ],
164 [ "dist", "distclean" ]
165 ]
166
167 def IsCleanRequested(pattern):
168 def CheckCleanProperty(step):
169 val = step.getProperty("clean")
170 if val and re.match(pattern, val):
171 return True
172 else:
173 return False
174
175 return CheckCleanProperty
176
177
178 c['builders'] = []
179
180 dlLock = locks.SlaveLock("slave_dl")
181
182 checkBuiltin = re.sub('[\t\n ]+', ' ', """
183 checkBuiltin() {
184 local symbol op path file;
185 for file in $CHANGED_FILES; do
186 case "$file" in
187 package/*/*) : ;;
188 *) return 0 ;;
189 esac;
190 done;
191 while read symbol op path; do
192 case "$symbol" in package-*)
193 symbol="${symbol##*(}";
194 symbol="${symbol%)}";
195 for file in $CHANGED_FILES; do
196 case "$file" in "package/$path/"*)
197 grep -qsx "$symbol=y" .config && return 0
198 ;; esac;
199 done;
200 esac;
201 done < tmp/.packagedeps;
202 return 1;
203 }
204 """).strip()
205
206
207 class IfBuiltinShellCommand(ShellCommand):
208 def _quote(self, str):
209 if re.search("[^a-zA-Z0-9/_.-]", str):
210 return "'%s'" %(re.sub("'", "'\"'\"'", str))
211 return str
212
213 def setCommand(self, command):
214 if not isinstance(command, (str, unicode)):
215 command = ' '.join(map(self._quote, command))
216 self.command = [
217 '/bin/sh', '-c',
218 '%s; if checkBuiltin; then %s; else exit 0; fi' %(checkBuiltin, command)
219 ]
220
221 def setupEnvironment(self, cmd):
222 slaveEnv = self.slaveEnvironment
223 if slaveEnv is None:
224 slaveEnv = { }
225 changedFiles = { }
226 for request in self.build.requests:
227 for source in request.sources:
228 for change in source.changes:
229 for file in change.files:
230 changedFiles[file] = True
231 fullSlaveEnv = slaveEnv.copy()
232 fullSlaveEnv['CHANGED_FILES'] = ' '.join(changedFiles.keys())
233 cmd.args['env'] = fullSlaveEnv
234
235 slaveNames = [ ]
236
237 for slave in c['slaves']:
238 slaveNames.append(slave.slavename)
239
240 for target in targets:
241 ts = target.split('/')
242
243 factory = BuildFactory()
244
245 # find number of cores
246 factory.addStep(SetProperty(
247 name = "nproc",
248 property = "nproc",
249 description = "Finding number of CPUs",
250 command = ["nproc"]))
251
252 # expire tree if needed
253 if tree_expire > 0:
254 factory.addStep(FileDownload(
255 mastersrc = "expire.sh",
256 slavedest = "../expire.sh",
257 mode = 0755))
258
259 factory.addStep(ShellCommand(
260 name = "expire",
261 description = "Checking for build tree expiry",
262 command = ["./expire.sh", str(tree_expire)],
263 workdir = ".",
264 haltOnFailure = True,
265 timeout = 2400))
266
267 # user-requested clean targets
268 for tuple in CleanTargetMap:
269 factory.addStep(ShellCommand(
270 name = tuple[1],
271 description = 'User-requested "make %s"' % tuple[1],
272 command = ["make", tuple[1], "V=s"],
273 doStepIf = IsCleanRequested(tuple[0])
274 ))
275
276 # check out the source
277 factory.addStep(Git(
278 repourl = repo_url,
279 branch = repo_branch,
280 mode = 'incremental',
281 method = 'clean'))
282
283 factory.addStep(ShellCommand(
284 name = "rmtmp",
285 description = "Remove tmp folder",
286 command=["rm", "-rf", "tmp/"]))
287
288 # feed
289 # factory.addStep(ShellCommand(
290 # name = "feedsconf",
291 # description = "Copy the feeds.conf",
292 # command='''cp ~/feeds.conf ./feeds.conf''' ))
293
294 # feed
295 factory.addStep(ShellCommand(
296 name = "rmfeedlinks",
297 description = "Remove feed symlinks",
298 command=["rm", "-rf", "package/feeds/"]))
299
300 # feed
301 factory.addStep(ShellCommand(
302 name = "updatefeeds",
303 description = "Updating feeds",
304 command=["./scripts/feeds", "update"]))
305
306 # feed
307 factory.addStep(ShellCommand(
308 name = "installfeeds",
309 description = "Installing feeds",
310 command=["./scripts/feeds", "install", "-a"]))
311
312 # seed config
313 factory.addStep(FileDownload(
314 mastersrc = "config.seed",
315 slavedest = ".config",
316 mode = 0644
317 ))
318
319 # configure
320 factory.addStep(ShellCommand(
321 name = "newconfig",
322 description = "Seeding .config",
323 command = "printf 'CONFIG_TARGET_%s=y\\nCONFIG_TARGET_%s_%s=y\\n' >> .config" %(ts[0], ts[0], ts[1])
324 ))
325
326 factory.addStep(ShellCommand(
327 name = "delbin",
328 description = "Removing output directory",
329 command = ["rm", "-rf", "bin/"]
330 ))
331
332 factory.addStep(ShellCommand(
333 name = "defconfig",
334 description = "Populating .config",
335 command = ["make", "defconfig"]
336 ))
337
338 # check arch
339 factory.addStep(ShellCommand(
340 name = "checkarch",
341 description = "Checking architecture",
342 command = ["grep", "-sq", "CONFIG_TARGET_%s=y" %(ts[0]), ".config"],
343 logEnviron = False,
344 want_stdout = False,
345 want_stderr = False,
346 haltOnFailure = True
347 ))
348
349 # find libc suffix
350 factory.addStep(SetProperty(
351 name = "libc",
352 property = "libc",
353 description = "Finding libc suffix",
354 command = ["sed", "-ne", '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }', ".config"]))
355
356 # install build key
357 factory.addStep(FileDownload(mastersrc=home_dir+'/key-build', slavedest="key-build", mode=0600))
358 factory.addStep(FileDownload(mastersrc=home_dir+'/key-build.pub', slavedest="key-build.pub", mode=0600))
359
360 # prepare dl
361 factory.addStep(ShellCommand(
362 name = "dldir",
363 description = "Preparing dl/",
364 command = "mkdir -p $HOME/dl && rm -rf ./dl && ln -sf $HOME/dl ./dl",
365 logEnviron = False,
366 want_stdout = False
367 ))
368
369 # prepare tar
370 factory.addStep(ShellCommand(
371 name = "dltar",
372 description = "Building GNU tar",
373 command = ["make", WithProperties("-j%(nproc:~4)s"), "tools/tar/install", "V=s"],
374 haltOnFailure = True
375 ))
376
377 # populate dl
378 factory.addStep(ShellCommand(
379 name = "dlrun",
380 description = "Populating dl/",
381 command = ["make", WithProperties("-j%(nproc:~4)s"), "download", "V=s"],
382 logEnviron = False,
383 locks = [dlLock.access('exclusive')]
384 ))
385
386 factory.addStep(ShellCommand(
387 name = "cleanbase",
388 description = "Cleaning base-files",
389 command=["make", "package/base-files/clean", "V=s"]
390 ))
391
392 # build
393 factory.addStep(ShellCommand(
394 name = "tools",
395 description = "Building tools",
396 command = ["make", WithProperties("-j%(nproc:~4)s"), "tools/install", "V=s"],
397 haltOnFailure = True
398 ))
399
400 factory.addStep(ShellCommand(
401 name = "toolchain",
402 description = "Building toolchain",
403 command=["make", WithProperties("-j%(nproc:~4)s"), "toolchain/install", "V=s"],
404 haltOnFailure = True
405 ))
406
407 factory.addStep(ShellCommand(
408 name = "kmods",
409 description = "Building kmods",
410 command=["make", WithProperties("-j%(nproc:~4)s"), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
411 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
412 haltOnFailure = True
413 ))
414
415 factory.addStep(ShellCommand(
416 name = "pkgbuild",
417 description = "Building packages",
418 command=["make", WithProperties("-j%(nproc:~4)s"), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
419 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
420 haltOnFailure = True
421 ))
422
423 # factory.addStep(IfBuiltinShellCommand(
424 factory.addStep(ShellCommand(
425 name = "pkginstall",
426 description = "Installing packages",
427 command=["make", WithProperties("-j%(nproc:~4)s"), "package/install", "V=s"],
428 haltOnFailure = True
429 ))
430
431 factory.addStep(ShellCommand(
432 name = "pkgindex",
433 description = "Indexing packages",
434 command=["make", WithProperties("-j%(nproc:~4)s"), "package/index", "V=s"],
435 haltOnFailure = True
436 ))
437
438 #factory.addStep(IfBuiltinShellCommand(
439 factory.addStep(ShellCommand(
440 name = "images",
441 description = "Building images",
442 command=["make", WithProperties("-j%(nproc:~4)s"), "target/install", "V=s"],
443 haltOnFailure = True
444 ))
445
446 factory.addStep(ShellCommand(
447 name = "checksums",
448 description = "Calculating checksums",
449 command=["make", "-j1", "checksum", "V=s"],
450 haltOnFailure = True
451 ))
452
453 # sign
454 if gpg_keyid is not None:
455 factory.addStep(MasterShellCommand(
456 name = "signprepare",
457 description = "Preparing temporary signing directory",
458 command = ["mkdir", "-p", "%s/signing" %(home_dir)],
459 haltOnFailure = True
460 ))
461
462 factory.addStep(ShellCommand(
463 name = "signpack",
464 description = "Packing files to sign",
465 command = WithProperties("find bin/targets/%s/%s%%(libc)s/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz" %(ts[0], ts[1])),
466 haltOnFailure = True
467 ))
468
469 factory.addStep(FileUpload(
470 slavesrc = "sign.tar.gz",
471 masterdest = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
472 haltOnFailure = True
473 ))
474
475 factory.addStep(MasterShellCommand(
476 name = "signfiles",
477 description = "Signing files",
478 command = ["%s/signall.sh" %(home_dir), "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]), gpg_keyid, gpg_comment],
479 env = {'GNUPGHOME': gpg_home, 'PASSFILE': gpg_passfile},
480 haltOnFailure = True
481 ))
482
483 factory.addStep(FileDownload(
484 mastersrc = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
485 slavedest = "sign.tar.gz",
486 haltOnFailure = True
487 ))
488
489 factory.addStep(ShellCommand(
490 name = "signunpack",
491 description = "Unpacking signed files",
492 command = ["tar", "-xzf", "sign.tar.gz"],
493 haltOnFailure = True
494 ))
495
496 # upload
497 factory.addStep(ShellCommand(
498 name = "uploadprepare",
499 description = "Preparing target directory",
500 command=["rsync", "-av", "--include", "/%s/" %(ts[0]), "--include", "/%s/%s/" %(ts[0], ts[1]), "--exclude", "/*", "--exclude", "/*/*", "--exclude", "/%s/%s/*" %(ts[0], ts[1]), "bin/targets/", "%s/targets/" %(rsync_bin_url)],
501 env={'RSYNC_PASSWORD': rsync_bin_key},
502 haltOnFailure = True,
503 logEnviron = False
504 ))
505
506 factory.addStep(ShellCommand(
507 name = "targetupload",
508 description = "Uploading target files",
509 command=["rsync", "--delete", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", WithProperties("bin/targets/%s/%s%%(libc)s/" %(ts[0], ts[1])), "%s/targets/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
510 env={'RSYNC_PASSWORD': rsync_bin_key},
511 haltOnFailure = True,
512 logEnviron = False
513 ))
514
515 if rsync_src_url is not None:
516 factory.addStep(ShellCommand(
517 name = "sourceupload",
518 description = "Uploading source archives",
519 command=["rsync", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "dl/", "%s/" %(rsync_src_url)],
520 env={'RSYNC_PASSWORD': rsync_src_key},
521 haltOnFailure = True,
522 logEnviron = False
523 ))
524
525 if False:
526 factory.addStep(ShellCommand(
527 name = "packageupload",
528 description = "Uploading package files",
529 command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "bin/packages/", "%s/packages/" %(rsync_bin_url)],
530 env={'RSYNC_PASSWORD': rsync_bin_key},
531 haltOnFailure = False,
532 logEnviron = False
533 ))
534
535 # logs
536 if False:
537 factory.addStep(ShellCommand(
538 name = "upload",
539 description = "Uploading logs",
540 command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "logs/", "%s/logs/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
541 env={'RSYNC_PASSWORD': rsync_bin_key},
542 haltOnFailure = False,
543 alwaysRun = True,
544 logEnviron = False
545 ))
546
547 from buildbot.config import BuilderConfig
548
549 c['builders'].append(BuilderConfig(name=target, slavenames=slaveNames, factory=factory))
550
551
552 ####### STATUS TARGETS
553
554 # 'status' is a list of Status Targets. The results of each build will be
555 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
556 # including web pages, email senders, and IRC bots.
557
558 c['status'] = []
559
560 from buildbot.status import html
561 from buildbot.status.web import authz, auth
562
563 if ini.has_option("status", "bind"):
564 if ini.has_option("status", "user") and ini.has_option("status", "password"):
565 authz_cfg=authz.Authz(
566 # change any of these to True to enable; see the manual for more
567 # options
568 auth=auth.BasicAuth([(ini.get("status", "user"), ini.get("status", "password"))]),
569 gracefulShutdown = 'auth',
570 forceBuild = 'auth', # use this to test your slave once it is set up
571 forceAllBuilds = 'auth',
572 pingBuilder = False,
573 stopBuild = 'auth',
574 stopAllBuilds = 'auth',
575 cancelPendingBuild = 'auth',
576 )
577 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind"), authz=authz_cfg))
578 else:
579 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind")))
580
581
582 from buildbot.status import words
583
584 if ini.has_option("irc", "host") and ini.has_option("irc", "nickname") and ini.has_option("irc", "channel"):
585 irc_host = ini.get("irc", "host")
586 irc_port = 6667
587 irc_chan = ini.get("irc", "channel")
588 irc_nick = ini.get("irc", "nickname")
589 irc_pass = None
590
591 if ini.has_option("irc", "port"):
592 irc_port = ini.getint("irc", "port")
593
594 if ini.has_option("irc", "password"):
595 irc_pass = ini.get("irc", "password")
596
597 irc = words.IRC(irc_host, irc_nick, port = irc_port, password = irc_pass,
598 channels = [{ "channel": irc_chan }],
599 notify_events = {
600 'exception': 1,
601 'successToFailure': 1,
602 'failureToSuccess': 1
603 }
604 )
605
606 c['status'].append(irc)
607
608
609 ####### PROJECT IDENTITY
610
611 # the 'title' string will appear at the top of this buildbot
612 # installation's html.WebStatus home page (linked to the
613 # 'titleURL') and is embedded in the title of the waterfall HTML page.
614
615 c['title'] = ini.get("general", "title")
616 c['titleURL'] = ini.get("general", "title_url")
617
618 # the 'buildbotURL' string should point to the location where the buildbot's
619 # internal web server (usually the html.WebStatus page) is visible. This
620 # typically uses the port number set in the Waterfall 'status' entry, but
621 # with an externally-visible host name which the buildbot cannot figure out
622 # without some help.
623
624 c['buildbotURL'] = ini.get("general", "buildbot_url")
625
626 ####### DB URL
627
628 c['db'] = {
629 # This specifies what database buildbot uses to store its state. You can leave
630 # this at its default for all but the largest installations.
631 'db_url' : "sqlite:///state.sqlite",
632 }