phase1: avoid creating nested dl/ symlinks
[buildbot.git] / phase1 / master.cfg
1 # -*- python -*-
2 # ex: set syntax=python:
3
4 import os
5 import re
6 import subprocess
7 import ConfigParser
8
9 from buildbot import locks
10
11 # This is a sample buildmaster config file. It must be installed as
12 # 'master.cfg' in your buildmaster's base directory.
13
14 ini = ConfigParser.ConfigParser()
15 ini.read("./config.ini")
16
17 # This is the dictionary that the buildmaster pays attention to. We also use
18 # a shorter alias to save typing.
19 c = BuildmasterConfig = {}
20
21 ####### BUILDSLAVES
22
23 # The 'slaves' list defines the set of recognized buildslaves. Each element is
24 # a BuildSlave object, specifying a unique slave name and password. The same
25 # slave name and password must be configured on the slave.
26 from buildbot.buildslave import BuildSlave
27
28 slave_port = 9989
29
30 if ini.has_option("general", "port"):
31 slave_port = ini.getint("general", "port")
32
33 c['slaves'] = []
34
35 for section in ini.sections():
36 if section.startswith("slave "):
37 if ini.has_option(section, "name") and ini.has_option(section, "password"):
38 name = ini.get(section, "name")
39 password = ini.get(section, "password")
40 max_builds = 1
41 if ini.has_option(section, "builds"):
42 max_builds = ini.getint(section, "builds")
43 c['slaves'].append(BuildSlave(name, password, max_builds = max_builds))
44
45 # 'slavePortnum' defines the TCP port to listen on for connections from slaves.
46 # This must match the value configured into the buildslaves (with their
47 # --master option)
48 c['slavePortnum'] = slave_port
49
50 # coalesce builds
51 c['mergeRequests'] = True
52
53 ####### CHANGESOURCES
54
55 home_dir = os.path.abspath(ini.get("general", "homedir"))
56 tree_expire = 0
57
58 if ini.has_option("general", "expire"):
59 tree_expire = ini.getint("general", "expire")
60
61 repo_url = ini.get("repo", "url")
62
63 rsync_bin_url = ini.get("rsync", "binary_url")
64 rsync_bin_key = ini.get("rsync", "binary_password")
65
66 rsync_src_url = None
67 rsync_src_key = None
68
69 if ini.has_option("rsync", "source_url"):
70 rsync_src_url = ini.get("rsync", "source_url")
71 rsync_src_key = ini.get("rsync", "source_password")
72
73 gpg_keyid = None
74 gpg_comment = "Unattended build signature"
75 gpg_passfile = "/dev/null"
76
77 if ini.has_option("gpg", "keyid"):
78 gpg_keyid = ini.get("gpg", "keyid")
79
80 if ini.has_option("gpg", "comment"):
81 gpg_comment = ini.get("gpg", "comment")
82
83 if ini.has_option("gpg", "passfile"):
84 gpg_passfile = ini.get("gpg", "passfile")
85
86
87 # find targets
88 targets = [ ]
89
90 if not os.path.isdir(home_dir+'/source.git'):
91 subprocess.call(["git", "clone", "--depth=1", repo_url, home_dir+'/source.git'])
92
93 findtargets = subprocess.Popen([home_dir+'/dumpinfo.pl', 'targets'],
94 stdout = subprocess.PIPE, cwd = home_dir+'/source.git')
95
96 while True:
97 line = findtargets.stdout.readline()
98 if not line:
99 break
100 ta = line.strip().split(' ')
101 targets.append(ta[0])
102
103
104 # the 'change_source' setting tells the buildmaster how it should find out
105 # about source code changes. Here we point to the buildbot clone of pyflakes.
106
107 from buildbot.changes.gitpoller import GitPoller
108 c['change_source'] = []
109 c['change_source'].append(GitPoller(
110 repo_url,
111 workdir=home_dir+'/source.git', branch='master',
112 pollinterval=300))
113
114 ####### SCHEDULERS
115
116 # Configure the Schedulers, which decide how to react to incoming changes. In this
117 # case, just kick off a 'basebuild' build
118
119 from buildbot.schedulers.basic import SingleBranchScheduler
120 from buildbot.schedulers.forcesched import ForceScheduler
121 from buildbot.changes import filter
122 c['schedulers'] = []
123 c['schedulers'].append(SingleBranchScheduler(
124 name="all",
125 change_filter=filter.ChangeFilter(branch='master'),
126 treeStableTimer=60,
127 builderNames=targets))
128
129 c['schedulers'].append(ForceScheduler(
130 name="force",
131 builderNames=targets))
132
133 ####### BUILDERS
134
135 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
136 # what steps, and which slaves can execute them. Note that any particular build will
137 # only take place on one slave.
138
139 from buildbot.process.factory import BuildFactory
140 from buildbot.steps.source import Git
141 from buildbot.steps.shell import ShellCommand
142 from buildbot.steps.shell import SetProperty
143 from buildbot.steps.transfer import FileUpload
144 from buildbot.steps.transfer import FileDownload
145 from buildbot.steps.master import MasterShellCommand
146 from buildbot.process.properties import WithProperties
147
148
149 CleanTargetMap = [
150 [ "tools", "tools/clean" ],
151 [ "chain", "toolchain/clean" ],
152 [ "linux", "target/linux/clean" ],
153 [ "dir", "dirclean" ],
154 [ "dist", "distclean" ]
155 ]
156
157 def IsCleanRequested(pattern):
158 def CheckCleanProperty(step):
159 val = step.getProperty("clean")
160 if val and re.match(pattern, val):
161 return True
162 else:
163 return False
164
165 return CheckCleanProperty
166
167
168 c['builders'] = []
169
170 dlLock = locks.SlaveLock("slave_dl")
171
172 checkBuiltin = re.sub('[\t\n ]+', ' ', """
173 checkBuiltin() {
174 local symbol op path file;
175 for file in $CHANGED_FILES; do
176 case "$file" in
177 package/*/*) : ;;
178 *) return 0 ;;
179 esac;
180 done;
181 while read symbol op path; do
182 case "$symbol" in package-*)
183 symbol="${symbol##*(}";
184 symbol="${symbol%)}";
185 for file in $CHANGED_FILES; do
186 case "$file" in "package/$path/"*)
187 grep -qsx "$symbol=y" .config && return 0
188 ;; esac;
189 done;
190 esac;
191 done < tmp/.packagedeps;
192 return 1;
193 }
194 """).strip()
195
196
197 class IfBuiltinShellCommand(ShellCommand):
198 def _quote(self, str):
199 if re.search("[^a-zA-Z0-9/_.-]", str):
200 return "'%s'" %(re.sub("'", "'\"'\"'", str))
201 return str
202
203 def setCommand(self, command):
204 if not isinstance(command, (str, unicode)):
205 command = ' '.join(map(self._quote, command))
206 self.command = [
207 '/bin/sh', '-c',
208 '%s; if checkBuiltin; then %s; else exit 0; fi' %(checkBuiltin, command)
209 ]
210
211 def setupEnvironment(self, cmd):
212 slaveEnv = self.slaveEnvironment
213 if slaveEnv is None:
214 slaveEnv = { }
215 changedFiles = { }
216 for request in self.build.requests:
217 for source in request.sources:
218 for change in source.changes:
219 for file in change.files:
220 changedFiles[file] = True
221 fullSlaveEnv = slaveEnv.copy()
222 fullSlaveEnv['CHANGED_FILES'] = ' '.join(changedFiles.keys())
223 cmd.args['env'] = fullSlaveEnv
224
225 slaveNames = [ ]
226
227 for slave in c['slaves']:
228 slaveNames.append(slave.slavename)
229
230 for target in targets:
231 ts = target.split('/')
232
233 factory = BuildFactory()
234
235 # find number of cores
236 factory.addStep(SetProperty(
237 name = "nproc",
238 property = "nproc",
239 description = "Finding number of CPUs",
240 command = ["nproc"]))
241
242 # expire tree if needed
243 if tree_expire > 0:
244 factory.addStep(FileDownload(
245 mastersrc = "expire.sh",
246 slavedest = "../expire.sh",
247 mode = 0755))
248
249 factory.addStep(ShellCommand(
250 name = "expire",
251 description = "Checking for build tree expiry",
252 command = ["./expire.sh", str(tree_expire)],
253 workdir = ".",
254 haltOnFailure = True,
255 timeout = 2400))
256
257 # user-requested clean targets
258 for tuple in CleanTargetMap:
259 factory.addStep(ShellCommand(
260 name = tuple[1],
261 description = 'User-requested "make %s"' % tuple[1],
262 command = ["make", tuple[1], "V=s"],
263 doStepIf = IsCleanRequested(tuple[0])
264 ))
265
266 # check out the source
267 factory.addStep(Git(repourl=repo_url, mode='update'))
268
269 factory.addStep(ShellCommand(
270 name = "rmtmp",
271 description = "Remove tmp folder",
272 command=["rm", "-rf", "tmp/"]))
273
274 # feed
275 # factory.addStep(ShellCommand(
276 # name = "feedsconf",
277 # description = "Copy the feeds.conf",
278 # command='''cp ~/feeds.conf ./feeds.conf''' ))
279
280 # feed
281 factory.addStep(ShellCommand(
282 name = "rmfeedlinks",
283 description = "Remove feed symlinks",
284 command=["rm", "-rf", "package/feeds/"]))
285
286 # feed
287 factory.addStep(ShellCommand(
288 name = "updatefeeds",
289 description = "Updating feeds",
290 command=["./scripts/feeds", "update"]))
291
292 # feed
293 factory.addStep(ShellCommand(
294 name = "installfeeds",
295 description = "Installing feeds",
296 command=["./scripts/feeds", "install", "-a"]))
297
298 # configure
299 factory.addStep(ShellCommand(
300 name = "newconfig",
301 description = "Seeding .config",
302 command='''cat <<EOT > .config
303 CONFIG_TARGET_%s=y
304 CONFIG_TARGET_%s_%s=y
305 CONFIG_ALL_NONSHARED=y
306 CONFIG_SDK=y
307 CONFIG_IB=y
308 # CONFIG_IB_STANDALONE is not set
309 CONFIG_DEVEL=y
310 CONFIG_CCACHE=y
311 CONFIG_SIGNED_PACKAGES=y
312 # CONFIG_PER_FEED_REPO_ADD_COMMENTED is not set
313 CONFIG_KERNEL_KALLSYMS=y
314 CONFIG_COLLECT_KERNEL_DEBUG=y
315 CONFIG_TARGET_ALL_PROFILES=y
316 CONFIG_TARGET_MULTI_PROFILE=y
317 CONFIG_TARGET_PER_DEVICE_ROOTFS=y
318 EOT''' %(ts[0], ts[0], ts[1]) ))
319
320 factory.addStep(ShellCommand(
321 name = "delbin",
322 description = "Removing output directory",
323 command = ["rm", "-rf", "bin/"]
324 ))
325
326 factory.addStep(ShellCommand(
327 name = "defconfig",
328 description = "Populating .config",
329 command = ["make", "defconfig"]
330 ))
331
332 # check arch
333 factory.addStep(ShellCommand(
334 name = "checkarch",
335 description = "Checking architecture",
336 command = ["grep", "-sq", "CONFIG_TARGET_%s=y" %(ts[0]), ".config"],
337 logEnviron = False,
338 want_stdout = False,
339 want_stderr = False,
340 haltOnFailure = True
341 ))
342
343 # find libc suffix
344 factory.addStep(SetProperty(
345 name = "libc",
346 property = "libc",
347 description = "Finding libc suffix",
348 command = ["sed", "-ne", '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }', ".config"]))
349
350 # install build key
351 factory.addStep(FileDownload(mastersrc=home_dir+'/key-build', slavedest="key-build", mode=0600))
352 factory.addStep(FileDownload(mastersrc=home_dir+'/key-build.pub', slavedest="key-build.pub", mode=0600))
353
354 # prepare dl
355 factory.addStep(ShellCommand(
356 name = "dldir",
357 description = "Preparing dl/",
358 command = "mkdir -p $HOME/dl && rm -rf ./dl && ln -sf $HOME/dl ./dl",
359 logEnviron = False,
360 want_stdout = False
361 ))
362
363 # prepare tar
364 factory.addStep(ShellCommand(
365 name = "dltar",
366 description = "Building GNU tar",
367 command = ["make", WithProperties("-j%(nproc:~4)s"), "tools/tar/install", "V=s"],
368 haltOnFailure = True
369 ))
370
371 # populate dl
372 factory.addStep(ShellCommand(
373 name = "dlrun",
374 description = "Populating dl/",
375 command = ["make", WithProperties("-j%(nproc:~4)s"), "download", "V=s"],
376 logEnviron = False,
377 locks = [dlLock.access('exclusive')]
378 ))
379
380 factory.addStep(ShellCommand(
381 name = "cleanbase",
382 description = "Cleaning base-files",
383 command=["make", "package/base-files/clean", "V=s"]
384 ))
385
386 # build
387 factory.addStep(ShellCommand(
388 name = "tools",
389 description = "Building tools",
390 command = ["make", WithProperties("-j%(nproc:~4)s"), "tools/install", "V=s"],
391 haltOnFailure = True
392 ))
393
394 factory.addStep(ShellCommand(
395 name = "toolchain",
396 description = "Building toolchain",
397 command=["make", WithProperties("-j%(nproc:~4)s"), "toolchain/install", "V=s"],
398 haltOnFailure = True
399 ))
400
401 factory.addStep(ShellCommand(
402 name = "kmods",
403 description = "Building kmods",
404 command=["make", WithProperties("-j%(nproc:~4)s"), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
405 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
406 haltOnFailure = True
407 ))
408
409 factory.addStep(ShellCommand(
410 name = "pkgbuild",
411 description = "Building packages",
412 command=["make", WithProperties("-j%(nproc:~4)s"), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
413 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
414 haltOnFailure = True
415 ))
416
417 # factory.addStep(IfBuiltinShellCommand(
418 factory.addStep(ShellCommand(
419 name = "pkginstall",
420 description = "Installing packages",
421 command=["make", WithProperties("-j%(nproc:~4)s"), "package/install", "V=s"],
422 haltOnFailure = True
423 ))
424
425 factory.addStep(ShellCommand(
426 name = "pkgindex",
427 description = "Indexing packages",
428 command=["make", WithProperties("-j%(nproc:~4)s"), "package/index", "V=s"],
429 haltOnFailure = True
430 ))
431
432 #factory.addStep(IfBuiltinShellCommand(
433 factory.addStep(ShellCommand(
434 name = "images",
435 description = "Building images",
436 command=["make", WithProperties("-j%(nproc:~4)s"), "target/install", "V=s"],
437 haltOnFailure = True
438 ))
439
440 factory.addStep(ShellCommand(
441 name = "checksums",
442 description = "Calculating checksums",
443 command=["make", "-j1", "checksum", "V=s"],
444 haltOnFailure = True
445 ))
446
447 # sign
448 if gpg_keyid is not None:
449 factory.addStep(MasterShellCommand(
450 name = "signprepare",
451 description = "Preparing temporary signing directory",
452 command = ["mkdir", "-p", "%s/signing" %(home_dir)],
453 haltOnFailure = True
454 ))
455
456 factory.addStep(ShellCommand(
457 name = "signpack",
458 description = "Packing files to sign",
459 command = WithProperties("find bin/targets/%s/%s%%(libc)s/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz" %(ts[0], ts[1])),
460 haltOnFailure = True
461 ))
462
463 factory.addStep(FileUpload(
464 slavesrc = "sign.tar.gz",
465 masterdest = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
466 haltOnFailure = True
467 ))
468
469 factory.addStep(MasterShellCommand(
470 name = "signfiles",
471 description = "Signing files",
472 command = ["%s/signall.sh" %(home_dir), "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]), gpg_keyid, gpg_passfile, gpg_comment],
473 haltOnFailure = True
474 ))
475
476 factory.addStep(FileDownload(
477 mastersrc = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
478 slavedest = "sign.tar.gz",
479 haltOnFailure = True
480 ))
481
482 factory.addStep(ShellCommand(
483 name = "signunpack",
484 description = "Unpacking signed files",
485 command = ["tar", "-xzf", "sign.tar.gz"],
486 haltOnFailure = True
487 ))
488
489 # upload
490 factory.addStep(ShellCommand(
491 name = "uploadprepare",
492 description = "Preparing target directory",
493 command=["rsync", "-av", "--include", "/%s/" %(ts[0]), "--include", "/%s/%s/" %(ts[0], ts[1]), "--exclude", "/*", "--exclude", "/*/*", "--exclude", "/%s/%s/*" %(ts[0], ts[1]), "bin/targets/", "%s/targets/" %(rsync_bin_url)],
494 env={'RSYNC_PASSWORD': rsync_bin_key},
495 haltOnFailure = True,
496 logEnviron = False
497 ))
498
499 factory.addStep(ShellCommand(
500 name = "targetupload",
501 description = "Uploading target files",
502 command=["rsync", "--delete", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", WithProperties("bin/targets/%s/%s%%(libc)s/" %(ts[0], ts[1])), "%s/targets/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
503 env={'RSYNC_PASSWORD': rsync_bin_key},
504 haltOnFailure = True,
505 logEnviron = False
506 ))
507
508 if rsync_src_url is not None:
509 factory.addStep(ShellCommand(
510 name = "sourceupload",
511 description = "Uploading source archives",
512 command=["rsync", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "dl/", "%s/" %(rsync_src_url)],
513 env={'RSYNC_PASSWORD': rsync_src_key},
514 haltOnFailure = True,
515 logEnviron = False
516 ))
517
518 if False:
519 factory.addStep(ShellCommand(
520 name = "packageupload",
521 description = "Uploading package files",
522 command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "bin/packages/", "%s/packages/" %(rsync_bin_url)],
523 env={'RSYNC_PASSWORD': rsync_bin_key},
524 haltOnFailure = False,
525 logEnviron = False
526 ))
527
528 # logs
529 if False:
530 factory.addStep(ShellCommand(
531 name = "upload",
532 description = "Uploading logs",
533 command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "logs/", "%s/logs/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
534 env={'RSYNC_PASSWORD': rsync_bin_key},
535 haltOnFailure = False,
536 alwaysRun = True,
537 logEnviron = False
538 ))
539
540 from buildbot.config import BuilderConfig
541
542 c['builders'].append(BuilderConfig(name=target, slavenames=slaveNames, factory=factory))
543
544
545 ####### STATUS TARGETS
546
547 # 'status' is a list of Status Targets. The results of each build will be
548 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
549 # including web pages, email senders, and IRC bots.
550
551 c['status'] = []
552
553 from buildbot.status import html
554 from buildbot.status.web import authz, auth
555
556 if ini.has_option("status", "bind"):
557 if ini.has_option("status", "user") and ini.has_option("status", "password"):
558 authz_cfg=authz.Authz(
559 # change any of these to True to enable; see the manual for more
560 # options
561 auth=auth.BasicAuth([(ini.get("status", "user"), ini.get("status", "password"))]),
562 gracefulShutdown = 'auth',
563 forceBuild = 'auth', # use this to test your slave once it is set up
564 forceAllBuilds = 'auth',
565 pingBuilder = False,
566 stopBuild = 'auth',
567 stopAllBuilds = 'auth',
568 cancelPendingBuild = 'auth',
569 )
570 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind"), authz=authz_cfg))
571 else:
572 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind")))
573
574
575 from buildbot.status import words
576
577 if ini.has_option("irc", "host") and ini.has_option("irc", "nickname") and ini.has_option("irc", "channel"):
578 irc_host = ini.get("irc", "host")
579 irc_port = 6667
580 irc_chan = ini.get("irc", "channel")
581 irc_nick = ini.get("irc", "nickname")
582 irc_pass = None
583
584 if ini.has_option("irc", "port"):
585 irc_port = ini.getint("irc", "port")
586
587 if ini.has_option("irc", "password"):
588 irc_pass = ini.get("irc", "password")
589
590 irc = words.IRC(irc_host, irc_nick, port = irc_port, password = irc_pass,
591 channels = [{ "channel": irc_chan }],
592 notify_events = {
593 'exception': 1,
594 'successToFailure': 1,
595 'failureToSuccess': 1
596 }
597 )
598
599 c['status'].append(irc)
600
601
602 ####### PROJECT IDENTITY
603
604 # the 'title' string will appear at the top of this buildbot
605 # installation's html.WebStatus home page (linked to the
606 # 'titleURL') and is embedded in the title of the waterfall HTML page.
607
608 c['title'] = ini.get("general", "title")
609 c['titleURL'] = ini.get("general", "title_url")
610
611 # the 'buildbotURL' string should point to the location where the buildbot's
612 # internal web server (usually the html.WebStatus page) is visible. This
613 # typically uses the port number set in the Waterfall 'status' entry, but
614 # with an externally-visible host name which the buildbot cannot figure out
615 # without some help.
616
617 c['buildbotURL'] = ini.get("general", "buildbot_url")
618
619 ####### DB URL
620
621 c['db'] = {
622 # This specifies what database buildbot uses to store its state. You can leave
623 # this at its default for all but the largest installations.
624 'db_url' : "sqlite:///state.sqlite",
625 }