phase1: improve Git cloning on the master
[buildbot.git] / phase1 / master.cfg
1 # -*- python -*-
2 # ex: set syntax=python:
3
4 import os
5 import re
6 import subprocess
7 import ConfigParser
8
9 from buildbot import locks
10
11 # This is a sample buildmaster config file. It must be installed as
12 # 'master.cfg' in your buildmaster's base directory.
13
14 ini = ConfigParser.ConfigParser()
15 ini.read("./config.ini")
16
17 # This is the dictionary that the buildmaster pays attention to. We also use
18 # a shorter alias to save typing.
19 c = BuildmasterConfig = {}
20
21 ####### BUILDSLAVES
22
23 # The 'slaves' list defines the set of recognized buildslaves. Each element is
24 # a BuildSlave object, specifying a unique slave name and password. The same
25 # slave name and password must be configured on the slave.
26 from buildbot.buildslave import BuildSlave
27
28 slave_port = 9989
29
30 if ini.has_option("general", "port"):
31 slave_port = ini.getint("general", "port")
32
33 c['slaves'] = []
34
35 for section in ini.sections():
36 if section.startswith("slave "):
37 if ini.has_option(section, "name") and ini.has_option(section, "password"):
38 name = ini.get(section, "name")
39 password = ini.get(section, "password")
40 max_builds = 1
41 if ini.has_option(section, "builds"):
42 max_builds = ini.getint(section, "builds")
43 c['slaves'].append(BuildSlave(name, password, max_builds = max_builds))
44
45 # 'slavePortnum' defines the TCP port to listen on for connections from slaves.
46 # This must match the value configured into the buildslaves (with their
47 # --master option)
48 c['slavePortnum'] = slave_port
49
50 # coalesce builds
51 c['mergeRequests'] = True
52
53 ####### CHANGESOURCES
54
55 home_dir = os.path.abspath(ini.get("general", "homedir"))
56 tree_expire = 0
57
58 if ini.has_option("general", "expire"):
59 tree_expire = ini.getint("general", "expire")
60
61 repo_url = ini.get("repo", "url")
62 repo_branch = "master"
63
64 if ini.has_option("repo", "branch"):
65 repo_branch = ini.get("repo", "branch")
66
67 rsync_bin_url = ini.get("rsync", "binary_url")
68 rsync_bin_key = ini.get("rsync", "binary_password")
69
70 rsync_src_url = None
71 rsync_src_key = None
72
73 if ini.has_option("rsync", "source_url"):
74 rsync_src_url = ini.get("rsync", "source_url")
75 rsync_src_key = ini.get("rsync", "source_password")
76
77 gpg_home = "~/.gnupg"
78 gpg_keyid = None
79 gpg_comment = "Unattended build signature"
80 gpg_passfile = "/dev/null"
81
82 if ini.has_option("gpg", "home"):
83 gpg_home = ini.get("gpg", "home")
84
85 if ini.has_option("gpg", "keyid"):
86 gpg_keyid = ini.get("gpg", "keyid")
87
88 if ini.has_option("gpg", "comment"):
89 gpg_comment = ini.get("gpg", "comment")
90
91 if ini.has_option("gpg", "passfile"):
92 gpg_passfile = ini.get("gpg", "passfile")
93
94
95 # find targets
96 targets = [ ]
97
98 if not os.path.isdir(home_dir+'/source.git'):
99 subprocess.call(["git", "clone", "--depth=1", "--branch="+repo_branch, repo_url, home_dir+'/source.git'])
100 else:
101 subprocess.call(["git", "pull"], cwd = home_dir+'/source.git')
102
103 findtargets = subprocess.Popen([home_dir+'/dumpinfo.pl', 'targets'],
104 stdout = subprocess.PIPE, cwd = home_dir+'/source.git')
105
106 while True:
107 line = findtargets.stdout.readline()
108 if not line:
109 break
110 ta = line.strip().split(' ')
111 targets.append(ta[0])
112
113
114 # the 'change_source' setting tells the buildmaster how it should find out
115 # about source code changes. Here we point to the buildbot clone of pyflakes.
116
117 from buildbot.changes.gitpoller import GitPoller
118 c['change_source'] = []
119 c['change_source'].append(GitPoller(
120 repo_url,
121 workdir=home_dir+'/work.git', branch='master',
122 pollinterval=300))
123
124 ####### SCHEDULERS
125
126 # Configure the Schedulers, which decide how to react to incoming changes. In this
127 # case, just kick off a 'basebuild' build
128
129 from buildbot.schedulers.basic import SingleBranchScheduler
130 from buildbot.schedulers.forcesched import ForceScheduler
131 from buildbot.changes import filter
132 c['schedulers'] = []
133 c['schedulers'].append(SingleBranchScheduler(
134 name="all",
135 change_filter=filter.ChangeFilter(branch='master'),
136 treeStableTimer=60,
137 builderNames=targets))
138
139 c['schedulers'].append(ForceScheduler(
140 name="force",
141 builderNames=targets))
142
143 ####### BUILDERS
144
145 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
146 # what steps, and which slaves can execute them. Note that any particular build will
147 # only take place on one slave.
148
149 from buildbot.process.factory import BuildFactory
150 from buildbot.steps.source import Git
151 from buildbot.steps.shell import ShellCommand
152 from buildbot.steps.shell import SetProperty
153 from buildbot.steps.transfer import FileUpload
154 from buildbot.steps.transfer import FileDownload
155 from buildbot.steps.master import MasterShellCommand
156 from buildbot.process.properties import WithProperties
157
158
159 CleanTargetMap = [
160 [ "tools", "tools/clean" ],
161 [ "chain", "toolchain/clean" ],
162 [ "linux", "target/linux/clean" ],
163 [ "dir", "dirclean" ],
164 [ "dist", "distclean" ]
165 ]
166
167 def IsCleanRequested(pattern):
168 def CheckCleanProperty(step):
169 val = step.getProperty("clean")
170 if val and re.match(pattern, val):
171 return True
172 else:
173 return False
174
175 return CheckCleanProperty
176
177
178 c['builders'] = []
179
180 dlLock = locks.SlaveLock("slave_dl")
181
182 checkBuiltin = re.sub('[\t\n ]+', ' ', """
183 checkBuiltin() {
184 local symbol op path file;
185 for file in $CHANGED_FILES; do
186 case "$file" in
187 package/*/*) : ;;
188 *) return 0 ;;
189 esac;
190 done;
191 while read symbol op path; do
192 case "$symbol" in package-*)
193 symbol="${symbol##*(}";
194 symbol="${symbol%)}";
195 for file in $CHANGED_FILES; do
196 case "$file" in "package/$path/"*)
197 grep -qsx "$symbol=y" .config && return 0
198 ;; esac;
199 done;
200 esac;
201 done < tmp/.packagedeps;
202 return 1;
203 }
204 """).strip()
205
206
207 class IfBuiltinShellCommand(ShellCommand):
208 def _quote(self, str):
209 if re.search("[^a-zA-Z0-9/_.-]", str):
210 return "'%s'" %(re.sub("'", "'\"'\"'", str))
211 return str
212
213 def setCommand(self, command):
214 if not isinstance(command, (str, unicode)):
215 command = ' '.join(map(self._quote, command))
216 self.command = [
217 '/bin/sh', '-c',
218 '%s; if checkBuiltin; then %s; else exit 0; fi' %(checkBuiltin, command)
219 ]
220
221 def setupEnvironment(self, cmd):
222 slaveEnv = self.slaveEnvironment
223 if slaveEnv is None:
224 slaveEnv = { }
225 changedFiles = { }
226 for request in self.build.requests:
227 for source in request.sources:
228 for change in source.changes:
229 for file in change.files:
230 changedFiles[file] = True
231 fullSlaveEnv = slaveEnv.copy()
232 fullSlaveEnv['CHANGED_FILES'] = ' '.join(changedFiles.keys())
233 cmd.args['env'] = fullSlaveEnv
234
235 slaveNames = [ ]
236
237 for slave in c['slaves']:
238 slaveNames.append(slave.slavename)
239
240 for target in targets:
241 ts = target.split('/')
242
243 factory = BuildFactory()
244
245 # find number of cores
246 factory.addStep(SetProperty(
247 name = "nproc",
248 property = "nproc",
249 description = "Finding number of CPUs",
250 command = ["nproc"]))
251
252 # expire tree if needed
253 if tree_expire > 0:
254 factory.addStep(FileDownload(
255 mastersrc = "expire.sh",
256 slavedest = "../expire.sh",
257 mode = 0755))
258
259 factory.addStep(ShellCommand(
260 name = "expire",
261 description = "Checking for build tree expiry",
262 command = ["./expire.sh", str(tree_expire)],
263 workdir = ".",
264 haltOnFailure = True,
265 timeout = 2400))
266
267 # user-requested clean targets
268 for tuple in CleanTargetMap:
269 factory.addStep(ShellCommand(
270 name = tuple[1],
271 description = 'User-requested "make %s"' % tuple[1],
272 command = ["make", tuple[1], "V=s"],
273 doStepIf = IsCleanRequested(tuple[0])
274 ))
275
276 # check out the source
277 factory.addStep(Git(repourl=repo_url, branch=repo_branch, mode='update'))
278
279 factory.addStep(ShellCommand(
280 name = "rmtmp",
281 description = "Remove tmp folder",
282 command=["rm", "-rf", "tmp/"]))
283
284 # feed
285 # factory.addStep(ShellCommand(
286 # name = "feedsconf",
287 # description = "Copy the feeds.conf",
288 # command='''cp ~/feeds.conf ./feeds.conf''' ))
289
290 # feed
291 factory.addStep(ShellCommand(
292 name = "rmfeedlinks",
293 description = "Remove feed symlinks",
294 command=["rm", "-rf", "package/feeds/"]))
295
296 # feed
297 factory.addStep(ShellCommand(
298 name = "updatefeeds",
299 description = "Updating feeds",
300 command=["./scripts/feeds", "update"]))
301
302 # feed
303 factory.addStep(ShellCommand(
304 name = "installfeeds",
305 description = "Installing feeds",
306 command=["./scripts/feeds", "install", "-a"]))
307
308 # seed config
309 factory.addStep(FileDownload(
310 mastersrc = "config.seed",
311 slavedest = ".config",
312 mode = 0644
313 ))
314
315 # configure
316 factory.addStep(ShellCommand(
317 name = "newconfig",
318 description = "Seeding .config",
319 command = "printf 'CONFIG_TARGET_%s=y\\nCONFIG_TARGET_%s_%s=y\\n' >> .config" %(ts[0], ts[0], ts[1])
320 ))
321
322 factory.addStep(ShellCommand(
323 name = "delbin",
324 description = "Removing output directory",
325 command = ["rm", "-rf", "bin/"]
326 ))
327
328 factory.addStep(ShellCommand(
329 name = "defconfig",
330 description = "Populating .config",
331 command = ["make", "defconfig"]
332 ))
333
334 # check arch
335 factory.addStep(ShellCommand(
336 name = "checkarch",
337 description = "Checking architecture",
338 command = ["grep", "-sq", "CONFIG_TARGET_%s=y" %(ts[0]), ".config"],
339 logEnviron = False,
340 want_stdout = False,
341 want_stderr = False,
342 haltOnFailure = True
343 ))
344
345 # find libc suffix
346 factory.addStep(SetProperty(
347 name = "libc",
348 property = "libc",
349 description = "Finding libc suffix",
350 command = ["sed", "-ne", '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }', ".config"]))
351
352 # install build key
353 factory.addStep(FileDownload(mastersrc=home_dir+'/key-build', slavedest="key-build", mode=0600))
354 factory.addStep(FileDownload(mastersrc=home_dir+'/key-build.pub', slavedest="key-build.pub", mode=0600))
355
356 # prepare dl
357 factory.addStep(ShellCommand(
358 name = "dldir",
359 description = "Preparing dl/",
360 command = "mkdir -p $HOME/dl && rm -rf ./dl && ln -sf $HOME/dl ./dl",
361 logEnviron = False,
362 want_stdout = False
363 ))
364
365 # prepare tar
366 factory.addStep(ShellCommand(
367 name = "dltar",
368 description = "Building GNU tar",
369 command = ["make", WithProperties("-j%(nproc:~4)s"), "tools/tar/install", "V=s"],
370 haltOnFailure = True
371 ))
372
373 # populate dl
374 factory.addStep(ShellCommand(
375 name = "dlrun",
376 description = "Populating dl/",
377 command = ["make", WithProperties("-j%(nproc:~4)s"), "download", "V=s"],
378 logEnviron = False,
379 locks = [dlLock.access('exclusive')]
380 ))
381
382 factory.addStep(ShellCommand(
383 name = "cleanbase",
384 description = "Cleaning base-files",
385 command=["make", "package/base-files/clean", "V=s"]
386 ))
387
388 # build
389 factory.addStep(ShellCommand(
390 name = "tools",
391 description = "Building tools",
392 command = ["make", WithProperties("-j%(nproc:~4)s"), "tools/install", "V=s"],
393 haltOnFailure = True
394 ))
395
396 factory.addStep(ShellCommand(
397 name = "toolchain",
398 description = "Building toolchain",
399 command=["make", WithProperties("-j%(nproc:~4)s"), "toolchain/install", "V=s"],
400 haltOnFailure = True
401 ))
402
403 factory.addStep(ShellCommand(
404 name = "kmods",
405 description = "Building kmods",
406 command=["make", WithProperties("-j%(nproc:~4)s"), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
407 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
408 haltOnFailure = True
409 ))
410
411 factory.addStep(ShellCommand(
412 name = "pkgbuild",
413 description = "Building packages",
414 command=["make", WithProperties("-j%(nproc:~4)s"), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
415 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
416 haltOnFailure = True
417 ))
418
419 # factory.addStep(IfBuiltinShellCommand(
420 factory.addStep(ShellCommand(
421 name = "pkginstall",
422 description = "Installing packages",
423 command=["make", WithProperties("-j%(nproc:~4)s"), "package/install", "V=s"],
424 haltOnFailure = True
425 ))
426
427 factory.addStep(ShellCommand(
428 name = "pkgindex",
429 description = "Indexing packages",
430 command=["make", WithProperties("-j%(nproc:~4)s"), "package/index", "V=s"],
431 haltOnFailure = True
432 ))
433
434 #factory.addStep(IfBuiltinShellCommand(
435 factory.addStep(ShellCommand(
436 name = "images",
437 description = "Building images",
438 command=["make", WithProperties("-j%(nproc:~4)s"), "target/install", "V=s"],
439 haltOnFailure = True
440 ))
441
442 factory.addStep(ShellCommand(
443 name = "checksums",
444 description = "Calculating checksums",
445 command=["make", "-j1", "checksum", "V=s"],
446 haltOnFailure = True
447 ))
448
449 # sign
450 if gpg_keyid is not None:
451 factory.addStep(MasterShellCommand(
452 name = "signprepare",
453 description = "Preparing temporary signing directory",
454 command = ["mkdir", "-p", "%s/signing" %(home_dir)],
455 haltOnFailure = True
456 ))
457
458 factory.addStep(ShellCommand(
459 name = "signpack",
460 description = "Packing files to sign",
461 command = WithProperties("find bin/targets/%s/%s%%(libc)s/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz" %(ts[0], ts[1])),
462 haltOnFailure = True
463 ))
464
465 factory.addStep(FileUpload(
466 slavesrc = "sign.tar.gz",
467 masterdest = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
468 haltOnFailure = True
469 ))
470
471 factory.addStep(MasterShellCommand(
472 name = "signfiles",
473 description = "Signing files",
474 command = ["%s/signall.sh" %(home_dir), "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]), gpg_keyid, gpg_comment],
475 env = {'GNUPGHOME': gpg_home, 'PASSFILE': gpg_passfile},
476 haltOnFailure = True
477 ))
478
479 factory.addStep(FileDownload(
480 mastersrc = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
481 slavedest = "sign.tar.gz",
482 haltOnFailure = True
483 ))
484
485 factory.addStep(ShellCommand(
486 name = "signunpack",
487 description = "Unpacking signed files",
488 command = ["tar", "-xzf", "sign.tar.gz"],
489 haltOnFailure = True
490 ))
491
492 # upload
493 factory.addStep(ShellCommand(
494 name = "uploadprepare",
495 description = "Preparing target directory",
496 command=["rsync", "-av", "--include", "/%s/" %(ts[0]), "--include", "/%s/%s/" %(ts[0], ts[1]), "--exclude", "/*", "--exclude", "/*/*", "--exclude", "/%s/%s/*" %(ts[0], ts[1]), "bin/targets/", "%s/targets/" %(rsync_bin_url)],
497 env={'RSYNC_PASSWORD': rsync_bin_key},
498 haltOnFailure = True,
499 logEnviron = False
500 ))
501
502 factory.addStep(ShellCommand(
503 name = "targetupload",
504 description = "Uploading target files",
505 command=["rsync", "--delete", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", WithProperties("bin/targets/%s/%s%%(libc)s/" %(ts[0], ts[1])), "%s/targets/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
506 env={'RSYNC_PASSWORD': rsync_bin_key},
507 haltOnFailure = True,
508 logEnviron = False
509 ))
510
511 if rsync_src_url is not None:
512 factory.addStep(ShellCommand(
513 name = "sourceupload",
514 description = "Uploading source archives",
515 command=["rsync", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "dl/", "%s/" %(rsync_src_url)],
516 env={'RSYNC_PASSWORD': rsync_src_key},
517 haltOnFailure = True,
518 logEnviron = False
519 ))
520
521 if False:
522 factory.addStep(ShellCommand(
523 name = "packageupload",
524 description = "Uploading package files",
525 command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "bin/packages/", "%s/packages/" %(rsync_bin_url)],
526 env={'RSYNC_PASSWORD': rsync_bin_key},
527 haltOnFailure = False,
528 logEnviron = False
529 ))
530
531 # logs
532 if False:
533 factory.addStep(ShellCommand(
534 name = "upload",
535 description = "Uploading logs",
536 command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "logs/", "%s/logs/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
537 env={'RSYNC_PASSWORD': rsync_bin_key},
538 haltOnFailure = False,
539 alwaysRun = True,
540 logEnviron = False
541 ))
542
543 from buildbot.config import BuilderConfig
544
545 c['builders'].append(BuilderConfig(name=target, slavenames=slaveNames, factory=factory))
546
547
548 ####### STATUS TARGETS
549
550 # 'status' is a list of Status Targets. The results of each build will be
551 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
552 # including web pages, email senders, and IRC bots.
553
554 c['status'] = []
555
556 from buildbot.status import html
557 from buildbot.status.web import authz, auth
558
559 if ini.has_option("status", "bind"):
560 if ini.has_option("status", "user") and ini.has_option("status", "password"):
561 authz_cfg=authz.Authz(
562 # change any of these to True to enable; see the manual for more
563 # options
564 auth=auth.BasicAuth([(ini.get("status", "user"), ini.get("status", "password"))]),
565 gracefulShutdown = 'auth',
566 forceBuild = 'auth', # use this to test your slave once it is set up
567 forceAllBuilds = 'auth',
568 pingBuilder = False,
569 stopBuild = 'auth',
570 stopAllBuilds = 'auth',
571 cancelPendingBuild = 'auth',
572 )
573 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind"), authz=authz_cfg))
574 else:
575 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind")))
576
577
578 from buildbot.status import words
579
580 if ini.has_option("irc", "host") and ini.has_option("irc", "nickname") and ini.has_option("irc", "channel"):
581 irc_host = ini.get("irc", "host")
582 irc_port = 6667
583 irc_chan = ini.get("irc", "channel")
584 irc_nick = ini.get("irc", "nickname")
585 irc_pass = None
586
587 if ini.has_option("irc", "port"):
588 irc_port = ini.getint("irc", "port")
589
590 if ini.has_option("irc", "password"):
591 irc_pass = ini.get("irc", "password")
592
593 irc = words.IRC(irc_host, irc_nick, port = irc_port, password = irc_pass,
594 channels = [{ "channel": irc_chan }],
595 notify_events = {
596 'exception': 1,
597 'successToFailure': 1,
598 'failureToSuccess': 1
599 }
600 )
601
602 c['status'].append(irc)
603
604
605 ####### PROJECT IDENTITY
606
607 # the 'title' string will appear at the top of this buildbot
608 # installation's html.WebStatus home page (linked to the
609 # 'titleURL') and is embedded in the title of the waterfall HTML page.
610
611 c['title'] = ini.get("general", "title")
612 c['titleURL'] = ini.get("general", "title_url")
613
614 # the 'buildbotURL' string should point to the location where the buildbot's
615 # internal web server (usually the html.WebStatus page) is visible. This
616 # typically uses the port number set in the Waterfall 'status' entry, but
617 # with an externally-visible host name which the buildbot cannot figure out
618 # without some help.
619
620 c['buildbotURL'] = ini.get("general", "buildbot_url")
621
622 ####### DB URL
623
624 c['db'] = {
625 # This specifies what database buildbot uses to store its state. You can leave
626 # this at its default for all but the largest installations.
627 'db_url' : "sqlite:///state.sqlite",
628 }