2 # ex: set syntax=python:
10 from dateutil.tz import tzutc
11 from datetime import datetime, timedelta
13 from twisted.internet import defer
14 from twisted.python import log
16 from buildbot import locks
17 from buildbot.data import resultspec
18 from buildbot.changes.gitpoller import GitPoller
19 from buildbot.config import BuilderConfig
20 from buildbot.plugins import reporters
21 from buildbot.plugins import schedulers
22 from buildbot.plugins import steps
23 from buildbot.plugins import util
24 from buildbot.process import properties
25 from buildbot.process import results
26 from buildbot.process.factory import BuildFactory
27 from buildbot.process.properties import Interpolate
28 from buildbot.process.properties import Property
29 from buildbot.schedulers.basic import AnyBranchScheduler
30 from buildbot.schedulers.forcesched import BaseParameter
31 from buildbot.schedulers.forcesched import ForceScheduler
32 from buildbot.schedulers.forcesched import ValidationError
33 from buildbot.steps.master import MasterShellCommand
34 from buildbot.steps.shell import SetPropertyFromCommand
35 from buildbot.steps.shell import ShellCommand
36 from buildbot.steps.source.git import Git
37 from buildbot.steps.transfer import FileDownload
38 from buildbot.steps.transfer import FileUpload
39 from buildbot.steps.transfer import StringDownload
40 from buildbot.worker import Worker
41 from buildbot.worker.local import LocalWorker
44 if not os.path.exists("twistd.pid"):
45 with open("twistd.pid", "w") as pidfile:
46 pidfile.write("{}".format(os.getpid()))
48 # This is a sample buildmaster config file. It must be installed as
49 # 'master.cfg' in your buildmaster's base directory.
51 ini = configparser.ConfigParser()
52 ini.read(os.getenv("BUILDMASTER_CONFIG", "./config.ini"))
54 if "general" not in ini or "phase1" not in ini:
55 raise ValueError("Fix your configuration")
60 work_dir = os.path.abspath(ini['general'].get("workdir", "."))
61 scripts_dir = os.path.abspath("../scripts")
63 repo_url = ini['repo'].get("url")
65 rsync_defopts = ["-v", "-4", "--timeout=120"]
67 #if rsync_bin_url.find("::") > 0 or rsync_bin_url.find("rsync://") == 0:
68 # rsync_bin_defopts += ["--contimeout=20"]
72 def ini_parse_branch(section):
74 name = section.get("name")
77 raise ValueError("missing 'name' in " + repr(section))
79 raise ValueError("duplicate branch name in " + repr(section))
82 b["bin_url"] = section.get("binary_url")
83 b["bin_key"] = section.get("binary_password")
85 b["src_url"] = section.get("source_url")
86 b["src_key"] = section.get("source_password")
88 b["gpg_key"] = section.get("gpg_key")
90 b["usign_key"] = section.get("usign_key")
91 usign_comment = "untrusted comment: " + name.replace("-", " ").title() + " key"
92 b["usign_comment"] = section.get("usign_comment", usign_comment)
94 b["config_seed"] = section.get("config_seed")
96 b["kmod_archive"] = section.getboolean("kmod_archive", False)
99 log.msg("Configured branch: {}".format(name))
101 # PB port can be either a numeric port or a connection string
102 pb_port = inip1.get("port") or 9989
104 # This is the dictionary that the buildmaster pays attention to. We also use
105 # a shorter alias to save typing.
106 c = BuildmasterConfig = {}
108 ####### PROJECT IDENTITY
110 # the 'title' string will appear at the top of this buildbot
111 # installation's html.WebStatus home page (linked to the
112 # 'titleURL') and is embedded in the title of the waterfall HTML page.
114 c['title'] = ini['general'].get("title")
115 c['titleURL'] = ini['general'].get("title_url")
117 # the 'buildbotURL' string should point to the location where the buildbot's
118 # internal web server (usually the html.WebStatus page) is visible. This
119 # typically uses the port number set in the Waterfall 'status' entry, but
120 # with an externally-visible host name which the buildbot cannot figure out
123 c['buildbotURL'] = inip1.get("buildbot_url")
127 # The 'workers' list defines the set of recognized buildworkers. Each element is
128 # a Worker object, specifying a unique worker name and password. The same
129 # worker name and password must be configured on the worker.
134 def ini_parse_workers(section):
135 name = section.get("name")
136 password = section.get("password")
137 phase = section.getint("phase")
139 if not name or not password or not phase == 1:
140 log.msg("invalid worker configuration ignored: {}".format(repr(section)))
143 sl_props = { 'dl_lock':None, 'ul_lock':None }
144 if "dl_lock" in section:
145 lockname = section.get("dl_lock")
146 sl_props['dl_lock'] = lockname
147 if lockname not in NetLocks:
148 NetLocks[lockname] = locks.MasterLock(lockname)
149 if "ul_lock" in section:
150 lockname = section.get("ul_lock")
151 sl_props['ul_lock'] = lockname
152 if lockname not in NetLocks:
153 NetLocks[lockname] = locks.MasterLock(lockname)
155 log.msg("Configured worker: {}".format(name))
156 c['workers'].append(Worker(name, password, max_builds = 1, properties = sl_props))
159 for section in ini.sections():
160 if section.startswith("branch "):
161 ini_parse_branch(ini[section])
163 if section.startswith("worker "):
164 ini_parse_workers(ini[section])
166 branchNames = [branches[b]["name"] for b in branches]
168 c['protocols'] = {'pb': {'port': pb_port}}
171 c['collapseRequests'] = True
173 # Reduce amount of backlog data
174 c['configurators'] = [util.JanitorConfigurator(
175 logHorizon=timedelta(days=3),
179 @defer.inlineCallbacks
180 def getNewestCompleteTime(bldr):
181 """Returns the complete_at of the latest completed and not SKIPPED
182 build request for this builder, or None if there are no such build
183 requests. We need to filter out SKIPPED requests because we're
184 using collapseRequests=True which is unfortunately marking all
185 previous requests as complete when new buildset is created.
187 @returns: datetime instance or None, via Deferred
190 bldrid = yield bldr.getBuilderId()
191 completed = yield bldr.master.data.get(
192 ('builders', bldrid, 'buildrequests'),
194 resultspec.Filter('complete', 'eq', [True]),
195 resultspec.Filter('results', 'ne', [results.SKIPPED]),
197 order=['-complete_at'], limit=1)
201 complete_at = completed[0]['complete_at']
203 last_build = yield bldr.master.data.get(
206 resultspec.Filter('builderid', 'eq', [bldrid]),
208 order=['-started_at'], limit=1)
210 if last_build and last_build[0]:
211 last_complete_at = last_build[0]['complete_at']
212 if last_complete_at and (last_complete_at > complete_at):
213 return last_complete_at
217 @defer.inlineCallbacks
218 def prioritizeBuilders(master, builders):
219 """Returns sorted list of builders by their last timestamp of completed and
220 not skipped build, ordered first by branch name.
222 @returns: list of sorted builders
225 bldrNamePrio = { "__Janitor": 0, "00_force_build": 0 }
227 for bname in branchNames:
228 bldrNamePrio[bname] = i
231 def is_building(bldr):
232 return bool(bldr.building) or bool(bldr.old_building)
235 d = defer.maybeDeferred(getNewestCompleteTime, bldr)
236 d.addCallback(lambda complete_at: (complete_at, bldr))
240 (complete_at, bldr) = item
243 for (name, prio) in bldrNamePrio.items():
244 if bldr.name.startswith(name):
250 complete_at = date.replace(tzinfo=tzutc())
252 if is_building(bldr):
254 complete_at = date.replace(tzinfo=tzutc())
256 return (pos, complete_at, bldr.name)
258 results = yield defer.gatherResults([bldr_info(bldr) for bldr in builders])
259 results.sort(key=bldr_sort)
262 # log.msg("prioritizeBuilders: {:>20} complete_at: {}".format(r[1].name, r[0]))
264 return [r[1] for r in results]
266 c['prioritizeBuilders'] = prioritizeBuilders
268 ####### CHANGESOURCES
273 def populateTargets():
274 log.msg("Populating targets, this will take time")
275 sourcegit = work_dir + '/source.git'
276 for branch in branchNames:
277 if os.path.isdir(sourcegit):
278 subprocess.call(["rm", "-rf", sourcegit])
280 subprocess.call(["git", "clone", "-q", "--depth=1", "--branch="+branch, repo_url, sourcegit])
282 os.makedirs(sourcegit + '/tmp', exist_ok=True)
283 findtargets = subprocess.Popen(['./scripts/dump-target-info.pl', 'targets'],
284 stdout = subprocess.PIPE, stderr = subprocess.DEVNULL, cwd = sourcegit)
287 line = findtargets.stdout.readline()
290 ta = line.decode().strip().split(' ')
293 subprocess.call(["rm", "-rf", sourcegit])
297 # the 'change_source' setting tells the buildmaster how it should find out
298 # about source code changes. Here we point to the buildbot clone of pyflakes.
300 c['change_source'] = []
301 c['change_source'].append(GitPoller(
303 workdir=work_dir+'/work.git', branches=branchNames,
304 pollAtLaunch=True, pollinterval=300))
308 # Configure the Schedulers, which decide how to react to incoming changes. In this
309 # case, just kick off a 'basebuild' build
311 class TagChoiceParameter(BaseParameter):
312 spec_attributes = ["strict", "choices"]
316 def __init__(self, name, label=None, **kw):
317 super().__init__(name, label, **kw)
318 self._choice_list = []
320 def getRevTags(self, findtag=None):
324 for b in branchNames:
325 basever = re.search(r'-([0-9]+\.[0-9]+)$', b)
327 branchvers.append(basever[1])
329 alltags = subprocess.Popen(
330 ['git', 'ls-remote', '--tags', repo_url],
331 stdout = subprocess.PIPE)
334 line = alltags.stdout.readline()
339 (rev, tag) = line.split()
341 tagver = re.search(r'\brefs/tags/(v[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?)$', tag.decode().strip())
343 # only list tags matching configured branches
344 if tagver and any(tagver[1][1:].startswith(b) for b in branchvers):
345 if findtag and findtag != tagver[1]:
347 taglist.append({'rev': rev.decode().strip(), 'tag': tagver[1]})
353 taglist = [rt['tag'] for rt in self.getRevTags()]
354 taglist.sort(reverse=True, key=lambda tag: tag if re.search(r'-rc[0-9]+$', tag) else tag + '-z')
355 taglist.insert(0, '')
357 self._choice_list = taglist
359 return self._choice_list
361 def updateFromKwargs(self, properties, kwargs, **unused):
362 tag = self.getFromKwargs(kwargs)
363 properties[self.name] = tag
365 # find the commit matching the tag
366 findtag = self.getRevTags(tag)
369 raise ValidationError("Couldn't find tag")
371 properties['force_revision'] = findtag[0]['rev']
373 # find the branch matching the tag
375 branchver = re.search(r'v([0-9]+\.[0-9]+)', tag)
376 for b in branchNames:
377 if b.endswith(branchver[1]):
381 raise ValidationError("Couldn't find branch")
383 properties['force_branch'] = branch
385 def parse_from_arg(self, s):
386 if self.strict and s not in self._choice_list:
387 raise ValidationError("'%s' does not belong to list of available choices '%s'" % (s, self._choice_list))
391 @defer.inlineCallbacks
392 def builderNames(props):
393 """ since we have per branch and per target builders,
394 address the relevant builder for each new buildrequest
395 based on the request's desired branch and target.
397 branch = props.getProperty("branch")
398 target = props.getProperty("target", "")
403 # if that didn't work, try sourcestamp to find a branch
405 # match builders with target branch
406 ss = props.sourcestamps[0]
408 branch = ss['branch']
410 log.msg("couldn't find builder")
411 return [] # nothing works
413 bname = branch + "_" + target
416 for b in (yield props.master.data.get(('builders',))):
417 if not b['name'].startswith(bname):
419 builders.append(b['name'])
424 c['schedulers'].append(AnyBranchScheduler(
426 change_filter = util.ChangeFilter(branch=branchNames),
427 treeStableTimer = 15*60,
428 builderNames = builderNames))
430 c['schedulers'].append(ForceScheduler(
432 buttonName = "Force builds",
433 label = "Force build details",
434 builderNames = [ "00_force_build" ],
437 util.CodebaseParameter(
439 label = "Repository",
440 branch = util.FixedParameter(name = "branch", default = ""),
441 revision = util.FixedParameter(name = "revision", default = ""),
442 repository = util.FixedParameter(name = "repository", default = ""),
443 project = util.FixedParameter(name = "project", default = "")
447 reason = util.StringParameter(
450 default = "Trigger build",
456 util.ChoiceStringParameter(
458 label = "Build target",
460 choices = [ "all" ] + list(targets)
470 c['schedulers'].append(schedulers.Triggerable(name="trigger", builderNames=builderNames))
474 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
475 # what steps, and which workers can execute them. Note that any particular build will
476 # only take place on one worker.
478 def IsNoMasterBuild(step):
479 return step.getProperty("branch") != "master"
481 def IsUsignEnabled(step):
482 branch = step.getProperty("branch")
483 return branch and branches[branch].get("usign_key")
485 def IsSignEnabled(step):
486 branch = step.getProperty("branch")
487 return IsUsignEnabled(step) or branch and branches[branch].get("gpg_key")
489 def IsKmodArchiveEnabled(step):
490 branch = step.getProperty("branch")
491 return branch and branches[branch].get("kmod_archive")
493 def GetBaseVersion(branch):
494 if re.match(r"^[^-]+-[0-9]+\.[0-9]+$", branch):
495 return branch.split('-')[1]
500 def GetVersionPrefix(props):
501 branch = props.getProperty("branch")
502 basever = GetBaseVersion(branch)
503 if props.hasProperty("tag") and re.match(r"^v[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", props["tag"]):
504 return "%s/" % props["tag"][1:]
505 elif basever != "master":
506 return "%s-SNAPSHOT/" % basever
511 def GetConfigSeed(props):
512 branch = props.getProperty("branch")
513 return branch and branches[branch].get("config_seed") or ""
516 def GetRsyncParams(props, srcorbin, urlorkey):
517 # srcorbin: 'bin' or 'src'; urlorkey: 'url' or 'key'
518 branch = props.getProperty("branch")
519 opt = srcorbin + "_" + urlorkey
520 return branch and branches[branch].get(opt)
523 def GetUsignKey(props):
524 branch = props.getProperty("branch")
525 return branch and branches[branch].get("usign_key")
527 def GetNextBuild(builder, requests):
530 # order tagged build first
531 if r.properties.hasProperty("tag"):
535 #log.msg("GetNextBuild: {:>20} id: {} bsid: {}".format(builder.name, r.id, r.bsid))
538 def MakeEnv(overrides=None, tryccache=False):
540 'CCC': Interpolate("%(prop:cc_command:-gcc)s"),
541 'CCXX': Interpolate("%(prop:cxx_command:-g++)s"),
544 env['CC'] = Interpolate("%(prop:builddir)s/ccache_cc.sh")
545 env['CXX'] = Interpolate("%(prop:builddir)s/ccache_cxx.sh")
546 env['CCACHE'] = Interpolate("%(prop:ccache_command:-)s")
548 env['CC'] = env['CCC']
549 env['CXX'] = env['CCXX']
551 if overrides is not None:
552 env.update(overrides)
556 def NetLockDl(props, extralock=None):
558 if props.hasProperty("dl_lock"):
559 lock = NetLocks[props["dl_lock"]]
561 return [lock.access('exclusive')]
566 def NetLockUl(props):
568 if props.hasProperty("ul_lock"):
569 lock = NetLocks[props["ul_lock"]]
571 return [lock.access('exclusive')]
575 def IsTargetSelected(target):
576 def CheckTargetProperty(step):
577 selected_target = step.getProperty("target", "all")
578 if selected_target != "all" and selected_target != target:
582 return CheckTargetProperty
585 def UsignSec2Pub(props):
586 branch = props.getProperty("branch")
588 comment = branches[branch].get("usign_comment") or "untrusted comment: secret key"
589 seckey = branches[branch].get("usign_key")
590 seckey = base64.b64decode(seckey)
594 return "{}\n{}".format(re.sub(r"\bsecret key$", "public key", comment),
595 base64.b64encode(seckey[0:2] + seckey[32:40] + seckey[72:]))
602 for worker in c['workers']:
603 workerNames.append(worker.workername)
605 # add a single LocalWorker to handle the forcebuild builder
606 c['workers'].append(LocalWorker("__local_force_build", max_builds=1))
608 force_factory = BuildFactory()
609 force_factory.addStep(steps.Trigger(
610 name = "trigger_build",
611 schedulerNames = [ "trigger" ],
612 sourceStamps = [{ "codebase": "", "branch": Property("force_branch"), "revision": Property("force_revision"), "repository": repo_url, "project": "" }],
613 set_properties = { "reason": Property("reason"), "tag": Property("tag"), "target": Property("target") },
616 c['builders'].append(BuilderConfig(
617 name = "00_force_build",
618 workername = "__local_force_build",
619 factory = force_factory))
621 for target in targets:
622 ts = target.split('/')
624 factory = BuildFactory()
626 # setup shared work directory if required
627 factory.addStep(ShellCommand(
629 descriptionDone = "Shared work directory set up",
630 command = 'test -L "$PWD" || (mkdir -p ../shared-workdir && rm -rf "$PWD" && ln -s shared-workdir "$PWD")',
632 haltOnFailure = True,
635 # find number of cores
636 factory.addStep(SetPropertyFromCommand(
639 description = "Finding number of CPUs",
643 # find gcc and g++ compilers
644 factory.addStep(FileDownload(
645 name = "dlfindbinpl",
646 mastersrc = scripts_dir + '/findbin.pl',
647 workerdest = "../findbin.pl",
651 factory.addStep(SetPropertyFromCommand(
653 property = "cc_command",
654 description = "Finding gcc command",
655 command = ["../findbin.pl", "gcc", "", ""],
656 haltOnFailure = True,
659 factory.addStep(SetPropertyFromCommand(
661 property = "cxx_command",
662 description = "Finding g++ command",
663 command = ["../findbin.pl", "g++", "", ""],
664 haltOnFailure = True,
667 # see if ccache is available
668 factory.addStep(SetPropertyFromCommand(
670 property = "ccache_command",
671 description = "Testing for ccache command",
672 command = ["which", "ccache"],
673 haltOnFailure = False,
674 flunkOnFailure = False,
675 warnOnFailure = False,
676 hideStepIf = lambda r, s: r==results.FAILURE,
679 # check out the source
681 # if repo doesn't exist: 'git clone repourl'
682 # method 'clean' runs 'git clean -d -f', method fresh runs 'git clean -f -f -d -x'. Only works with mode='full'
683 # git cat-file -e <commit>
684 # git checkout -f <commit>
685 # git checkout -B <branch>
693 haltOnFailure = True,
697 factory.addStep(ShellCommand(
699 description = "Fetching Git remote refs",
700 command = ["git", "fetch", "origin", Interpolate("+refs/heads/%(prop:branch)s:refs/remotes/origin/%(prop:branch)s")],
701 haltOnFailure = True,
704 # Verify that Git HEAD points to a tag or branch
705 # Ref: https://web.archive.org/web/20190729224316/http://lists.infradead.org/pipermail/openwrt-devel/2019-June/017809.html
706 factory.addStep(ShellCommand(
708 description = "Ensure that Git HEAD is pointing to a branch or tag",
709 command = 'git rev-parse --abbrev-ref HEAD | grep -vxqF HEAD || git show-ref --tags --dereference 2>/dev/null | sed -ne "/^$(git rev-parse HEAD) / { s|^.*/||; s|\\^.*||; p }" | grep -qE "^v[0-9][0-9]\\."',
710 haltOnFailure = True,
713 factory.addStep(ShellCommand(
715 description = "Remove tmp folder",
716 command=["rm", "-rf", "tmp/"],
720 factory.addStep(ShellCommand(
721 name = "rmfeedlinks",
722 description = "Remove feed symlinks",
723 command=["rm", "-rf", "package/feeds/"],
726 factory.addStep(StringDownload(
728 s = '#!/bin/sh\nexec ${CCACHE} ${CCC} "$@"\n',
729 workerdest = "../ccache_cc.sh",
733 factory.addStep(StringDownload(
735 s = '#!/bin/sh\nexec ${CCACHE} ${CCXX} "$@"\n',
736 workerdest = "../ccache_cxx.sh",
741 factory.addStep(ShellCommand(
742 name = "updatefeeds",
743 description = "Updating feeds",
744 command=["./scripts/feeds", "update"],
745 env = MakeEnv(tryccache=True),
746 haltOnFailure = True,
751 factory.addStep(ShellCommand(
752 name = "installfeeds",
753 description = "Installing feeds",
754 command=["./scripts/feeds", "install", "-a"],
755 env = MakeEnv(tryccache=True),
756 haltOnFailure = True,
760 factory.addStep(StringDownload(
761 name = "dlconfigseed",
762 s = Interpolate("%(kw:seed)s\n", seed=GetConfigSeed),
763 workerdest = ".config",
768 factory.addStep(ShellCommand(
770 descriptionDone = ".config seeded",
771 command = Interpolate("printf 'CONFIG_TARGET_%(kw:target)s=y\\nCONFIG_TARGET_%(kw:target)s_%(kw:subtarget)s=y\\nCONFIG_SIGNED_PACKAGES=%(kw:usign:#?|y|n)s\\n' >> .config", target=ts[0], subtarget=ts[1], usign=GetUsignKey),
774 factory.addStep(ShellCommand(
776 description = "Removing output directory",
777 command = ["rm", "-rf", "bin/"],
780 factory.addStep(ShellCommand(
782 description = "Populating .config",
783 command = ["make", "defconfig"],
787 # check arch - exit early if does not exist - NB: some targets do not define CONFIG_TARGET_target_subtarget
788 factory.addStep(ShellCommand(
790 description = "Checking architecture",
791 descriptionDone = "Architecture validated",
792 command = 'grep -sq CONFIG_TARGET_%s=y .config && grep -sq CONFIG_TARGET_SUBTARGET=\\"%s\\" .config' %(ts[0], ts[1]),
796 haltOnFailure = True,
797 flunkOnFailure = False, # this is not a build FAILURE
801 factory.addStep(SetPropertyFromCommand(
804 description = "Finding libc suffix",
805 command = ["sed", "-ne", '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }', ".config"],
809 factory.addStep(StringDownload(
810 name = "dlkeybuildpub",
811 s = Interpolate("%(kw:sec2pub)s", sec2pub=UsignSec2Pub),
812 workerdest = "key-build.pub",
814 doStepIf = IsUsignEnabled,
817 factory.addStep(StringDownload(
819 s = "# fake private key",
820 workerdest = "key-build",
822 doStepIf = IsUsignEnabled,
825 factory.addStep(StringDownload(
826 name = "dlkeybuilducert",
827 s = "# fake certificate",
828 workerdest = "key-build.ucert",
830 doStepIf = IsUsignEnabled,
834 factory.addStep(ShellCommand(
836 description = "Preparing dl/",
837 descriptionDone = "dl/ prepared",
838 command = 'mkdir -p ../dl && rm -rf "build/dl" && ln -s ../../dl "build/dl"',
839 workdir = Property("builddir"),
845 factory.addStep(ShellCommand(
847 description = "Pruning dl/",
848 descriptionDone = "dl/ pruned",
849 command = 'find dl/ -atime +15 -delete -print',
854 factory.addStep(ShellCommand(
856 description = "Building and installing GNU tar",
857 descriptionDone = "GNU tar built and installed",
858 command = ["make", Interpolate("-j%(prop:nproc:-1)s"), "tools/tar/compile", "V=s"],
859 env = MakeEnv(tryccache=True),
860 haltOnFailure = True,
864 factory.addStep(ShellCommand(
866 description = "Populating dl/",
867 descriptionDone = "dl/ populated",
868 command = ["make", Interpolate("-j%(prop:nproc:-1)s"), "download", "V=s"],
874 factory.addStep(ShellCommand(
876 description = "Cleaning base-files",
877 command=["make", "package/base-files/clean", "V=s"],
881 factory.addStep(ShellCommand(
883 description = "Building and installing tools",
884 descriptionDone = "Tools built and installed",
885 command = ["make", Interpolate("-j%(prop:nproc:-1)s"), "tools/install", "V=s"],
886 env = MakeEnv(tryccache=True),
887 haltOnFailure = True,
890 factory.addStep(ShellCommand(
892 description = "Building and installing toolchain",
893 descriptionDone = "Toolchain built and installed",
894 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "toolchain/install", "V=s"],
896 haltOnFailure = True,
899 factory.addStep(ShellCommand(
901 description = "Building kmods",
902 descriptionDone = "Kmods built",
903 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
905 haltOnFailure = True,
908 # find kernel version
909 factory.addStep(SetPropertyFromCommand(
910 name = "kernelversion",
911 property = "kernelversion",
912 description = "Finding the effective Kernel version",
913 command = "make --no-print-directory -C target/linux/ val.LINUX_VERSION val.LINUX_RELEASE val.LINUX_VERMAGIC | xargs printf '%s-%s-%s\\n'",
914 env = { 'TOPDIR': Interpolate("%(prop:builddir)s/build") },
917 factory.addStep(ShellCommand(
919 description = "Cleaning up package build",
920 descriptionDone = "Package build cleaned up",
921 command=["make", "package/cleanup", "V=s"],
924 factory.addStep(ShellCommand(
926 description = "Building packages",
927 descriptionDone = "Packages built",
928 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
930 haltOnFailure = True,
933 factory.addStep(ShellCommand(
935 description = "Installing packages",
936 descriptionDone = "Packages installed",
937 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/install", "V=s"],
939 haltOnFailure = True,
942 factory.addStep(ShellCommand(
944 description = "Indexing packages",
945 descriptionDone = "Packages indexed",
946 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/index", "V=s", "CONFIG_SIGNED_PACKAGES="],
948 haltOnFailure = True,
951 factory.addStep(ShellCommand(
953 description = "Building and installing images",
954 descriptionDone = "Images built and installed",
955 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "target/install", "V=s"],
957 haltOnFailure = True,
960 factory.addStep(ShellCommand(
962 description = "Generating config.buildinfo, version.buildinfo and feeds.buildinfo",
963 command = "make -j1 buildinfo V=s || true",
965 haltOnFailure = True,
968 factory.addStep(ShellCommand(
969 name = "json_overview_image_info",
970 description = "Generating profiles.json in target folder",
971 command = "make -j1 json_overview_image_info V=s || true",
973 haltOnFailure = True,
976 factory.addStep(ShellCommand(
978 description = "Calculating checksums",
979 descriptionDone = "Checksums calculated",
980 command=["make", "-j1", "checksum", "V=s"],
982 haltOnFailure = True,
985 factory.addStep(ShellCommand(
987 descriptionDone = "Kmod directory created",
988 command=["mkdir", "-p", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s", target=ts[0], subtarget=ts[1])],
989 haltOnFailure = True,
990 doStepIf = IsKmodArchiveEnabled,
993 factory.addStep(ShellCommand(
994 name = "kmodprepare",
995 description = "Preparing kmod archive",
996 descriptionDone = "Kmod archive prepared",
997 command=["rsync", "--include=/kmod-*.ipk", "--exclude=*", "-va",
998 Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/packages/", target=ts[0], subtarget=ts[1]),
999 Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1])],
1000 haltOnFailure = True,
1001 doStepIf = IsKmodArchiveEnabled,
1004 factory.addStep(ShellCommand(
1006 description = "Indexing kmod archive",
1007 descriptionDone = "Kmod archive indexed",
1008 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/index", "V=s", "CONFIG_SIGNED_PACKAGES=",
1009 Interpolate("PACKAGE_SUBDIRS=bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1])],
1011 haltOnFailure = True,
1012 doStepIf = IsKmodArchiveEnabled,
1016 factory.addStep(MasterShellCommand(
1017 name = "signprepare",
1018 descriptionDone = "Temporary signing directory prepared",
1019 command = ["mkdir", "-p", "%s/signing" %(work_dir)],
1020 haltOnFailure = True,
1021 doStepIf = IsSignEnabled,
1025 factory.addStep(ShellCommand(
1027 description = "Packing files to sign",
1028 descriptionDone = "Files to sign packed",
1029 command = Interpolate("find bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/ bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz", target=ts[0], subtarget=ts[1]),
1030 haltOnFailure = True,
1031 doStepIf = IsSignEnabled,
1034 factory.addStep(FileUpload(
1035 workersrc = "sign.tar.gz",
1036 masterdest = "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1]),
1037 haltOnFailure = True,
1038 doStepIf = IsSignEnabled,
1041 factory.addStep(MasterShellCommand(
1043 description = "Signing files",
1044 descriptionDone = "Files signed",
1045 command = ["%s/signall.sh" %(scripts_dir), "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1]), Interpolate("%(prop:branch)s")],
1046 env = { 'CONFIG_INI': os.getenv("BUILDMASTER_CONFIG", "./config.ini") },
1047 haltOnFailure = True,
1048 doStepIf = IsSignEnabled,
1051 factory.addStep(FileDownload(
1052 name = "dlsigntargz",
1053 mastersrc = "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1]),
1054 workerdest = "sign.tar.gz",
1055 haltOnFailure = True,
1056 doStepIf = IsSignEnabled,
1059 factory.addStep(ShellCommand(
1060 name = "signunpack",
1061 description = "Unpacking signed files",
1062 descriptionDone = "Signed files unpacked",
1063 command = ["tar", "-xzf", "sign.tar.gz"],
1064 haltOnFailure = True,
1065 doStepIf = IsSignEnabled,
1069 factory.addStep(ShellCommand(
1070 name = "dirprepare",
1071 descriptionDone = "Upload directory structure prepared",
1072 command = ["mkdir", "-p", Interpolate("tmp/upload/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s", target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1073 haltOnFailure = True,
1076 factory.addStep(ShellCommand(
1077 name = "linkprepare",
1078 descriptionDone = "Repository symlink prepared",
1079 command = ["ln", "-s", "-f", Interpolate("../packages-%(kw:basever)s", basever=util.Transform(GetBaseVersion, Property("branch"))), Interpolate("tmp/upload/%(kw:prefix)spackages", prefix=GetVersionPrefix)],
1080 doStepIf = IsNoMasterBuild,
1081 haltOnFailure = True,
1084 factory.addStep(ShellCommand(
1085 name = "kmoddirprepare",
1086 descriptionDone = "Kmod archive upload directory prepared",
1087 command = ["mkdir", "-p", Interpolate("tmp/upload/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s", target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1088 haltOnFailure = True,
1089 doStepIf = IsKmodArchiveEnabled,
1092 factory.addStep(ShellCommand(
1094 description = "Uploading directory structure",
1095 descriptionDone = "Directory structure uploaded",
1096 command = ["rsync", "-az"] + rsync_defopts + ["tmp/upload/", Interpolate("%(kw:url)s/", url=GetRsyncParams.withArgs("bin", "url"))],
1097 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1098 haltOnFailure = True,
1101 doStepIf = util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
1104 # download remote sha256sums to 'target-sha256sums'
1105 factory.addStep(ShellCommand(
1106 name = "target-sha256sums",
1107 description = "Fetching remote sha256sums for target",
1108 descriptionDone = "Remote sha256sums for target fetched",
1109 command = ["rsync", "-z"] + rsync_defopts + [Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/sha256sums", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix), "target-sha256sums"],
1110 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1112 haltOnFailure = False,
1113 flunkOnFailure = False,
1114 warnOnFailure = False,
1115 doStepIf = util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
1118 # build list of files to upload
1119 factory.addStep(FileDownload(
1120 name = "dlsha2rsyncpl",
1121 mastersrc = scripts_dir + '/sha2rsync.pl',
1122 workerdest = "../sha2rsync.pl",
1126 factory.addStep(ShellCommand(
1128 description = "Building list of files to upload",
1129 descriptionDone = "List of files to upload built",
1130 command = ["../sha2rsync.pl", "target-sha256sums", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/sha256sums", target=ts[0], subtarget=ts[1]), "rsynclist"],
1131 haltOnFailure = True,
1134 factory.addStep(FileDownload(
1135 name = "dlrsync.sh",
1136 mastersrc = scripts_dir + '/rsync.sh',
1137 workerdest = "../rsync.sh",
1141 # upload new files and update existing ones
1142 factory.addStep(ShellCommand(
1143 name = "targetupload",
1144 description = "Uploading target files",
1145 descriptionDone = "Target files uploaded",
1146 command=["../rsync.sh", "--exclude=/kmods/", "--files-from=rsynclist", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_defopts +
1147 ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]),
1148 Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1149 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1150 haltOnFailure = True,
1152 doStepIf = util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
1155 # delete files which don't exist locally
1156 factory.addStep(ShellCommand(
1157 name = "targetprune",
1158 description = "Pruning target files",
1159 descriptionDone = "Target files pruned",
1160 command=["../rsync.sh", "--exclude=/kmods/", "--delete", "--existing", "--ignore-existing", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_defopts +
1161 ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]),
1162 Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1163 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1164 haltOnFailure = True,
1167 doStepIf = util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
1170 factory.addStep(ShellCommand(
1171 name = "kmodupload",
1172 description = "Uploading kmod archive",
1173 descriptionDone = "Kmod archive uploaded",
1174 command=["../rsync.sh", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_defopts +
1175 ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1]),
1176 Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1177 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1178 haltOnFailure = True,
1181 doStepIf = util.Transform(lambda a, b: bool(a and b), IsKmodArchiveEnabled, GetRsyncParams.withArgs("bin", "url")),
1184 factory.addStep(ShellCommand(
1185 name = "sourcelist",
1186 description = "Finding source archives to upload",
1187 descriptionDone = "Source archives to upload found",
1188 command = "find dl/ -maxdepth 1 -type f -not -size 0 -not -name '.*' -not -name '*.hash' -not -name '*.dl' -newer .config -printf '%f\\n' > sourcelist",
1189 haltOnFailure = True,
1192 factory.addStep(ShellCommand(
1193 name = "sourceupload",
1194 description = "Uploading source archives",
1195 descriptionDone = "Source archives uploaded",
1196 command=["../rsync.sh", "--files-from=sourcelist", "--size-only", "--delay-updates"] + rsync_defopts +
1197 [Interpolate("--partial-dir=.~tmp~%(kw:target)s~%(kw:subtarget)s~%(prop:workername)s", target=ts[0], subtarget=ts[1]), "-a", "dl/", Interpolate("%(kw:url)s/", url=GetRsyncParams.withArgs("src", "url"))],
1198 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("src", "key")) },
1199 haltOnFailure = True,
1202 doStepIf = util.Transform(bool, GetRsyncParams.withArgs("src", "url")),
1205 factory.addStep(ShellCommand(
1207 description = "Reporting disk usage",
1208 command=["df", "-h", "."],
1209 env={'LC_ALL': 'C'},
1211 haltOnFailure = False,
1212 flunkOnFailure = False,
1213 warnOnFailure = False,
1217 factory.addStep(ShellCommand(
1219 description = "Reporting estimated file space usage",
1220 command=["du", "-sh", "."],
1221 env={'LC_ALL': 'C'},
1223 haltOnFailure = False,
1224 flunkOnFailure = False,
1225 warnOnFailure = False,
1229 factory.addStep(ShellCommand(
1230 name = "ccachestat",
1231 description = "Reporting ccache stats",
1232 command=["ccache", "-s"],
1233 env = MakeEnv(overrides={ 'PATH': ["${PATH}", "./staging_dir/host/bin"] }),
1235 want_stderr = False,
1236 haltOnFailure = False,
1237 flunkOnFailure = False,
1238 warnOnFailure = False,
1239 hideStepIf = lambda r, s: r==results.FAILURE,
1242 for brname in branchNames:
1243 bldrname = brname + "_" + target
1244 c['builders'].append(BuilderConfig(name=bldrname, workernames=workerNames, factory=factory, nextBuild=GetNextBuild))
1247 ####### STATUS TARGETS
1249 # 'status' is a list of Status Targets. The results of each build will be
1250 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
1251 # including web pages, email senders, and IRC bots.
1253 if "status_bind" in inip1:
1255 'port': inip1.get("status_bind"),
1257 'waterfall_view': True,
1258 'console_view': True,
1263 if "status_user" in inip1 and "status_password" in inip1:
1264 c['www']['auth'] = util.UserPasswordAuth([
1265 (inip1.get("status_user"), inip1.get("status_password"))
1267 c['www']['authz'] = util.Authz(
1268 allowRules=[ util.AnyControlEndpointMatcher(role="admins") ],
1269 roleMatchers=[ util.RolesFromUsername(roles=["admins"], usernames=[inip1.get("status_user")]) ]
1273 if ini.has_section("irc"):
1275 irc_host = iniirc.get("host", None)
1276 irc_port = iniirc.getint("port", 6667)
1277 irc_chan = iniirc.get("channel", None)
1278 irc_nick = iniirc.get("nickname", None)
1279 irc_pass = iniirc.get("password", None)
1281 if irc_host and irc_nick and irc_chan:
1282 irc = reporters.IRC(irc_host, irc_nick,
1284 password = irc_pass,
1285 channels = [ irc_chan ],
1286 notify_events = [ 'exception', 'problem', 'recovery' ]
1289 c['services'].append(irc)
1291 c['revlink'] = util.RevlinkMatch([
1292 r'https://git.openwrt.org/openwrt/(.*).git'
1294 r'https://git.openwrt.org/?p=openwrt/\1.git;a=commit;h=%s')
1299 # This specifies what database buildbot uses to store its state. You can leave
1300 # this at its default for all but the largest installations.
1301 'db_url' : "sqlite:///state.sqlite",
1304 c['buildbotNetUsageData'] = None