from buildbot import locks
from buildbot.data import resultspec
-from buildbot.changes import filter
from buildbot.changes.gitpoller import GitPoller
from buildbot.config import BuilderConfig
from buildbot.plugins import reporters
from buildbot.process.factory import BuildFactory
from buildbot.process.properties import Interpolate
from buildbot.process.properties import Property
-from buildbot.schedulers.basic import SingleBranchScheduler
+from buildbot.schedulers.basic import AnyBranchScheduler
from buildbot.schedulers.forcesched import BaseParameter
from buildbot.schedulers.forcesched import ForceScheduler
from buildbot.schedulers.forcesched import ValidationError
ini = configparser.ConfigParser()
ini.read(os.getenv("BUILDMASTER_CONFIG", "./config.ini"))
-if "general" not in ini or "phase1" not in ini or "rsync" not in ini:
+if "general" not in ini or "phase1" not in ini:
raise ValueError("Fix your configuration")
inip1 = ini['phase1']
+# Globals
+work_dir = os.path.abspath(ini['general'].get("workdir", "."))
+scripts_dir = os.path.abspath("../scripts")
+
+repo_url = ini['repo'].get("url")
+
+rsync_defopts = ["-v", "-4", "--timeout=120"]
+
+#if rsync_bin_url.find("::") > 0 or rsync_bin_url.find("rsync://") == 0:
+# rsync_bin_defopts += ["--contimeout=20"]
+
+branches = {}
+
+def ini_parse_branch(section):
+ b = {}
+ name = section.get("name")
+
+ if not name:
+ raise ValueError("missing 'name' in " + repr(section))
+ if name in branches:
+ raise ValueError("duplicate branch name in " + repr(section))
+
+ b["name"] = name
+ b["bin_url"] = section.get("binary_url")
+ b["bin_key"] = section.get("binary_password")
+
+ b["src_url"] = section.get("source_url")
+ b["src_key"] = section.get("source_password")
+
+ b["gpg_key"] = section.get("gpg_key")
+
+ b["usign_key"] = section.get("usign_key")
+ usign_comment = "untrusted comment: " + name.replace("-", " ").title() + " key"
+ b["usign_comment"] = section.get("usign_comment", usign_comment)
+
+ b["config_seed"] = section.get("config_seed")
+
+ b["kmod_archive"] = section.getboolean("kmod_archive", False)
+
+ branches[name] = b
+ log.msg("Configured branch: {}".format(name))
+
+# PB port can be either a numeric port or a connection string
+pb_port = inip1.get("port") or 9989
+
# This is the dictionary that the buildmaster pays attention to. We also use
# a shorter alias to save typing.
c = BuildmasterConfig = {}
NetLocks = dict()
for section in ini.sections():
+ if section.startswith("branch "):
+ ini_parse_branch(ini[section])
+
if section.startswith("worker "):
if ini.has_option(section, "name") and ini.has_option(section, "password") and \
(not ini.has_option(section, "phase") or ini.getint(section, "phase") == 1):
- sl_props = { 'dl_lock':None, 'ul_lock':None, 'do_cleanup':False, 'max_builds':1, 'shared_wd':False }
+ sl_props = { 'dl_lock':None, 'ul_lock':None }
name = ini.get(section, "name")
password = ini.get(section, "password")
- max_builds = 1
- if ini.has_option(section, "builds"):
- max_builds = ini.getint(section, "builds")
- sl_props['max_builds'] = max_builds
- if max_builds == 1:
- sl_props['shared_wd'] = True
- if ini.has_option(section, "cleanup"):
- sl_props['do_cleanup'] = ini.getboolean(section, "cleanup")
if ini.has_option(section, "dl_lock"):
lockname = ini.get(section, "dl_lock")
sl_props['dl_lock'] = lockname
if lockname not in NetLocks:
NetLocks[lockname] = locks.MasterLock(lockname)
if ini.has_option(section, "ul_lock"):
- lockname = ini.get(section, "dl_lock")
+ lockname = ini.get(section, "ul_lock")
sl_props['ul_lock'] = lockname
if lockname not in NetLocks:
NetLocks[lockname] = locks.MasterLock(lockname)
- if ini.has_option(section, "shared_wd"):
- shared_wd = ini.getboolean(section, "shared_wd")
- sl_props['shared_wd'] = shared_wd
- if shared_wd and (max_builds != 1):
- raise ValueError('max_builds must be 1 with shared workdir!')
- c['workers'].append(Worker(name, password, max_builds = max_builds, properties = sl_props))
+ c['workers'].append(Worker(name, password, max_builds = 1, properties = sl_props))
-# PB port can be either a numeric port or a connection string
-pb_port = inip1.get("port") or 9989
c['protocols'] = {'pb': {'port': pb_port}}
# coalesce builds
####### CHANGESOURCES
-work_dir = os.path.abspath(ini['general'].get("workdir", "."))
-scripts_dir = os.path.abspath("../scripts")
-
-tree_expire = inip1.getint("expire", 0)
-config_seed = inip1.get("config_seed", "")
-
-repo_url = ini['repo'].get("url")
-repo_branch = ini['repo'].get("branch", "master")
-
-rsync_bin_url = ini['rsync'].get("binary_url")
-rsync_bin_key = ini['rsync'].get("binary_password")
-rsync_bin_defopts = ["-v", "-4", "--timeout=120"]
-
-if rsync_bin_url.find("::") > 0 or rsync_bin_url.find("rsync://") == 0:
- rsync_bin_defopts += ["--contimeout=20"]
-
-rsync_src_url = ini['rsync'].get("source_url")
-rsync_src_key = ini['rsync'].get("source_password")
-rsync_src_defopts = ["-v", "-4", "--timeout=120"]
-
-if rsync_src_url.find("::") > 0 or rsync_src_url.find("rsync://") == 0:
- rsync_src_defopts += ["--contimeout=20"]
-
-usign_key = None
-usign_comment = "untrusted comment: " + repo_branch.replace("-", " ").title() + " key"
+branchNames = [branches[b]["name"] for b in branches]
-if ini.has_section("usign"):
- usign_key = ini['usign'].get("key")
- usign_comment = ini['usign'].get("comment", usign_comment)
-
-enable_kmod_archive = inip1.getboolean("kmod_archive", False)
+# find targets
+targets = set()
+def populateTargets():
+ log.msg("Populating targets, this will take time")
+ sourcegit = work_dir + '/source.git'
+ for branch in branchNames:
+ if os.path.isdir(sourcegit):
+ subprocess.call(["rm", "-rf", sourcegit])
-# find targets
-targets = [ ]
+ subprocess.call(["git", "clone", "-q", "--depth=1", "--branch="+branch, repo_url, sourcegit])
-if not os.path.isdir(work_dir+'/source.git'):
- subprocess.call(["git", "clone", "--depth=1", "--branch="+repo_branch, repo_url, work_dir+'/source.git'])
-else:
- subprocess.call(["git", "pull"], cwd = work_dir+'/source.git')
+ os.makedirs(sourcegit + '/tmp', exist_ok=True)
+ findtargets = subprocess.Popen(['./scripts/dump-target-info.pl', 'targets'],
+ stdout = subprocess.PIPE, stderr = subprocess.DEVNULL, cwd = sourcegit)
-os.makedirs(work_dir+'/source.git/tmp', exist_ok=True)
-findtargets = subprocess.Popen(['./scripts/dump-target-info.pl', 'targets'],
- stdout = subprocess.PIPE, cwd = work_dir+'/source.git')
+ while True:
+ line = findtargets.stdout.readline()
+ if not line:
+ break
+ ta = line.decode().strip().split(' ')
+ targets.add(ta[0])
-while True:
- line = findtargets.stdout.readline()
- if not line:
- break
- ta = line.decode().strip().split(' ')
- targets.append(ta[0])
+ subprocess.call(["rm", "-rf", sourcegit])
+populateTargets()
# the 'change_source' setting tells the buildmaster how it should find out
# about source code changes. Here we point to the buildbot clone of pyflakes.
c['change_source'] = []
c['change_source'].append(GitPoller(
repo_url,
- workdir=work_dir+'/work.git', branch=repo_branch,
- pollinterval=300))
+ workdir=work_dir+'/work.git', branches=branchNames,
+ pollAtLaunch=True, pollinterval=300))
####### SCHEDULERS
@property
def choices(self):
taglist = []
- basever = re.search(r'-([0-9]+\.[0-9]+)$', repo_branch)
+ basever = re.search(r'-([0-9]+\.[0-9]+)$', "master") # XXX FIXME
if basever:
findtags = subprocess.Popen(
return s
c['schedulers'] = []
-c['schedulers'].append(SingleBranchScheduler(
+c['schedulers'].append(AnyBranchScheduler(
name = "all",
- change_filter = filter.ChangeFilter(branch=repo_branch),
- treeStableTimer = 60,
- builderNames = targets))
+ change_filter = util.ChangeFilter(branch=branchNames),
+ treeStableTimer = 15*60,
+ builderNames = list(targets)))
c['schedulers'].append(ForceScheduler(
name = "force",
name = "target",
label = "Build target",
default = "all",
- choices = [ "all" ] + targets
+ choices = set( "all" ) | targets
),
TagChoiceParameter(
name = "tag",
# what steps, and which workers can execute them. Note that any particular build will
# only take place on one worker.
-def IsSharedWorkdir(step):
- return bool(step.getProperty("shared_wd"))
+def IsTaggingRequested(step):
+ tag = step.getProperty("tag")
+ return tag and re.match(r"^[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", tag)
-def IsCleanupRequested(step):
- if IsSharedWorkdir(step):
- return False
- do_cleanup = step.getProperty("do_cleanup")
- if do_cleanup:
- return True
- else:
- return False
+def IsNoMasterBuild(step):
+ return step.getProperty("branch") != "master"
-def IsExpireRequested(step):
- if IsSharedWorkdir(step):
- return False
- else:
- return not IsCleanupRequested(step)
+def IsUsignEnabled(step):
+ branch = step.getProperty("branch")
+ return branch and branches[branch].get("usign_key")
-def IsTaggingRequested(step):
- val = step.getProperty("tag")
- if val and re.match(r"^[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", val):
- return True
- else:
- return False
+def IsSignEnabled(step):
+ branch = step.getProperty("branch")
+ return IsUsignEnabled(step) or branch and branches[branch].get("gpg_key")
-def IsNoMasterBuild(step):
- return repo_branch != "master"
+def IsKmodArchiveEnabled(step):
+ branch = step.getProperty("branch")
+ return branch and branches[branch].get("kmod_archive")
-def GetBaseVersion():
- if re.match(r"^[^-]+-[0-9]+\.[0-9]+$", repo_branch):
- return repo_branch.split('-')[1]
+def GetBaseVersion(branch):
+ if re.match(r"^[^-]+-[0-9]+\.[0-9]+$", branch):
+ return branch.split('-')[1]
else:
return "master"
@properties.renderer
def GetVersionPrefix(props):
- basever = GetBaseVersion()
+ branch = props.getProperty("branch")
+ basever = GetBaseVersion(branch)
if props.hasProperty("tag") and re.match(r"^[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", props["tag"]):
return "%s/" % props["tag"]
elif basever != "master":
else:
return ""
-@properties.renderer
-def GetNumJobs(props):
- if props.hasProperty("max_builds") and props.hasProperty("nproc"):
- return str(int(int(props["nproc"]) / props["max_builds"]))
- else:
- return "1"
+@util.renderer
+def GetConfigSeed(props):
+ branch = props.getProperty("branch")
+ return branch and branches[branch].get("config_seed") or ""
-@properties.renderer
-def GetCCache(props):
- if props.hasProperty("ccache_command") and "ccache" in props["ccache_command"]:
- return props["ccache_command"]
- else:
- return ""
+@util.renderer
+def GetRsyncParams(props, srcorbin, urlorkey):
+ # srcorbin: 'bin' or 'src'; urlorkey: 'url' or 'key'
+ branch = props.getProperty("branch")
+ opt = srcorbin + "_" + urlorkey
+ return branch and branches[branch].get(opt)
+
+@util.renderer
+def GetUsignKey(props):
+ branch = props.getProperty("branch")
+ return branch and branches[branch].get("usign_key")
def GetNextBuild(builder, requests):
for r in requests:
- if r.properties and r.properties.hasProperty("tag"):
- return r
+ if r.properties:
+ # order tagged build first
+ if r.properties.hasProperty("tag"):
+ return r
+ # then order by branch order
+ pbranch = r.properties.getProperty("branch")
+ for name in branchNames:
+ if pbranch == name:
+ return r
r = requests[0]
log.msg("GetNextBuild: {:>20} id: {} bsid: {}".format(builder.name, r.id, r.bsid))
if tryccache:
env['CC'] = Interpolate("%(prop:builddir)s/ccache_cc.sh")
env['CXX'] = Interpolate("%(prop:builddir)s/ccache_cxx.sh")
- env['CCACHE'] = Interpolate("%(kw:ccache)s", ccache=GetCCache)
+ env['CCACHE'] = Interpolate("%(prop:ccache_command:-)s")
else:
env['CC'] = env['CCC']
env['CXX'] = env['CCXX']
return env
@properties.renderer
-def NetLockDl(props):
+def NetLockDl(props, extralock=None):
lock = None
+ locks = []
if props.hasProperty("dl_lock"):
lock = NetLocks[props["dl_lock"]]
if lock is not None:
- return [lock.access('exclusive')]
- else:
- return []
+ locks.append(lock.access('exclusive'))
+ if extralock is not None:
+ locks.append(extralock)
+ return locks
@properties.renderer
def NetLockUl(props):
return CheckTargetProperty
-def UsignSec2Pub(seckey, comment="untrusted comment: secret key"):
+@util.renderer
+def UsignSec2Pub(props):
+ branch = props.getProperty("branch")
try:
+ comment = branches[branch].get("usign_comment") or "untrusted comment: secret key"
+ seckey = branches[branch].get("usign_key")
seckey = base64.b64decode(seckey)
except:
return None
# setup shared work directory if required
factory.addStep(ShellCommand(
name = "sharedwd",
- description = "Setting up shared work directory",
+ descriptionDone = "Shared work directory set up",
command = 'test -L "$PWD" || (mkdir -p ../shared-workdir && rm -rf "$PWD" && ln -s shared-workdir "$PWD")',
workdir = ".",
- haltOnFailure = True,
- doStepIf = IsSharedWorkdir))
+ haltOnFailure = True))
# find number of cores
factory.addStep(SetPropertyFromCommand(
# see if ccache is available
factory.addStep(SetPropertyFromCommand(
+ name = "ccache",
property = "ccache_command",
command = ["which", "ccache"],
description = "Testing for ccache command",
haltOnFailure = False,
flunkOnFailure = False,
warnOnFailure = False,
+ hideStepIf = lambda r, s: r==results.FAILURE,
))
- # expire tree if needed
- if tree_expire > 0:
- factory.addStep(FileDownload(
- name = "dlexpiresh",
- doStepIf = IsExpireRequested,
- mastersrc = scripts_dir + '/expire.sh',
- workerdest = "../expire.sh",
- mode = 0o755))
-
- factory.addStep(ShellCommand(
- name = "expire",
- description = "Checking for build tree expiry",
- command = ["./expire.sh", str(tree_expire)],
- workdir = ".",
- haltOnFailure = True,
- doStepIf = IsExpireRequested,
- timeout = 2400))
-
- # cleanup.sh if needed
- factory.addStep(FileDownload(
- name = "dlcleanupsh",
- mastersrc = scripts_dir + '/cleanup.sh',
- workerdest = "../cleanup.sh",
- mode = 0o755,
- doStepIf = IsCleanupRequested))
-
- factory.addStep(ShellCommand(
- name = "cleanold",
- description = "Cleaning previous builds",
- command = ["./cleanup.sh", c['buildbotURL'], Interpolate("%(prop:workername)s"), Interpolate("%(prop:buildername)s"), "full"],
- workdir = ".",
- haltOnFailure = True,
- doStepIf = IsCleanupRequested,
- timeout = 2400))
-
- factory.addStep(ShellCommand(
- name = "cleanup",
- description = "Cleaning work area",
- command = ["./cleanup.sh", c['buildbotURL'], Interpolate("%(prop:workername)s"), Interpolate("%(prop:buildername)s"), "single"],
- workdir = ".",
- haltOnFailure = True,
- doStepIf = IsCleanupRequested,
- timeout = 2400))
-
# Workaround bug when switching from a checked out tag back to a branch
# Ref: http://lists.infradead.org/pipermail/openwrt-devel/2019-June/017809.html
factory.addStep(ShellCommand(
name = "gitcheckout",
description = "Ensure that Git HEAD is sane",
- command = "if [ -d .git ]; then git checkout -f %s && git branch --set-upstream-to origin/%s || rm -fr .git; else exit 0; fi" %(repo_branch, repo_branch),
+ command = Interpolate("if [ -d .git ]; then git checkout -f %(prop:branch)s && git branch --set-upstream-to origin/%(prop:branch)s || rm -fr .git; else exit 0; fi"),
haltOnFailure = True))
# check out the source
factory.addStep(Git(
name = "git",
repourl = repo_url,
- branch = repo_branch,
mode = 'full',
- method = Interpolate("%(prop:do_cleanup:#?|fresh|clean)s"),
+ method = 'fresh',
locks = NetLockDl,
haltOnFailure = True,
))
factory.addStep(ShellCommand(
name = "fetchrefs",
description = "Fetching Git remote refs",
- command = ["git", "fetch", "origin", "+refs/heads/%s:refs/remotes/origin/%s" %(repo_branch, repo_branch)],
+ command = ["git", "fetch", "origin", Interpolate("+refs/heads/%(prop:branch)s:refs/remotes/origin/%(prop:branch)s")],
haltOnFailure = True
))
))
# seed config
- if config_seed is not None:
- factory.addStep(StringDownload(
- name = "dlconfigseed",
- s = config_seed + '\n',
- workerdest = ".config",
- mode = 0o644
- ))
+ factory.addStep(StringDownload(
+ name = "dlconfigseed",
+ s = Interpolate("%(kw:seed)s\n", seed=GetConfigSeed),
+ workerdest = ".config",
+ mode = 0o644
+ ))
# configure
factory.addStep(ShellCommand(
name = "newconfig",
- description = "Seeding .config",
- command = "printf 'CONFIG_TARGET_%s=y\\nCONFIG_TARGET_%s_%s=y\\nCONFIG_SIGNED_PACKAGES=%s\\n' >> .config" %(ts[0], ts[0], ts[1], 'y' if usign_key is not None else 'n')
+ descriptionDone = ".config seeded",
+ command = Interpolate("printf 'CONFIG_TARGET_%(kw:target)s=y\\nCONFIG_TARGET_%(kw:target)s_%(kw:subtarget)s=y\\nCONFIG_SIGNED_PACKAGES=%(kw:usign:#?|y|n)s\\n' >> .config", target=ts[0], subtarget=ts[1], usign=GetUsignKey)
))
factory.addStep(ShellCommand(
env = MakeEnv()
))
- # check arch
+ # check arch - exit early if does not exist - NB: some targets do not define CONFIG_TARGET_target_subtarget
factory.addStep(ShellCommand(
name = "checkarch",
description = "Checking architecture",
- command = ["grep", "-sq", "CONFIG_TARGET_%s=y" %(ts[0]), ".config"],
+ descriptionDone = "Architecture validated",
+ command = 'grep -sq CONFIG_TARGET_%s=y .config && grep -sq CONFIG_TARGET_SUBTARGET=\\"%s\\" .config' %(ts[0], ts[1]),
logEnviron = False,
want_stdout = False,
want_stderr = False,
- haltOnFailure = True
+ haltOnFailure = True,
+ flunkOnFailure = False, # this is not a build FAILURE
))
# find libc suffix
command = ["sed", "-ne", '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }', ".config"]))
# install build key
- if usign_key is not None:
- factory.addStep(StringDownload(
- name = "dlkeybuildpub",
- s = UsignSec2Pub(usign_key, usign_comment),
- workerdest = "key-build.pub",
- mode = 0o600,
- ))
-
- factory.addStep(StringDownload(
- name = "dlkeybuild",
- s = "# fake private key",
- workerdest = "key-build",
- mode = 0o600,
- ))
-
- factory.addStep(StringDownload(
- name = "dlkeybuilducert",
- s = "# fake certificate",
- workerdest = "key-build.ucert",
- mode = 0o600,
- ))
+ factory.addStep(StringDownload(
+ name = "dlkeybuildpub",
+ s = Interpolate("%(kw:sec2pub)s", sec2pub=UsignSec2Pub),
+ workerdest = "key-build.pub",
+ mode = 0o600,
+ doStepIf = IsUsignEnabled,
+ ))
+
+ factory.addStep(StringDownload(
+ name = "dlkeybuild",
+ s = "# fake private key",
+ workerdest = "key-build",
+ mode = 0o600,
+ doStepIf = IsUsignEnabled,
+ ))
+
+ factory.addStep(StringDownload(
+ name = "dlkeybuilducert",
+ s = "# fake certificate",
+ workerdest = "key-build.ucert",
+ mode = 0o600,
+ doStepIf = IsUsignEnabled,
+ ))
# prepare dl
factory.addStep(ShellCommand(
name = "dldir",
description = "Preparing dl/",
+ descriptionDone = "dl/ prepared",
command = "mkdir -p $HOME/dl && rm -rf ./dl && ln -sf $HOME/dl ./dl",
logEnviron = False,
want_stdout = False
factory.addStep(ShellCommand(
name = "dltar",
description = "Building and installing GNU tar",
- command = ["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "tools/tar/compile", "V=s"],
+ descriptionDone = "GNU tar built and installed",
+ command = ["make", Interpolate("-j%(prop:nproc:-1)s"), "tools/tar/compile", "V=s"],
env = MakeEnv(tryccache=True),
haltOnFailure = True
))
factory.addStep(ShellCommand(
name = "dlrun",
description = "Populating dl/",
- command = ["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "download", "V=s"],
+ descriptionDone = "dl/ populated",
+ command = ["make", Interpolate("-j%(prop:nproc:-1)s"), "download", "V=s"],
env = MakeEnv(),
logEnviron = False,
- locks = properties.FlattenList(NetLockDl, [dlLock.access('exclusive')]),
+ locks = NetLockDl.withArgs(dlLock.access('exclusive')),
))
factory.addStep(ShellCommand(
factory.addStep(ShellCommand(
name = "tools",
description = "Building and installing tools",
- command = ["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "tools/install", "V=s"],
+ descriptionDone = "Tools built and installed",
+ command = ["make", Interpolate("-j%(prop:nproc:-1)s"), "tools/install", "V=s"],
env = MakeEnv(tryccache=True),
haltOnFailure = True
))
factory.addStep(ShellCommand(
name = "toolchain",
description = "Building and installing toolchain",
- command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "toolchain/install", "V=s"],
+ descriptionDone = "Toolchain built and installed",
+ command=["make", Interpolate("-j%(prop:nproc:-1)s"), "toolchain/install", "V=s"],
env = MakeEnv(),
haltOnFailure = True
))
factory.addStep(ShellCommand(
name = "kmods",
description = "Building kmods",
- command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
+ descriptionDone = "Kmods built",
+ command=["make", Interpolate("-j%(prop:nproc:-1)s"), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
env = MakeEnv(),
haltOnFailure = True
))
factory.addStep(ShellCommand(
name = "pkgclean",
description = "Cleaning up package build",
+ descriptionDone = "Package build cleaned up",
command=["make", "package/cleanup", "V=s"]
))
factory.addStep(ShellCommand(
name = "pkgbuild",
description = "Building packages",
- command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
+ descriptionDone = "Packages built",
+ command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
env = MakeEnv(),
haltOnFailure = True
))
factory.addStep(ShellCommand(
name = "pkginstall",
description = "Installing packages",
- command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "package/install", "V=s"],
+ descriptionDone = "Packages installed",
+ command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/install", "V=s"],
env = MakeEnv(),
haltOnFailure = True
))
factory.addStep(ShellCommand(
name = "pkgindex",
description = "Indexing packages",
- command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "package/index", "V=s", "CONFIG_SIGNED_PACKAGES="],
+ descriptionDone = "Packages indexed",
+ command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/index", "V=s", "CONFIG_SIGNED_PACKAGES="],
env = MakeEnv(),
haltOnFailure = True
))
factory.addStep(ShellCommand(
name = "images",
description = "Building and installing images",
- command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "target/install", "V=s"],
+ descriptionDone = "Images built and installed",
+ command=["make", Interpolate("-j%(prop:nproc:-1)s"), "target/install", "V=s"],
env = MakeEnv(),
haltOnFailure = True
))
factory.addStep(ShellCommand(
name = "json_overview_image_info",
- description = "Generate profiles.json in target folder",
+ description = "Generating profiles.json in target folder",
command = "make -j1 json_overview_image_info V=s || true",
env = MakeEnv(),
haltOnFailure = True
factory.addStep(ShellCommand(
name = "checksums",
description = "Calculating checksums",
+ descriptionDone = "Checksums calculated",
command=["make", "-j1", "checksum", "V=s"],
env = MakeEnv(),
haltOnFailure = True
))
- if enable_kmod_archive:
- factory.addStep(ShellCommand(
- name = "kmoddir",
- description = "Creating kmod directory",
- command=["mkdir", "-p", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s", target=ts[0], subtarget=ts[1])],
- haltOnFailure = True
- ))
-
- factory.addStep(ShellCommand(
- name = "kmodprepare",
- description = "Preparing kmod archive",
- command=["rsync", "--include=/kmod-*.ipk", "--exclude=*", "-va",
- Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/packages/", target=ts[0], subtarget=ts[1]),
- Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1])],
- haltOnFailure = True
- ))
-
- factory.addStep(ShellCommand(
- name = "kmodindex",
- description = "Indexing kmod archive",
- command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "package/index", "V=s", "CONFIG_SIGNED_PACKAGES=",
- Interpolate("PACKAGE_SUBDIRS=bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1])],
- env = MakeEnv(),
- haltOnFailure = True
- ))
+ factory.addStep(ShellCommand(
+ name = "kmoddir",
+ descriptionDone = "Kmod directory created",
+ command=["mkdir", "-p", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s", target=ts[0], subtarget=ts[1])],
+ haltOnFailure = True,
+ doStepIf = IsKmodArchiveEnabled,
+ ))
+
+ factory.addStep(ShellCommand(
+ name = "kmodprepare",
+ description = "Preparing kmod archive",
+ descriptionDone = "Kmod archive prepared",
+ command=["rsync", "--include=/kmod-*.ipk", "--exclude=*", "-va",
+ Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/packages/", target=ts[0], subtarget=ts[1]),
+ Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1])],
+ haltOnFailure = True,
+ doStepIf = IsKmodArchiveEnabled,
+ ))
+
+ factory.addStep(ShellCommand(
+ name = "kmodindex",
+ description = "Indexing kmod archive",
+ descriptionDone = "Kmod archive indexed",
+ command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/index", "V=s", "CONFIG_SIGNED_PACKAGES=",
+ Interpolate("PACKAGE_SUBDIRS=bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1])],
+ env = MakeEnv(),
+ haltOnFailure = True,
+ doStepIf = IsKmodArchiveEnabled,
+ ))
# sign
- if ini.has_option("gpg", "key") or usign_key is not None:
- factory.addStep(MasterShellCommand(
- name = "signprepare",
- description = "Preparing temporary signing directory",
- command = ["mkdir", "-p", "%s/signing" %(work_dir)],
- haltOnFailure = True
- ))
-
- factory.addStep(ShellCommand(
- name = "signpack",
- description = "Packing files to sign",
- command = Interpolate("find bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/ bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz", target=ts[0], subtarget=ts[1]),
- haltOnFailure = True
- ))
-
- factory.addStep(FileUpload(
- workersrc = "sign.tar.gz",
- masterdest = "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1]),
- haltOnFailure = True
- ))
-
- factory.addStep(MasterShellCommand(
- name = "signfiles",
- description = "Signing files",
- command = ["%s/signall.sh" %(scripts_dir), "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1])],
- env = { 'CONFIG_INI': os.getenv("BUILDMASTER_CONFIG", "./config.ini") },
- haltOnFailure = True
- ))
-
- factory.addStep(FileDownload(
- name = "dlsigntargz",
- mastersrc = "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1]),
- workerdest = "sign.tar.gz",
- haltOnFailure = True
- ))
-
- factory.addStep(ShellCommand(
- name = "signunpack",
- description = "Unpacking signed files",
- command = ["tar", "-xzf", "sign.tar.gz"],
- haltOnFailure = True
- ))
+ factory.addStep(MasterShellCommand(
+ name = "signprepare",
+ descriptionDone = "Temporary signing directory prepared",
+ command = ["mkdir", "-p", "%s/signing" %(work_dir)],
+ haltOnFailure = True,
+ doStepIf = IsSignEnabled,
+
+ ))
+
+ factory.addStep(ShellCommand(
+ name = "signpack",
+ description = "Packing files to sign",
+ descriptionDone = "Files to sign packed",
+ command = Interpolate("find bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/ bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz", target=ts[0], subtarget=ts[1]),
+ haltOnFailure = True,
+ doStepIf = IsSignEnabled,
+ ))
+
+ factory.addStep(FileUpload(
+ workersrc = "sign.tar.gz",
+ masterdest = "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1]),
+ haltOnFailure = True,
+ doStepIf = IsSignEnabled,
+ ))
+
+ factory.addStep(MasterShellCommand(
+ name = "signfiles",
+ description = "Signing files",
+ descriptionDone = "Files signed",
+ command = ["%s/signall.sh" %(scripts_dir), "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1]), Interpolate("%(prop:branch)s")],
+ env = { 'CONFIG_INI': os.getenv("BUILDMASTER_CONFIG", "./config.ini") },
+ haltOnFailure = True,
+ doStepIf = IsSignEnabled,
+ ))
+
+ factory.addStep(FileDownload(
+ name = "dlsigntargz",
+ mastersrc = "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1]),
+ workerdest = "sign.tar.gz",
+ haltOnFailure = True,
+ doStepIf = IsSignEnabled,
+ ))
+
+ factory.addStep(ShellCommand(
+ name = "signunpack",
+ description = "Unpacking signed files",
+ descriptionDone = "Signed files unpacked",
+ command = ["tar", "-xzf", "sign.tar.gz"],
+ haltOnFailure = True,
+ doStepIf = IsSignEnabled,
+ ))
# upload
factory.addStep(ShellCommand(
name = "dirprepare",
- description = "Preparing upload directory structure",
+ descriptionDone = "Upload directory structure prepared",
command = ["mkdir", "-p", Interpolate("tmp/upload/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s", target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
haltOnFailure = True
))
factory.addStep(ShellCommand(
name = "linkprepare",
- description = "Preparing repository symlink",
- command = ["ln", "-s", "-f", Interpolate("../packages-%(kw:basever)s", basever=GetBaseVersion()), Interpolate("tmp/upload/%(kw:prefix)spackages", prefix=GetVersionPrefix)],
+ descriptionDone = "Repository symlink prepared",
+ command = ["ln", "-s", "-f", Interpolate("../packages-%(kw:basever)s", basever=util.Transform(GetBaseVersion, Property("branch"))), Interpolate("tmp/upload/%(kw:prefix)spackages", prefix=GetVersionPrefix)],
doStepIf = IsNoMasterBuild,
haltOnFailure = True
))
- if enable_kmod_archive:
- factory.addStep(ShellCommand(
- name = "kmoddirprepare",
- description = "Preparing kmod archive upload directory",
- command = ["mkdir", "-p", Interpolate("tmp/upload/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s", target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
- haltOnFailure = True
- ))
+ factory.addStep(ShellCommand(
+ name = "kmoddirprepare",
+ descriptionDone = "Kmod archive upload directory prepared",
+ command = ["mkdir", "-p", Interpolate("tmp/upload/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s", target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
+ haltOnFailure = True,
+ doStepIf = IsKmodArchiveEnabled,
+ ))
factory.addStep(ShellCommand(
name = "dirupload",
description = "Uploading directory structure",
- command = ["rsync", "-az"] + rsync_bin_defopts + ["tmp/upload/", "%s/" %(rsync_bin_url)],
- env={'RSYNC_PASSWORD': rsync_bin_key},
+ descriptionDone = "Directory structure uploaded",
+ command = ["rsync", "-az"] + rsync_defopts + ["tmp/upload/", Interpolate("%(kw:url)s/", url=GetRsyncParams.withArgs("bin", "url"))],
+ env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
haltOnFailure = True,
logEnviron = False,
locks = NetLockUl,
factory.addStep(ShellCommand(
name = "target-sha256sums",
description = "Fetching remote sha256sums for target",
- command = ["rsync", "-z"] + rsync_bin_defopts + [Interpolate("%(kw:rsyncbinurl)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/sha256sums", rsyncbinurl=rsync_bin_url, target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix), "target-sha256sums"],
- env={'RSYNC_PASSWORD': rsync_bin_key},
+ descriptionDone = "Remote sha256sums for target fetched",
+ command = ["rsync", "-z"] + rsync_defopts + [Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/sha256sums", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix), "target-sha256sums"],
+ env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
logEnviron = False,
haltOnFailure = False,
flunkOnFailure = False,
factory.addStep(ShellCommand(
name = "buildlist",
description = "Building list of files to upload",
+ descriptionDone = "List of files to upload built",
command = ["../sha2rsync.pl", "target-sha256sums", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/sha256sums", target=ts[0], subtarget=ts[1]), "rsynclist"],
haltOnFailure = True,
))
factory.addStep(ShellCommand(
name = "targetupload",
description = "Uploading target files",
- command=["../rsync.sh", "--exclude=/kmods/", "--files-from=rsynclist", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_bin_defopts +
+ descriptionDone = "Target files uploaded",
+ command=["../rsync.sh", "--exclude=/kmods/", "--files-from=rsynclist", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_defopts +
["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]),
- Interpolate("%(kw:rsyncbinurl)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", rsyncbinurl=rsync_bin_url, target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
- env={'RSYNC_PASSWORD': rsync_bin_key},
+ Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
+ env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
haltOnFailure = True,
logEnviron = False,
))
factory.addStep(ShellCommand(
name = "targetprune",
description = "Pruning target files",
- command=["../rsync.sh", "--exclude=/kmods/", "--delete", "--existing", "--ignore-existing", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_bin_defopts +
+ descriptionDone = "Target files pruned",
+ command=["../rsync.sh", "--exclude=/kmods/", "--delete", "--existing", "--ignore-existing", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_defopts +
["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]),
- Interpolate("%(kw:rsyncbinurl)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", rsyncbinurl=rsync_bin_url, target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
- env={'RSYNC_PASSWORD': rsync_bin_key},
+ Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
+ env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
+ haltOnFailure = True,
+ logEnviron = False,
+ locks = NetLockUl,
+ ))
+
+ factory.addStep(ShellCommand(
+ name = "kmodupload",
+ description = "Uploading kmod archive",
+ descriptionDone = "Kmod archive uploaded",
+ command=["../rsync.sh", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_defopts +
+ ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1]),
+ Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
+ env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
haltOnFailure = True,
logEnviron = False,
locks = NetLockUl,
+ doStepIf = IsKmodArchiveEnabled,
+ ))
+
+ factory.addStep(ShellCommand(
+ name = "sourcelist",
+ description = "Finding source archives to upload",
+ descriptionDone = "Source archives to upload found",
+ command = "find dl/ -maxdepth 1 -type f -not -size 0 -not -name '.*' -not -name '*.hash' -not -name '*.dl' -newer .config -printf '%f\\n' > sourcelist",
+ haltOnFailure = True
))
- if enable_kmod_archive:
- factory.addStep(ShellCommand(
- name = "kmodupload",
- description = "Uploading kmod archive",
- command=["../rsync.sh", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_bin_defopts +
- ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1]),
- Interpolate("%(kw:rsyncbinurl)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s/", rsyncbinurl=rsync_bin_url, target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
- env={'RSYNC_PASSWORD': rsync_bin_key},
- haltOnFailure = True,
- logEnviron = False,
- locks = NetLockUl,
- ))
-
- if rsync_src_url is not None:
- factory.addStep(ShellCommand(
- name = "sourcelist",
- description = "Finding source archives to upload",
- command = "find dl/ -maxdepth 1 -type f -not -size 0 -not -name '.*' -not -name '*.hash' -not -name '*.dl' -newer .config -printf '%f\\n' > sourcelist",
- haltOnFailure = True
- ))
-
- factory.addStep(ShellCommand(
- name = "sourceupload",
- description = "Uploading source archives",
- command=["../rsync.sh", "--files-from=sourcelist", "--size-only", "--delay-updates"] + rsync_src_defopts +
- [Interpolate("--partial-dir=.~tmp~%(kw:target)s~%(kw:subtarget)s~%(prop:workername)s", target=ts[0], subtarget=ts[1]), "-a", "dl/", "%s/" %(rsync_src_url)],
- env={'RSYNC_PASSWORD': rsync_src_key},
- haltOnFailure = True,
- logEnviron = False,
- locks = NetLockUl,
- ))
+ factory.addStep(ShellCommand(
+ name = "sourceupload",
+ description = "Uploading source archives",
+ descriptionDone = "Source archives uploaded",
+ command=["../rsync.sh", "--files-from=sourcelist", "--size-only", "--delay-updates"] + rsync_defopts +
+ [Interpolate("--partial-dir=.~tmp~%(kw:target)s~%(kw:subtarget)s~%(prop:workername)s", target=ts[0], subtarget=ts[1]), "-a", "dl/", Interpolate("%(kw:url)s/", url=GetRsyncParams.withArgs("src", "url"))],
+ env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("src", "key")) },
+ haltOnFailure = True,
+ logEnviron = False,
+ locks = NetLockUl,
+ ))
factory.addStep(ShellCommand(
name = "df",
haltOnFailure = False,
flunkOnFailure = False,
warnOnFailure = False,
- alwaysRun = True,
+ hideStepIf = lambda r, s: r==results.FAILURE,
))
c['builders'].append(BuilderConfig(name=target, workernames=workerNames, factory=factory, nextBuild=GetNextBuild))
projects / buildbot.git / blobdiff