def bldr_sort(item):
(complete_at, bldr) = item
+ if bldr.name == "00_force_build":
+ date = datetime.min
+ complete_at = date.replace(tzinfo=tzutc())
+ return (complete_at, bldr.name)
+
if not complete_at:
date = datetime.min
complete_at = date.replace(tzinfo=tzutc())
if os.path.isdir(sourcegit):
subprocess.call(["rm", "-rf", sourcegit])
- subprocess.call(["git", "clone", "--depth=1", "--branch="+branch, repo_url, sourcegit])
+ subprocess.call(["git", "clone", "-q", "--depth=1", "--branch="+branch, repo_url, sourcegit])
os.makedirs(sourcegit + '/tmp', exist_ok=True)
findtargets = subprocess.Popen(['./scripts/dump-target-info.pl', 'targets'],
@property
def choices(self):
taglist = []
- basever = re.search(r'-([0-9]+\.[0-9]+)$', "master") # XXX FIXME
+ branchvers = []
- if basever:
- findtags = subprocess.Popen(
- ['git', 'ls-remote', '--tags', repo_url],
- stdout = subprocess.PIPE)
+ for b in branchNames:
+ basever = re.search(r'-([0-9]+\.[0-9]+)$', b)
+ if basever:
+ branchvers.append(basever[1])
- while True:
- line = findtags.stdout.readline()
+ alltags = subprocess.Popen(
+ ['git', 'ls-remote', '--tags', repo_url],
+ stdout = subprocess.PIPE)
- if not line:
- break
+ while True:
+ line = alltags.stdout.readline()
- tagver = re.search(r'\brefs/tags/v([0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?)$', line.decode().strip())
+ if not line:
+ break
+
+ (ref, tag) = line.split()
- if tagver and tagver[1].find(basever[1]) == 0:
- taglist.append(tagver[1])
+ tagver = re.search(r'\brefs/tags/(v[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?)$', tag.decode().strip())
+
+ # only list tags matching configured branches
+ if tagver and any(tagver[1][1:].startswith(b) for b in branchvers):
+ taglist.append(tagver[1])
taglist.sort(reverse=True, key=lambda tag: tag if re.search(r'-rc[0-9]+$', tag) else tag + '-z')
taglist.insert(0, '')
return self._choice_list
+ def updateFromKwargs(self, properties, kwargs, **unused):
+ tag = self.getFromKwargs(kwargs)
+ properties[self.name] = tag
+
+ # find the commit matching the tag
+ findrev = subprocess.Popen(['git', 'rev-parse', 'tags/'+tag], stdout=subprocess.PIPE, cwd=work_dir+'/work.git')
+ findrev.wait(timeout=10)
+ line = findrev.stdout.readline()
+
+ if findrev.returncode!=0 or not line:
+ raise ValidationError("Couldn't find tag")
+
+ properties['force_revision'] = line.decode().strip()
+
+ # find the branch matching the tag
+ branch = None
+ branchver = re.search(r'v([0-9]+\.[0-9]+)', tag)
+ for b in branchNames:
+ if b.endswith(branchver[1]):
+ branch = b
+
+ if not branch:
+ raise ValidationError("Couldn't find branch")
+
+ properties['force_branch'] = branch
+
def parse_from_arg(self, s):
if self.strict and s not in self._choice_list:
raise ValidationError("'%s' does not belong to list of available choices '%s'" % (s, self._choice_list))
),
properties = [
- util.NestedParameter(
- name="options",
- label="Build Options",
- layout="vertical",
- fields=[
- util.ChoiceStringParameter(
- name = "target",
- label = "Build target",
- default = "all",
- choices = set( "all" ) | targets
- ),
- TagChoiceParameter(
- name = "tag",
- label = "Build tag",
- default = ""
- )
- ]
+ util.ChoiceStringParameter(
+ name = "target",
+ label = "Build target",
+ default = "all",
+ choices = [ "all" ] + list(targets)
+ ),
+ TagChoiceParameter(
+ name = "tag",
+ label = "Build tag",
+ default = ""
)
]
))
# what steps, and which workers can execute them. Note that any particular build will
# only take place on one worker.
-def IsTaggingRequested(step):
- tag = step.getProperty("tag")
- return tag and re.match(r"^[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", tag)
-
def IsNoMasterBuild(step):
return step.getProperty("branch") != "master"
def GetVersionPrefix(props):
branch = props.getProperty("branch")
basever = GetBaseVersion(branch)
- if props.hasProperty("tag") and re.match(r"^[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", props["tag"]):
- return "%s/" % props["tag"]
+ if props.hasProperty("tag") and re.match(r"^v[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", props["tag"]):
+ return "%s/" % props["tag"][1:]
elif basever != "master":
return "%s-SNAPSHOT/" % basever
else:
return env
@properties.renderer
-def NetLockDl(props):
+def NetLockDl(props, extralock=None):
lock = None
+ locks = []
if props.hasProperty("dl_lock"):
lock = NetLocks[props["dl_lock"]]
if lock is not None:
- return [lock.access('exclusive')]
- else:
- return []
+ locks.append(lock.access('exclusive'))
+ if extralock is not None:
+ locks.append(extralock)
+ return locks
@properties.renderer
def NetLockUl(props):
else:
return []
-@util.renderer
-def TagPropertyValue(props):
- if props.hasProperty("options"):
- options = props.getProperty("options")
- if type(options) is dict:
- return options.get("tag")
- return None
-
def IsTargetSelected(target):
def CheckTargetProperty(step):
- try:
- options = step.getProperty("options")
- if type(options) is dict:
- selected_target = options.get("target", "all")
- if selected_target != "all" and selected_target != target:
- return False
- except KeyError:
- pass
-
+ selected_target = step.getProperty("target", "all")
+ if selected_target != "all" and selected_target != target:
+ return False
return True
return CheckTargetProperty
# setup shared work directory if required
factory.addStep(ShellCommand(
name = "sharedwd",
- description = "Setting up shared work directory",
+ descriptionDone = "Shared work directory set up",
command = 'test -L "$PWD" || (mkdir -p ../shared-workdir && rm -rf "$PWD" && ln -s shared-workdir "$PWD")',
workdir = ".",
- haltOnFailure = True))
+ haltOnFailure = True,
+ ))
# find number of cores
factory.addStep(SetPropertyFromCommand(
name = "nproc",
property = "nproc",
description = "Finding number of CPUs",
- command = ["nproc"]))
+ command = ["nproc"],
+ ))
# find gcc and g++ compilers
factory.addStep(FileDownload(
name = "dlfindbinpl",
mastersrc = scripts_dir + '/findbin.pl',
workerdest = "../findbin.pl",
- mode = 0o755))
+ mode = 0o755,
+ ))
factory.addStep(SetPropertyFromCommand(
name = "gcc",
property = "cc_command",
description = "Finding gcc command",
- command = [
- "../findbin.pl", "gcc", "", "",
- ],
- haltOnFailure = True))
+ command = ["../findbin.pl", "gcc", "", ""],
+ haltOnFailure = True,
+ ))
factory.addStep(SetPropertyFromCommand(
name = "g++",
property = "cxx_command",
description = "Finding g++ command",
- command = [
- "../findbin.pl", "g++", "", "",
- ],
- haltOnFailure = True))
+ command = ["../findbin.pl", "g++", "", ""],
+ haltOnFailure = True,
+ ))
# see if ccache is available
factory.addStep(SetPropertyFromCommand(
+ name = "ccache",
property = "ccache_command",
- command = ["which", "ccache"],
description = "Testing for ccache command",
+ command = ["which", "ccache"],
haltOnFailure = False,
flunkOnFailure = False,
warnOnFailure = False,
+ hideStepIf = lambda r, s: r==results.FAILURE,
))
- # Workaround bug when switching from a checked out tag back to a branch
- # Ref: http://lists.infradead.org/pipermail/openwrt-devel/2019-June/017809.html
- factory.addStep(ShellCommand(
- name = "gitcheckout",
- description = "Ensure that Git HEAD is sane",
- command = Interpolate("if [ -d .git ]; then git checkout -f %(prop:branch)s && git branch --set-upstream-to origin/%(prop:branch)s || rm -fr .git; else exit 0; fi"),
- haltOnFailure = True))
-
# check out the source
# Git() runs:
# if repo doesn't exist: 'git clone repourl'
- # method 'clean' runs 'git clean -d -f', method fresh runs 'git clean -d -f x'. Only works with mode='full'
- # 'git fetch -t repourl branch; git reset --hard revision'
+ # method 'clean' runs 'git clean -d -f', method fresh runs 'git clean -f -f -d -x'. Only works with mode='full'
+ # git cat-file -e <commit>
+ # git checkout -f <commit>
+ # git checkout -B <branch>
+ # git rev-parse HEAD
factory.addStep(Git(
name = "git",
repourl = repo_url,
name = "fetchrefs",
description = "Fetching Git remote refs",
command = ["git", "fetch", "origin", Interpolate("+refs/heads/%(prop:branch)s:refs/remotes/origin/%(prop:branch)s")],
- haltOnFailure = True
- ))
-
- # switch to tag
- factory.addStep(ShellCommand(
- name = "switchtag",
- description = "Checking out Git tag",
- command = ["git", "checkout", Interpolate("tags/v%(prop:tag:-)s")],
haltOnFailure = True,
- doStepIf = IsTaggingRequested
))
# Verify that Git HEAD points to a tag or branch
- # Ref: http://lists.infradead.org/pipermail/openwrt-devel/2019-June/017809.html
+ # Ref: https://web.archive.org/web/20190729224316/http://lists.infradead.org/pipermail/openwrt-devel/2019-June/017809.html
factory.addStep(ShellCommand(
name = "gitverify",
description = "Ensure that Git HEAD is pointing to a branch or tag",
command = 'git rev-parse --abbrev-ref HEAD | grep -vxqF HEAD || git show-ref --tags --dereference 2>/dev/null | sed -ne "/^$(git rev-parse HEAD) / { s|^.*/||; s|\\^.*||; p }" | grep -qE "^v[0-9][0-9]\\."',
- haltOnFailure = True))
+ haltOnFailure = True,
+ ))
factory.addStep(ShellCommand(
name = "rmtmp",
description = "Remove tmp folder",
- command=["rm", "-rf", "tmp/"]))
+ command=["rm", "-rf", "tmp/"],
+ ))
# feed
factory.addStep(ShellCommand(
name = "rmfeedlinks",
description = "Remove feed symlinks",
- command=["rm", "-rf", "package/feeds/"]))
+ command=["rm", "-rf", "package/feeds/"],
+ ))
factory.addStep(StringDownload(
name = "ccachecc",
description = "Installing feeds",
command=["./scripts/feeds", "install", "-a"],
env = MakeEnv(tryccache=True),
- haltOnFailure = True
+ haltOnFailure = True,
))
# seed config
name = "dlconfigseed",
s = Interpolate("%(kw:seed)s\n", seed=GetConfigSeed),
workerdest = ".config",
- mode = 0o644
+ mode = 0o644,
))
# configure
factory.addStep(ShellCommand(
name = "newconfig",
- description = "Seeding .config",
- command = Interpolate("printf 'CONFIG_TARGET_%(kw:target)s=y\\nCONFIG_TARGET_%(kw:target)s_%(kw:subtarget)s=y\\nCONFIG_SIGNED_PACKAGES=%(kw:usign:#?|y|n)s\\n' >> .config", target=ts[0], subtarget=ts[1], usign=GetUsignKey)
+ descriptionDone = ".config seeded",
+ command = Interpolate("printf 'CONFIG_TARGET_%(kw:target)s=y\\nCONFIG_TARGET_%(kw:target)s_%(kw:subtarget)s=y\\nCONFIG_SIGNED_PACKAGES=%(kw:usign:#?|y|n)s\\n' >> .config", target=ts[0], subtarget=ts[1], usign=GetUsignKey),
))
factory.addStep(ShellCommand(
name = "delbin",
description = "Removing output directory",
- command = ["rm", "-rf", "bin/"]
+ command = ["rm", "-rf", "bin/"],
))
factory.addStep(ShellCommand(
name = "defconfig",
description = "Populating .config",
command = ["make", "defconfig"],
- env = MakeEnv()
+ env = MakeEnv(),
))
# check arch - exit early if does not exist - NB: some targets do not define CONFIG_TARGET_target_subtarget
factory.addStep(ShellCommand(
name = "checkarch",
description = "Checking architecture",
+ descriptionDone = "Architecture validated",
command = 'grep -sq CONFIG_TARGET_%s=y .config && grep -sq CONFIG_TARGET_SUBTARGET=\\"%s\\" .config' %(ts[0], ts[1]),
logEnviron = False,
want_stdout = False,
name = "libc",
property = "libc",
description = "Finding libc suffix",
- command = ["sed", "-ne", '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }', ".config"]))
+ command = ["sed", "-ne", '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }', ".config"],
+ ))
# install build key
factory.addStep(StringDownload(
factory.addStep(ShellCommand(
name = "dldir",
description = "Preparing dl/",
- command = "mkdir -p $HOME/dl && rm -rf ./dl && ln -sf $HOME/dl ./dl",
+ descriptionDone = "dl/ prepared",
+ command = 'mkdir -p ../dl && rm -rf "build/dl" && ln -s ../../dl "build/dl"',
+ workdir = Property("builddir"),
logEnviron = False,
- want_stdout = False
+ want_stdout = False,
))
# prepare tar
factory.addStep(ShellCommand(
name = "dltar",
description = "Building and installing GNU tar",
+ descriptionDone = "GNU tar built and installed",
command = ["make", Interpolate("-j%(prop:nproc:-1)s"), "tools/tar/compile", "V=s"],
env = MakeEnv(tryccache=True),
- haltOnFailure = True
+ haltOnFailure = True,
))
# populate dl
factory.addStep(ShellCommand(
name = "dlrun",
description = "Populating dl/",
+ descriptionDone = "dl/ populated",
command = ["make", Interpolate("-j%(prop:nproc:-1)s"), "download", "V=s"],
env = MakeEnv(),
logEnviron = False,
- locks = properties.FlattenList(NetLockDl, [dlLock.access('exclusive')]),
+ locks = NetLockDl.withArgs(dlLock.access('exclusive')),
))
factory.addStep(ShellCommand(
name = "cleanbase",
description = "Cleaning base-files",
- command=["make", "package/base-files/clean", "V=s"]
+ command=["make", "package/base-files/clean", "V=s"],
))
# build
factory.addStep(ShellCommand(
name = "tools",
description = "Building and installing tools",
+ descriptionDone = "Tools built and installed",
command = ["make", Interpolate("-j%(prop:nproc:-1)s"), "tools/install", "V=s"],
env = MakeEnv(tryccache=True),
- haltOnFailure = True
+ haltOnFailure = True,
))
factory.addStep(ShellCommand(
name = "toolchain",
description = "Building and installing toolchain",
+ descriptionDone = "Toolchain built and installed",
command=["make", Interpolate("-j%(prop:nproc:-1)s"), "toolchain/install", "V=s"],
env = MakeEnv(),
- haltOnFailure = True
+ haltOnFailure = True,
))
factory.addStep(ShellCommand(
name = "kmods",
description = "Building kmods",
+ descriptionDone = "Kmods built",
command=["make", Interpolate("-j%(prop:nproc:-1)s"), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
env = MakeEnv(),
- haltOnFailure = True
+ haltOnFailure = True,
))
# find kernel version
property = "kernelversion",
description = "Finding the effective Kernel version",
command = "make --no-print-directory -C target/linux/ val.LINUX_VERSION val.LINUX_RELEASE val.LINUX_VERMAGIC | xargs printf '%s-%s-%s\\n'",
- env = { 'TOPDIR': Interpolate("%(prop:builddir)s/build") }
+ env = { 'TOPDIR': Interpolate("%(prop:builddir)s/build") },
))
factory.addStep(ShellCommand(
name = "pkgclean",
description = "Cleaning up package build",
- command=["make", "package/cleanup", "V=s"]
+ descriptionDone = "Package build cleaned up",
+ command=["make", "package/cleanup", "V=s"],
))
factory.addStep(ShellCommand(
name = "pkgbuild",
description = "Building packages",
+ descriptionDone = "Packages built",
command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
env = MakeEnv(),
- haltOnFailure = True
+ haltOnFailure = True,
))
factory.addStep(ShellCommand(
name = "pkginstall",
description = "Installing packages",
+ descriptionDone = "Packages installed",
command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/install", "V=s"],
env = MakeEnv(),
- haltOnFailure = True
+ haltOnFailure = True,
))
factory.addStep(ShellCommand(
name = "pkgindex",
description = "Indexing packages",
+ descriptionDone = "Packages indexed",
command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/index", "V=s", "CONFIG_SIGNED_PACKAGES="],
env = MakeEnv(),
- haltOnFailure = True
+ haltOnFailure = True,
))
factory.addStep(ShellCommand(
name = "images",
description = "Building and installing images",
+ descriptionDone = "Images built and installed",
command=["make", Interpolate("-j%(prop:nproc:-1)s"), "target/install", "V=s"],
env = MakeEnv(),
- haltOnFailure = True
+ haltOnFailure = True,
))
factory.addStep(ShellCommand(
description = "Generating config.buildinfo, version.buildinfo and feeds.buildinfo",
command = "make -j1 buildinfo V=s || true",
env = MakeEnv(),
- haltOnFailure = True
+ haltOnFailure = True,
))
factory.addStep(ShellCommand(
name = "json_overview_image_info",
- description = "Generate profiles.json in target folder",
+ description = "Generating profiles.json in target folder",
command = "make -j1 json_overview_image_info V=s || true",
env = MakeEnv(),
- haltOnFailure = True
+ haltOnFailure = True,
))
factory.addStep(ShellCommand(
name = "checksums",
description = "Calculating checksums",
+ descriptionDone = "Checksums calculated",
command=["make", "-j1", "checksum", "V=s"],
env = MakeEnv(),
- haltOnFailure = True
+ haltOnFailure = True,
))
factory.addStep(ShellCommand(
name = "kmoddir",
- description = "Creating kmod directory",
+ descriptionDone = "Kmod directory created",
command=["mkdir", "-p", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s", target=ts[0], subtarget=ts[1])],
haltOnFailure = True,
doStepIf = IsKmodArchiveEnabled,
factory.addStep(ShellCommand(
name = "kmodprepare",
description = "Preparing kmod archive",
+ descriptionDone = "Kmod archive prepared",
command=["rsync", "--include=/kmod-*.ipk", "--exclude=*", "-va",
Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/packages/", target=ts[0], subtarget=ts[1]),
Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1])],
factory.addStep(ShellCommand(
name = "kmodindex",
description = "Indexing kmod archive",
+ descriptionDone = "Kmod archive indexed",
command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/index", "V=s", "CONFIG_SIGNED_PACKAGES=",
Interpolate("PACKAGE_SUBDIRS=bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1])],
env = MakeEnv(),
# sign
factory.addStep(MasterShellCommand(
name = "signprepare",
- description = "Preparing temporary signing directory",
+ descriptionDone = "Temporary signing directory prepared",
command = ["mkdir", "-p", "%s/signing" %(work_dir)],
haltOnFailure = True,
doStepIf = IsSignEnabled,
factory.addStep(ShellCommand(
name = "signpack",
description = "Packing files to sign",
+ descriptionDone = "Files to sign packed",
command = Interpolate("find bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/ bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz", target=ts[0], subtarget=ts[1]),
haltOnFailure = True,
doStepIf = IsSignEnabled,
factory.addStep(MasterShellCommand(
name = "signfiles",
description = "Signing files",
+ descriptionDone = "Files signed",
command = ["%s/signall.sh" %(scripts_dir), "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1]), Interpolate("%(prop:branch)s")],
env = { 'CONFIG_INI': os.getenv("BUILDMASTER_CONFIG", "./config.ini") },
haltOnFailure = True,
factory.addStep(ShellCommand(
name = "signunpack",
description = "Unpacking signed files",
+ descriptionDone = "Signed files unpacked",
command = ["tar", "-xzf", "sign.tar.gz"],
haltOnFailure = True,
doStepIf = IsSignEnabled,
# upload
factory.addStep(ShellCommand(
name = "dirprepare",
- description = "Preparing upload directory structure",
+ descriptionDone = "Upload directory structure prepared",
command = ["mkdir", "-p", Interpolate("tmp/upload/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s", target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
- haltOnFailure = True
+ haltOnFailure = True,
))
factory.addStep(ShellCommand(
name = "linkprepare",
- description = "Preparing repository symlink",
+ descriptionDone = "Repository symlink prepared",
command = ["ln", "-s", "-f", Interpolate("../packages-%(kw:basever)s", basever=util.Transform(GetBaseVersion, Property("branch"))), Interpolate("tmp/upload/%(kw:prefix)spackages", prefix=GetVersionPrefix)],
doStepIf = IsNoMasterBuild,
- haltOnFailure = True
+ haltOnFailure = True,
))
factory.addStep(ShellCommand(
name = "kmoddirprepare",
- description = "Preparing kmod archive upload directory",
+ descriptionDone = "Kmod archive upload directory prepared",
command = ["mkdir", "-p", Interpolate("tmp/upload/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s", target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
haltOnFailure = True,
doStepIf = IsKmodArchiveEnabled,
factory.addStep(ShellCommand(
name = "dirupload",
description = "Uploading directory structure",
+ descriptionDone = "Directory structure uploaded",
command = ["rsync", "-az"] + rsync_defopts + ["tmp/upload/", Interpolate("%(kw:url)s/", url=GetRsyncParams.withArgs("bin", "url"))],
env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
haltOnFailure = True,
logEnviron = False,
locks = NetLockUl,
+ doStepIf = util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
))
# download remote sha256sums to 'target-sha256sums'
factory.addStep(ShellCommand(
name = "target-sha256sums",
description = "Fetching remote sha256sums for target",
+ descriptionDone = "Remote sha256sums for target fetched",
command = ["rsync", "-z"] + rsync_defopts + [Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/sha256sums", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix), "target-sha256sums"],
env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
logEnviron = False,
haltOnFailure = False,
flunkOnFailure = False,
warnOnFailure = False,
+ doStepIf = util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
))
# build list of files to upload
factory.addStep(ShellCommand(
name = "buildlist",
description = "Building list of files to upload",
+ descriptionDone = "List of files to upload built",
command = ["../sha2rsync.pl", "target-sha256sums", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/sha256sums", target=ts[0], subtarget=ts[1]), "rsynclist"],
haltOnFailure = True,
))
name = "dlrsync.sh",
mastersrc = scripts_dir + '/rsync.sh',
workerdest = "../rsync.sh",
- mode = 0o755
+ mode = 0o755,
))
# upload new files and update existing ones
factory.addStep(ShellCommand(
name = "targetupload",
description = "Uploading target files",
+ descriptionDone = "Target files uploaded",
command=["../rsync.sh", "--exclude=/kmods/", "--files-from=rsynclist", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_defopts +
["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]),
Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
haltOnFailure = True,
logEnviron = False,
+ doStepIf = util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
))
# delete files which don't exist locally
factory.addStep(ShellCommand(
name = "targetprune",
description = "Pruning target files",
+ descriptionDone = "Target files pruned",
command=["../rsync.sh", "--exclude=/kmods/", "--delete", "--existing", "--ignore-existing", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_defopts +
["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]),
Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
haltOnFailure = True,
logEnviron = False,
locks = NetLockUl,
+ doStepIf = util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
))
factory.addStep(ShellCommand(
name = "kmodupload",
description = "Uploading kmod archive",
+ descriptionDone = "Kmod archive uploaded",
command=["../rsync.sh", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_defopts +
["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1]),
Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
haltOnFailure = True,
logEnviron = False,
locks = NetLockUl,
- doStepIf = IsKmodArchiveEnabled,
+ doStepIf = util.Transform(lambda a, b: bool(a and b), IsKmodArchiveEnabled, GetRsyncParams.withArgs("bin", "url")),
))
factory.addStep(ShellCommand(
name = "sourcelist",
description = "Finding source archives to upload",
+ descriptionDone = "Source archives to upload found",
command = "find dl/ -maxdepth 1 -type f -not -size 0 -not -name '.*' -not -name '*.hash' -not -name '*.dl' -newer .config -printf '%f\\n' > sourcelist",
- haltOnFailure = True
+ haltOnFailure = True,
))
factory.addStep(ShellCommand(
name = "sourceupload",
description = "Uploading source archives",
+ descriptionDone = "Source archives uploaded",
command=["../rsync.sh", "--files-from=sourcelist", "--size-only", "--delay-updates"] + rsync_defopts +
[Interpolate("--partial-dir=.~tmp~%(kw:target)s~%(kw:subtarget)s~%(prop:workername)s", target=ts[0], subtarget=ts[1]), "-a", "dl/", Interpolate("%(kw:url)s/", url=GetRsyncParams.withArgs("src", "url"))],
env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("src", "key")) },
haltOnFailure = True,
logEnviron = False,
locks = NetLockUl,
+ doStepIf = util.Transform(bool, GetRsyncParams.withArgs("src", "url")),
))
factory.addStep(ShellCommand(
description = "Reporting disk usage",
command=["df", "-h", "."],
env={'LC_ALL': 'C'},
+ logEnviron = False,
haltOnFailure = False,
flunkOnFailure = False,
warnOnFailure = False,
- alwaysRun = True
+ alwaysRun = True,
))
factory.addStep(ShellCommand(
description = "Reporting estimated file space usage",
command=["du", "-sh", "."],
env={'LC_ALL': 'C'},
+ logEnviron = False,
haltOnFailure = False,
flunkOnFailure = False,
warnOnFailure = False,
- alwaysRun = True
+ alwaysRun = True,
))
factory.addStep(ShellCommand(
description = "Reporting ccache stats",
command=["ccache", "-s"],
env = MakeEnv(overrides={ 'PATH': ["${PATH}", "./staging_dir/host/bin"] }),
+ logEnviron = False,
want_stderr = False,
haltOnFailure = False,
flunkOnFailure = False,
warnOnFailure = False,
- alwaysRun = True,
+ hideStepIf = lambda r, s: r==results.FAILURE,
))
c['builders'].append(BuilderConfig(name=target, workernames=workerNames, factory=factory, nextBuild=GetNextBuild))
name = "trigger_%s" % target,
description = "Triggering %s build" % target,
schedulerNames = [ "trigger_%s" % target ],
- set_properties = { "reason": Property("reason"), "tag": TagPropertyValue },
- doStepIf = IsTargetSelected(target)
+ sourceStamps = [{ "codebase": "", "branch": Property("force_branch"), "revision": Property("force_revision"), "repository": repo_url, "project": "" }],
+ set_properties = { "reason": Property("reason"), "tag": Property("tag"), },
+ doStepIf = IsTargetSelected(target),
))
projects / buildbot.git / blobdiff