return env
@properties.renderer
-def NetLockDl(props):
+def NetLockDl(props, extralock=None):
lock = None
+ locks = []
if props.hasProperty("dl_lock"):
lock = NetLocks[props["dl_lock"]]
if lock is not None:
- return [lock.access('exclusive')]
- else:
- return []
+ locks.append(lock.access('exclusive'))
+ if extralock is not None:
+ locks.append(extralock)
+ return locks
@properties.renderer
def NetLockUl(props):
# setup shared work directory if required
factory.addStep(ShellCommand(
name = "sharedwd",
- description = "Setting up shared work directory",
+ descriptionDone = "Shared work directory set up",
command = 'test -L "$PWD" || (mkdir -p ../shared-workdir && rm -rf "$PWD" && ln -s shared-workdir "$PWD")',
workdir = ".",
haltOnFailure = True))
# see if ccache is available
factory.addStep(SetPropertyFromCommand(
+ name = "ccache",
property = "ccache_command",
command = ["which", "ccache"],
description = "Testing for ccache command",
haltOnFailure = False,
flunkOnFailure = False,
warnOnFailure = False,
+ hideStepIf = lambda r, s: r==results.FAILURE,
))
# Workaround bug when switching from a checked out tag back to a branch
# configure
factory.addStep(ShellCommand(
name = "newconfig",
- description = "Seeding .config",
+ descriptionDone = ".config seeded",
command = Interpolate("printf 'CONFIG_TARGET_%(kw:target)s=y\\nCONFIG_TARGET_%(kw:target)s_%(kw:subtarget)s=y\\nCONFIG_SIGNED_PACKAGES=%(kw:usign:#?|y|n)s\\n' >> .config", target=ts[0], subtarget=ts[1], usign=GetUsignKey)
))
factory.addStep(ShellCommand(
name = "checkarch",
description = "Checking architecture",
+ descriptionDone = "Architecture validated",
command = 'grep -sq CONFIG_TARGET_%s=y .config && grep -sq CONFIG_TARGET_SUBTARGET=\\"%s\\" .config' %(ts[0], ts[1]),
logEnviron = False,
want_stdout = False,
factory.addStep(ShellCommand(
name = "dldir",
description = "Preparing dl/",
+ descriptionDone = "dl/ prepared",
command = "mkdir -p $HOME/dl && rm -rf ./dl && ln -sf $HOME/dl ./dl",
logEnviron = False,
want_stdout = False
factory.addStep(ShellCommand(
name = "dltar",
description = "Building and installing GNU tar",
+ descriptionDone = "GNU tar built and installed",
command = ["make", Interpolate("-j%(prop:nproc:-1)s"), "tools/tar/compile", "V=s"],
env = MakeEnv(tryccache=True),
haltOnFailure = True
factory.addStep(ShellCommand(
name = "dlrun",
description = "Populating dl/",
+ descriptionDone = "dl/ populated",
command = ["make", Interpolate("-j%(prop:nproc:-1)s"), "download", "V=s"],
env = MakeEnv(),
logEnviron = False,
- locks = properties.FlattenList(NetLockDl, [dlLock.access('exclusive')]),
+ locks = NetLockDl.withArgs(dlLock.access('exclusive')),
))
factory.addStep(ShellCommand(
factory.addStep(ShellCommand(
name = "tools",
description = "Building and installing tools",
+ descriptionDone = "Tools built and installed",
command = ["make", Interpolate("-j%(prop:nproc:-1)s"), "tools/install", "V=s"],
env = MakeEnv(tryccache=True),
haltOnFailure = True
factory.addStep(ShellCommand(
name = "toolchain",
description = "Building and installing toolchain",
+ descriptionDone = "Toolchain built and installed",
command=["make", Interpolate("-j%(prop:nproc:-1)s"), "toolchain/install", "V=s"],
env = MakeEnv(),
haltOnFailure = True
factory.addStep(ShellCommand(
name = "kmods",
description = "Building kmods",
+ descriptionDone = "Kmods built",
command=["make", Interpolate("-j%(prop:nproc:-1)s"), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
env = MakeEnv(),
haltOnFailure = True
factory.addStep(ShellCommand(
name = "pkgclean",
description = "Cleaning up package build",
+ descriptionDone = "Package build cleaned up",
command=["make", "package/cleanup", "V=s"]
))
factory.addStep(ShellCommand(
name = "pkgbuild",
description = "Building packages",
+ descriptionDone = "Packages built",
command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
env = MakeEnv(),
haltOnFailure = True
factory.addStep(ShellCommand(
name = "pkginstall",
description = "Installing packages",
+ descriptionDone = "Packages installed",
command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/install", "V=s"],
env = MakeEnv(),
haltOnFailure = True
factory.addStep(ShellCommand(
name = "pkgindex",
description = "Indexing packages",
+ descriptionDone = "Packages indexed",
command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/index", "V=s", "CONFIG_SIGNED_PACKAGES="],
env = MakeEnv(),
haltOnFailure = True
factory.addStep(ShellCommand(
name = "images",
description = "Building and installing images",
+ descriptionDone = "Images built and installed",
command=["make", Interpolate("-j%(prop:nproc:-1)s"), "target/install", "V=s"],
env = MakeEnv(),
haltOnFailure = True
factory.addStep(ShellCommand(
name = "json_overview_image_info",
- description = "Generate profiles.json in target folder",
+ description = "Generating profiles.json in target folder",
command = "make -j1 json_overview_image_info V=s || true",
env = MakeEnv(),
haltOnFailure = True
factory.addStep(ShellCommand(
name = "checksums",
description = "Calculating checksums",
+ descriptionDone = "Checksums calculated",
command=["make", "-j1", "checksum", "V=s"],
env = MakeEnv(),
haltOnFailure = True
factory.addStep(ShellCommand(
name = "kmoddir",
- description = "Creating kmod directory",
+ descriptionDone = "Kmod directory created",
command=["mkdir", "-p", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s", target=ts[0], subtarget=ts[1])],
haltOnFailure = True,
doStepIf = IsKmodArchiveEnabled,
factory.addStep(ShellCommand(
name = "kmodprepare",
description = "Preparing kmod archive",
+ descriptionDone = "Kmod archive prepared",
command=["rsync", "--include=/kmod-*.ipk", "--exclude=*", "-va",
Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/packages/", target=ts[0], subtarget=ts[1]),
Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1])],
factory.addStep(ShellCommand(
name = "kmodindex",
description = "Indexing kmod archive",
+ descriptionDone = "Kmod archive indexed",
command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/index", "V=s", "CONFIG_SIGNED_PACKAGES=",
Interpolate("PACKAGE_SUBDIRS=bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1])],
env = MakeEnv(),
# sign
factory.addStep(MasterShellCommand(
name = "signprepare",
- description = "Preparing temporary signing directory",
+ descriptionDone = "Temporary signing directory prepared",
command = ["mkdir", "-p", "%s/signing" %(work_dir)],
haltOnFailure = True,
doStepIf = IsSignEnabled,
factory.addStep(ShellCommand(
name = "signpack",
description = "Packing files to sign",
+ descriptionDone = "Files to sign packed",
command = Interpolate("find bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/ bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz", target=ts[0], subtarget=ts[1]),
haltOnFailure = True,
doStepIf = IsSignEnabled,
factory.addStep(MasterShellCommand(
name = "signfiles",
description = "Signing files",
+ descriptionDone = "Files signed",
command = ["%s/signall.sh" %(scripts_dir), "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1]), Interpolate("%(prop:branch)s")],
env = { 'CONFIG_INI': os.getenv("BUILDMASTER_CONFIG", "./config.ini") },
haltOnFailure = True,
factory.addStep(ShellCommand(
name = "signunpack",
description = "Unpacking signed files",
+ descriptionDone = "Signed files unpacked",
command = ["tar", "-xzf", "sign.tar.gz"],
haltOnFailure = True,
doStepIf = IsSignEnabled,
# upload
factory.addStep(ShellCommand(
name = "dirprepare",
- description = "Preparing upload directory structure",
+ descriptionDone = "Upload directory structure prepared",
command = ["mkdir", "-p", Interpolate("tmp/upload/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s", target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
haltOnFailure = True
))
factory.addStep(ShellCommand(
name = "linkprepare",
- description = "Preparing repository symlink",
+ descriptionDone = "Repository symlink prepared",
command = ["ln", "-s", "-f", Interpolate("../packages-%(kw:basever)s", basever=util.Transform(GetBaseVersion, Property("branch"))), Interpolate("tmp/upload/%(kw:prefix)spackages", prefix=GetVersionPrefix)],
doStepIf = IsNoMasterBuild,
haltOnFailure = True
factory.addStep(ShellCommand(
name = "kmoddirprepare",
- description = "Preparing kmod archive upload directory",
+ descriptionDone = "Kmod archive upload directory prepared",
command = ["mkdir", "-p", Interpolate("tmp/upload/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s", target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
haltOnFailure = True,
doStepIf = IsKmodArchiveEnabled,
factory.addStep(ShellCommand(
name = "dirupload",
description = "Uploading directory structure",
+ descriptionDone = "Directory structure uploaded",
command = ["rsync", "-az"] + rsync_defopts + ["tmp/upload/", Interpolate("%(kw:url)s/", url=GetRsyncParams.withArgs("bin", "url"))],
env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
haltOnFailure = True,
factory.addStep(ShellCommand(
name = "target-sha256sums",
description = "Fetching remote sha256sums for target",
+ descriptionDone = "Remote sha256sums for target fetched",
command = ["rsync", "-z"] + rsync_defopts + [Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/sha256sums", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix), "target-sha256sums"],
env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
logEnviron = False,
factory.addStep(ShellCommand(
name = "buildlist",
description = "Building list of files to upload",
+ descriptionDone = "List of files to upload built",
command = ["../sha2rsync.pl", "target-sha256sums", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/sha256sums", target=ts[0], subtarget=ts[1]), "rsynclist"],
haltOnFailure = True,
))
factory.addStep(ShellCommand(
name = "targetupload",
description = "Uploading target files",
+ descriptionDone = "Target files uploaded",
command=["../rsync.sh", "--exclude=/kmods/", "--files-from=rsynclist", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_defopts +
["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]),
Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
factory.addStep(ShellCommand(
name = "targetprune",
description = "Pruning target files",
+ descriptionDone = "Target files pruned",
command=["../rsync.sh", "--exclude=/kmods/", "--delete", "--existing", "--ignore-existing", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_defopts +
["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]),
Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
factory.addStep(ShellCommand(
name = "kmodupload",
description = "Uploading kmod archive",
+ descriptionDone = "Kmod archive uploaded",
command=["../rsync.sh", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_defopts +
["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1]),
Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
factory.addStep(ShellCommand(
name = "sourcelist",
description = "Finding source archives to upload",
+ descriptionDone = "Source archives to upload found",
command = "find dl/ -maxdepth 1 -type f -not -size 0 -not -name '.*' -not -name '*.hash' -not -name '*.dl' -newer .config -printf '%f\\n' > sourcelist",
haltOnFailure = True
))
factory.addStep(ShellCommand(
name = "sourceupload",
description = "Uploading source archives",
+ descriptionDone = "Source archives uploaded",
command=["../rsync.sh", "--files-from=sourcelist", "--size-only", "--delay-updates"] + rsync_defopts +
[Interpolate("--partial-dir=.~tmp~%(kw:target)s~%(kw:subtarget)s~%(prop:workername)s", target=ts[0], subtarget=ts[1]), "-a", "dl/", Interpolate("%(kw:url)s/", url=GetRsyncParams.withArgs("src", "url"))],
env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("src", "key")) },
haltOnFailure = False,
flunkOnFailure = False,
warnOnFailure = False,
- alwaysRun = True,
+ hideStepIf = lambda r, s: r==results.FAILURE,
))
c['builders'].append(BuilderConfig(name=target, workernames=workerNames, factory=factory, nextBuild=GetNextBuild))
projects / buildbot.git / blobdiff