worker_port = 9990
persistent = False
-tree_expire = 0
-git_ssh = False
-git_ssh_key = None
if ini.has_option("phase2", "port"):
worker_port = ini.get("phase2", "port")
if ini.has_option("phase2", "persistent"):
persistent = ini.getboolean("phase2", "persistent")
-if ini.has_option("phase2", "expire"):
- tree_expire = ini.getint("phase2", "expire")
-
-if ini.has_option("general", "git_ssh"):
- git_ssh = ini.getboolean("general", "git_ssh")
-
-if ini.has_option("general", "git_ssh_key"):
- git_ssh_key = ini.get("general", "git_ssh_key")
-else:
- git_ssh = False
-
c['workers'] = []
-max_builds = dict()
for section in ini.sections():
if section.startswith("worker "):
ini.has_option(section, "phase") and ini.getint(section, "phase") == 2:
name = ini.get(section, "name")
password = ini.get(section, "password")
- sl_props = { 'shared_wd': False }
- max_builds[name] = 1
-
- if ini.has_option(section, "builds"):
- max_builds[name] = ini.getint(section, "builds")
-
- if max_builds[name] == 1:
- sl_props['shared_wd'] = True
+ sl_props = { 'shared_wd': True }
if ini.has_option(section, "shared_wd"):
sl_props['shared_wd'] = ini.getboolean(section, "shared_wd")
- if sl_props['shared_wd'] and (max_builds != 1):
- raise ValueError('max_builds must be 1 with shared workdir!')
- c['workers'].append(Worker(name, password, max_builds = max_builds[name], properties = sl_props))
+ c['workers'].append(Worker(name, password, max_builds = 1, properties = sl_props))
# 'workerPortnum' defines the TCP port to listen on for connections from workers.
# This must match the value configured into the buildworkers (with their
rsync_sdk_url = None
rsync_sdk_key = None
-rsync_sdk_pat = "openwrt-sdk-*.tar.xz"
+rsync_sdk_pat = "openwrt-sdk-*.tar.*"
if ini.has_option("rsync", "sdk_url"):
rsync_sdk_url = ini.get("rsync", "sdk_url")
if ini.has_option("rsync", "sdk_pattern"):
rsync_sdk_pat = ini.get("rsync", "sdk_pattern")
+rsync_defopts = ["-4", "-v", "--timeout=120"]
+
repo_url = ini.get("repo", "url")
-repo_branch = "master"
+repo_branch = "main"
if ini.has_option("repo", "branch"):
repo_branch = ini.get("repo", "branch")
if parts[0].startswith("src-git"):
feeds.append(parts)
url = parts[2].strip().split(';')
- branch = url[1] if len(url) > 1 else 'master'
+ branch = url[1] if len(url) > 1 else 'main'
feedbranches[url[0]] = branch
c['change_source'].append(GitPoller(url[0], branch=branch, workdir='%s/%s.git' %(os.getcwd(), parts[1]), pollinterval=300))
return "-%02d.%02d" %(int(m.group(1)), int(m.group(2)))
return ""
-@properties.renderer
-def GetNumJobs(props):
- if props.hasProperty("workername") and props.hasProperty("nproc"):
- return str(int(props["nproc"]) / max_builds[props["workername"]])
- else:
- return "1"
-
@properties.renderer
def GetCwd(props):
if props.hasProperty("builddir"):
def CheckArchitectureProperty(step):
try:
options = step.getProperty("options")
- if type(options) is dict:
+ if isinstance(options, dict):
selected_arch = options.get("architecture", "all")
if selected_arch != "all" and selected_arch != target:
return False
haltOnFailure = True,
timeout = 2400))
- # expire tree if needed
- elif tree_expire > 0:
- factory.addStep(FileDownload(
- mastersrc = scripts_dir + '/expire.sh',
- workerdest = "../expire.sh",
- mode = 0o755))
-
- factory.addStep(ShellCommand(
- name = "expire",
- description = "Checking for build tree expiry",
- command = ["./expire.sh", str(tree_expire)],
- workdir = ".",
- haltOnFailure = True,
- timeout = 2400))
-
factory.addStep(ShellCommand(
name = "mksdkdir",
description = "Preparing SDK directory",
factory.addStep(ShellCommand(
name = "downloadsdk",
description = "Downloading SDK archive",
- command = ["rsync", "-4", "-va", "%s/%s/%s/%s" %(rsync_sdk_url, ts[0], ts[1], rsync_sdk_pat), "sdk.archive"],
+ command = ["rsync"] + rsync_defopts + ["-a", "%s/%s/%s/%s" %(rsync_sdk_url, ts[0], ts[1], rsync_sdk_pat), "sdk.archive"],
env={'RSYNC_PASSWORD': rsync_sdk_key},
haltOnFailure = True,
logEnviron = False))
factory.addStep(ShellCommand(
name = "updatesdk",
description = "Updating SDK",
- command = "rsync --checksum -av sdk_update/ sdk/ && rm -rf sdk_update",
+ command = "rsync " + (" ").join(rsync_defopts) + " --checksum -a sdk_update/ sdk/ && rm -rf sdk_update",
haltOnFailure = True))
factory.addStep(ShellCommand(
command = ["./ccache.sh"],
haltOnFailure = True))
- factory.addStep(ShellCommand(
- name = "patchfeedsconfgitfull",
- description = "Patching feeds.conf to use src-git-full",
- workdir = "build/sdk",
- command = "sed -e 's#^src-git #src-git-full #g' feeds.conf.default > feeds.conf",
- haltOnFailure = True))
-
- if git_ssh:
- factory.addStep(StringDownload(
- name = "dlgitclonekey",
- s = git_ssh_key,
- workerdest = "../git-clone.key",
- mode = 0o600))
-
- factory.addStep(ShellCommand(
- name = "patchfeedsconf",
- description = "Patching feeds.conf to use SSH cloning",
- workdir = "build/sdk",
- command = "sed -i -e 's#https://#ssh://git@#g' feeds.conf",
- haltOnFailure = True))
-
factory.addStep(ShellCommand(
name = "updatefeeds",
description = "Updating feeds",
workdir = "build/sdk",
command = ["./scripts/feeds", "update", "-f"],
- env = {'GIT_SSH_COMMAND': Interpolate("ssh -o IdentitiesOnly=yes -o IdentityFile=%(kw:cwd)s/git-clone.key -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no", cwd=GetCwd)} if git_ssh else {},
haltOnFailure = True))
- if git_ssh:
- factory.addStep(ShellCommand(
- name = "rmfeedsconf",
- description = "Removing feeds.conf",
- workdir = "build/sdk",
- command=["rm", "feeds.conf"],
- haltOnFailure = True))
-
factory.addStep(ShellCommand(
name = "installfeeds",
description = "Installing feeds",
description = "Building packages",
workdir = "build/sdk",
timeout = 3600,
- command = ["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "IGNORE_ERRORS=n m y", "BUILD_LOG=1", "CONFIG_AUTOREMOVE=y", "CONFIG_SIGNED_PACKAGES="],
+ command = ["make", Interpolate("-j%(prop:nproc:-1)s"), "IGNORE_ERRORS=n m y", "BUILD_LOG=1", "CONFIG_AUTOREMOVE=y", "CONFIG_SIGNED_PACKAGES="],
env = {'CCACHE_BASEDIR': Interpolate("%(kw:cwd)s", cwd=GetCwd)},
haltOnFailure = True))
workdir = "build/sdk",
command = "./scripts/feeds list -s -f > bin/packages/%s/feeds.conf" %(arch[0])))
+ factory.addStep(ShellCommand(
+ name = "checksums",
+ description = "Calculating checksums",
+ descriptionDone="Checksums calculated",
+ workdir = "build/sdk",
+ command = "cd bin/packages/%s; " %(arch[0])
+ + "find . -type f -not -name 'sha256sums' -printf \"%P\n\" | "
+ + "sort | xargs -r ../../../staging_dir/host/bin/mkhash -n sha256 | "
+ + r"sed -ne 's!^\(.*\) \(.*\)$!\1 *\2!p' > sha256sums",
+ haltOnFailure = True
+ ))
+
if ini.has_option("gpg", "key") or usign_key is not None:
factory.addStep(MasterShellCommand(
name = "signprepare",
haltOnFailure = True
))
+ # download remote sha256sums to 'target-sha256sums'
+ factory.addStep(ShellCommand(
+ name = "target-sha256sums",
+ description = "Fetching remote sha256sums for arch",
+ command = ["rsync"] + rsync_defopts + ["-z", Interpolate("%(kw:rsyncbinurl)s/packages%(kw:suffix)s/%(kw:archname)s/sha256sums", rsyncbinurl=rsync_bin_url, suffix=GetDirectorySuffix, archname=arch[0]), "arch-sha256sums"],
+ env={'RSYNC_PASSWORD': rsync_bin_key},
+ logEnviron = False,
+ haltOnFailure = False,
+ flunkOnFailure = False,
+ warnOnFailure = False,
+ ))
+
+ factory.addStep(FileDownload(
+ name="dlrsync.sh",
+ mastersrc = scripts_dir + "/rsync.sh",
+ workerdest = "../rsync.sh",
+ mode = 0o755
+ ))
+
+ factory.addStep(FileDownload(
+ name = "dlsha2rsyncpl",
+ mastersrc = scripts_dir + "/sha2rsync.pl",
+ workerdest = "../sha2rsync.pl",
+ mode = 0o755,
+ ))
+
+ factory.addStep(ShellCommand(
+ name = "buildlist",
+ description = "Building list of files to upload",
+ workdir = "build/sdk",
+ command = ["../../sha2rsync.pl", "../arch-sha256sums", "bin/packages/%s/sha256sums" %(arch[0]), "rsynclist"],
+ haltOnFailure = True,
+ ))
+
factory.addStep(ShellCommand(
name = "uploadprepare",
description = "Preparing package directory",
workdir = "build/sdk",
- command = ["rsync", "-4", "-av", "--include", "/%s/" %(arch[0]), "--exclude", "/*", "--exclude", "/%s/*" %(arch[0]), "bin/packages/", Interpolate("%(kw:rsyncbinurl)s/packages%(kw:suffix)s/", rsyncbinurl=rsync_bin_url, suffix=GetDirectorySuffix)],
+ command = ["rsync"] + rsync_defopts + ["-a", "--include", "/%s/" %(arch[0]), "--exclude", "/*", "--exclude", "/%s/*" %(arch[0]), "bin/packages/", Interpolate("%(kw:rsyncbinurl)s/packages%(kw:suffix)s/", rsyncbinurl=rsync_bin_url, suffix=GetDirectorySuffix)],
env={'RSYNC_PASSWORD': rsync_bin_key},
haltOnFailure = True,
logEnviron = False
name = "packageupload",
description = "Uploading package files",
workdir = "build/sdk",
- command = ["rsync", "-4", "--progress", "--delete", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-avz", "bin/packages/%s/" %(arch[0]), Interpolate("%(kw:rsyncbinurl)s/packages%(kw:suffix)s/%(kw:archname)s/", rsyncbinurl=rsync_bin_url, suffix=GetDirectorySuffix, archname=arch[0])],
+ command = ["../../rsync.sh"] + rsync_defopts + ["--files-from=rsynclist", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-a", "bin/packages/%s/" %(arch[0]), Interpolate("%(kw:rsyncbinurl)s/packages%(kw:suffix)s/%(kw:archname)s/", rsyncbinurl=rsync_bin_url, suffix=GetDirectorySuffix, archname=arch[0])],
+ env={'RSYNC_PASSWORD': rsync_bin_key},
+ haltOnFailure = True,
+ logEnviron = False
+ ))
+
+ factory.addStep(ShellCommand(
+ name = "packageprune",
+ description = "Pruning package files",
+ workdir = "build/sdk",
+ command = ["../../rsync.sh"] + rsync_defopts + ["--delete", "--existing", "--ignore-existing", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-a", "bin/packages/%s/" %(arch[0]), Interpolate("%(kw:rsyncbinurl)s/packages%(kw:suffix)s/%(kw:archname)s/", rsyncbinurl=rsync_bin_url, suffix=GetDirectorySuffix, archname=arch[0])],
env={'RSYNC_PASSWORD': rsync_bin_key},
haltOnFailure = True,
logEnviron = False
name = "logprepare",
description = "Preparing log directory",
workdir = "build/sdk",
- command = ["rsync", "-4", "-av", "--include", "/%s/" %(arch[0]), "--exclude", "/*", "--exclude", "/%s/*" %(arch[0]), "bin/packages/", Interpolate("%(kw:rsyncbinurl)s/faillogs%(kw:suffix)s/", rsyncbinurl=rsync_bin_url, suffix=GetDirectorySuffix)],
+ command = ["rsync"] + rsync_defopts + ["-a", "--include", "/%s/" %(arch[0]), "--exclude", "/*", "--exclude", "/%s/*" %(arch[0]), "bin/packages/", Interpolate("%(kw:rsyncbinurl)s/faillogs%(kw:suffix)s/", rsyncbinurl=rsync_bin_url, suffix=GetDirectorySuffix)],
env={'RSYNC_PASSWORD': rsync_bin_key},
haltOnFailure = True,
logEnviron = False
name = "logcollect",
description = "Collecting failure logs",
workdir = "build/sdk",
- command = ["rsync", "-av", "--files-from=logs.txt", "logs/package/feeds/", "faillogs/"],
+ command = ["rsync"] + rsync_defopts + ["-a", "--files-from=logs.txt", "logs/package/feeds/", "faillogs/"],
haltOnFailure = False,
flunkOnFailure = False,
warnOnFailure = True,
name = "logupload",
description = "Uploading failure logs",
workdir = "build/sdk",
- command = ["rsync", "-4", "--progress", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-avz", "faillogs/", Interpolate("%(kw:rsyncbinurl)s/faillogs%(kw:suffix)s/%(kw:archname)s/", rsyncbinurl=rsync_bin_url, suffix=GetDirectorySuffix, archname=arch[0])],
+ command = ["../../rsync.sh"] + rsync_defopts + ["--delete", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-az", "faillogs/", Interpolate("%(kw:rsyncbinurl)s/faillogs%(kw:suffix)s/%(kw:archname)s/", rsyncbinurl=rsync_bin_url, suffix=GetDirectorySuffix, archname=arch[0])],
env={'RSYNC_PASSWORD': rsync_bin_key},
haltOnFailure = False,
flunkOnFailure = False,
name = "sourceupload",
description = "Uploading source archives",
workdir = "build/sdk",
- command = ["rsync", "--files-from=sourcelist", "-4", "--progress", "--checksum", "--delay-updates",
- Interpolate("--partial-dir=.~tmp~%(kw:archname)s~%(prop:workername)s", archname=arch[0]), "-avz", "dl/", "%s/" %(rsync_src_url)],
+ command = ["../../rsync.sh"] + rsync_defopts + ["--files-from=sourcelist", "--size-only", "--delay-updates",
+ Interpolate("--partial-dir=.~tmp~%(kw:archname)s~%(prop:workername)s", archname=arch[0]), "-a", "dl/", "%s/" %(rsync_src_url)],
env={'RSYNC_PASSWORD': rsync_src_key},
haltOnFailure = False,
flunkOnFailure = False,
alwaysRun = True
))
+ factory.addStep(ShellCommand(
+ name = "ccachestat",
+ description = "Reporting ccache stats",
+ command=["ccache", "-s"],
+ want_stderr = False,
+ haltOnFailure = False,
+ flunkOnFailure = False,
+ warnOnFailure = False,
+ alwaysRun = True,
+ ))
+
c['builders'].append(BuilderConfig(name=arch[0], workernames=workerNames, factory=factory))
c['schedulers'].append(schedulers.Triggerable(name="trigger_%s" % arch[0], builderNames=[ arch[0] ]))
projects / buildbot.git / blobdiff