X-Git-Url: http://git.openwrt.org/?a=blobdiff_plain;f=phase2%2Fmaster.cfg;h=c399c66eb0fc9467e250e85d6b2652d99388abab;hb=HEAD;hp=14378259c0c2cbaf2ea1e606345bf0bf9970e320;hpb=32c36de026a501af18384f20a316f76f07229c06;p=buildbot.git diff --git a/phase2/master.cfg b/phase2/master.cfg index 1437825..c399c66 100644 --- a/phase2/master.cfg +++ b/phase2/master.cfg @@ -62,9 +62,6 @@ c = BuildmasterConfig = {} worker_port = 9990 persistent = False -tree_expire = 0 -git_ssh = False -git_ssh_key = None if ini.has_option("phase2", "port"): worker_port = ini.get("phase2", "port") @@ -72,19 +69,7 @@ if ini.has_option("phase2", "port"): if ini.has_option("phase2", "persistent"): persistent = ini.getboolean("phase2", "persistent") -if ini.has_option("phase2", "expire"): - tree_expire = ini.getint("phase2", "expire") - -if ini.has_option("general", "git_ssh"): - git_ssh = ini.getboolean("general", "git_ssh") - -if ini.has_option("general", "git_ssh_key"): - git_ssh_key = ini.get("general", "git_ssh_key") -else: - git_ssh = False - c['workers'] = [] -max_builds = dict() for section in ini.sections(): if section.startswith("worker "): @@ -92,21 +77,12 @@ for section in ini.sections(): ini.has_option(section, "phase") and ini.getint(section, "phase") == 2: name = ini.get(section, "name") password = ini.get(section, "password") - sl_props = { 'shared_wd': False } - max_builds[name] = 1 - - if ini.has_option(section, "builds"): - max_builds[name] = ini.getint(section, "builds") - - if max_builds[name] == 1: - sl_props['shared_wd'] = True + sl_props = { 'shared_wd': True } if ini.has_option(section, "shared_wd"): sl_props['shared_wd'] = ini.getboolean(section, "shared_wd") - if sl_props['shared_wd'] and (max_builds != 1): - raise ValueError('max_builds must be 1 with shared workdir!') - c['workers'].append(Worker(name, password, max_builds = max_builds[name], properties = sl_props)) + c['workers'].append(Worker(name, password, max_builds = 1, properties = sl_props)) # 'workerPortnum' defines the TCP port to listen on for connections from workers. # This must match the value configured into the buildworkers (with their @@ -139,7 +115,7 @@ if ini.has_option("rsync", "source_url"): rsync_sdk_url = None rsync_sdk_key = None -rsync_sdk_pat = "openwrt-sdk-*.tar.xz" +rsync_sdk_pat = "openwrt-sdk-*.tar.*" if ini.has_option("rsync", "sdk_url"): rsync_sdk_url = ini.get("rsync", "sdk_url") @@ -150,8 +126,10 @@ if ini.has_option("rsync", "sdk_password"): if ini.has_option("rsync", "sdk_pattern"): rsync_sdk_pat = ini.get("rsync", "sdk_pattern") +rsync_defopts = ["-4", "-v", "--timeout=120"] + repo_url = ini.get("repo", "url") -repo_branch = "master" +repo_branch = "main" if ini.has_option("repo", "branch"): repo_branch = ini.get("repo", "branch") @@ -199,7 +177,7 @@ def parse_feed_entry(line): if parts[0].startswith("src-git"): feeds.append(parts) url = parts[2].strip().split(';') - branch = url[1] if len(url) > 1 else 'master' + branch = url[1] if len(url) > 1 else 'main' feedbranches[url[0]] = branch c['change_source'].append(GitPoller(url[0], branch=branch, workdir='%s/%s.git' %(os.getcwd(), parts[1]), pollinterval=300)) @@ -289,13 +267,6 @@ def GetDirectorySuffix(props): return "-%02d.%02d" %(int(m.group(1)), int(m.group(2))) return "" -@properties.renderer -def GetNumJobs(props): - if props.hasProperty("workername") and props.hasProperty("nproc"): - return str(int(props["nproc"]) / max_builds[props["workername"]]) - else: - return "1" - @properties.renderer def GetCwd(props): if props.hasProperty("builddir"): @@ -309,7 +280,7 @@ def IsArchitectureSelected(target): def CheckArchitectureProperty(step): try: options = step.getProperty("options") - if type(options) is dict: + if isinstance(options, dict): selected_arch = options.get("architecture", "all") if selected_arch != "all" and selected_arch != target: return False @@ -468,21 +439,6 @@ for arch in arches: haltOnFailure = True, timeout = 2400)) - # expire tree if needed - elif tree_expire > 0: - factory.addStep(FileDownload( - mastersrc = scripts_dir + '/expire.sh', - workerdest = "../expire.sh", - mode = 0o755)) - - factory.addStep(ShellCommand( - name = "expire", - description = "Checking for build tree expiry", - command = ["./expire.sh", str(tree_expire)], - workdir = ".", - haltOnFailure = True, - timeout = 2400)) - factory.addStep(ShellCommand( name = "mksdkdir", description = "Preparing SDK directory", @@ -492,7 +448,7 @@ for arch in arches: factory.addStep(ShellCommand( name = "downloadsdk", description = "Downloading SDK archive", - command = ["rsync", "-4", "-va", "%s/%s/%s/%s" %(rsync_sdk_url, ts[0], ts[1], rsync_sdk_pat), "sdk.archive"], + command = ["rsync"] + rsync_defopts + ["-a", "%s/%s/%s/%s" %(rsync_sdk_url, ts[0], ts[1], rsync_sdk_pat), "sdk.archive"], env={'RSYNC_PASSWORD': rsync_sdk_key}, haltOnFailure = True, logEnviron = False)) @@ -506,7 +462,7 @@ for arch in arches: factory.addStep(ShellCommand( name = "updatesdk", description = "Updating SDK", - command = "rsync --checksum -av sdk_update/ sdk/ && rm -rf sdk_update", + command = "rsync " + (" ").join(rsync_defopts) + " --checksum -a sdk_update/ sdk/ && rm -rf sdk_update", haltOnFailure = True)) factory.addStep(ShellCommand( @@ -572,36 +528,13 @@ for arch in arches: command = ["./ccache.sh"], haltOnFailure = True)) - if git_ssh: - factory.addStep(StringDownload( - name = "dlgitclonekey", - s = git_ssh_key, - workerdest = "../git-clone.key", - mode = 0o600)) - - factory.addStep(ShellCommand( - name = "patchfeedsconf", - description = "Patching feeds.conf", - workdir = "build/sdk", - command = "sed -e 's#https://#ssh://git@#g' feeds.conf.default > feeds.conf", - haltOnFailure = True)) - factory.addStep(ShellCommand( name = "updatefeeds", description = "Updating feeds", workdir = "build/sdk", command = ["./scripts/feeds", "update", "-f"], - env = {'GIT_SSH_COMMAND': Interpolate("ssh -o IdentitiesOnly=yes -o IdentityFile=%(kw:cwd)s/git-clone.key -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no", cwd=GetCwd)} if git_ssh else {}, haltOnFailure = True)) - if git_ssh: - factory.addStep(ShellCommand( - name = "rmfeedsconf", - description = "Removing feeds.conf", - workdir = "build/sdk", - command=["rm", "feeds.conf"], - haltOnFailure = True)) - factory.addStep(ShellCommand( name = "installfeeds", description = "Installing feeds", @@ -624,7 +557,7 @@ for arch in arches: description = "Building packages", workdir = "build/sdk", timeout = 3600, - command = ["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "IGNORE_ERRORS=n m y", "BUILD_LOG=1", "CONFIG_AUTOREMOVE=y", "CONFIG_SIGNED_PACKAGES="], + command = ["make", Interpolate("-j%(prop:nproc:-1)s"), "IGNORE_ERRORS=n m y", "BUILD_LOG=1", "CONFIG_AUTOREMOVE=y", "CONFIG_SIGNED_PACKAGES="], env = {'CCACHE_BASEDIR': Interpolate("%(kw:cwd)s", cwd=GetCwd)}, haltOnFailure = True)) @@ -634,6 +567,18 @@ for arch in arches: workdir = "build/sdk", command = "./scripts/feeds list -s -f > bin/packages/%s/feeds.conf" %(arch[0]))) + factory.addStep(ShellCommand( + name = "checksums", + description = "Calculating checksums", + descriptionDone="Checksums calculated", + workdir = "build/sdk", + command = "cd bin/packages/%s; " %(arch[0]) + + "find . -type f -not -name 'sha256sums' -printf \"%P\n\" | " + + "sort | xargs -r ../../../staging_dir/host/bin/mkhash -n sha256 | " + + r"sed -ne 's!^\(.*\) \(.*\)$!\1 *\2!p' > sha256sums", + haltOnFailure = True + )) + if ini.has_option("gpg", "key") or usign_key is not None: factory.addStep(MasterShellCommand( name = "signprepare", @@ -678,11 +623,45 @@ for arch in arches: haltOnFailure = True )) + # download remote sha256sums to 'target-sha256sums' + factory.addStep(ShellCommand( + name = "target-sha256sums", + description = "Fetching remote sha256sums for arch", + command = ["rsync"] + rsync_defopts + ["-z", Interpolate("%(kw:rsyncbinurl)s/packages%(kw:suffix)s/%(kw:archname)s/sha256sums", rsyncbinurl=rsync_bin_url, suffix=GetDirectorySuffix, archname=arch[0]), "arch-sha256sums"], + env={'RSYNC_PASSWORD': rsync_bin_key}, + logEnviron = False, + haltOnFailure = False, + flunkOnFailure = False, + warnOnFailure = False, + )) + + factory.addStep(FileDownload( + name="dlrsync.sh", + mastersrc = scripts_dir + "/rsync.sh", + workerdest = "../rsync.sh", + mode = 0o755 + )) + + factory.addStep(FileDownload( + name = "dlsha2rsyncpl", + mastersrc = scripts_dir + "/sha2rsync.pl", + workerdest = "../sha2rsync.pl", + mode = 0o755, + )) + + factory.addStep(ShellCommand( + name = "buildlist", + description = "Building list of files to upload", + workdir = "build/sdk", + command = ["../../sha2rsync.pl", "../arch-sha256sums", "bin/packages/%s/sha256sums" %(arch[0]), "rsynclist"], + haltOnFailure = True, + )) + factory.addStep(ShellCommand( name = "uploadprepare", description = "Preparing package directory", workdir = "build/sdk", - command = ["rsync", "-4", "-av", "--include", "/%s/" %(arch[0]), "--exclude", "/*", "--exclude", "/%s/*" %(arch[0]), "bin/packages/", Interpolate("%(kw:rsyncbinurl)s/packages%(kw:suffix)s/", rsyncbinurl=rsync_bin_url, suffix=GetDirectorySuffix)], + command = ["rsync"] + rsync_defopts + ["-a", "--include", "/%s/" %(arch[0]), "--exclude", "/*", "--exclude", "/%s/*" %(arch[0]), "bin/packages/", Interpolate("%(kw:rsyncbinurl)s/packages%(kw:suffix)s/", rsyncbinurl=rsync_bin_url, suffix=GetDirectorySuffix)], env={'RSYNC_PASSWORD': rsync_bin_key}, haltOnFailure = True, logEnviron = False @@ -692,7 +671,17 @@ for arch in arches: name = "packageupload", description = "Uploading package files", workdir = "build/sdk", - command = ["rsync", "-4", "--progress", "--delete", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-avz", "bin/packages/%s/" %(arch[0]), Interpolate("%(kw:rsyncbinurl)s/packages%(kw:suffix)s/%(kw:archname)s/", rsyncbinurl=rsync_bin_url, suffix=GetDirectorySuffix, archname=arch[0])], + command = ["../../rsync.sh"] + rsync_defopts + ["--files-from=rsynclist", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-a", "bin/packages/%s/" %(arch[0]), Interpolate("%(kw:rsyncbinurl)s/packages%(kw:suffix)s/%(kw:archname)s/", rsyncbinurl=rsync_bin_url, suffix=GetDirectorySuffix, archname=arch[0])], + env={'RSYNC_PASSWORD': rsync_bin_key}, + haltOnFailure = True, + logEnviron = False + )) + + factory.addStep(ShellCommand( + name = "packageprune", + description = "Pruning package files", + workdir = "build/sdk", + command = ["../../rsync.sh"] + rsync_defopts + ["--delete", "--existing", "--ignore-existing", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-a", "bin/packages/%s/" %(arch[0]), Interpolate("%(kw:rsyncbinurl)s/packages%(kw:suffix)s/%(kw:archname)s/", rsyncbinurl=rsync_bin_url, suffix=GetDirectorySuffix, archname=arch[0])], env={'RSYNC_PASSWORD': rsync_bin_key}, haltOnFailure = True, logEnviron = False @@ -702,7 +691,7 @@ for arch in arches: name = "logprepare", description = "Preparing log directory", workdir = "build/sdk", - command = ["rsync", "-4", "-av", "--include", "/%s/" %(arch[0]), "--exclude", "/*", "--exclude", "/%s/*" %(arch[0]), "bin/packages/", Interpolate("%(kw:rsyncbinurl)s/faillogs%(kw:suffix)s/", rsyncbinurl=rsync_bin_url, suffix=GetDirectorySuffix)], + command = ["rsync"] + rsync_defopts + ["-a", "--include", "/%s/" %(arch[0]), "--exclude", "/*", "--exclude", "/%s/*" %(arch[0]), "bin/packages/", Interpolate("%(kw:rsyncbinurl)s/faillogs%(kw:suffix)s/", rsyncbinurl=rsync_bin_url, suffix=GetDirectorySuffix)], env={'RSYNC_PASSWORD': rsync_bin_key}, haltOnFailure = True, logEnviron = False @@ -722,7 +711,7 @@ for arch in arches: name = "logcollect", description = "Collecting failure logs", workdir = "build/sdk", - command = ["rsync", "-av", "--files-from=logs.txt", "logs/package/feeds/", "faillogs/"], + command = ["rsync"] + rsync_defopts + ["-a", "--files-from=logs.txt", "logs/package/feeds/", "faillogs/"], haltOnFailure = False, flunkOnFailure = False, warnOnFailure = True, @@ -732,7 +721,7 @@ for arch in arches: name = "logupload", description = "Uploading failure logs", workdir = "build/sdk", - command = ["rsync", "-4", "--progress", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-avz", "faillogs/", Interpolate("%(kw:rsyncbinurl)s/faillogs%(kw:suffix)s/%(kw:archname)s/", rsyncbinurl=rsync_bin_url, suffix=GetDirectorySuffix, archname=arch[0])], + command = ["../../rsync.sh"] + rsync_defopts + ["--delete", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-az", "faillogs/", Interpolate("%(kw:rsyncbinurl)s/faillogs%(kw:suffix)s/%(kw:archname)s/", rsyncbinurl=rsync_bin_url, suffix=GetDirectorySuffix, archname=arch[0])], env={'RSYNC_PASSWORD': rsync_bin_key}, haltOnFailure = False, flunkOnFailure = False, @@ -753,8 +742,8 @@ for arch in arches: name = "sourceupload", description = "Uploading source archives", workdir = "build/sdk", - command = ["rsync", "--files-from=sourcelist", "-4", "--progress", "--checksum", "--delay-updates", - Interpolate("--partial-dir=.~tmp~%(kw:archname)s~%(prop:workername)s", archname=arch[0]), "-avz", "dl/", "%s/" %(rsync_src_url)], + command = ["../../rsync.sh"] + rsync_defopts + ["--files-from=sourcelist", "--size-only", "--delay-updates", + Interpolate("--partial-dir=.~tmp~%(kw:archname)s~%(prop:workername)s", archname=arch[0]), "-a", "dl/", "%s/" %(rsync_src_url)], env={'RSYNC_PASSWORD': rsync_src_key}, haltOnFailure = False, flunkOnFailure = False,