inip1 = ini['phase1']
+# Globals
+work_dir = os.path.abspath(ini['general'].get("workdir", "."))
+scripts_dir = os.path.abspath("../scripts")
+
+config_seed = inip1.get("config_seed", "")
+
+repo_url = ini['repo'].get("url")
+repo_branch = ini['repo'].get("branch", "master")
+
+rsync_bin_url = ini['rsync'].get("binary_url")
+rsync_bin_key = ini['rsync'].get("binary_password")
+rsync_bin_defopts = ["-v", "-4", "--timeout=120"]
+
+if rsync_bin_url.find("::") > 0 or rsync_bin_url.find("rsync://") == 0:
+ rsync_bin_defopts += ["--contimeout=20"]
+
+rsync_src_url = ini['rsync'].get("source_url")
+rsync_src_key = ini['rsync'].get("source_password")
+rsync_src_defopts = ["-v", "-4", "--timeout=120"]
+
+if rsync_src_url.find("::") > 0 or rsync_src_url.find("rsync://") == 0:
+ rsync_src_defopts += ["--contimeout=20"]
+
+usign_key = None
+usign_comment = "untrusted comment: " + repo_branch.replace("-", " ").title() + " key"
+
+if ini.has_section("usign"):
+ usign_key = ini['usign'].get("key")
+ usign_comment = ini['usign'].get("comment", usign_comment)
+
+enable_kmod_archive = inip1.getboolean("kmod_archive", False)
+
+# PB port can be either a numeric port or a connection string
+pb_port = inip1.get("port") or 9989
+
# This is the dictionary that the buildmaster pays attention to. We also use
# a shorter alias to save typing.
c = BuildmasterConfig = {}
if section.startswith("worker "):
if ini.has_option(section, "name") and ini.has_option(section, "password") and \
(not ini.has_option(section, "phase") or ini.getint(section, "phase") == 1):
- sl_props = { 'dl_lock':None, 'ul_lock':None, 'do_cleanup':False, 'max_builds':1, 'shared_wd':False }
+ sl_props = { 'dl_lock':None, 'ul_lock':None }
name = ini.get(section, "name")
password = ini.get(section, "password")
- max_builds = 1
- if ini.has_option(section, "builds"):
- max_builds = ini.getint(section, "builds")
- sl_props['max_builds'] = max_builds
- if max_builds == 1:
- sl_props['shared_wd'] = True
- if ini.has_option(section, "cleanup"):
- sl_props['do_cleanup'] = ini.getboolean(section, "cleanup")
if ini.has_option(section, "dl_lock"):
lockname = ini.get(section, "dl_lock")
sl_props['dl_lock'] = lockname
if lockname not in NetLocks:
NetLocks[lockname] = locks.MasterLock(lockname)
if ini.has_option(section, "ul_lock"):
- lockname = ini.get(section, "dl_lock")
+ lockname = ini.get(section, "ul_lock")
sl_props['ul_lock'] = lockname
if lockname not in NetLocks:
NetLocks[lockname] = locks.MasterLock(lockname)
- if ini.has_option(section, "shared_wd"):
- shared_wd = ini.getboolean(section, "shared_wd")
- sl_props['shared_wd'] = shared_wd
- if shared_wd and (max_builds != 1):
- raise ValueError('max_builds must be 1 with shared workdir!')
- c['workers'].append(Worker(name, password, max_builds = max_builds, properties = sl_props))
+ c['workers'].append(Worker(name, password, max_builds = 1, properties = sl_props))
-# PB port can be either a numeric port or a connection string
-pb_port = inip1.get("port") or 9989
c['protocols'] = {'pb': {'port': pb_port}}
# coalesce builds
####### CHANGESOURCES
-work_dir = os.path.abspath(ini['general'].get("workdir", "."))
-scripts_dir = os.path.abspath("../scripts")
-
-tree_expire = inip1.getint("expire", 0)
-config_seed = inip1.get("config_seed", "")
-
-repo_url = ini['repo'].get("url")
-repo_branch = ini['repo'].get("branch", "master")
-
-rsync_bin_url = ini['rsync'].get("binary_url")
-rsync_bin_key = ini['rsync'].get("binary_password")
-rsync_bin_defopts = ["-v", "-4", "--timeout=120"]
-
-if rsync_bin_url.find("::") > 0 or rsync_bin_url.find("rsync://") == 0:
- rsync_bin_defopts += ["--contimeout=20"]
-
-rsync_src_url = ini['rsync'].get("source_url")
-rsync_src_key = ini['rsync'].get("source_password")
-rsync_src_defopts = ["-v", "-4", "--timeout=120"]
-
-if rsync_src_url.find("::") > 0 or rsync_src_url.find("rsync://") == 0:
- rsync_src_defopts += ["--contimeout=20"]
-
-usign_key = None
-usign_comment = "untrusted comment: " + repo_branch.replace("-", " ").title() + " key"
-
-if ini.has_section("usign"):
- usign_key = ini['usign'].get("key")
- usign_comment = ini['usign'].get("comment", usign_comment)
-
-enable_kmod_archive = inip1.getboolean("kmod_archive", False)
-
# find targets
targets = [ ]
-if not os.path.isdir(work_dir+'/source.git'):
- subprocess.call(["git", "clone", "--depth=1", "--branch="+repo_branch, repo_url, work_dir+'/source.git'])
-else:
- subprocess.call(["git", "pull"], cwd = work_dir+'/source.git')
+def populateTargets():
+ sourcegit = work_dir + '/source.git'
+ if os.path.isdir(sourcegit):
+ subprocess.call(["rm", "-rf", sourcegit])
-os.makedirs(work_dir+'/source.git/tmp', exist_ok=True)
-findtargets = subprocess.Popen(['./scripts/dump-target-info.pl', 'targets'],
- stdout = subprocess.PIPE, cwd = work_dir+'/source.git')
+ subprocess.call(["git", "clone", "--depth=1", "--branch="+repo_branch, repo_url, sourcegit])
-while True:
- line = findtargets.stdout.readline()
- if not line:
- break
- ta = line.decode().strip().split(' ')
- targets.append(ta[0])
+ os.makedirs(sourcegit + '/tmp', exist_ok=True)
+ findtargets = subprocess.Popen(['./scripts/dump-target-info.pl', 'targets'],
+ stdout = subprocess.PIPE, stderr = subprocess.DEVNULL, cwd = sourcegit)
+ while True:
+ line = findtargets.stdout.readline()
+ if not line:
+ break
+ ta = line.decode().strip().split(' ')
+ targets.append(ta[0])
+
+ subprocess.call(["rm", "-rf", sourcegit])
+
+populateTargets()
# the 'change_source' setting tells the buildmaster how it should find out
# about source code changes. Here we point to the buildbot clone of pyflakes.
# what steps, and which workers can execute them. Note that any particular build will
# only take place on one worker.
-CleanTargetMap = [
- [ "tools", "tools/clean" ],
- [ "chain", "toolchain/clean" ],
- [ "linux", "target/linux/clean" ],
- [ "dir", "dirclean" ],
- [ "dist", "distclean" ]
-]
-
-def IsMakeCleanRequested(pattern):
- def CheckCleanProperty(step):
- val = step.getProperty("clean")
- if val and re.match(pattern, val):
- return True
- else:
- return False
-
- return CheckCleanProperty
-
-def IsSharedWorkdir(step):
- return bool(step.getProperty("shared_wd"))
-
-def IsCleanupRequested(step):
- if IsSharedWorkdir(step):
- return False
- do_cleanup = step.getProperty("do_cleanup")
- if do_cleanup:
- return True
- else:
- return False
-
-def IsExpireRequested(step):
- if IsSharedWorkdir(step):
- return False
- else:
- return not IsCleanupRequested(step)
-
-def IsGitFreshRequested(step):
- do_cleanup = step.getProperty("do_cleanup")
- if do_cleanup:
- return True
- else:
- return False
-
-def IsGitCleanRequested(step):
- return not IsGitFreshRequested(step)
-
def IsTaggingRequested(step):
val = step.getProperty("tag")
if val and re.match(r"^[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", val):
else:
return False
-def IsNoTaggingRequested(step):
- return not IsTaggingRequested(step)
-
def IsNoMasterBuild(step):
- return repo_branch != "master"
+ return step.getProperty("branch") != "master"
-def GetBaseVersion():
- if re.match(r"^[^-]+-[0-9]+\.[0-9]+$", repo_branch):
- return repo_branch.split('-')[1]
+def GetBaseVersion(branch):
+ if re.match(r"^[^-]+-[0-9]+\.[0-9]+$", branch):
+ return branch.split('-')[1]
else:
return "master"
@properties.renderer
def GetVersionPrefix(props):
- basever = GetBaseVersion()
+ branch = props.getProperty("branch")
+ basever = GetBaseVersion(branch)
if props.hasProperty("tag") and re.match(r"^[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", props["tag"]):
return "%s/" % props["tag"]
elif basever != "master":
else:
return ""
-@properties.renderer
-def GetNumJobs(props):
- if props.hasProperty("max_builds") and props.hasProperty("nproc"):
- return str(int(int(props["nproc"]) / props["max_builds"]))
- else:
- return "1"
-
-@properties.renderer
-def GetCCache(props):
- if props.hasProperty("ccache_command") and "ccache" in props["ccache_command"]:
- return props["ccache_command"]
- else:
- return ""
-
def GetNextBuild(builder, requests):
for r in requests:
if r.properties and r.properties.hasProperty("tag"):
if tryccache:
env['CC'] = Interpolate("%(prop:builddir)s/ccache_cc.sh")
env['CXX'] = Interpolate("%(prop:builddir)s/ccache_cxx.sh")
- env['CCACHE'] = Interpolate("%(kw:ccache)s", ccache=GetCCache)
+ env['CCACHE'] = Interpolate("%(prop:ccache_command:-)s")
else:
env['CC'] = env['CCC']
env['CXX'] = env['CCXX']
description = "Setting up shared work directory",
command = 'test -L "$PWD" || (mkdir -p ../shared-workdir && rm -rf "$PWD" && ln -s shared-workdir "$PWD")',
workdir = ".",
- haltOnFailure = True,
- doStepIf = IsSharedWorkdir))
+ haltOnFailure = True))
# find number of cores
factory.addStep(SetPropertyFromCommand(
warnOnFailure = False,
))
- # expire tree if needed
- if tree_expire > 0:
- factory.addStep(FileDownload(
- name = "dlexpiresh",
- doStepIf = IsExpireRequested,
- mastersrc = scripts_dir + '/expire.sh',
- workerdest = "../expire.sh",
- mode = 0o755))
-
- factory.addStep(ShellCommand(
- name = "expire",
- description = "Checking for build tree expiry",
- command = ["./expire.sh", str(tree_expire)],
- workdir = ".",
- haltOnFailure = True,
- doStepIf = IsExpireRequested,
- timeout = 2400))
-
- # cleanup.sh if needed
- factory.addStep(FileDownload(
- name = "dlcleanupsh",
- mastersrc = scripts_dir + '/cleanup.sh',
- workerdest = "../cleanup.sh",
- mode = 0o755,
- doStepIf = IsCleanupRequested))
-
- factory.addStep(ShellCommand(
- name = "cleanold",
- description = "Cleaning previous builds",
- command = ["./cleanup.sh", c['buildbotURL'], Interpolate("%(prop:workername)s"), Interpolate("%(prop:buildername)s"), "full"],
- workdir = ".",
- haltOnFailure = True,
- doStepIf = IsCleanupRequested,
- timeout = 2400))
-
- factory.addStep(ShellCommand(
- name = "cleanup",
- description = "Cleaning work area",
- command = ["./cleanup.sh", c['buildbotURL'], Interpolate("%(prop:workername)s"), Interpolate("%(prop:buildername)s"), "single"],
- workdir = ".",
- haltOnFailure = True,
- doStepIf = IsCleanupRequested,
- timeout = 2400))
-
- # user-requested clean targets
- for tuple in CleanTargetMap:
- factory.addStep(ShellCommand(
- name = tuple[1],
- description = 'User-requested "make %s"' % tuple[1],
- command = ["make", tuple[1], "V=s"],
- env = MakeEnv(),
- doStepIf = IsMakeCleanRequested(tuple[0])
- ))
-
# Workaround bug when switching from a checked out tag back to a branch
# Ref: http://lists.infradead.org/pipermail/openwrt-devel/2019-June/017809.html
factory.addStep(ShellCommand(
name = "gitcheckout",
description = "Ensure that Git HEAD is sane",
- command = "if [ -d .git ]; then git checkout -f %s && git branch --set-upstream-to origin/%s || rm -fr .git; else exit 0; fi" %(repo_branch, repo_branch),
+ command = Interpolate("if [ -d .git ]; then git checkout -f %(prop:branch)s && git branch --set-upstream-to origin/%(prop:branch)s || rm -fr .git; else exit 0; fi"),
haltOnFailure = True))
# check out the source
# if repo doesn't exist: 'git clone repourl'
# method 'clean' runs 'git clean -d -f', method fresh runs 'git clean -d -f x'. Only works with mode='full'
# 'git fetch -t repourl branch; git reset --hard revision'
- # Git() parameters can't take a renderer until buildbot 0.8.10, so we have to split the fresh and clean cases
- # if buildbot is updated, one can use: method = Interpolate('%(prop:do_cleanup:#?|fresh|clean)s')
- factory.addStep(Git(
- name = "gitclean",
- repourl = repo_url,
- branch = repo_branch,
- mode = 'full',
- method = 'clean',
- locks = NetLockDl,
- haltOnFailure = True,
- doStepIf = IsGitCleanRequested,
- ))
-
factory.addStep(Git(
- name = "gitfresh",
+ name = "git",
repourl = repo_url,
- branch = repo_branch,
mode = 'full',
method = 'fresh',
locks = NetLockDl,
haltOnFailure = True,
- doStepIf = IsGitFreshRequested,
))
# update remote refs
factory.addStep(ShellCommand(
name = "fetchrefs",
description = "Fetching Git remote refs",
- command = ["git", "fetch", "origin", "+refs/heads/%s:refs/remotes/origin/%s" %(repo_branch, repo_branch)],
+ command = ["git", "fetch", "origin", Interpolate("+refs/heads/%(prop:branch)s:refs/remotes/origin/%(prop:branch)s")],
haltOnFailure = True
))
factory.addStep(ShellCommand(
name = "dltar",
description = "Building and installing GNU tar",
- command = ["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "tools/tar/compile", "V=s"],
+ command = ["make", Interpolate("-j%(prop:nproc:-1)s"), "tools/tar/compile", "V=s"],
env = MakeEnv(tryccache=True),
haltOnFailure = True
))
factory.addStep(ShellCommand(
name = "dlrun",
description = "Populating dl/",
- command = ["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "download", "V=s"],
+ command = ["make", Interpolate("-j%(prop:nproc:-1)s"), "download", "V=s"],
env = MakeEnv(),
logEnviron = False,
locks = properties.FlattenList(NetLockDl, [dlLock.access('exclusive')]),
factory.addStep(ShellCommand(
name = "tools",
description = "Building and installing tools",
- command = ["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "tools/install", "V=s"],
+ command = ["make", Interpolate("-j%(prop:nproc:-1)s"), "tools/install", "V=s"],
env = MakeEnv(tryccache=True),
haltOnFailure = True
))
factory.addStep(ShellCommand(
name = "toolchain",
description = "Building and installing toolchain",
- command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "toolchain/install", "V=s"],
+ command=["make", Interpolate("-j%(prop:nproc:-1)s"), "toolchain/install", "V=s"],
env = MakeEnv(),
haltOnFailure = True
))
factory.addStep(ShellCommand(
name = "kmods",
description = "Building kmods",
- command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
+ command=["make", Interpolate("-j%(prop:nproc:-1)s"), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
env = MakeEnv(),
haltOnFailure = True
))
factory.addStep(ShellCommand(
name = "pkgbuild",
description = "Building packages",
- command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
+ command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
env = MakeEnv(),
haltOnFailure = True
))
factory.addStep(ShellCommand(
name = "pkginstall",
description = "Installing packages",
- command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "package/install", "V=s"],
+ command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/install", "V=s"],
env = MakeEnv(),
haltOnFailure = True
))
factory.addStep(ShellCommand(
name = "pkgindex",
description = "Indexing packages",
- command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "package/index", "V=s", "CONFIG_SIGNED_PACKAGES="],
+ command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/index", "V=s", "CONFIG_SIGNED_PACKAGES="],
env = MakeEnv(),
haltOnFailure = True
))
factory.addStep(ShellCommand(
name = "images",
description = "Building and installing images",
- command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "target/install", "V=s"],
+ command=["make", Interpolate("-j%(prop:nproc:-1)s"), "target/install", "V=s"],
env = MakeEnv(),
haltOnFailure = True
))
factory.addStep(ShellCommand(
name = "kmodindex",
description = "Indexing kmod archive",
- command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "package/index", "V=s", "CONFIG_SIGNED_PACKAGES=",
+ command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/index", "V=s", "CONFIG_SIGNED_PACKAGES=",
Interpolate("PACKAGE_SUBDIRS=bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1])],
env = MakeEnv(),
haltOnFailure = True
factory.addStep(ShellCommand(
name = "linkprepare",
description = "Preparing repository symlink",
- command = ["ln", "-s", "-f", Interpolate("../packages-%(kw:basever)s", basever=GetBaseVersion()), Interpolate("tmp/upload/%(kw:prefix)spackages", prefix=GetVersionPrefix)],
+ command = ["ln", "-s", "-f", Interpolate("../packages-%(kw:basever)s", basever=util.Transform(GetBaseVersion, Property("branch"))), Interpolate("tmp/upload/%(kw:prefix)spackages", prefix=GetVersionPrefix)],
doStepIf = IsNoMasterBuild,
haltOnFailure = True
))
projects / buildbot.git / blobdiff