from buildbot.buildslave import BuildSlave
slave_port = 9990
+persistent = False
+other_builds = 0
+tree_expire = 0
if ini.has_option("general", "port"):
slave_port = ini.getint("general", "port")
+if ini.has_option("general", "persistent"):
+ persistent = ini.getboolean("general", "persistent")
+
+if ini.has_option("general", "other_builds"):
+ other_builds = ini.getint("general", "other_builds")
+
+if ini.has_option("general", "expire"):
+ tree_expire = ini.getint("general", "expire")
+
c['slaves'] = []
+max_builds = dict()
for section in ini.sections():
if section.startswith("slave "):
if ini.has_option(section, "name") and ini.has_option(section, "password"):
name = ini.get(section, "name")
password = ini.get(section, "password")
- max_builds = 1
+ max_builds[name] = 1
if ini.has_option(section, "builds"):
- max_builds = ini.getint(section, "builds")
- c['slaves'].append(BuildSlave(name, password, max_builds = max_builds))
+ max_builds[name] = ini.getint(section, "builds")
+ c['slaves'].append(BuildSlave(name, password, max_builds = max_builds[name]))
# 'slavePortnum' defines the TCP port to listen on for connections from slaves.
# This must match the value configured into the buildslaves (with their
# coalesce builds
c['mergeRequests'] = True
+# Reduce amount of backlog data
+c['buildHorizon'] = 30
+c['logHorizon'] = 20
+
####### CHANGESOURCES
home_dir = os.path.abspath(ini.get("general", "homedir"))
if ini.has_option("rsync", "sdk_pattern"):
rsync_sdk_pat = ini.get("rsync", "sdk_pattern")
+gpg_home = "~/.gnupg"
gpg_keyid = None
gpg_comment = "Unattended build signature"
gpg_passfile = "/dev/null"
+if ini.has_option("gpg", "home"):
+ gpg_home = ini.get("gpg", "home")
+
if ini.has_option("gpg", "keyid"):
gpg_keyid = ini.get("gpg", "keyid")
# find feeds
feeds = []
+feedbranches = dict()
from buildbot.changes.gitpoller import GitPoller
c['change_source'] = []
feeds.append(parts)
url = parts[2].strip().split(';')
branch = url[1] if len(url) > 1 else 'master'
- c['change_source'].append(GitPoller(url[0], branch=branch, workdir='%s/%s.git' %(os.getcwd(), parts[1]), branch='master', pollinterval=300))
+ feedbranches[url[0]] = branch
+ c['change_source'].append(GitPoller(url[0], branch=branch, workdir='%s/%s.git' %(os.getcwd(), parts[1]), pollinterval=300))
####### SCHEDULERS
# Configure the Schedulers, which decide how to react to incoming changes. In this
# case, just kick off a 'basebuild' build
+def branch_change_filter(change):
+ return change.branch == feedbranches[change.repository]
+
from buildbot.schedulers.basic import SingleBranchScheduler
from buildbot.schedulers.forcesched import ForceScheduler
from buildbot.changes import filter
c['schedulers'] = []
c['schedulers'].append(SingleBranchScheduler(
name="all",
- change_filter=filter.ChangeFilter(branch='master'),
+ change_filter=filter.ChangeFilter(filter_fn=branch_change_filter),
treeStableTimer=60,
builderNames=archnames))
from buildbot.steps.master import MasterShellCommand
from buildbot.process.properties import WithProperties
+
+def GetDirectorySuffix(props):
+ if props.hasProperty("slavename") and re.match("^[^-]+-[0-9]+\.[0-9]+-[^-]+$", props["slavename"]):
+ return "-%s" % props["slavename"].split('-')[1]
+ else:
+ return ""
+
+def GetNumJobs(props):
+ if props.hasProperty("slavename") and props.hasProperty("nproc"):
+ return ((int(props["nproc"]) / (max_builds[props["slavename"]] + other_builds)) + 1)
+ else:
+ return 1
+
+
c['builders'] = []
dlLock = locks.SlaveLock("slave_dl")
# prepare workspace
factory.addStep(FileDownload(mastersrc="cleanup.sh", slavedest="cleanup.sh", mode=0755))
- factory.addStep(ShellCommand(
- name = "cleanold",
- description = "Cleaning previous builds",
- command = ["./cleanup.sh", buildbot_url, WithProperties("%(slavename)s"), WithProperties("%(buildername)s"), "full"],
- haltOnFailure = True,
- timeout = 2400))
+ if not persistent:
+ factory.addStep(ShellCommand(
+ name = "cleanold",
+ description = "Cleaning previous builds",
+ command = ["./cleanup.sh", buildbot_url, WithProperties("%(slavename)s"), WithProperties("%(buildername)s"), "full"],
+ haltOnFailure = True,
+ timeout = 2400))
- factory.addStep(ShellCommand(
- name = "cleanup",
- description = "Cleaning work area",
- command = ["./cleanup.sh", buildbot_url, WithProperties("%(slavename)s"), WithProperties("%(buildername)s"), "single"],
- haltOnFailure = True,
- timeout = 2400))
+ factory.addStep(ShellCommand(
+ name = "cleanup",
+ description = "Cleaning work area",
+ command = ["./cleanup.sh", buildbot_url, WithProperties("%(slavename)s"), WithProperties("%(buildername)s"), "single"],
+ haltOnFailure = True,
+ timeout = 2400))
+
+ # expire tree if needed
+ elif tree_expire > 0:
+ factory.addStep(FileDownload(
+ mastersrc = home_dir+"/expire.sh",
+ slavedest = "../expire.sh",
+ mode = 0755))
+
+ factory.addStep(ShellCommand(
+ name = "expire",
+ description = "Checking for build tree expiry",
+ command = ["./expire.sh", str(tree_expire)],
+ workdir = ".",
+ haltOnFailure = True,
+ timeout = 2400))
factory.addStep(ShellCommand(
name = "mksdkdir",
description = "Preparing SDK directory",
- command = ["mkdir", "sdk"],
+ command = ["mkdir", "-p", "sdk"],
haltOnFailure = True))
factory.addStep(ShellCommand(
factory.addStep(ShellCommand(
name = "unpacksdk",
description = "Unpacking SDK archive",
- command = ["tar", "--strip-components=1", "-C", "sdk/", "-vxf", "sdk.archive"],
+ command = ["tar", "--keep-newer-files", "--no-overwrite-dir", "--strip-components=1", "-C", "sdk/", "-vxf", "sdk.archive"],
haltOnFailure = True))
factory.addStep(FileDownload(mastersrc=home_dir+'/key-build', slavedest="sdk/key-build", mode=0600))
workdir = "build/sdk",
command = ["sh", "-c", "rm -f .config && make defconfig"]))
+ factory.addStep(FileDownload(
+ mastersrc = home_dir+'/ccache.sh',
+ slavedest = 'sdk/ccache.sh',
+ mode = 0755))
+
+ factory.addStep(ShellCommand(
+ name = "prepccache",
+ description = "Preparing ccache",
+ workdir = "build/sdk",
+ command = ["./ccache.sh"]))
+
factory.addStep(ShellCommand(
name = "updatefeeds",
description = "Updating feeds",
name = "compile",
description = "Building packages",
workdir = "build/sdk",
- command = ["make", WithProperties("-j%(nproc:~4)s"), "V=s", "IGNORE_ERRORS=n m y", "BUILD_LOG=1", "CONFIG_SIGNED_PACKAGES=y"]))
+ command = ["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "IGNORE_ERRORS=n m y", "BUILD_LOG=1", "CONFIG_SIGNED_PACKAGES=y", "CONFIG_AUTOREMOVE=y"]))
+
+ factory.addStep(ShellCommand(
+ name = "mkfeedsconf",
+ description = "Generating pinned feeds.conf",
+ workdir = "build/sdk",
+ command = "./scripts/feeds list -s -f > bin/packages/%s/feeds.conf" %(arch[0])))
if gpg_keyid is not None:
factory.addStep(MasterShellCommand(
factory.addStep(MasterShellCommand(
name = "signfiles",
description = "Signing files",
- command = ["%s/signall.sh" %(home_dir), "%s/signing/%s.tar.gz" %(home_dir, arch[0]), gpg_keyid, gpg_passfile, gpg_comment],
+ command = ["%s/signall.sh" %(home_dir), "%s/signing/%s.tar.gz" %(home_dir, arch[0]), gpg_keyid, gpg_comment],
+ env = {'GNUPGHOME': gpg_home, 'PASSFILE': gpg_passfile},
haltOnFailure = True
))
name = "uploadprepare",
description = "Preparing package directory",
workdir = "build/sdk",
- command = ["rsync", "-av", "--include", "/%s/" %(arch[0]), "--exclude", "/*", "--exclude", "/%s/*" %(arch[0]), "bin/packages/", "%s/packages/" %(rsync_bin_url)],
+ command = ["rsync", "-av", "--include", "/%s/" %(arch[0]), "--exclude", "/*", "--exclude", "/%s/*" %(arch[0]), "bin/packages/", WithProperties("%s/packages%%(suffix)s/" %(rsync_bin_url), suffix=GetDirectorySuffix)],
env={'RSYNC_PASSWORD': rsync_bin_key},
haltOnFailure = True,
logEnviron = False
name = "packageupload",
description = "Uploading package files",
workdir = "build/sdk",
- command = ["rsync", "--delete", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-avz", "bin/packages/%s/" %(arch[0]), "%s/packages/%s/" %(rsync_bin_url, arch[0])],
+ command = ["rsync", "--progress", "--delete", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-avz", "bin/packages/%s/" %(arch[0]), WithProperties("%s/packages%%(suffix)s/%s/" %(rsync_bin_url, arch[0]), suffix=GetDirectorySuffix)],
env={'RSYNC_PASSWORD': rsync_bin_key},
haltOnFailure = True,
logEnviron = False
name = "logupload",
description = "Uploading failure logs",
workdir = "build/sdk",
- command = ["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-avz", "faillogs/", "%s/faillogs/%s/" %(rsync_bin_url, arch[0])],
+ command = ["rsync", "--progress", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-avz", "faillogs/", "%s/faillogs/%s/" %(rsync_bin_url, arch[0])],
env={'RSYNC_PASSWORD': rsync_bin_key},
haltOnFailure = False,
logEnviron = False
name = "sourceupload",
description = "Uploading source archives",
workdir = "build/sdk",
- command = ["rsync", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-avz", "dl/", "%s/" %(rsync_src_url)],
+ command = ["rsync", "--progress", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-avz", "dl/", "%s/" %(rsync_src_url)],
env={'RSYNC_PASSWORD': rsync_src_key},
haltOnFailure = False,
logEnviron = False
projects / buildbot.git / blobdiff