# slave name and password must be configured on the slave.
from buildbot.buildslave import BuildSlave
+slave_port = 9989
+
+if ini.has_option("general", "port"):
+ slave_port = ini.getint("general", "port")
+
c['slaves'] = []
+max_builds = dict()
for section in ini.sections():
if section.startswith("slave "):
if ini.has_option(section, "name") and ini.has_option(section, "password"):
name = ini.get(section, "name")
password = ini.get(section, "password")
- max_builds = 1
+ max_builds[name] = 1
if ini.has_option(section, "builds"):
- max_builds = ini.getint(section, "builds")
- c['slaves'].append(BuildSlave(name, password, max_builds = max_builds))
+ max_builds[name] = ini.getint(section, "builds")
+ c['slaves'].append(BuildSlave(name, password, max_builds = max_builds[name]))
# 'slavePortnum' defines the TCP port to listen on for connections from slaves.
# This must match the value configured into the buildslaves (with their
# --master option)
-c['slavePortnum'] = 9989
+c['slavePortnum'] = slave_port
# coalesce builds
c['mergeRequests'] = True
####### CHANGESOURCES
home_dir = os.path.abspath(ini.get("general", "homedir"))
+tree_expire = 0
+other_builds = 0
+
+if ini.has_option("general", "expire"):
+ tree_expire = ini.getint("general", "expire")
+
+if ini.has_option("general", "other_builds"):
+ other_builds = ini.getint("general", "other_builds")
repo_url = ini.get("repo", "url")
+repo_branch = "master"
+
+if ini.has_option("repo", "branch"):
+ repo_branch = ini.get("repo", "branch")
+
+rsync_bin_url = ini.get("rsync", "binary_url")
+rsync_bin_key = ini.get("rsync", "binary_password")
+
+rsync_src_url = None
+rsync_src_key = None
+
+if ini.has_option("rsync", "source_url"):
+ rsync_src_url = ini.get("rsync", "source_url")
+ rsync_src_key = ini.get("rsync", "source_password")
+
+gpg_home = "~/.gnupg"
+gpg_keyid = None
+gpg_comment = "Unattended build signature"
+gpg_passfile = "/dev/null"
+
+if ini.has_option("gpg", "home"):
+ gpg_home = ini.get("gpg", "home")
+
+if ini.has_option("gpg", "keyid"):
+ gpg_keyid = ini.get("gpg", "keyid")
+
+if ini.has_option("gpg", "comment"):
+ gpg_comment = ini.get("gpg", "comment")
+
+if ini.has_option("gpg", "passfile"):
+ gpg_passfile = ini.get("gpg", "passfile")
-rsync_url = ini.get("rsync", "url")
-rsync_key = ini.get("rsync", "password")
# find targets
targets = [ ]
+if not os.path.isdir(home_dir+'/source.git'):
+ subprocess.call(["git", "clone", "--depth=1", "--branch="+repo_branch, repo_url, home_dir+'/source.git'])
+else:
+ subprocess.call(["git", "pull"], cwd = home_dir+'/source.git')
+
findtargets = subprocess.Popen([home_dir+'/dumpinfo.pl', 'targets'],
stdout = subprocess.PIPE, cwd = home_dir+'/source.git')
c['change_source'] = []
c['change_source'].append(GitPoller(
repo_url,
- workdir=home_dir+'/source.git', branch='master',
+ workdir=home_dir+'/work.git', branch=repo_branch,
pollinterval=300))
####### SCHEDULERS
c['schedulers'] = []
c['schedulers'].append(SingleBranchScheduler(
name="all",
- change_filter=filter.ChangeFilter(branch='master'),
+ change_filter=filter.ChangeFilter(branch=repo_branch),
treeStableTimer=60,
builderNames=targets))
# only take place on one slave.
from buildbot.process.factory import BuildFactory
-from buildbot.steps.source import Git
+from buildbot.steps.source.git import Git
from buildbot.steps.shell import ShellCommand
+from buildbot.steps.shell import SetProperty
+from buildbot.steps.transfer import FileUpload
from buildbot.steps.transfer import FileDownload
+from buildbot.steps.master import MasterShellCommand
+from buildbot.process.properties import WithProperties
+
+
+CleanTargetMap = [
+ [ "tools", "tools/clean" ],
+ [ "chain", "toolchain/clean" ],
+ [ "linux", "target/linux/clean" ],
+ [ "dir", "dirclean" ],
+ [ "dist", "distclean" ]
+]
+
+def IsCleanRequested(pattern):
+ def CheckCleanProperty(step):
+ val = step.getProperty("clean")
+ if val and re.match(pattern, val):
+ return True
+ else:
+ return False
+ return CheckCleanProperty
-MakeTargetMap = {
- "^tools/": "tools/clean",
- "^toolchain/": "toolchain/clean",
- "^target/linux/": "target/linux/clean",
- "^(config|include)/": "dirclean"
-}
-
-def IsAffected(pattern):
- def CheckAffected(change):
- for request in change.build.requests:
- for source in request.sources:
- for change in source.changes:
- for file in change.files:
- if re.match(pattern, file):
- return True
+def IsTaggingRequested(step):
+ val = step.getProperty("tag")
+ if val and re.match("^[0-9]+\.[0-9]+\.[0-9]+$", val):
+ return True
+ else:
return False
- return CheckAffected
-def isPathBuiltin(path):
- incl = {}
- pkgs = {}
- conf = open(".config", "r")
+def IsNoTaggingRequested(step):
+ return not IsTaggingRequested(step)
- while True:
- line = conf.readline()
- if line == '':
- break
- m = re.match("^(CONFIG_PACKAGE_.+?)=y", line)
- if m:
- incl[m.group(1)] = True
+def IsNoMasterBuild(step):
+ return repo_branch != "master"
- conf.close()
-
- deps = open("tmp/.packagedeps", "r")
-
- while True:
- line = deps.readline()
- if line == '':
- break
- m = re.match("^package-\$\((CONFIG_PACKAGE_.+?)\) \+= (\S+)", line)
- if m and incl.get(m.group(1)) == True:
- pkgs["package/%s" % m.group(2)] = True
-
- deps.close()
-
- while path != '':
- if pkgs.get(path) == True:
- return True
- path = os.path.dirname(path)
-
- return False
+def GetBaseVersion(props):
+ if re.match("^[^-]+-[0-9]+\.[0-9]+$", repo_branch):
+ return repo_branch.split('-')[1]
+ else:
+ return "master"
+
+def GetVersionPrefix(props):
+ basever = GetBaseVersion(props)
+ if props.hasProperty("tag") and re.match("^[0-9]+\.[0-9]+\.[0-9]+$", props["tag"]):
+ return "%s/" % props["tag"]
+ elif basever != "master":
+ return "%s-SNAPSHOT/" % basever
+ else:
+ return ""
-def isChangeBuiltin(change):
- return True
-# for request in change.build.requests:
-# for source in request.sources:
-# for change in source.changes:
-# for file in change.files:
-# if isPathBuiltin(file):
-# return True
-# return False
+def GetNumJobs(props):
+ if props.hasProperty("slavename") and props.hasProperty("nproc"):
+ return ((int(props["nproc"]) / (max_builds[props["slavename"]] + other_builds)) + 1)
+ else:
+ return 1
c['builders'] = []
dlLock = locks.SlaveLock("slave_dl")
+tagLock = locks.MasterLock("make_tag")
checkBuiltin = re.sub('[\t\n ]+', ' ', """
checkBuiltin() {
factory = BuildFactory()
+ # find number of cores
+ factory.addStep(SetProperty(
+ name = "nproc",
+ property = "nproc",
+ description = "Finding number of CPUs",
+ command = ["nproc"]))
+
+ # expire tree if needed
+ if tree_expire > 0:
+ factory.addStep(FileDownload(
+ mastersrc = "expire.sh",
+ slavedest = "../expire.sh",
+ mode = 0755))
+
+ factory.addStep(ShellCommand(
+ name = "expire",
+ description = "Checking for build tree expiry",
+ command = ["./expire.sh", str(tree_expire)],
+ workdir = ".",
+ haltOnFailure = True,
+ timeout = 2400))
+
+ # user-requested clean targets
+ for tuple in CleanTargetMap:
+ factory.addStep(ShellCommand(
+ name = tuple[1],
+ description = 'User-requested "make %s"' % tuple[1],
+ command = ["make", tuple[1], "V=s"],
+ doStepIf = IsCleanRequested(tuple[0])
+ ))
+
+ factory.addStep(MasterShellCommand(
+ name = "maketag",
+ description = "Tagging Git repository",
+ command = [home_dir+'/maketag.sh', '-i', '-k', str(gpg_keyid or ''),
+ '-p', str(gpg_passfile or ''), '-v', WithProperties("%(tag:-)s")],
+ path = home_dir+'/source.git',
+ env = {'GNUPGHOME': gpg_home},
+ haltOnFailure = True,
+ doStepIf = IsTaggingRequested,
+ locks = [tagLock.access('exclusive')]
+ ))
+
+ # switch to branch
+ factory.addStep(ShellCommand(
+ name = "switchbranch",
+ description = "Checking out Git branch",
+ command = "if [ -d .git ]; then git checkout '%s'; else exit 0; fi" % repo_branch,
+ haltOnFailure = True,
+ doStepIf = IsNoTaggingRequested
+ ))
+
# check out the source
- factory.addStep(Git(repourl=repo_url, mode='update'))
+ factory.addStep(Git(
+ repourl = repo_url,
+ branch = repo_branch,
+ mode = 'incremental',
+ method = 'clean'))
+
+ # update remote refs
+ factory.addStep(ShellCommand(
+ name = "fetchrefs",
+ description = "Fetching Git remote refs",
+ command = ["git", "fetch", "origin", "+refs/heads/%s:refs/remotes/origin/%s" %(repo_branch, repo_branch)],
+ haltOnFailure = True
+ ))
+
+ # fetch tags
+ factory.addStep(ShellCommand(
+ name = "fetchtag",
+ description = "Fetching Git tags",
+ command = ["git", "fetch", "--tags", "--", repo_url],
+ haltOnFailure = True,
+ doStepIf = IsTaggingRequested
+ ))
+
+ # switch to tag
+ factory.addStep(ShellCommand(
+ name = "switchtag",
+ description = "Checking out Git tag",
+ command = ["git", "checkout", WithProperties("tags/v%(tag:-)s")],
+ haltOnFailure = True,
+ doStepIf = IsTaggingRequested
+ ))
factory.addStep(ShellCommand(
name = "rmtmp",
# description = "Copy the feeds.conf",
# command='''cp ~/feeds.conf ./feeds.conf''' ))
+ # feed
+ factory.addStep(ShellCommand(
+ name = "rmfeedlinks",
+ description = "Remove feed symlinks",
+ command=["rm", "-rf", "package/feeds/"]))
+
# feed
factory.addStep(ShellCommand(
name = "updatefeeds",
description = "Installing feeds",
command=["./scripts/feeds", "install", "-a"]))
+ # seed config
+ factory.addStep(FileDownload(
+ mastersrc = "config.seed",
+ slavedest = ".config",
+ mode = 0644
+ ))
+
# configure
factory.addStep(ShellCommand(
name = "newconfig",
description = "Seeding .config",
- command='''cat <<EOT > .config
-CONFIG_TARGET_%s=y
-CONFIG_TARGET_%s_%s=y
-CONFIG_ALL_NONSHARED=y
-CONFIG_SDK=y
-CONFIG_IB=y
-# CONFIG_IB_STANDALONE is not set
-CONFIG_DEVEL=y
-CONFIG_CCACHE=y
-CONFIG_SIGNED_PACKAGES=y
-# CONFIG_PER_FEED_REPO_ADD_COMMENTED is not set
-CONFIG_KERNEL_KALLSYMS=y
-CONFIG_COLLECT_KERNEL_DEBUG=y
-EOT''' %(ts[0], ts[0], ts[1]) ))
+ command = "printf 'CONFIG_TARGET_%s=y\\nCONFIG_TARGET_%s_%s=y\\n' >> .config" %(ts[0], ts[0], ts[1])
+ ))
factory.addStep(ShellCommand(
name = "delbin",
haltOnFailure = True
))
+ # find libc suffix
+ factory.addStep(SetProperty(
+ name = "libc",
+ property = "libc",
+ description = "Finding libc suffix",
+ command = ["sed", "-ne", '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }', ".config"]))
+
# install build key
factory.addStep(FileDownload(mastersrc=home_dir+'/key-build', slavedest="key-build", mode=0600))
factory.addStep(FileDownload(mastersrc=home_dir+'/key-build.pub', slavedest="key-build.pub", mode=0600))
factory.addStep(ShellCommand(
name = "dldir",
description = "Preparing dl/",
- command = "mkdir -p $HOME/dl && ln -sf $HOME/dl ./dl",
+ command = "mkdir -p $HOME/dl && rm -rf ./dl && ln -sf $HOME/dl ./dl",
logEnviron = False,
want_stdout = False
))
+ # prepare tar
+ factory.addStep(ShellCommand(
+ name = "dltar",
+ description = "Building GNU tar",
+ command = ["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "tools/tar/install", "V=s"],
+ haltOnFailure = True
+ ))
+
# populate dl
factory.addStep(ShellCommand(
name = "dlrun",
description = "Populating dl/",
- command = ["make", "-j4", "download", "V=s"],
+ command = ["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "download", "V=s"],
logEnviron = False,
locks = [dlLock.access('exclusive')]
))
command=["make", "package/base-files/clean", "V=s"]
))
- # optional clean steps
- for pattern, maketarget in MakeTargetMap.items():
- factory.addStep(ShellCommand(
- name = maketarget,
- description = maketarget,
- command=["make", maketarget, "V=s"], doStepIf=IsAffected(pattern)
- ))
-
# build
factory.addStep(ShellCommand(
name = "tools",
description = "Building tools",
- command = ["make", "-j4", "tools/install", "V=s"],
+ command = ["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "tools/install", "V=s"],
haltOnFailure = True
))
factory.addStep(ShellCommand(
name = "toolchain",
description = "Building toolchain",
- command=["make", "-j4", "toolchain/install", "V=s"],
+ command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "toolchain/install", "V=s"],
haltOnFailure = True
))
factory.addStep(ShellCommand(
name = "kmods",
description = "Building kmods",
- command=["make", "-j4", "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
+ command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
#env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
haltOnFailure = True
))
factory.addStep(ShellCommand(
name = "pkgbuild",
description = "Building packages",
- command=["make", "-j4", "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
+ command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
#env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
haltOnFailure = True
))
factory.addStep(ShellCommand(
name = "pkginstall",
description = "Installing packages",
- command=["make", "-j4", "package/install", "V=s"],
- doStepIf = isChangeBuiltin,
+ command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "package/install", "V=s"],
haltOnFailure = True
))
factory.addStep(ShellCommand(
name = "pkgindex",
description = "Indexing packages",
- command=["make", "-j4", "package/index", "V=s"],
+ command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "package/index", "V=s"],
haltOnFailure = True
))
factory.addStep(ShellCommand(
name = "images",
description = "Building images",
- command=["make", "-j1", "target/install", "V=s"],
- doStepIf = isChangeBuiltin,
+ command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "target/install", "V=s"],
+ haltOnFailure = True
+ ))
+
+ factory.addStep(ShellCommand(
+ name = "diffconfig",
+ description = "Generating config.seed",
+ command=["make", "-j1", "diffconfig", "V=s"],
+ haltOnFailure = True
+ ))
+
+ factory.addStep(ShellCommand(
+ name = "checksums",
+ description = "Calculating checksums",
+ command=["make", "-j1", "checksum", "V=s"],
haltOnFailure = True
))
+ # sign
+ if gpg_keyid is not None:
+ factory.addStep(MasterShellCommand(
+ name = "signprepare",
+ description = "Preparing temporary signing directory",
+ command = ["mkdir", "-p", "%s/signing" %(home_dir)],
+ haltOnFailure = True
+ ))
+
+ factory.addStep(ShellCommand(
+ name = "signpack",
+ description = "Packing files to sign",
+ command = WithProperties("find bin/targets/%s/%s%%(libc)s/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz" %(ts[0], ts[1])),
+ haltOnFailure = True
+ ))
+
+ factory.addStep(FileUpload(
+ slavesrc = "sign.tar.gz",
+ masterdest = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
+ haltOnFailure = True
+ ))
+
+ factory.addStep(MasterShellCommand(
+ name = "signfiles",
+ description = "Signing files",
+ command = ["%s/signall.sh" %(home_dir), "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]), gpg_keyid, gpg_comment],
+ env = {'GNUPGHOME': gpg_home, 'PASSFILE': gpg_passfile},
+ haltOnFailure = True
+ ))
+
+ factory.addStep(FileDownload(
+ mastersrc = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
+ slavedest = "sign.tar.gz",
+ haltOnFailure = True
+ ))
+
+ factory.addStep(ShellCommand(
+ name = "signunpack",
+ description = "Unpacking signed files",
+ command = ["tar", "-xzf", "sign.tar.gz"],
+ haltOnFailure = True
+ ))
+
# upload
factory.addStep(ShellCommand(
- name = "uploadprepare",
- description = "Preparing target directory",
- command=["rsync", "-av", "--include", "/%s/" %(ts[0]), "--include", "/%s/%s/" %(ts[0], ts[1]), "--exclude", "/*", "--exclude", "/*/*", "--exclude", "/%s/%s/*" %(ts[0], ts[1]), "bin/targets/", "%s/targets/" %(rsync_url)],
- env={'RSYNC_PASSWORD': rsync_key},
+ name = "dirprepare",
+ description = "Preparing upload directory structure",
+ command = ["mkdir", "-p", WithProperties("tmp/upload/%%(prefix)stargets/%s/%s" %(ts[0], ts[1]), prefix=GetVersionPrefix)],
+ haltOnFailure = True
+ ))
+
+ factory.addStep(ShellCommand(
+ name = "linkprepare",
+ description = "Preparing repository symlink",
+ command = ["ln", "-s", "-f", WithProperties("../packages-%(basever)s", basever=GetBaseVersion), WithProperties("tmp/upload/%(prefix)spackages", prefix=GetVersionPrefix)],
+ doStepIf = IsNoMasterBuild,
+ haltOnFailure = True
+ ))
+
+ factory.addStep(ShellCommand(
+ name = "dirupload",
+ description = "Uploading directory structure",
+ command = ["rsync", "-avz", "tmp/upload/", "%s/" %(rsync_bin_url)],
+ env={'RSYNC_PASSWORD': rsync_bin_key},
haltOnFailure = True,
logEnviron = False
))
factory.addStep(ShellCommand(
name = "targetupload",
description = "Uploading target files",
- command=["rsync", "--delete", "--delay-updates", "-avz", "bin/targets/%s/%s/" %(ts[0], ts[1]), "%s/targets/%s/%s/" %(rsync_url, ts[0], ts[1])],
- env={'RSYNC_PASSWORD': rsync_key},
+ command=["rsync", "--delete", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]),
+ "-avz", WithProperties("bin/targets/%s/%s%%(libc)s/" %(ts[0], ts[1])),
+ WithProperties("%s/%%(prefix)stargets/%s/%s/" %(rsync_bin_url, ts[0], ts[1]), prefix=GetVersionPrefix)],
+ env={'RSYNC_PASSWORD': rsync_bin_key},
haltOnFailure = True,
logEnviron = False
))
+ if rsync_src_url is not None:
+ factory.addStep(ShellCommand(
+ name = "sourceupload",
+ description = "Uploading source archives",
+ command=["rsync", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "dl/", "%s/" %(rsync_src_url)],
+ env={'RSYNC_PASSWORD': rsync_src_key},
+ haltOnFailure = True,
+ logEnviron = False
+ ))
+
if False:
factory.addStep(ShellCommand(
name = "packageupload",
description = "Uploading package files",
- command=["rsync", "--delete", "--delay-updates", "-avz", "bin/packages/", "%s/packages/" %(rsync_url)],
- env={'RSYNC_PASSWORD': rsync_key},
+ command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "bin/packages/", "%s/packages/" %(rsync_bin_url)],
+ env={'RSYNC_PASSWORD': rsync_bin_key},
haltOnFailure = False,
logEnviron = False
))
factory.addStep(ShellCommand(
name = "upload",
description = "Uploading logs",
- command=["rsync", "--delete", "--delay-updates", "-avz", "logs/", "%s/logs/%s/%s/" %(rsync_url, ts[0], ts[1])],
- env={'RSYNC_PASSWORD': rsync_key},
+ command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "logs/", "%s/logs/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
+ env={'RSYNC_PASSWORD': rsync_bin_key},
haltOnFailure = False,
alwaysRun = True,
logEnviron = False
# change any of these to True to enable; see the manual for more
# options
auth=auth.BasicAuth([(ini.get("status", "user"), ini.get("status", "password"))]),
- gracefulShutdown = False,
+ gracefulShutdown = 'auth',
forceBuild = 'auth', # use this to test your slave once it is set up
forceAllBuilds = 'auth',
pingBuilder = False,
else:
c['status'].append(html.WebStatus(http_port=ini.get("status", "bind")))
+
+from buildbot.status import words
+
+if ini.has_option("irc", "host") and ini.has_option("irc", "nickname") and ini.has_option("irc", "channel"):
+ irc_host = ini.get("irc", "host")
+ irc_port = 6667
+ irc_chan = ini.get("irc", "channel")
+ irc_nick = ini.get("irc", "nickname")
+ irc_pass = None
+
+ if ini.has_option("irc", "port"):
+ irc_port = ini.getint("irc", "port")
+
+ if ini.has_option("irc", "password"):
+ irc_pass = ini.get("irc", "password")
+
+ irc = words.IRC(irc_host, irc_nick, port = irc_port, password = irc_pass,
+ channels = [{ "channel": irc_chan }],
+ notify_events = {
+ 'exception': 1,
+ 'successToFailure': 1,
+ 'failureToSuccess': 1
+ }
+ )
+
+ c['status'].append(irc)
+
+
####### PROJECT IDENTITY
# the 'title' string will appear at the top of this buildbot
# this at its default for all but the largest installations.
'db_url' : "sqlite:///state.sqlite",
}
-
projects / buildbot.git / blobdiff