if not os.path.exists("twistd.pid"):
- with open("twistd.pid", "w") as pidfile:
- pidfile.write("{}".format(os.getpid()))
+ with open("twistd.pid", "w") as pidfile:
+ pidfile.write("{}".format(os.getpid()))
# This is a sample buildmaster config file. It must be installed as
# 'master.cfg' in your buildmaster's base directory.
ini = configparser.ConfigParser()
ini.read(os.getenv("BUILDMASTER_CONFIG", "./config.ini"))
+if "general" not in ini or "phase1" not in ini or "rsync" not in ini:
+ raise ValueError("Fix your configuration")
+
+inip1 = ini['phase1']
+
+# Globals
+work_dir = os.path.abspath(ini['general'].get("workdir", "."))
+scripts_dir = os.path.abspath("../scripts")
+
+config_seed = inip1.get("config_seed", "")
+
+repo_url = ini['repo'].get("url")
+repo_branch = ini['repo'].get("branch", "master")
+
+rsync_bin_url = ini['rsync'].get("binary_url")
+rsync_bin_key = ini['rsync'].get("binary_password")
+rsync_bin_defopts = ["-v", "-4", "--timeout=120"]
+
+if rsync_bin_url.find("::") > 0 or rsync_bin_url.find("rsync://") == 0:
+ rsync_bin_defopts += ["--contimeout=20"]
+
+rsync_src_url = ini['rsync'].get("source_url")
+rsync_src_key = ini['rsync'].get("source_password")
+rsync_src_defopts = ["-v", "-4", "--timeout=120"]
+
+if rsync_src_url.find("::") > 0 or rsync_src_url.find("rsync://") == 0:
+ rsync_src_defopts += ["--contimeout=20"]
+
+usign_key = None
+usign_comment = "untrusted comment: " + repo_branch.replace("-", " ").title() + " key"
+
+if ini.has_section("usign"):
+ usign_key = ini['usign'].get("key")
+ usign_comment = ini['usign'].get("comment", usign_comment)
+
+enable_kmod_archive = inip1.getboolean("kmod_archive", False)
+
+# PB port can be either a numeric port or a connection string
+pb_port = inip1.get("port") or 9989
+
# This is the dictionary that the buildmaster pays attention to. We also use
# a shorter alias to save typing.
c = BuildmasterConfig = {}
# installation's html.WebStatus home page (linked to the
# 'titleURL') and is embedded in the title of the waterfall HTML page.
-c['title'] = ini.get("general", "title")
-c['titleURL'] = ini.get("general", "title_url")
+c['title'] = ini['general'].get("title")
+c['titleURL'] = ini['general'].get("title_url")
# the 'buildbotURL' string should point to the location where the buildbot's
# internal web server (usually the html.WebStatus page) is visible. This
# with an externally-visible host name which the buildbot cannot figure out
# without some help.
-c['buildbotURL'] = ini.get("phase1", "buildbot_url")
+c['buildbotURL'] = inip1.get("buildbot_url")
####### BUILDWORKERS
# a Worker object, specifying a unique worker name and password. The same
# worker name and password must be configured on the worker.
-worker_port = 9989
-
-if ini.has_option("phase1", "port"):
- worker_port = ini.get("phase1", "port")
-
c['workers'] = []
NetLocks = dict()
if section.startswith("worker "):
if ini.has_option(section, "name") and ini.has_option(section, "password") and \
(not ini.has_option(section, "phase") or ini.getint(section, "phase") == 1):
- sl_props = { 'dl_lock':None, 'ul_lock':None, 'do_cleanup':False, 'max_builds':1, 'shared_wd':False }
+ sl_props = { 'dl_lock':None, 'ul_lock':None }
name = ini.get(section, "name")
password = ini.get(section, "password")
- max_builds = 1
- if ini.has_option(section, "builds"):
- max_builds = ini.getint(section, "builds")
- sl_props['max_builds'] = max_builds
- if max_builds == 1:
- sl_props['shared_wd'] = True
- if ini.has_option(section, "cleanup"):
- sl_props['do_cleanup'] = ini.getboolean(section, "cleanup")
if ini.has_option(section, "dl_lock"):
lockname = ini.get(section, "dl_lock")
sl_props['dl_lock'] = lockname
if lockname not in NetLocks:
NetLocks[lockname] = locks.MasterLock(lockname)
if ini.has_option(section, "ul_lock"):
- lockname = ini.get(section, "dl_lock")
+ lockname = ini.get(section, "ul_lock")
sl_props['ul_lock'] = lockname
if lockname not in NetLocks:
NetLocks[lockname] = locks.MasterLock(lockname)
- if ini.has_option(section, "shared_wd"):
- shared_wd = ini.getboolean(section, "shared_wd")
- sl_props['shared_wd'] = shared_wd
- if shared_wd and (max_builds != 1):
- raise ValueError('max_builds must be 1 with shared workdir!')
- c['workers'].append(Worker(name, password, max_builds = max_builds, properties = sl_props))
-
-# 'workerPortnum' defines the TCP port to listen on for connections from workers.
-# This must match the value configured into the buildworkers (with their
-# --master option)
-c['protocols'] = {'pb': {'port': worker_port}}
+ c['workers'].append(Worker(name, password, max_builds = 1, properties = sl_props))
+
+c['protocols'] = {'pb': {'port': pb_port}}
# coalesce builds
c['collapseRequests'] = True
# Reduce amount of backlog data
c['configurators'] = [util.JanitorConfigurator(
- logHorizon=timedelta(days=3),
- hour=6,
+ logHorizon=timedelta(days=3),
+ hour=6,
)]
@defer.inlineCallbacks
if not completed:
return
- return completed[0]['complete_at']
+ complete_at = completed[0]['complete_at']
+
+ last_build = yield bldr.master.data.get(
+ ('builds', ),
+ [
+ resultspec.Filter('builderid', 'eq', [bldrid]),
+ ],
+ order=['-started_at'], limit=1)
+
+ if last_build and last_build[0]:
+ last_complete_at = last_build[0]['complete_at']
+ if last_complete_at and (last_complete_at > complete_at):
+ return last_complete_at
+
+ return complete_at
@defer.inlineCallbacks
def prioritizeBuilders(master, builders):
####### CHANGESOURCES
-work_dir = os.path.abspath(ini.get("general", "workdir") or ".")
-scripts_dir = os.path.abspath("../scripts")
-tree_expire = 0
-other_builds = 0
-cc_version = None
-
-cc_command = "gcc"
-cxx_command = "g++"
-
-config_seed = ""
-
-git_ssh = False
-git_ssh_key = None
-
-if ini.has_option("phase1", "expire"):
- tree_expire = ini.getint("phase1", "expire")
-
-if ini.has_option("phase1", "other_builds"):
- other_builds = ini.getint("phase1", "other_builds")
-
-if ini.has_option("phase1", "cc_version"):
- cc_version = ini.get("phase1", "cc_version").split()
- if len(cc_version) == 1:
- cc_version = ["eq", cc_version[0]]
-
-if ini.has_option("general", "git_ssh"):
- git_ssh = ini.getboolean("general", "git_ssh")
-
-if ini.has_option("general", "git_ssh_key"):
- git_ssh_key = ini.get("general", "git_ssh_key")
-else:
- git_ssh = False
-
-if ini.has_option("phase1", "config_seed"):
- config_seed = ini.get("phase1", "config_seed")
-
-repo_url = ini.get("repo", "url")
-repo_branch = "master"
-
-if ini.has_option("repo", "branch"):
- repo_branch = ini.get("repo", "branch")
-
-rsync_bin_url = ini.get("rsync", "binary_url")
-rsync_bin_key = ini.get("rsync", "binary_password")
-rsync_bin_defopts = ["-v", "-4", "--timeout=120"]
-
-if rsync_bin_url.find("::") > 0 or rsync_bin_url.find("rsync://") == 0:
- rsync_bin_defopts += ["--contimeout=20"]
-
-rsync_src_url = None
-rsync_src_key = None
-rsync_src_defopts = ["-v", "-4", "--timeout=120"]
-
-if ini.has_option("rsync", "source_url"):
- rsync_src_url = ini.get("rsync", "source_url")
- rsync_src_key = ini.get("rsync", "source_password")
-
- if rsync_src_url.find("::") > 0 or rsync_src_url.find("rsync://") == 0:
- rsync_src_defopts += ["--contimeout=20"]
-
-usign_key = None
-usign_comment = "untrusted comment: " + repo_branch.replace("-", " ").title() + " key"
-
-if ini.has_option("usign", "key"):
- usign_key = ini.get("usign", "key")
-
-if ini.has_option("usign", "comment"):
- usign_comment = ini.get("usign", "comment")
-
-enable_kmod_archive = False
-embed_kmod_repository = False
-
-if ini.has_option("phase1", "kmod_archive"):
- enable_kmod_archive = ini.getboolean("phase1", "kmod_archive")
-
-if ini.has_option("phase1", "kmod_repository"):
- embed_kmod_repository = ini.getboolean("phase1", "kmod_repository")
-
# find targets
targets = [ ]
-if not os.path.isdir(work_dir+'/source.git'):
- subprocess.call(["git", "clone", "--depth=1", "--branch="+repo_branch, repo_url, work_dir+'/source.git'])
-else:
- subprocess.call(["git", "pull"], cwd = work_dir+'/source.git')
+def populateTargets():
+ sourcegit = work_dir + '/source.git'
+ if os.path.isdir(sourcegit):
+ subprocess.call(["rm", "-rf", sourcegit])
+
+ subprocess.call(["git", "clone", "--depth=1", "--branch="+repo_branch, repo_url, sourcegit])
+
+ os.makedirs(sourcegit + '/tmp', exist_ok=True)
+ findtargets = subprocess.Popen(['./scripts/dump-target-info.pl', 'targets'],
+ stdout = subprocess.PIPE, stderr = subprocess.DEVNULL, cwd = sourcegit)
-os.makedirs(work_dir+'/source.git/tmp', exist_ok=True)
-findtargets = subprocess.Popen(['./scripts/dump-target-info.pl', 'targets'],
- stdout = subprocess.PIPE, cwd = work_dir+'/source.git')
+ while True:
+ line = findtargets.stdout.readline()
+ if not line:
+ break
+ ta = line.decode().strip().split(' ')
+ targets.append(ta[0])
-while True:
- line = findtargets.stdout.readline()
- if not line:
- break
- ta = line.decode().strip().split(' ')
- targets.append(ta[0])
+ subprocess.call(["rm", "-rf", sourcegit])
+populateTargets()
# the 'change_source' setting tells the buildmaster how it should find out
# about source code changes. Here we point to the buildbot clone of pyflakes.
# what steps, and which workers can execute them. Note that any particular build will
# only take place on one worker.
-CleanTargetMap = [
- [ "tools", "tools/clean" ],
- [ "chain", "toolchain/clean" ],
- [ "linux", "target/linux/clean" ],
- [ "dir", "dirclean" ],
- [ "dist", "distclean" ]
-]
-
-def IsMakeCleanRequested(pattern):
- def CheckCleanProperty(step):
- val = step.getProperty("clean")
- if val and re.match(pattern, val):
- return True
- else:
- return False
-
- return CheckCleanProperty
-
-def IsSharedWorkdir(step):
- return bool(step.getProperty("shared_wd"))
-
-def IsCleanupRequested(step):
- if IsSharedWorkdir(step):
- return False
- do_cleanup = step.getProperty("do_cleanup")
- if do_cleanup:
- return True
- else:
- return False
-
-def IsExpireRequested(step):
- if IsSharedWorkdir(step):
- return False
- else:
- return not IsCleanupRequested(step)
-
-def IsGitFreshRequested(step):
- do_cleanup = step.getProperty("do_cleanup")
- if do_cleanup:
- return True
- else:
- return False
-
-def IsGitCleanRequested(step):
- return not IsGitFreshRequested(step)
-
def IsTaggingRequested(step):
val = step.getProperty("tag")
if val and re.match(r"^[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", val):
else:
return False
-def IsNoTaggingRequested(step):
- return not IsTaggingRequested(step)
-
def IsNoMasterBuild(step):
- return repo_branch != "master"
+ return step.getProperty("branch") != "master"
-def GetBaseVersion():
- if re.match(r"^[^-]+-[0-9]+\.[0-9]+$", repo_branch):
- return repo_branch.split('-')[1]
+def GetBaseVersion(branch):
+ if re.match(r"^[^-]+-[0-9]+\.[0-9]+$", branch):
+ return branch.split('-')[1]
else:
return "master"
@properties.renderer
def GetVersionPrefix(props):
- basever = GetBaseVersion()
+ branch = props.getProperty("branch")
+ basever = GetBaseVersion(branch)
if props.hasProperty("tag") and re.match(r"^[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", props["tag"]):
return "%s/" % props["tag"]
elif basever != "master":
else:
return ""
-@properties.renderer
-def GetNumJobs(props):
- if props.hasProperty("max_builds") and props.hasProperty("nproc"):
- return str(int(int(props["nproc"]) / (props["max_builds"] + other_builds)))
- else:
- return "1"
-
-@properties.renderer
-def GetCC(props):
- if props.hasProperty("cc_command"):
- return props["cc_command"]
- else:
- return "gcc"
-
-@properties.renderer
-def GetCXX(props):
- if props.hasProperty("cxx_command"):
- return props["cxx_command"]
- else:
- return "g++"
-
-@properties.renderer
-def GetCwd(props):
- if props.hasProperty("builddir"):
- return props["builddir"]
- elif props.hasProperty("workdir"):
- return props["workdir"]
- else:
- return "/"
-
-@properties.renderer
-def GetCCache(props):
- if props.hasProperty("ccache_command") and "ccache" in props["ccache_command"]:
- return props["ccache_command"]
- else:
- return ""
-
def GetNextBuild(builder, requests):
for r in requests:
if r.properties and r.properties.hasProperty("tag"):
def MakeEnv(overrides=None, tryccache=False):
env = {
- 'CCC': Interpolate("%(kw:cc)s", cc=GetCC),
- 'CCXX': Interpolate("%(kw:cxx)s", cxx=GetCXX),
+ 'CCC': Interpolate("%(prop:cc_command:-gcc)s"),
+ 'CCXX': Interpolate("%(prop:cxx_command:-g++)s"),
}
if tryccache:
- env['CC'] = Interpolate("%(kw:cwd)s/ccache_cc.sh", cwd=GetCwd)
- env['CXX'] = Interpolate("%(kw:cwd)s/ccache_cxx.sh", cwd=GetCwd)
- env['CCACHE'] = Interpolate("%(kw:ccache)s", ccache=GetCCache)
+ env['CC'] = Interpolate("%(prop:builddir)s/ccache_cc.sh")
+ env['CXX'] = Interpolate("%(prop:builddir)s/ccache_cxx.sh")
+ env['CCACHE'] = Interpolate("%(prop:ccache_command:-)s")
else:
env['CC'] = env['CCC']
env['CXX'] = env['CCXX']
dlLock = locks.WorkerLock("worker_dl")
-checkBuiltin = re.sub('[\t\n ]+', ' ', """
- checkBuiltin() {
- local symbol op path file;
- for file in $CHANGED_FILES; do
- case "$file" in
- package/*/*) : ;;
- *) return 0 ;;
- esac;
- done;
- while read symbol op path; do
- case "$symbol" in package-*)
- symbol="${symbol##*(}";
- symbol="${symbol%)}";
- for file in $CHANGED_FILES; do
- case "$file" in "package/$path/"*)
- grep -qsx "$symbol=y" .config && return 0
- ;; esac;
- done;
- esac;
- done < tmp/.packagedeps;
- return 1;
- }
-""").strip()
-
-
-class IfBuiltinShellCommand(ShellCommand):
- def _quote(self, str):
- if re.search("[^a-zA-Z0-9/_.-]", str):
- return "'%s'" %(re.sub("'", "'\"'\"'", str))
- return str
-
- def setCommand(self, command):
- if not isinstance(command, (str, unicode)):
- command = ' '.join(map(self._quote, command))
- self.command = [
- '/bin/sh', '-c',
- '%s; if checkBuiltin; then %s; else exit 0; fi' %(checkBuiltin, command)
- ]
-
- def setupEnvironment(self, cmd):
- workerEnv = self.workerEnvironment
- if workerEnv is None:
- workerEnv = { }
- changedFiles = { }
- for request in self.build.requests:
- for source in request.sources:
- for change in source.changes:
- for file in change.files:
- changedFiles[file] = True
- fullSlaveEnv = workerEnv.copy()
- fullSlaveEnv['CHANGED_FILES'] = ' '.join(changedFiles.keys())
- cmd.args['env'] = fullSlaveEnv
-
workerNames = [ ]
for worker in c['workers']:
description = "Setting up shared work directory",
command = 'test -L "$PWD" || (mkdir -p ../shared-workdir && rm -rf "$PWD" && ln -s shared-workdir "$PWD")',
workdir = ".",
- haltOnFailure = True,
- doStepIf = IsSharedWorkdir))
+ haltOnFailure = True))
# find number of cores
factory.addStep(SetPropertyFromCommand(
property = "cc_command",
description = "Finding gcc command",
command = [
- "../findbin.pl", "gcc",
- cc_version[0] if cc_version is not None else '',
- cc_version[1] if cc_version is not None else ''
+ "../findbin.pl", "gcc", "", "",
],
haltOnFailure = True))
property = "cxx_command",
description = "Finding g++ command",
command = [
- "../findbin.pl", "g++",
- cc_version[0] if cc_version is not None else '',
- cc_version[1] if cc_version is not None else ''
+ "../findbin.pl", "g++", "", "",
],
haltOnFailure = True))
warnOnFailure = False,
))
- # expire tree if needed
- if tree_expire > 0:
- factory.addStep(FileDownload(
- name = "dlexpiresh",
- doStepIf = IsExpireRequested,
- mastersrc = scripts_dir + '/expire.sh',
- workerdest = "../expire.sh",
- mode = 0o755))
-
- factory.addStep(ShellCommand(
- name = "expire",
- description = "Checking for build tree expiry",
- command = ["./expire.sh", str(tree_expire)],
- workdir = ".",
- haltOnFailure = True,
- doStepIf = IsExpireRequested,
- timeout = 2400))
-
- # cleanup.sh if needed
- factory.addStep(FileDownload(
- name = "dlcleanupsh",
- mastersrc = scripts_dir + '/cleanup.sh',
- workerdest = "../cleanup.sh",
- mode = 0o755,
- doStepIf = IsCleanupRequested))
-
- factory.addStep(ShellCommand(
- name = "cleanold",
- description = "Cleaning previous builds",
- command = ["./cleanup.sh", c['buildbotURL'], Interpolate("%(prop:workername)s"), Interpolate("%(prop:buildername)s"), "full"],
- workdir = ".",
- haltOnFailure = True,
- doStepIf = IsCleanupRequested,
- timeout = 2400))
-
- factory.addStep(ShellCommand(
- name = "cleanup",
- description = "Cleaning work area",
- command = ["./cleanup.sh", c['buildbotURL'], Interpolate("%(prop:workername)s"), Interpolate("%(prop:buildername)s"), "single"],
- workdir = ".",
- haltOnFailure = True,
- doStepIf = IsCleanupRequested,
- timeout = 2400))
-
- # user-requested clean targets
- for tuple in CleanTargetMap:
- factory.addStep(ShellCommand(
- name = tuple[1],
- description = 'User-requested "make %s"' % tuple[1],
- command = ["make", tuple[1], "V=s"],
- env = MakeEnv(),
- doStepIf = IsMakeCleanRequested(tuple[0])
- ))
-
# Workaround bug when switching from a checked out tag back to a branch
# Ref: http://lists.infradead.org/pipermail/openwrt-devel/2019-June/017809.html
factory.addStep(ShellCommand(
name = "gitcheckout",
description = "Ensure that Git HEAD is sane",
- command = "if [ -d .git ]; then git checkout -f %s && git branch --set-upstream-to origin/%s || rm -fr .git; else exit 0; fi" %(repo_branch, repo_branch),
+ command = Interpolate("if [ -d .git ]; then git checkout -f %(prop:branch)s && git branch --set-upstream-to origin/%(prop:branch)s || rm -fr .git; else exit 0; fi"),
haltOnFailure = True))
# check out the source
# if repo doesn't exist: 'git clone repourl'
# method 'clean' runs 'git clean -d -f', method fresh runs 'git clean -d -f x'. Only works with mode='full'
# 'git fetch -t repourl branch; git reset --hard revision'
- # Git() parameters can't take a renderer until buildbot 0.8.10, so we have to split the fresh and clean cases
- # if buildbot is updated, one can use: method = Interpolate('%(prop:do_cleanup:#?|fresh|clean)s')
factory.addStep(Git(
- name = "gitclean",
+ name = "git",
repourl = repo_url,
- branch = repo_branch,
- mode = 'full',
- method = 'clean',
- haltOnFailure = True,
- doStepIf = IsGitCleanRequested,
- ))
-
- factory.addStep(Git(
- name = "gitfresh",
- repourl = repo_url,
- branch = repo_branch,
mode = 'full',
method = 'fresh',
+ locks = NetLockDl,
haltOnFailure = True,
- doStepIf = IsGitFreshRequested,
))
# update remote refs
factory.addStep(ShellCommand(
name = "fetchrefs",
description = "Fetching Git remote refs",
- command = ["git", "fetch", "origin", "+refs/heads/%s:refs/remotes/origin/%s" %(repo_branch, repo_branch)],
+ command = ["git", "fetch", "origin", Interpolate("+refs/heads/%(prop:branch)s:refs/remotes/origin/%(prop:branch)s")],
haltOnFailure = True
))
command=["rm", "-rf", "tmp/"]))
# feed
-# factory.addStep(ShellCommand(
-# name = "feedsconf",
-# description = "Copy the feeds.conf",
-# command='''cp ~/feeds.conf ./feeds.conf''' ))
-
- # feed
factory.addStep(ShellCommand(
name = "rmfeedlinks",
description = "Remove feed symlinks",
mode = 0o755,
))
- # Git SSH
- if git_ssh:
- factory.addStep(StringDownload(
- name = "dlgitclonekey",
- s = git_ssh_key,
- workerdest = "../git-clone.key",
- mode = 0o600,
- ))
-
- factory.addStep(ShellCommand(
- name = "patchfeedsconf",
- description = "Patching feeds.conf",
- command="sed -e 's#https://#ssh://git@#g' feeds.conf.default > feeds.conf",
- haltOnFailure = True
- ))
-
# feed
factory.addStep(ShellCommand(
name = "updatefeeds",
description = "Updating feeds",
command=["./scripts/feeds", "update"],
- env = MakeEnv(tryccache=True, overrides={'GIT_SSH_COMMAND': Interpolate("ssh -o IdentitiesOnly=yes -o IdentityFile=%(kw:cwd)s/git-clone.key -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no", cwd=GetCwd)} if git_ssh else {}),
- haltOnFailure = True
+ env = MakeEnv(tryccache=True),
+ haltOnFailure = True,
+ locks = NetLockDl,
))
- # Git SSH
- if git_ssh:
- factory.addStep(ShellCommand(
- name = "rmfeedsconf",
- description = "Removing feeds.conf",
- command=["rm", "feeds.conf"],
- haltOnFailure = True
- ))
-
# feed
factory.addStep(ShellCommand(
name = "installfeeds",
factory.addStep(ShellCommand(
name = "dltar",
description = "Building and installing GNU tar",
- command = ["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "tools/tar/compile", "V=s"],
+ command = ["make", Interpolate("-j%(prop:nproc:-1)s"), "tools/tar/compile", "V=s"],
env = MakeEnv(tryccache=True),
haltOnFailure = True
))
factory.addStep(ShellCommand(
name = "dlrun",
description = "Populating dl/",
- command = ["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "download", "V=s"],
+ command = ["make", Interpolate("-j%(prop:nproc:-1)s"), "download", "V=s"],
env = MakeEnv(),
logEnviron = False,
- locks = [dlLock.access('exclusive')],
+ locks = properties.FlattenList(NetLockDl, [dlLock.access('exclusive')]),
))
factory.addStep(ShellCommand(
factory.addStep(ShellCommand(
name = "tools",
description = "Building and installing tools",
- command = ["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "tools/install", "V=s"],
+ command = ["make", Interpolate("-j%(prop:nproc:-1)s"), "tools/install", "V=s"],
env = MakeEnv(tryccache=True),
haltOnFailure = True
))
factory.addStep(ShellCommand(
name = "toolchain",
description = "Building and installing toolchain",
- command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "toolchain/install", "V=s"],
+ command=["make", Interpolate("-j%(prop:nproc:-1)s"), "toolchain/install", "V=s"],
env = MakeEnv(),
haltOnFailure = True
))
factory.addStep(ShellCommand(
name = "kmods",
description = "Building kmods",
- command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
+ command=["make", Interpolate("-j%(prop:nproc:-1)s"), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
env = MakeEnv(),
- #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
haltOnFailure = True
))
property = "kernelversion",
description = "Finding the effective Kernel version",
command = "make --no-print-directory -C target/linux/ val.LINUX_VERSION val.LINUX_RELEASE val.LINUX_VERMAGIC | xargs printf '%s-%s-%s\\n'",
- env = { 'TOPDIR': Interpolate("%(kw:cwd)s/build", cwd=GetCwd) }
+ env = { 'TOPDIR': Interpolate("%(prop:builddir)s/build") }
))
factory.addStep(ShellCommand(
factory.addStep(ShellCommand(
name = "pkgbuild",
description = "Building packages",
- command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
+ command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
env = MakeEnv(),
- #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
haltOnFailure = True
))
- # factory.addStep(IfBuiltinShellCommand(
factory.addStep(ShellCommand(
name = "pkginstall",
description = "Installing packages",
- command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "package/install", "V=s"],
+ command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/install", "V=s"],
env = MakeEnv(),
haltOnFailure = True
))
factory.addStep(ShellCommand(
name = "pkgindex",
description = "Indexing packages",
- command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "package/index", "V=s", "CONFIG_SIGNED_PACKAGES="],
+ command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/index", "V=s", "CONFIG_SIGNED_PACKAGES="],
env = MakeEnv(),
haltOnFailure = True
))
- if enable_kmod_archive and embed_kmod_repository:
- # embed kmod repository. Must happen before 'images'
-
- # find rootfs staging directory
- factory.addStep(SetPropertyFromCommand(
- name = "stageroot",
- property = "stageroot",
- description = "Finding the rootfs staging directory",
- command=["make", "--no-print-directory", "val.STAGING_DIR_ROOT"],
- env = { 'TOPDIR': Interpolate("%(kw:cwd)s/build", cwd=GetCwd) },
- want_stderr = False
- ))
-
- factory.addStep(ShellCommand(
- name = "filesdir",
- description = "Creating file overlay directory",
- command=["mkdir", "-p", "files/etc/opkg"],
- haltOnFailure = True
- ))
-
- factory.addStep(ShellCommand(
- name = "kmodconfig",
- description = "Embedding kmod repository configuration",
- command=Interpolate("sed -e 's#^\\(src/gz .*\\)_core \\(.*\\)/packages$#&\\n\\1_kmods \\2/kmods/%(prop:kernelversion)s#' " +
- "%(prop:stageroot)s/etc/opkg/distfeeds.conf > files/etc/opkg/distfeeds.conf"),
- haltOnFailure = True
- ))
-
- #factory.addStep(IfBuiltinShellCommand(
factory.addStep(ShellCommand(
name = "images",
description = "Building and installing images",
- command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "target/install", "V=s"],
+ command=["make", Interpolate("-j%(prop:nproc:-1)s"), "target/install", "V=s"],
env = MakeEnv(),
haltOnFailure = True
))
name = "kmodprepare",
description = "Preparing kmod archive",
command=["rsync", "--include=/kmod-*.ipk", "--exclude=*", "-va",
- Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/packages/", target=ts[0], subtarget=ts[1]),
- Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1])],
+ Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/packages/", target=ts[0], subtarget=ts[1]),
+ Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1])],
haltOnFailure = True
))
factory.addStep(ShellCommand(
name = "kmodindex",
description = "Indexing kmod archive",
- command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "package/index", "V=s", "CONFIG_SIGNED_PACKAGES=",
- Interpolate("PACKAGE_SUBDIRS=bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1])],
+ command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/index", "V=s", "CONFIG_SIGNED_PACKAGES=",
+ Interpolate("PACKAGE_SUBDIRS=bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1])],
env = MakeEnv(),
haltOnFailure = True
))
factory.addStep(ShellCommand(
name = "linkprepare",
description = "Preparing repository symlink",
- command = ["ln", "-s", "-f", Interpolate("../packages-%(kw:basever)s", basever=GetBaseVersion()), Interpolate("tmp/upload/%(kw:prefix)spackages", prefix=GetVersionPrefix)],
+ command = ["ln", "-s", "-f", Interpolate("../packages-%(kw:basever)s", basever=util.Transform(GetBaseVersion, Property("branch"))), Interpolate("tmp/upload/%(kw:prefix)spackages", prefix=GetVersionPrefix)],
doStepIf = IsNoMasterBuild,
haltOnFailure = True
))
env={'RSYNC_PASSWORD': rsync_bin_key},
haltOnFailure = True,
logEnviron = False,
+ locks = NetLockUl,
))
# download remote sha256sums to 'target-sha256sums'
name = "targetupload",
description = "Uploading target files",
command=["../rsync.sh", "--exclude=/kmods/", "--files-from=rsynclist", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_bin_defopts +
- ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]),
- Interpolate("%(kw:rsyncbinurl)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", rsyncbinurl=rsync_bin_url, target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
+ ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]),
+ Interpolate("%(kw:rsyncbinurl)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", rsyncbinurl=rsync_bin_url, target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
env={'RSYNC_PASSWORD': rsync_bin_key},
haltOnFailure = True,
logEnviron = False,
name = "targetprune",
description = "Pruning target files",
command=["../rsync.sh", "--exclude=/kmods/", "--delete", "--existing", "--ignore-existing", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_bin_defopts +
- ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]),
- Interpolate("%(kw:rsyncbinurl)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", rsyncbinurl=rsync_bin_url, target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
+ ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]),
+ Interpolate("%(kw:rsyncbinurl)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", rsyncbinurl=rsync_bin_url, target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
env={'RSYNC_PASSWORD': rsync_bin_key},
haltOnFailure = True,
logEnviron = False,
+ locks = NetLockUl,
))
if enable_kmod_archive:
name = "kmodupload",
description = "Uploading kmod archive",
command=["../rsync.sh", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_bin_defopts +
- ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1]),
- Interpolate("%(kw:rsyncbinurl)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s/", rsyncbinurl=rsync_bin_url, target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
+ ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1]),
+ Interpolate("%(kw:rsyncbinurl)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s/", rsyncbinurl=rsync_bin_url, target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
env={'RSYNC_PASSWORD': rsync_bin_key},
haltOnFailure = True,
logEnviron = False,
+ locks = NetLockUl,
))
if rsync_src_url is not None:
name = "sourceupload",
description = "Uploading source archives",
command=["../rsync.sh", "--files-from=sourcelist", "--size-only", "--delay-updates"] + rsync_src_defopts +
- [Interpolate("--partial-dir=.~tmp~%(kw:target)s~%(kw:subtarget)s~%(prop:workername)s", target=ts[0], subtarget=ts[1]), "-a", "dl/", "%s/" %(rsync_src_url)],
+ [Interpolate("--partial-dir=.~tmp~%(kw:target)s~%(kw:subtarget)s~%(prop:workername)s", target=ts[0], subtarget=ts[1]), "-a", "dl/", "%s/" %(rsync_src_url)],
env={'RSYNC_PASSWORD': rsync_src_key},
haltOnFailure = True,
logEnviron = False,
- ))
-
- if False:
- factory.addStep(ShellCommand(
- name = "packageupload",
- description = "Uploading package files",
- command=["../rsync.sh", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-a"] + rsync_bin_defopts + ["bin/packages/", "%s/packages/" %(rsync_bin_url)],
- env={'RSYNC_PASSWORD': rsync_bin_key},
- haltOnFailure = False,
- flunkOnFailure = False,
- warnOnFailure = True,
- logEnviron = False,
- ))
-
- # logs
- if False:
- factory.addStep(ShellCommand(
- name = "upload",
- description = "Uploading logs",
- command=["../rsync.sh", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-az"] + rsync_bin_defopts + ["logs/", "%s/logs/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
- env={'RSYNC_PASSWORD': rsync_bin_key},
- haltOnFailure = False,
- flunkOnFailure = False,
- warnOnFailure = True,
- alwaysRun = True,
- logEnviron = False,
+ locks = NetLockUl,
))
factory.addStep(ShellCommand(
alwaysRun = True
))
+ factory.addStep(ShellCommand(
+ name = "du",
+ description = "Reporting estimated file space usage",
+ command=["du", "-sh", "."],
+ env={'LC_ALL': 'C'},
+ haltOnFailure = False,
+ flunkOnFailure = False,
+ warnOnFailure = False,
+ alwaysRun = True
+ ))
+
factory.addStep(ShellCommand(
name = "ccachestat",
description = "Reporting ccache stats",
# pushed to these targets. buildbot/status/*.py has a variety to choose from,
# including web pages, email senders, and IRC bots.
-if ini.has_option("phase1", "status_bind"):
+if "status_bind" in inip1:
c['www'] = {
- 'port': ini.get("phase1", "status_bind"),
+ 'port': inip1.get("status_bind"),
'plugins': {
'waterfall_view': True,
'console_view': True,
}
}
- if ini.has_option("phase1", "status_user") and ini.has_option("phase1", "status_password"):
+ if "status_user" in inip1 and "status_password" in inip1:
c['www']['auth'] = util.UserPasswordAuth([
- (ini.get("phase1", "status_user"), ini.get("phase1", "status_password"))
+ (inip1.get("status_user"), inip1.get("status_password"))
])
c['www']['authz'] = util.Authz(
allowRules=[ util.AnyControlEndpointMatcher(role="admins") ],
- roleMatchers=[ util.RolesFromUsername(roles=["admins"], usernames=[ini.get("phase1", "status_user")]) ]
+ roleMatchers=[ util.RolesFromUsername(roles=["admins"], usernames=[inip1.get("status_user")]) ]
)
c['services'] = []
-if ini.has_option("irc", "host") and ini.has_option("irc", "nickname") and ini.has_option("irc", "channel"):
- irc_host = ini.get("irc", "host")
- irc_port = 6667
- irc_chan = ini.get("irc", "channel")
- irc_nick = ini.get("irc", "nickname")
- irc_pass = None
-
- if ini.has_option("irc", "port"):
- irc_port = ini.getint("irc", "port")
-
- if ini.has_option("irc", "password"):
- irc_pass = ini.get("irc", "password")
-
- irc = reporters.IRC(irc_host, irc_nick,
- port = irc_port,
- password = irc_pass,
- channels = [ irc_chan ],
- notify_events = [ 'exception', 'problem', 'recovery' ]
- )
-
- c['services'].append(irc)
+if ini.has_section("irc"):
+ iniirc = ini['irc']
+ irc_host = iniirc.get("host", None)
+ irc_port = iniirc.getint("port", 6667)
+ irc_chan = iniirc.get("channel", None)
+ irc_nick = iniirc.get("nickname", None)
+ irc_pass = iniirc.get("password", None)
+
+ if irc_host and irc_nick and irc_chan:
+ irc = reporters.IRC(irc_host, irc_nick,
+ port = irc_port,
+ password = irc_pass,
+ channels = [ irc_chan ],
+ notify_events = [ 'exception', 'problem', 'recovery' ]
+ )
+
+ c['services'].append(irc)
c['revlink'] = util.RevlinkMatch([
r'https://git.openwrt.org/openwrt/(.*).git'
projects / buildbot.git / blobdiff