import os
import re
+import sys
import base64
-import random
import subprocess
import configparser
from buildbot.schedulers.basic import SingleBranchScheduler
from buildbot.schedulers.forcesched import ForceScheduler
from buildbot.steps.master import MasterShellCommand
-from buildbot.steps.shell import SetProperty
+from buildbot.steps.shell import SetPropertyFromCommand
from buildbot.steps.shell import ShellCommand
from buildbot.steps.transfer import FileDownload
from buildbot.steps.transfer import FileUpload
worker_port = 9990
persistent = False
-other_builds = 0
tree_expire = 0
git_ssh = False
git_ssh_key = None
if ini.has_option("phase2", "persistent"):
persistent = ini.getboolean("phase2", "persistent")
-if ini.has_option("phase2", "other_builds"):
- other_builds = ini.getint("phase2", "other_builds")
-
if ini.has_option("phase2", "expire"):
tree_expire = ini.getint("phase2", "expire")
for section in ini.sections():
if section.startswith("worker "):
if ini.has_option(section, "name") and ini.has_option(section, "password") and \
- ini.has_option(section, "phase") and ini.getint(section, "phase") == 2:
+ ini.has_option(section, "phase") and ini.getint(section, "phase") == 2:
name = ini.get(section, "name")
password = ini.get(section, "password")
sl_props = { 'shared_wd': False }
def parse_feed_entry(line):
parts = line.strip().split()
- if parts[0] == "src-git":
+ if parts[0].startswith("src-git"):
feeds.append(parts)
url = parts[2].strip().split(';')
branch = url[1] if len(url) > 1 else 'master'
line = make.stdout.readline()
if line:
- parse_feed_entry(line)
+ parse_feed_entry(str(line, 'utf-8'))
-with open(work_dir+'/source.git/feeds.conf.default', 'r') as f:
+with open(work_dir+'/source.git/feeds.conf.default', 'r', encoding='utf-8') as f:
for line in f:
parse_feed_entry(line)
+if len(c['change_source']) == 0:
+ log.err("FATAL ERROR: no change_sources defined, aborting!")
+ sys.exit(-1)
####### SCHEDULERS
def GetNumJobs(props):
if props.hasProperty("workername") and props.hasProperty("nproc"):
- return ((int(props["nproc"]) / (max_builds[props["workername"]] + other_builds)) + 1)
+ return (int(props["nproc"]) / max_builds[props["workername"]])
else:
return 1
def UsignSec2Pub(seckey, comment="untrusted comment: secret key"):
try:
seckey = base64.b64decode(seckey)
- except:
+ except Exception:
return None
return "{}\n{}".format(re.sub(r"\bsecret key$", "public key", comment),
if not completed:
return
- return completed[0]['complete_at']
+ complete_at = completed[0]['complete_at']
+
+ last_build = yield bldr.master.data.get(
+ ('builds', ),
+ [
+ resultspec.Filter('builderid', 'eq', [bldrid]),
+ ],
+ order=['-started_at'], limit=1)
+
+ if last_build and last_build[0]:
+ last_complete_at = last_build[0]['complete_at']
+ if last_complete_at and (last_complete_at > complete_at):
+ return last_complete_at
+
+ return complete_at
@defer.inlineCallbacks
def prioritizeBuilders(master, builders):
doStepIf = IsSharedWorkdir))
# find number of cores
- factory.addStep(SetProperty(
+ factory.addStep(SetPropertyFromCommand(
name = "nproc",
property = "nproc",
description = "Finding number of CPUs",
workerdest = "sdk/getversion.mk",
mode = 0o755))
- factory.addStep(SetProperty(
+ factory.addStep(SetPropertyFromCommand(
name = "getversion",
property = "release_version",
description = "Finding SDK release version",
description = "Clearing failure logs",
workdir = "build/sdk",
command = ["rm", "-rf", "logs/package/error.txt", "faillogs/"],
- haltOnFailure = False
+ haltOnFailure = False,
+ flunkOnFailure = False,
+ warnOnFailure = True,
))
factory.addStep(ShellCommand(
description = "Finding failure logs",
workdir = "build/sdk/logs/package/feeds",
command = ["sh", "-c", "sed -ne 's!^ *ERROR: package/feeds/\\([^ ]*\\) .*$!\\1!p' ../error.txt | sort -u | xargs -r find > ../../../logs.txt"],
- haltOnFailure = False
+ haltOnFailure = False,
+ flunkOnFailure = False,
+ warnOnFailure = True,
))
factory.addStep(ShellCommand(
description = "Collecting failure logs",
workdir = "build/sdk",
command = ["rsync", "-av", "--files-from=logs.txt", "logs/package/feeds/", "faillogs/"],
- haltOnFailure = False
+ haltOnFailure = False,
+ flunkOnFailure = False,
+ warnOnFailure = True,
))
factory.addStep(ShellCommand(
command = ["rsync", "-4", "--progress", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-avz", "faillogs/", WithProperties("%s/faillogs%%(suffix)s/%s/" %(rsync_bin_url, arch[0]), suffix=GetDirectorySuffix)],
env={'RSYNC_PASSWORD': rsync_bin_key},
haltOnFailure = False,
+ flunkOnFailure = False,
+ warnOnFailure = True,
logEnviron = False
))
name = "sourcelist",
description = "Finding source archives to upload",
workdir = "build/sdk",
- command = "find dl/ -maxdepth 1 -type f -not -size 0 -not -name '.*' -newer ../sdk.archive -printf '%f\\n' > sourcelist",
+ command = "find dl/ -maxdepth 1 -type f -not -size 0 -not -name '.*' -not -name '*.hash' -not -name '*.dl' -newer ../sdk.archive -printf '%f\\n' > sourcelist",
haltOnFailure = True
))
description = "Uploading source archives",
workdir = "build/sdk",
command = ["rsync", "--files-from=sourcelist", "-4", "--progress", "--checksum", "--delay-updates",
- WithProperties("--partial-dir=.~tmp~%s~%%(workername)s" %(arch[0])), "-avz", "dl/", "%s/" %(rsync_src_url)],
+ WithProperties("--partial-dir=.~tmp~%s~%%(workername)s" %(arch[0])), "-avz", "dl/", "%s/" %(rsync_src_url)],
env={'RSYNC_PASSWORD': rsync_src_key},
haltOnFailure = False,
+ flunkOnFailure = False,
+ warnOnFailure = True,
logEnviron = False
))
command=["df", "-h", "."],
env={'LC_ALL': 'C'},
haltOnFailure = False,
+ flunkOnFailure = False,
+ warnOnFailure = False,
+ alwaysRun = True
+ ))
+
+ factory.addStep(ShellCommand(
+ name = "du",
+ description = "Reporting estimated file space usage",
+ command=["du", "-sh", "."],
+ env={'LC_ALL': 'C'},
+ haltOnFailure = False,
+ flunkOnFailure = False,
+ warnOnFailure = False,
alwaysRun = True
))
projects / buildbot.git / blobdiff