# -*- python -*- # ex: set syntax=python: import os import re import base64 import subprocess import configparser from buildbot import locks from buildbot.changes import filter from buildbot.changes.gitpoller import GitPoller from buildbot.config import BuilderConfig from buildbot.plugins import schedulers from buildbot.plugins import steps from buildbot.plugins import util from buildbot.process.factory import BuildFactory from buildbot.process.properties import Property from buildbot.process.properties import WithProperties from buildbot.schedulers.basic import SingleBranchScheduler from buildbot.schedulers.forcesched import ForceScheduler from buildbot.steps.master import MasterShellCommand from buildbot.steps.shell import SetProperty from buildbot.steps.shell import ShellCommand from buildbot.steps.transfer import FileDownload from buildbot.steps.transfer import FileUpload from buildbot.steps.transfer import StringDownload from buildbot.worker import Worker ini = configparser.ConfigParser() ini.read(os.getenv("BUILDMASTER_CONFIG", "./config.ini")) buildbot_url = ini.get("phase2", "buildbot_url") # This is a sample buildmaster config file. It must be installed as # 'master.cfg' in your buildmaster's base directory. # This is the dictionary that the buildmaster pays attention to. We also use # a shorter alias to save typing. c = BuildmasterConfig = {} ####### BUILDSLAVES # The 'workers' list defines the set of recognized buildslaves. Each element is # a Worker object, specifying a unique slave name and password. The same # slave name and password must be configured on the slave. slave_port = 9990 persistent = False other_builds = 0 tree_expire = 0 git_ssh = False git_ssh_key = None if ini.has_option("phase2", "port"): slave_port = ini.get("phase2", "port") if ini.has_option("phase2", "persistent"): persistent = ini.getboolean("phase2", "persistent") if ini.has_option("phase2", "other_builds"): other_builds = ini.getint("phase2", "other_builds") if ini.has_option("phase2", "expire"): tree_expire = ini.getint("phase2", "expire") if ini.has_option("general", "git_ssh"): git_ssh = ini.getboolean("general", "git_ssh") if ini.has_option("general", "git_ssh_key"): git_ssh_key = ini.get("general", "git_ssh_key") else: git_ssh = False c['workers'] = [] max_builds = dict() for section in ini.sections(): if section.startswith("slave "): if ini.has_option(section, "name") and ini.has_option(section, "password") and \ ini.has_option(section, "phase") and ini.getint(section, "phase") == 2: name = ini.get(section, "name") password = ini.get(section, "password") sl_props = { 'shared_wd': False } max_builds[name] = 1 if ini.has_option(section, "builds"): max_builds[name] = ini.getint(section, "builds") if max_builds[name] == 1: sl_props['shared_wd'] = True if ini.has_option(section, "shared_wd"): sl_props['shared_wd'] = ini.getboolean(section, "shared_wd") if sl_props['shared_wd'] and (max_builds != 1): raise ValueError('max_builds must be 1 with shared workdir!') c['workers'].append(Worker(name, password, max_builds = max_builds[name], properties = sl_props)) # 'slavePortnum' defines the TCP port to listen on for connections from workers. # This must match the value configured into the buildslaves (with their # --master option) c['protocols'] = {'pb': {'port': slave_port}} # coalesce builds c['collapseRequests'] = True # Reduce amount of backlog data c['buildHorizon'] = 30 c['logHorizon'] = 20 ####### CHANGESOURCES work_dir = os.path.abspath(ini.get("general", "workdir") or ".") scripts_dir = os.path.abspath("../scripts") rsync_bin_url = ini.get("rsync", "binary_url") rsync_bin_key = ini.get("rsync", "binary_password") rsync_src_url = None rsync_src_key = None if ini.has_option("rsync", "source_url"): rsync_src_url = ini.get("rsync", "source_url") rsync_src_key = ini.get("rsync", "source_password") rsync_sdk_url = None rsync_sdk_key = None rsync_sdk_pat = "openwrt-sdk-*.tar.xz" if ini.has_option("rsync", "sdk_url"): rsync_sdk_url = ini.get("rsync", "sdk_url") if ini.has_option("rsync", "sdk_password"): rsync_sdk_key = ini.get("rsync", "sdk_password") if ini.has_option("rsync", "sdk_pattern"): rsync_sdk_pat = ini.get("rsync", "sdk_pattern") repo_url = ini.get("repo", "url") repo_branch = "master" if ini.has_option("repo", "branch"): repo_branch = ini.get("repo", "branch") usign_key = None usign_comment = "untrusted comment: " + repo_branch.replace("-", " ").title() + " key" if ini.has_option("usign", "key"): usign_key = ini.get("usign", "key") if ini.has_option("usign", "comment"): usign_comment = ini.get("usign", "comment") # find arches arches = [ ] archnames = [ ] if not os.path.isdir(work_dir+'/source.git'): subprocess.call(["git", "clone", "--depth=1", "--branch="+repo_branch, repo_url, work_dir+'/source.git']) else: subprocess.call(["git", "pull"], cwd = work_dir+'/source.git') findarches = subprocess.Popen([scripts_dir + '/dumpinfo.pl', 'architectures'], stdout = subprocess.PIPE, cwd = work_dir+'/source.git') while True: line = findarches.stdout.readline() if not line: break at = line.decode().strip().split() arches.append(at) archnames.append(at[0]) # find feeds feeds = [] feedbranches = dict() c['change_source'] = [] def parse_feed_entry(line): parts = line.strip().split() if parts[0] == "src-git": feeds.append(parts) url = parts[2].strip().split(';') branch = url[1] if len(url) > 1 else 'master' feedbranches[url[0]] = branch c['change_source'].append(GitPoller(url[0], branch=branch, workdir='%s/%s.git' %(os.getcwd(), parts[1]), pollinterval=300)) make = subprocess.Popen(['make', '--no-print-directory', '-C', work_dir+'/source.git/target/sdk/', 'val.BASE_FEED'], env = dict(os.environ, TOPDIR=work_dir+'/source.git'), stdout = subprocess.PIPE) line = make.stdout.readline() if line: parse_feed_entry(line) with open(work_dir+'/source.git/feeds.conf.default', 'r') as f: for line in f: parse_feed_entry(line) ####### SCHEDULERS # Configure the Schedulers, which decide how to react to incoming changes. In this # case, just kick off a 'basebuild' build c['schedulers'] = [] c['schedulers'].append(SingleBranchScheduler( name = "all", change_filter = filter.ChangeFilter( filter_fn = lambda change: change.branch == feedbranches[change.repository] ), treeStableTimer = 60, builderNames = archnames)) c['schedulers'].append(ForceScheduler( name = "force", buttonName = "Force builds", label = "Force build details", builderNames = [ "00_force_build" ], codebases = [ util.CodebaseParameter( "", label = "Repository", branch = util.FixedParameter(name = "branch", default = ""), revision = util.FixedParameter(name = "revision", default = ""), repository = util.FixedParameter(name = "repository", default = ""), project = util.FixedParameter(name = "project", default = "") ) ], reason = util.StringParameter( name = "reason", label = "Reason", default = "Trigger build", required = True, size = 80 ), properties = [ util.NestedParameter( name="options", label="Build Options", layout="vertical", fields=[ util.ChoiceStringParameter( name = "architecture", label = "Build architecture", default = "all", choices = [ "all" ] + archnames ) ] ) ] )) ####### BUILDERS # The 'builders' list defines the Builders, which tell Buildbot how to perform a build: # what steps, and which workers can execute them. Note that any particular build will # only take place on one slave. def GetDirectorySuffix(props): verpat = re.compile(r'^([0-9]{2})\.([0-9]{2})(?:\.([0-9]+)(?:-rc([0-9]+))?|-(SNAPSHOT))$') if props.hasProperty("release_version"): m = verpat.match(props["release_version"]) if m is not None: return "-%02d.%02d" %(int(m.group(1)), int(m.group(2))) return "" def GetNumJobs(props): if props.hasProperty("workername") and props.hasProperty("nproc"): return ((int(props["nproc"]) / (max_builds[props["workername"]] + other_builds)) + 1) else: return 1 def GetCwd(props): if props.hasProperty("builddir"): return props["builddir"] elif props.hasProperty("workdir"): return props["workdir"] else: return "/" def IsArchitectureSelected(target): def CheckArchitectureProperty(step): try: options = step.getProperty("options") if type(options) is dict: selected_arch = options.get("architecture", "all") if selected_arch != "all" and selected_arch != target: return False except KeyError: pass return True return CheckArchitectureProperty def UsignSec2Pub(seckey, comment="untrusted comment: secret key"): try: seckey = base64.b64decode(seckey) except: return None return "{}\n{}".format(re.sub(r"\bsecret key$", "public key", comment), base64.b64encode(seckey[0:2] + seckey[32:40] + seckey[72:])) def IsSharedWorkdir(step): return bool(step.getProperty("shared_wd")) c['builders'] = [] dlLock = locks.WorkerLock("slave_dl") slaveNames = [ ] for slave in c['workers']: slaveNames.append(slave.workername) force_factory = BuildFactory() c['builders'].append(BuilderConfig( name = "00_force_build", workernames = slaveNames, factory = force_factory)) for arch in arches: ts = arch[1].split('/') factory = BuildFactory() # setup shared work directory if required factory.addStep(ShellCommand( name = "sharedwd", description = "Setting up shared work directory", command = 'test -L "$PWD" || (mkdir -p ../shared-workdir && rm -rf "$PWD" && ln -s shared-workdir "$PWD")', workdir = ".", haltOnFailure = True, doStepIf = IsSharedWorkdir)) # find number of cores factory.addStep(SetProperty( name = "nproc", property = "nproc", description = "Finding number of CPUs", command = ["nproc"])) # prepare workspace factory.addStep(FileDownload( mastersrc = scripts_dir + '/cleanup.sh', workerdest = "../cleanup.sh", mode = 0o755)) if not persistent: factory.addStep(ShellCommand( name = "cleanold", description = "Cleaning previous builds", command = ["./cleanup.sh", buildbot_url, WithProperties("%(workername)s"), WithProperties("%(buildername)s"), "full"], workdir = ".", haltOnFailure = True, timeout = 2400)) factory.addStep(ShellCommand( name = "cleanup", description = "Cleaning work area", command = ["./cleanup.sh", buildbot_url, WithProperties("%(workername)s"), WithProperties("%(buildername)s"), "single"], workdir = ".", haltOnFailure = True, timeout = 2400)) # expire tree if needed elif tree_expire > 0: factory.addStep(FileDownload( mastersrc = scripts_dir + '/expire.sh', workerdest = "../expire.sh", mode = 0o755)) factory.addStep(ShellCommand( name = "expire", description = "Checking for build tree expiry", command = ["./expire.sh", str(tree_expire)], workdir = ".", haltOnFailure = True, timeout = 2400)) factory.addStep(ShellCommand( name = "mksdkdir", description = "Preparing SDK directory", command = ["mkdir", "-p", "sdk"], haltOnFailure = True)) factory.addStep(ShellCommand( name = "downloadsdk", description = "Downloading SDK archive", command = ["rsync", "-4", "-va", "%s/%s/%s/%s" %(rsync_sdk_url, ts[0], ts[1], rsync_sdk_pat), "sdk.archive"], env={'RSYNC_PASSWORD': rsync_sdk_key}, haltOnFailure = True, logEnviron = False)) factory.addStep(ShellCommand( name = "unpacksdk", description = "Unpacking SDK archive", command = "rm -rf sdk_update && mkdir sdk_update && tar --strip-components=1 -C sdk_update/ -vxf sdk.archive", haltOnFailure = True)) factory.addStep(ShellCommand( name = "updatesdk", description = "Updating SDK", command = "rsync --checksum -av sdk_update/ sdk/ && rm -rf sdk_update", haltOnFailure = True)) factory.addStep(ShellCommand( name = "cleancmdlinks", description = "Sanitizing host command symlinks", command = "find sdk/staging_dir/host/bin/ -type l -exec sh -c 'case $(readlink {}) in /bin/*|/usr/bin/*) true;; /*) rm -vf {};; esac' \\;", haltOnFailure = True)) factory.addStep(StringDownload( name = "writeversionmk", s = 'TOPDIR:=${CURDIR}\n\ninclude $(TOPDIR)/include/version.mk\n\nversion:\n\t@echo $(VERSION_NUMBER)\n', workerdest = "sdk/getversion.mk", mode = 0o755)) factory.addStep(SetProperty( name = "getversion", property = "release_version", description = "Finding SDK release version", workdir = "build/sdk", command = ["make", "-f", "getversion.mk"])) # install build key if usign_key is not None: factory.addStep(StringDownload( name = "dlkeybuildpub", s = UsignSec2Pub(usign_key, usign_comment), workerdest = "sdk/key-build.pub", mode = 0o600)) factory.addStep(StringDownload( name = "dlkeybuild", s = "# fake private key", workerdest = "sdk/key-build", mode = 0o600)) factory.addStep(StringDownload( name = "dlkeybuilducert", s = "# fake certificate", workerdest = "sdk/key-build.ucert", mode = 0o600)) factory.addStep(ShellCommand( name = "mkdldir", description = "Preparing download directory", command = ["sh", "-c", "mkdir -p $HOME/dl && rm -rf ./sdk/dl && ln -sf $HOME/dl ./sdk/dl"], haltOnFailure = True)) factory.addStep(ShellCommand( name = "mkconf", description = "Preparing SDK configuration", workdir = "build/sdk", command = ["sh", "-c", "rm -f .config && make defconfig"])) factory.addStep(FileDownload( mastersrc = scripts_dir + '/ccache.sh', workerdest = 'sdk/ccache.sh', mode = 0o755)) factory.addStep(ShellCommand( name = "prepccache", description = "Preparing ccache", workdir = "build/sdk", command = ["./ccache.sh"], haltOnFailure = True)) factory.addStep(ShellCommand( name = "patchfeedsconfgitfull", description = "Patching feeds.conf to use src-git-full", workdir = "build/sdk", command = "sed -e 's#^src-git #src-git-full #g' feeds.conf.default > feeds.conf", haltOnFailure = True)) if git_ssh: factory.addStep(StringDownload( name = "dlgitclonekey", s = git_ssh_key, workerdest = "../git-clone.key", mode = 0o600)) factory.addStep(ShellCommand( name = "patchfeedsconf", description = "Patching feeds.conf to use SSH cloning", workdir = "build/sdk", command = "sed -i -e 's#https://#ssh://git@#g' feeds.conf", haltOnFailure = True)) factory.addStep(ShellCommand( name = "updatefeeds", description = "Updating feeds", workdir = "build/sdk", command = ["./scripts/feeds", "update", "-f"], env = {'GIT_SSH_COMMAND': WithProperties("ssh -o IdentitiesOnly=yes -o IdentityFile=%(cwd)s/git-clone.key -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no", cwd=GetCwd)} if git_ssh else {}, haltOnFailure = True)) if git_ssh: factory.addStep(ShellCommand( name = "rmfeedsconf", description = "Removing feeds.conf", workdir = "build/sdk", command=["rm", "feeds.conf"], haltOnFailure = True)) factory.addStep(ShellCommand( name = "installfeeds", description = "Installing feeds", workdir = "build/sdk", command = ["./scripts/feeds", "install", "-a"], haltOnFailure = True)) factory.addStep(ShellCommand( name = "logclear", description = "Clearing failure logs", workdir = "build/sdk", command = ["rm", "-rf", "logs/package/error.txt", "faillogs/"], haltOnFailure = False )) factory.addStep(ShellCommand( name = "compile", description = "Building packages", workdir = "build/sdk", timeout = 3600, command = ["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "IGNORE_ERRORS=n m y", "BUILD_LOG=1", "CONFIG_AUTOREMOVE=y", "CONFIG_SIGNED_PACKAGES="], env = {'CCACHE_BASEDIR': WithProperties("%(cwd)s", cwd=GetCwd)}, haltOnFailure = True)) factory.addStep(ShellCommand( name = "mkfeedsconf", description = "Generating pinned feeds.conf", workdir = "build/sdk", command = "./scripts/feeds list -s -f > bin/packages/%s/feeds.conf" %(arch[0]))) if ini.has_option("gpg", "key") or usign_key is not None: factory.addStep(MasterShellCommand( name = "signprepare", description = "Preparing temporary signing directory", command = ["mkdir", "-p", "%s/signing" %(work_dir)], haltOnFailure = True )) factory.addStep(ShellCommand( name = "signpack", description = "Packing files to sign", workdir = "build/sdk", command = "find bin/packages/%s/ -mindepth 2 -maxdepth 2 -type f -name Packages -print0 | xargs -0 tar -czf sign.tar.gz" %(arch[0]), haltOnFailure = True )) factory.addStep(FileUpload( workersrc = "sdk/sign.tar.gz", masterdest = "%s/signing/%s.tar.gz" %(work_dir, arch[0]), haltOnFailure = True )) factory.addStep(MasterShellCommand( name = "signfiles", description = "Signing files", command = ["%s/signall.sh" %(scripts_dir), "%s/signing/%s.tar.gz" %(work_dir, arch[0])], env = { 'CONFIG_INI': os.getenv("BUILDMASTER_CONFIG", "./config.ini") }, haltOnFailure = True )) factory.addStep(FileDownload( mastersrc = "%s/signing/%s.tar.gz" %(work_dir, arch[0]), workerdest = "sdk/sign.tar.gz", haltOnFailure = True )) factory.addStep(ShellCommand( name = "signunpack", description = "Unpacking signed files", workdir = "build/sdk", command = ["tar", "-xzf", "sign.tar.gz"], haltOnFailure = True )) factory.addStep(ShellCommand( name = "uploadprepare", description = "Preparing package directory", workdir = "build/sdk", command = ["rsync", "-4", "-av", "--include", "/%s/" %(arch[0]), "--exclude", "/*", "--exclude", "/%s/*" %(arch[0]), "bin/packages/", WithProperties("%s/packages%%(suffix)s/" %(rsync_bin_url), suffix=GetDirectorySuffix)], env={'RSYNC_PASSWORD': rsync_bin_key}, haltOnFailure = True, logEnviron = False )) factory.addStep(ShellCommand( name = "packageupload", description = "Uploading package files", workdir = "build/sdk", command = ["rsync", "-4", "--progress", "--delete", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-avz", "bin/packages/%s/" %(arch[0]), WithProperties("%s/packages%%(suffix)s/%s/" %(rsync_bin_url, arch[0]), suffix=GetDirectorySuffix)], env={'RSYNC_PASSWORD': rsync_bin_key}, haltOnFailure = True, logEnviron = False )) factory.addStep(ShellCommand( name = "logprepare", description = "Preparing log directory", workdir = "build/sdk", command = ["rsync", "-4", "-av", "--include", "/%s/" %(arch[0]), "--exclude", "/*", "--exclude", "/%s/*" %(arch[0]), "bin/packages/", WithProperties("%s/faillogs%%(suffix)s/" %(rsync_bin_url), suffix=GetDirectorySuffix)], env={'RSYNC_PASSWORD': rsync_bin_key}, haltOnFailure = True, logEnviron = False )) factory.addStep(ShellCommand( name = "logfind", description = "Finding failure logs", workdir = "build/sdk/logs/package/feeds", command = ["sh", "-c", "sed -ne 's!^ *ERROR: package/feeds/\\([^ ]*\\) .*$!\\1!p' ../error.txt | sort -u | xargs -r find > ../../../logs.txt"], haltOnFailure = False )) factory.addStep(ShellCommand( name = "logcollect", description = "Collecting failure logs", workdir = "build/sdk", command = ["rsync", "-av", "--files-from=logs.txt", "logs/package/feeds/", "faillogs/"], haltOnFailure = False )) factory.addStep(ShellCommand( name = "logupload", description = "Uploading failure logs", workdir = "build/sdk", command = ["rsync", "-4", "--progress", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-avz", "faillogs/", WithProperties("%s/faillogs%%(suffix)s/%s/" %(rsync_bin_url, arch[0]), suffix=GetDirectorySuffix)], env={'RSYNC_PASSWORD': rsync_bin_key}, haltOnFailure = False, logEnviron = False )) if rsync_src_url is not None: factory.addStep(ShellCommand( name = "sourcelist", description = "Finding source archives to upload", workdir = "build/sdk", command = "find dl/ -maxdepth 1 -type f -not -size 0 -not -name '.*' -newer ../sdk.archive -printf '%f\\n' > sourcelist", haltOnFailure = True )) factory.addStep(ShellCommand( name = "sourceupload", description = "Uploading source archives", workdir = "build/sdk", command = ["rsync", "--files-from=sourcelist", "-4", "--progress", "--checksum", "--delay-updates", WithProperties("--partial-dir=.~tmp~%s~%%(workername)s" %(arch[0])), "-avz", "dl/", "%s/" %(rsync_src_url)], env={'RSYNC_PASSWORD': rsync_src_key}, haltOnFailure = False, logEnviron = False )) factory.addStep(ShellCommand( name = "df", description = "Reporting disk usage", command=["df", "-h", "."], env={'LC_ALL': 'C'}, haltOnFailure = False, alwaysRun = True )) c['builders'].append(BuilderConfig(name=arch[0], workernames=slaveNames, factory=factory)) c['schedulers'].append(schedulers.Triggerable(name="trigger_%s" % arch[0], builderNames=[ arch[0] ])) force_factory.addStep(steps.Trigger( name = "trigger_%s" % arch[0], description = "Triggering %s build" % arch[0], schedulerNames = [ "trigger_%s" % arch[0] ], set_properties = { "reason": Property("reason") }, doStepIf = IsArchitectureSelected(arch[0]) )) ####### STATUS arches # 'status' is a list of Status arches. The results of each build will be # pushed to these arches. buildbot/status/*.py has a variety to choose from, # including web pages, email senders, and IRC bots. if ini.has_option("phase2", "status_bind"): c['www'] = { 'port': ini.get("phase2", "status_bind"), 'plugins': { 'waterfall_view': True, 'console_view': True, 'grid_view': True } } if ini.has_option("phase2", "status_user") and ini.has_option("phase2", "status_password"): c['www']['auth'] = util.UserPasswordAuth([ (ini.get("phase2", "status_user"), ini.get("phase2", "status_password")) ]) c['www']['authz'] = util.Authz( allowRules=[ util.AnyControlEndpointMatcher(role="admins") ], roleMatchers=[ util.RolesFromUsername(roles=["admins"], usernames=[ini.get("phase2", "status_user")]) ] ) ####### PROJECT IDENTITY # the 'title' string will appear at the top of this buildbot # installation's html.WebStatus home page (linked to the # 'titleURL') and is embedded in the title of the waterfall HTML page. c['title'] = ini.get("general", "title") c['titleURL'] = ini.get("general", "title_url") # the 'buildbotURL' string should point to the location where the buildbot's # internal web server (usually the html.WebStatus page) is visible. This # typically uses the port number set in the Waterfall 'status' entry, but # with an externally-visible host name which the buildbot cannot figure out # without some help. c['buildbotURL'] = buildbot_url ####### DB URL c['db'] = { # This specifies what database buildbot uses to store its state. You can leave # this at its default for all but the largest installations. 'db_url' : "sqlite:///state.sqlite", } c['buildbotNetUsageData'] = None