X-Git-Url: http://git.openwrt.org/?a=blobdiff_plain;f=phase2%2Fmaster.cfg;h=d6d2e224edd6f3387a42fe2e2b09f2eae31d3f54;hb=17b4c63900a461ef5ae3c5cdaed7e8d255847fde;hp=873757a3ca1a99fe08374c471c882db0379675bf;hpb=32bf5adcb47e8d889a3544642b78fe870b4a612d;p=buildbot.git diff --git a/phase2/master.cfg b/phase2/master.cfg index 873757a..d6d2e22 100644 --- a/phase2/master.cfg +++ b/phase2/master.cfg @@ -11,6 +11,8 @@ from buildbot import locks ini = ConfigParser.ConfigParser() ini.read("./config.ini") +buildbot_url = ini.get("general", "buildbot_url") + # This is a sample buildmaster config file. It must be installed as # 'master.cfg' in your buildmaster's base directory. @@ -25,32 +27,52 @@ c = BuildmasterConfig = {} # slave name and password must be configured on the slave. from buildbot.buildslave import BuildSlave +slave_port = 9990 +persistent = False +other_builds = 0 +tree_expire = 0 + +if ini.has_option("general", "port"): + slave_port = ini.getint("general", "port") + +if ini.has_option("general", "persistent"): + persistent = ini.getboolean("general", "persistent") + +if ini.has_option("general", "other_builds"): + other_builds = ini.getint("general", "other_builds") + +if ini.has_option("general", "expire"): + tree_expire = ini.getint("general", "expire") + c['slaves'] = [] +max_builds = dict() for section in ini.sections(): if section.startswith("slave "): if ini.has_option(section, "name") and ini.has_option(section, "password"): name = ini.get(section, "name") password = ini.get(section, "password") - max_builds = 1 + max_builds[name] = 1 if ini.has_option(section, "builds"): - max_builds = ini.getint(section, "builds") - c['slaves'].append(BuildSlave(name, password, max_builds = max_builds)) + max_builds[name] = ini.getint(section, "builds") + c['slaves'].append(BuildSlave(name, password, max_builds = max_builds[name])) # 'slavePortnum' defines the TCP port to listen on for connections from slaves. # This must match the value configured into the buildslaves (with their # --master option) -c['slavePortnum'] = 9990 +c['slavePortnum'] = slave_port # coalesce builds c['mergeRequests'] = True +# Reduce amount of backlog data +c['buildHorizon'] = 30 +c['logHorizon'] = 20 + ####### CHANGESOURCES home_dir = os.path.abspath(ini.get("general", "homedir")) -repo_url = ini.get("repo", "url") - rsync_bin_url = ini.get("rsync", "binary_url") rsync_bin_key = ini.get("rsync", "binary_password") @@ -61,6 +83,37 @@ if ini.has_option("rsync", "source_url"): rsync_src_url = ini.get("rsync", "source_url") rsync_src_key = ini.get("rsync", "source_password") +rsync_sdk_url = None +rsync_sdk_key = None +rsync_sdk_pat = "lede-sdk-*.tar.xz" + +if ini.has_option("rsync", "sdk_url"): + rsync_sdk_url = ini.get("rsync", "sdk_url") + +if ini.has_option("rsync", "sdk_password"): + rsync_sdk_key = ini.get("rsync", "sdk_password") + +if ini.has_option("rsync", "sdk_pattern"): + rsync_sdk_pat = ini.get("rsync", "sdk_pattern") + +gpg_home = "~/.gnupg" +gpg_keyid = None +gpg_comment = "Unattended build signature" +gpg_passfile = "/dev/null" + +if ini.has_option("gpg", "home"): + gpg_home = ini.get("gpg", "home") + +if ini.has_option("gpg", "keyid"): + gpg_keyid = ini.get("gpg", "keyid") + +if ini.has_option("gpg", "comment"): + gpg_comment = ini.get("gpg", "comment") + +if ini.has_option("gpg", "passfile"): + gpg_passfile = ini.get("gpg", "passfile") + + # find arches arches = [ ] archnames = [ ] @@ -79,6 +132,7 @@ while True: # find feeds feeds = [] +feedbranches = dict() from buildbot.changes.gitpoller import GitPoller c['change_source'] = [] @@ -88,7 +142,10 @@ with open(home_dir+'/source.git/feeds.conf.default', 'r') as f: parts = line.strip().split() if parts[0] == "src-git": feeds.append(parts) - c['change_source'].append(GitPoller(parts[2], workdir='%s/%s.git' %(os.getcwd(), parts[1]), branch='master', pollinterval=300)) + url = parts[2].strip().split(';') + branch = url[1] if len(url) > 1 else 'master' + feedbranches[url[0]] = branch + c['change_source'].append(GitPoller(url[0], branch=branch, workdir='%s/%s.git' %(os.getcwd(), parts[1]), pollinterval=300)) ####### SCHEDULERS @@ -96,13 +153,16 @@ with open(home_dir+'/source.git/feeds.conf.default', 'r') as f: # Configure the Schedulers, which decide how to react to incoming changes. In this # case, just kick off a 'basebuild' build +def branch_change_filter(change): + return change.branch == feedbranches[change.repository] + from buildbot.schedulers.basic import SingleBranchScheduler from buildbot.schedulers.forcesched import ForceScheduler from buildbot.changes import filter c['schedulers'] = [] c['schedulers'].append(SingleBranchScheduler( name="all", - change_filter=filter.ChangeFilter(branch='master'), + change_filter=filter.ChangeFilter(filter_fn=branch_change_filter), treeStableTimer=60, builderNames=archnames)) @@ -120,9 +180,25 @@ from buildbot.process.factory import BuildFactory from buildbot.steps.source import Git from buildbot.steps.shell import ShellCommand from buildbot.steps.shell import SetProperty +from buildbot.steps.transfer import FileUpload from buildbot.steps.transfer import FileDownload +from buildbot.steps.master import MasterShellCommand from buildbot.process.properties import WithProperties + +def GetDirectorySuffix(props): + if props.hasProperty("slavename") and re.match("^[^-]+-[0-9]+\.[0-9]+-[^-]+$", props["slavename"]): + return "-%s" % props["slavename"].split('-')[1] + else: + return "" + +def GetNumJobs(props): + if props.hasProperty("slavename") and props.hasProperty("nproc"): + return ((int(props["nproc"]) / (max_builds[props["slavename"]] + other_builds)) + 1) + else: + return 1 + + c['builders'] = [] dlLock = locks.SlaveLock("slave_dl") @@ -147,36 +223,54 @@ for arch in arches: # prepare workspace factory.addStep(FileDownload(mastersrc="cleanup.sh", slavedest="cleanup.sh", mode=0755)) - factory.addStep(ShellCommand( - name = "cleanold", - description = "Cleaning previous builds", - command = ["./cleanup.sh", WithProperties("%(slavename)s"), WithProperties("%(buildername)s"), "full"], - haltOnFailure = True, - timeout = 2400)) + if not persistent: + factory.addStep(ShellCommand( + name = "cleanold", + description = "Cleaning previous builds", + command = ["./cleanup.sh", buildbot_url, WithProperties("%(slavename)s"), WithProperties("%(buildername)s"), "full"], + haltOnFailure = True, + timeout = 2400)) - factory.addStep(ShellCommand( - name = "cleanup", - description = "Cleaning work area", - command = ["./cleanup.sh", WithProperties("%(slavename)s"), WithProperties("%(buildername)s"), "single"], - haltOnFailure = True, - timeout = 2400)) + factory.addStep(ShellCommand( + name = "cleanup", + description = "Cleaning work area", + command = ["./cleanup.sh", buildbot_url, WithProperties("%(slavename)s"), WithProperties("%(buildername)s"), "single"], + haltOnFailure = True, + timeout = 2400)) + + # expire tree if needed + elif tree_expire > 0: + factory.addStep(FileDownload( + mastersrc = home_dir+"/expire.sh", + slavedest = "../expire.sh", + mode = 0755)) + + factory.addStep(ShellCommand( + name = "expire", + description = "Checking for build tree expiry", + command = ["./expire.sh", str(tree_expire)], + workdir = ".", + haltOnFailure = True, + timeout = 2400)) factory.addStep(ShellCommand( name = "mksdkdir", description = "Preparing SDK directory", - command = ["mkdir", "sdk"], + command = ["mkdir", "-p", "sdk"], haltOnFailure = True)) factory.addStep(ShellCommand( name = "downloadsdk", description = "Downloading SDK archive", - command = ["rsync", "-va", "downloads.lede-project.org::downloads/snapshots/targets/%s/%s/[Ll][Ee][Dd][Ee]-[Ss][Dd][Kk]-*.tar.bz2" %(ts[0], ts[1]), "sdk.tar.bz2"], - haltOnFailure = True)) + command = ["rsync", "-va", "%s/%s/%s/%s" %(rsync_sdk_url, ts[0], ts[1], rsync_sdk_pat), "sdk.archive"], + env={'RSYNC_PASSWORD': rsync_sdk_key}, + haltOnFailure = True, + logEnviron = False)) factory.addStep(ShellCommand( name = "unpacksdk", description = "Unpacking SDK archive", - command = ["tar", "--strip-components=1", "-C", "sdk/", "-vxjf", "sdk.tar.bz2"], + command = ["tar", "--keep-newer-files", "--no-overwrite-dir", "--strip-components=1", "-C", "sdk/", "-vxf", "sdk.archive"], haltOnFailure = True)) factory.addStep(FileDownload(mastersrc=home_dir+'/key-build', slavedest="sdk/key-build", mode=0600)) @@ -185,7 +279,7 @@ for arch in arches: factory.addStep(ShellCommand( name = "mkdldir", description = "Preparing download directory", - command = ["sh", "-c", "mkdir -p $HOME/dl && rmdir ./sdk/dl && ln -sf $HOME/dl ./sdk/dl"])) + command = ["sh", "-c", "mkdir -p $HOME/dl && rm -rf ./sdk/dl && ln -sf $HOME/dl ./sdk/dl"])) factory.addStep(ShellCommand( name = "mkconf", @@ -193,6 +287,17 @@ for arch in arches: workdir = "build/sdk", command = ["sh", "-c", "rm -f .config && make defconfig"])) + factory.addStep(FileDownload( + mastersrc = home_dir+'/ccache.sh', + slavedest = 'sdk/ccache.sh', + mode = 0755)) + + factory.addStep(ShellCommand( + name = "prepccache", + description = "Preparing ccache", + workdir = "build/sdk", + command = ["./ccache.sh"])) + factory.addStep(ShellCommand( name = "updatefeeds", description = "Updating feeds", @@ -209,13 +314,63 @@ for arch in arches: name = "compile", description = "Building packages", workdir = "build/sdk", - command = ["make", WithProperties("-j%(nproc:~4)s"), "V=s", "IGNORE_ERRORS=n m y", "BUILD_LOG=1", "CONFIG_SIGNED_PACKAGES=y"])) + command = ["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "IGNORE_ERRORS=n m y", "BUILD_LOG=1", "CONFIG_SIGNED_PACKAGES=y", "CONFIG_AUTOREMOVE=y"])) + + factory.addStep(ShellCommand( + name = "mkfeedsconf", + description = "Generating pinned feeds.conf", + workdir = "build/sdk", + command = "./scripts/feeds list -s -f > bin/packages/%s/feeds.conf" %(arch[0]))) + + if gpg_keyid is not None: + factory.addStep(MasterShellCommand( + name = "signprepare", + description = "Preparing temporary signing directory", + command = ["mkdir", "-p", "%s/signing" %(home_dir)], + haltOnFailure = True + )) + + factory.addStep(ShellCommand( + name = "signpack", + description = "Packing files to sign", + workdir = "build/sdk", + command = "find bin/packages/%s/ -mindepth 2 -maxdepth 2 -type f -name Packages -print0 | xargs -0 tar -czf sign.tar.gz" %(arch[0]), + haltOnFailure = True + )) + + factory.addStep(FileUpload( + slavesrc = "sdk/sign.tar.gz", + masterdest = "%s/signing/%s.tar.gz" %(home_dir, arch[0]), + haltOnFailure = True + )) + + factory.addStep(MasterShellCommand( + name = "signfiles", + description = "Signing files", + command = ["%s/signall.sh" %(home_dir), "%s/signing/%s.tar.gz" %(home_dir, arch[0]), gpg_keyid, gpg_comment], + env = {'GNUPGHOME': gpg_home, 'PASSFILE': gpg_passfile}, + haltOnFailure = True + )) + + factory.addStep(FileDownload( + mastersrc = "%s/signing/%s.tar.gz" %(home_dir, arch[0]), + slavedest = "sdk/sign.tar.gz", + haltOnFailure = True + )) + + factory.addStep(ShellCommand( + name = "signunpack", + description = "Unpacking signed files", + workdir = "build/sdk", + command = ["tar", "-xzf", "sign.tar.gz"], + haltOnFailure = True + )) factory.addStep(ShellCommand( name = "uploadprepare", description = "Preparing package directory", workdir = "build/sdk", - command = ["rsync", "-av", "--include", "/%s/" %(arch[0]), "--exclude", "/*", "--exclude", "/%s/*" %(arch[0]), "bin/packages/", "%s/packages/" %(rsync_bin_url)], + command = ["rsync", "-av", "--include", "/%s/" %(arch[0]), "--exclude", "/*", "--exclude", "/%s/*" %(arch[0]), "bin/packages/", WithProperties("%s/packages%%(suffix)s/" %(rsync_bin_url), suffix=GetDirectorySuffix)], env={'RSYNC_PASSWORD': rsync_bin_key}, haltOnFailure = True, logEnviron = False @@ -225,7 +380,7 @@ for arch in arches: name = "packageupload", description = "Uploading package files", workdir = "build/sdk", - command = ["rsync", "--delete", "--delay-updates", "-avz", "bin/packages/%s/" %(arch[0]), "%s/packages/%s/" %(rsync_bin_url, arch[0])], + command = ["rsync", "--progress", "--delete", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-avz", "bin/packages/%s/" %(arch[0]), WithProperties("%s/packages%%(suffix)s/%s/" %(rsync_bin_url, arch[0]), suffix=GetDirectorySuffix)], env={'RSYNC_PASSWORD': rsync_bin_key}, haltOnFailure = True, logEnviron = False @@ -261,7 +416,7 @@ for arch in arches: name = "logupload", description = "Uploading failure logs", workdir = "build/sdk", - command = ["rsync", "--delete", "--delay-updates", "-avz", "faillogs/", "%s/faillogs/%s/" %(rsync_bin_url, arch[0])], + command = ["rsync", "--progress", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-avz", "faillogs/", "%s/faillogs/%s/" %(rsync_bin_url, arch[0])], env={'RSYNC_PASSWORD': rsync_bin_key}, haltOnFailure = False, logEnviron = False @@ -272,7 +427,7 @@ for arch in arches: name = "sourceupload", description = "Uploading source archives", workdir = "build/sdk", - command = ["rsync", "--delay-updates", "-avz", "dl/", "%s/" %(rsync_src_url)], + command = ["rsync", "--progress", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-avz", "dl/", "%s/" %(rsync_src_url)], env={'RSYNC_PASSWORD': rsync_src_key}, haltOnFailure = False, logEnviron = False @@ -300,7 +455,7 @@ if ini.has_option("status", "bind"): # change any of these to True to enable; see the manual for more # options auth=auth.BasicAuth([(ini.get("status", "user"), ini.get("status", "password"))]), - gracefulShutdown = False, + gracefulShutdown = 'auth', forceBuild = 'auth', # use this to test your slave once it is set up forceAllBuilds = 'auth', pingBuilder = False, @@ -327,7 +482,7 @@ c['titleURL'] = ini.get("general", "title_url") # with an externally-visible host name which the buildbot cannot figure out # without some help. -c['buildbotURL'] = ini.get("general", "buildbot_url") +c['buildbotURL'] = buildbot_url ####### DB URL