X-Git-Url: http://git.openwrt.org/?a=blobdiff_plain;f=phase1%2Fmaster.cfg;h=04879e800f6ebfa17470714bc679dc1ccfc178b7;hb=da4174f4bd4d3af7e10f3d9f3fcd5ac713b33b09;hp=b9bbf1214839ee08e410e7a3c48c1b66473f0632;hpb=eaf8408145a72e13fbb602c172487d644ab8cfc8;p=buildbot.git diff --git a/phase1/master.cfg b/phase1/master.cfg index b9bbf12..04879e8 100644 --- a/phase1/master.cfg +++ b/phase1/master.cfg @@ -31,16 +31,17 @@ if ini.has_option("general", "port"): slave_port = ini.getint("general", "port") c['slaves'] = [] +max_builds = dict() for section in ini.sections(): if section.startswith("slave "): if ini.has_option(section, "name") and ini.has_option(section, "password"): name = ini.get(section, "name") password = ini.get(section, "password") - max_builds = 1 + max_builds[name] = 1 if ini.has_option(section, "builds"): - max_builds = ini.getint(section, "builds") - c['slaves'].append(BuildSlave(name, password, max_builds = max_builds)) + max_builds[name] = ini.getint(section, "builds") + c['slaves'].append(BuildSlave(name, password, max_builds = max_builds[name])) # 'slavePortnum' defines the TCP port to listen on for connections from slaves. # This must match the value configured into the buildslaves (with their @@ -50,14 +51,31 @@ c['slavePortnum'] = slave_port # coalesce builds c['mergeRequests'] = True +# Reduce amount of backlog data +c['buildHorizon'] = 30 +c['logHorizon'] = 20 + ####### CHANGESOURCES home_dir = os.path.abspath(ini.get("general", "homedir")) tree_expire = 0 +other_builds = 0 +cc_version = None + +cc_command = "gcc" +cxx_command = "g++" if ini.has_option("general", "expire"): tree_expire = ini.getint("general", "expire") +if ini.has_option("general", "other_builds"): + other_builds = ini.getint("general", "other_builds") + +if ini.has_option("general", "cc_version"): + cc_version = ini.get("general", "cc_version").split() + if len(cc_version) == 1: + cc_version = ["eq", cc_version[0]] + repo_url = ini.get("repo", "url") repo_branch = "master" @@ -176,7 +194,7 @@ def IsCleanRequested(pattern): def IsTaggingRequested(step): val = step.getProperty("tag") - if val and re.match("^[0-9]+\.[0-9]+\.[0-9]+$", val): + if val and re.match("^[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", val): return True else: return False @@ -195,13 +213,55 @@ def GetBaseVersion(props): def GetVersionPrefix(props): basever = GetBaseVersion(props) - if props.hasProperty("tag") and re.match("^[0-9]+\.[0-9]+\.[0-9]+$", props["tag"]): + if props.hasProperty("tag") and re.match("^[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", props["tag"]): return "%s/" % props["tag"] elif basever != "master": return "%s-SNAPSHOT/" % basever else: return "" +def GetNumJobs(props): + if props.hasProperty("slavename") and props.hasProperty("nproc"): + return ((int(props["nproc"]) / (max_builds[props["slavename"]] + other_builds)) + 1) + else: + return 1 + +def GetCC(props): + if props.hasProperty("cc_command"): + return props["cc_command"] + else: + return "gcc" + +def GetCXX(props): + if props.hasProperty("cxx_command"): + return props["cxx_command"] + else: + return "g++" + +def GetCwd(props): + if props.hasProperty("builddir"): + return props["builddir"] + elif props.hasProperty("workdir"): + return props["workdir"] + else: + return "/" + +def GetNextBuild(builder, requests): + for r in requests: + if r.properties and r.properties.hasProperty("tag"): + return r + return requests[0] + +def MakeEnv(overrides=None): + env = { + 'CC': WithProperties("%(cc)s", cc=GetCC), + 'CXX': WithProperties("%(cxx)s", cxx=GetCXX), + 'CCACHE_BASEDIR': WithProperties("%(cwd)s", cwd=GetCwd) + } + if overrides is not None: + env.update(overrides) + return env + c['builders'] = [] @@ -278,6 +338,27 @@ for target in targets: description = "Finding number of CPUs", command = ["nproc"])) + # find gcc and g++ compilers + if cc_version is not None: + factory.addStep(FileDownload( + mastersrc = "findbin.pl", + slavedest = "../findbin.pl", + mode = 0755)) + + factory.addStep(SetProperty( + name = "gcc", + property = "cc_command", + description = "Finding gcc command", + command = ["../findbin.pl", "gcc", cc_version[0], cc_version[1]], + haltOnFailure = True)) + + factory.addStep(SetProperty( + name = "g++", + property = "cxx_command", + description = "Finding g++ command", + command = ["../findbin.pl", "g++", cc_version[0], cc_version[1]], + haltOnFailure = True)) + # expire tree if needed if tree_expire > 0: factory.addStep(FileDownload( @@ -299,6 +380,7 @@ for target in targets: name = tuple[1], description = 'User-requested "make %s"' % tuple[1], command = ["make", tuple[1], "V=s"], + env = MakeEnv(), doStepIf = IsCleanRequested(tuple[0]) )) @@ -318,7 +400,7 @@ for target in targets: factory.addStep(ShellCommand( name = "switchbranch", description = "Checking out Git branch", - command = "if [ -d .git ]; then git checkout '%s'; else exit 0; fi" % repo_branch, + command = "if [ -d .git ]; then git fetch && git checkout '%s'; else exit 0; fi" % repo_branch, haltOnFailure = True, doStepIf = IsNoTaggingRequested )) @@ -377,13 +459,15 @@ for target in targets: factory.addStep(ShellCommand( name = "updatefeeds", description = "Updating feeds", - command=["./scripts/feeds", "update"])) + command=["./scripts/feeds", "update"], + env = MakeEnv())) # feed factory.addStep(ShellCommand( name = "installfeeds", description = "Installing feeds", - command=["./scripts/feeds", "install", "-a"])) + command=["./scripts/feeds", "install", "-a"], + env = MakeEnv())) # seed config factory.addStep(FileDownload( @@ -408,7 +492,8 @@ for target in targets: factory.addStep(ShellCommand( name = "defconfig", description = "Populating .config", - command = ["make", "defconfig"] + command = ["make", "defconfig"], + env = MakeEnv() )) # check arch @@ -429,6 +514,20 @@ for target in targets: description = "Finding libc suffix", command = ["sed", "-ne", '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }', ".config"])) + # ccache helper + factory.addStep(FileDownload( + mastersrc = "ccache.sh", + slavedest = "ccache.sh", + mode = 0755 + )) + + # ccache prepare + factory.addStep(ShellCommand( + name = "prepccache", + description = "Preparing ccache", + command = ["./ccache.sh"] + )) + # install build key factory.addStep(FileDownload(mastersrc=home_dir+'/key-build', slavedest="key-build", mode=0600)) factory.addStep(FileDownload(mastersrc=home_dir+'/key-build.pub', slavedest="key-build.pub", mode=0600)) @@ -446,7 +545,8 @@ for target in targets: factory.addStep(ShellCommand( name = "dltar", description = "Building GNU tar", - command = ["make", WithProperties("-j%(nproc:~4)s"), "tools/tar/install", "V=s"], + command = ["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "tools/tar/compile", "V=s"], + env = MakeEnv(), haltOnFailure = True )) @@ -454,7 +554,8 @@ for target in targets: factory.addStep(ShellCommand( name = "dlrun", description = "Populating dl/", - command = ["make", WithProperties("-j%(nproc:~4)s"), "download", "V=s"], + command = ["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "download", "V=s"], + env = MakeEnv(), logEnviron = False, locks = [dlLock.access('exclusive')] )) @@ -469,29 +570,39 @@ for target in targets: factory.addStep(ShellCommand( name = "tools", description = "Building tools", - command = ["make", WithProperties("-j%(nproc:~4)s"), "tools/install", "V=s"], + command = ["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "tools/install", "V=s"], + env = MakeEnv(), haltOnFailure = True )) factory.addStep(ShellCommand( name = "toolchain", description = "Building toolchain", - command=["make", WithProperties("-j%(nproc:~4)s"), "toolchain/install", "V=s"], + command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "toolchain/install", "V=s"], + env = MakeEnv(), haltOnFailure = True )) factory.addStep(ShellCommand( name = "kmods", description = "Building kmods", - command=["make", WithProperties("-j%(nproc:~4)s"), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"], + command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"], + env = MakeEnv(), #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])}, haltOnFailure = True )) + factory.addStep(ShellCommand( + name = "pkgclean", + description = "Cleaning up package build", + command=["make", "package/cleanup", "V=s"] + )) + factory.addStep(ShellCommand( name = "pkgbuild", description = "Building packages", - command=["make", WithProperties("-j%(nproc:~4)s"), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"], + command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"], + env = MakeEnv(), #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])}, haltOnFailure = True )) @@ -500,14 +611,16 @@ for target in targets: factory.addStep(ShellCommand( name = "pkginstall", description = "Installing packages", - command=["make", WithProperties("-j%(nproc:~4)s"), "package/install", "V=s"], + command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "package/install", "V=s"], + env = MakeEnv(), haltOnFailure = True )) factory.addStep(ShellCommand( name = "pkgindex", description = "Indexing packages", - command=["make", WithProperties("-j%(nproc:~4)s"), "package/index", "V=s"], + command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "package/index", "V=s"], + env = MakeEnv(), haltOnFailure = True )) @@ -515,7 +628,16 @@ for target in targets: factory.addStep(ShellCommand( name = "images", description = "Building images", - command=["make", WithProperties("-j%(nproc:~4)s"), "target/install", "V=s"], + command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "target/install", "V=s"], + env = MakeEnv(), + haltOnFailure = True + )) + + factory.addStep(ShellCommand( + name = "diffconfig", + description = "Generating config.seed", + command=["make", "-j1", "diffconfig", "V=s"], + env = MakeEnv(), haltOnFailure = True )) @@ -523,6 +645,7 @@ for target in targets: name = "checksums", description = "Calculating checksums", command=["make", "-j1", "checksum", "V=s"], + env = MakeEnv(), haltOnFailure = True )) @@ -597,7 +720,7 @@ for target in targets: factory.addStep(ShellCommand( name = "targetupload", description = "Uploading target files", - command=["rsync", "--delete", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), + command=["rsync", "--progress", "--delete", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", WithProperties("bin/targets/%s/%s%%(libc)s/" %(ts[0], ts[1])), WithProperties("%s/%%(prefix)stargets/%s/%s/" %(rsync_bin_url, ts[0], ts[1]), prefix=GetVersionPrefix)], env={'RSYNC_PASSWORD': rsync_bin_key}, @@ -609,7 +732,7 @@ for target in targets: factory.addStep(ShellCommand( name = "sourceupload", description = "Uploading source archives", - command=["rsync", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "dl/", "%s/" %(rsync_src_url)], + command=["rsync", "--progress", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "dl/", "%s/" %(rsync_src_url)], env={'RSYNC_PASSWORD': rsync_src_key}, haltOnFailure = True, logEnviron = False @@ -639,7 +762,7 @@ for target in targets: from buildbot.config import BuilderConfig - c['builders'].append(BuilderConfig(name=target, slavenames=slaveNames, factory=factory)) + c['builders'].append(BuilderConfig(name=target, slavenames=slaveNames, factory=factory, nextBuild=GetNextBuild)) ####### STATUS TARGETS