phase1, phase2: reduce backlog size

[buildbot.git] / phase2 / master.cfg

diff --git a/phase2/master.cfg b/phase2/master.cfg
index ddcca29b123b6fab28a205edfa85e2b6b5b835f2..b1d3860627e5aef400e0fed9e87904d7178fa627 100644 (file)
--- a/phase2/master.cfg
+++ b/phase2/master.cfg
@@ -29,6 +29,7 @@ from buildbot.buildslave import BuildSlave
 
 slave_port = 9990
 persistent = False
+other_builds = 0
 
 if ini.has_option("general", "port"):
        slave_port = ini.getint("general", "port")
@@ -36,17 +37,21 @@ if ini.has_option("general", "port"):
 if ini.has_option("general", "persistent"):
        persistent = ini.getboolean("general", "persistent")
 
+if ini.has_option("general", "other_builds"):
+       other_builds = ini.getint("general", "other_builds")
+
 c['slaves'] = []
+max_builds = dict()
 
 for section in ini.sections():
        if section.startswith("slave "):
                if ini.has_option(section, "name") and ini.has_option(section, "password"):
                        name = ini.get(section, "name")
                        password = ini.get(section, "password")
-                       max_builds = 1
+                       max_builds[name] = 1
                        if ini.has_option(section, "builds"):
-                               max_builds = ini.getint(section, "builds")
-                       c['slaves'].append(BuildSlave(name, password, max_builds = max_builds))
+                               max_builds[name] = ini.getint(section, "builds")
+                       c['slaves'].append(BuildSlave(name, password, max_builds = max_builds[name]))
 
 # 'slavePortnum' defines the TCP port to listen on for connections from slaves.
 # This must match the value configured into the buildslaves (with their
@@ -56,6 +61,10 @@ c['slavePortnum'] = slave_port
 # coalesce builds
 c['mergeRequests'] = True
 
+# Reduce amount of backlog data
+c['buildHorizon'] = 30
+c['logHorizon'] = 20
+
 ####### CHANGESOURCES
 
 home_dir = os.path.abspath(ini.get("general", "homedir"))
@@ -119,6 +128,7 @@ while True:
 
 # find feeds
 feeds = []
+feedbranches = dict()
 
 from buildbot.changes.gitpoller import GitPoller
 c['change_source'] = []
@@ -130,6 +140,7 @@ with open(home_dir+'/source.git/feeds.conf.default', 'r') as f:
                        feeds.append(parts)
                        url = parts[2].strip().split(';')
                        branch = url[1] if len(url) > 1 else 'master'
+                       feedbranches[url[0]] = branch
                        c['change_source'].append(GitPoller(url[0], branch=branch, workdir='%s/%s.git' %(os.getcwd(), parts[1]), pollinterval=300))
 
 
@@ -138,13 +149,16 @@ with open(home_dir+'/source.git/feeds.conf.default', 'r') as f:
 # Configure the Schedulers, which decide how to react to incoming changes.  In this
 # case, just kick off a 'basebuild' build
 
+def branch_change_filter(change):
+       return change.branch == feedbranches[change.repository]
+
 from buildbot.schedulers.basic import SingleBranchScheduler
 from buildbot.schedulers.forcesched import ForceScheduler
 from buildbot.changes import filter
 c['schedulers'] = []
 c['schedulers'].append(SingleBranchScheduler(
        name="all",
-       change_filter=filter.ChangeFilter(branch='master'),
+       change_filter=filter.ChangeFilter(filter_fn=branch_change_filter),
        treeStableTimer=60,
        builderNames=archnames))
 
@@ -174,6 +188,12 @@ def GetDirectorySuffix(props):
        else:
                return ""
 
+def GetNumJobs(props):
+       if props.hasProperty("slavename") and props.hasProperty("nproc"):
+               return ((int(props["nproc"]) / (max_builds[props["slavename"]] + other_builds)) + 1)
+       else:
+               return 1
+
 
 c['builders'] = []
 
@@ -264,7 +284,13 @@ for arch in arches:
                name = "compile",
                description = "Building packages",
                workdir = "build/sdk",
-               command = ["make", WithProperties("-j%(nproc:~4)s"), "V=s", "IGNORE_ERRORS=n m y", "BUILD_LOG=1", "CONFIG_SIGNED_PACKAGES=y"]))
+               command = ["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "V=s", "IGNORE_ERRORS=n m y", "BUILD_LOG=1", "CONFIG_SIGNED_PACKAGES=y"]))
+
+       factory.addStep(ShellCommand(
+               name = "mkfeedsconf",
+               description = "Generating pinned feeds.conf",
+               workdir = "build/sdk",
+               command = "./scripts/feeds list -s -f > bin/packages/%s/feeds.conf" %(arch[0])))
 
        if gpg_keyid is not None:
                factory.addStep(MasterShellCommand(
@@ -324,7 +350,7 @@ for arch in arches:
                name = "packageupload",
                description = "Uploading package files",
                workdir = "build/sdk",
-               command = ["rsync", "--delete", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-avz", "bin/packages/%s/" %(arch[0]), WithProperties("%s/packages%%(suffix)s/%s/" %(rsync_bin_url, arch[0]), suffix=GetDirectorySuffix)],
+               command = ["rsync", "--progress", "--delete", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-avz", "bin/packages/%s/" %(arch[0]), WithProperties("%s/packages%%(suffix)s/%s/" %(rsync_bin_url, arch[0]), suffix=GetDirectorySuffix)],
                env={'RSYNC_PASSWORD': rsync_bin_key},
                haltOnFailure = True,
                logEnviron = False
@@ -360,7 +386,7 @@ for arch in arches:
                name = "logupload",
                description = "Uploading failure logs",
                workdir = "build/sdk",
-               command = ["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-avz", "faillogs/", "%s/faillogs/%s/" %(rsync_bin_url, arch[0])],
+               command = ["rsync", "--progress", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-avz", "faillogs/", "%s/faillogs/%s/" %(rsync_bin_url, arch[0])],
                env={'RSYNC_PASSWORD': rsync_bin_key},
                haltOnFailure = False,
                logEnviron = False
@@ -371,7 +397,7 @@ for arch in arches:
                        name = "sourceupload",
                        description = "Uploading source archives",
                        workdir = "build/sdk",
-                       command = ["rsync", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-avz", "dl/", "%s/" %(rsync_src_url)],
+                       command = ["rsync", "--progress", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-avz", "dl/", "%s/" %(rsync_src_url)],
                        env={'RSYNC_PASSWORD': rsync_src_key},
                        haltOnFailure = False,
                        logEnviron = False