global: calculate suitable number of jobs

[buildbot.git] / phase1 / master.cfg

diff --git a/phase1/master.cfg b/phase1/master.cfg
index b9bbf1214839ee08e410e7a3c48c1b66473f0632..554fe4ca39c25cfc3b5b26dd12ef20942ad2378e 100644 (file)
--- a/phase1/master.cfg
+++ b/phase1/master.cfg
@@ -31,16 +31,17 @@ if ini.has_option("general", "port"):
        slave_port = ini.getint("general", "port")
 
 c['slaves'] = []
+max_builds = dict()
 
 for section in ini.sections():
        if section.startswith("slave "):
                if ini.has_option(section, "name") and ini.has_option(section, "password"):
                        name = ini.get(section, "name")
                        password = ini.get(section, "password")
-                       max_builds = 1
+                       max_builds[name] = 1
                        if ini.has_option(section, "builds"):
-                               max_builds = ini.getint(section, "builds")
-                       c['slaves'].append(BuildSlave(name, password, max_builds = max_builds))
+                               max_builds[name] = ini.getint(section, "builds")
+                       c['slaves'].append(BuildSlave(name, password, max_builds = max_builds[name]))
 
 # 'slavePortnum' defines the TCP port to listen on for connections from slaves.
 # This must match the value configured into the buildslaves (with their
@@ -54,10 +55,14 @@ c['mergeRequests'] = True
 
 home_dir = os.path.abspath(ini.get("general", "homedir"))
 tree_expire = 0
+other_builds = 0
 
 if ini.has_option("general", "expire"):
        tree_expire = ini.getint("general", "expire")
 
+if ini.has_option("general", "other_builds"):
+       other_builds = ini.getint("general", "other_builds")
+
 repo_url = ini.get("repo", "url")
 repo_branch = "master"
 
@@ -202,6 +207,12 @@ def GetVersionPrefix(props):
        else:
                return ""
 
+def GetNumJobs(props):
+       if props.hasProperty("slavename") and props.hasProperty("nproc"):
+               return ((int(props["nproc"]) / (max_builds[props["slavename"]] + other_builds)) + 1)
+       else:
+               return 1
+
 
 c['builders'] = []
 
@@ -446,7 +457,7 @@ for target in targets:
        factory.addStep(ShellCommand(
                name = "dltar",
                description = "Building GNU tar",
-               command = ["make", WithProperties("-j%(nproc:~4)s"), "tools/tar/install", "V=s"],
+               command = ["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "tools/tar/install", "V=s"],
                haltOnFailure = True
        ))
 
@@ -454,7 +465,7 @@ for target in targets:
        factory.addStep(ShellCommand(
                name = "dlrun",
                description = "Populating dl/",
-               command = ["make", WithProperties("-j%(nproc:~4)s"), "download", "V=s"],
+               command = ["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "download", "V=s"],
                logEnviron = False,
                locks = [dlLock.access('exclusive')]
        ))
@@ -469,21 +480,21 @@ for target in targets:
        factory.addStep(ShellCommand(
                name = "tools",
                description = "Building tools",
-               command = ["make", WithProperties("-j%(nproc:~4)s"), "tools/install", "V=s"],
+               command = ["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "tools/install", "V=s"],
                haltOnFailure = True
        ))
 
        factory.addStep(ShellCommand(
                name = "toolchain",
                description = "Building toolchain",
-               command=["make", WithProperties("-j%(nproc:~4)s"), "toolchain/install", "V=s"],
+               command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "toolchain/install", "V=s"],
                haltOnFailure = True
        ))
 
        factory.addStep(ShellCommand(
                name = "kmods",
                description = "Building kmods",
-               command=["make", WithProperties("-j%(nproc:~4)s"), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
+               command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
                #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
                haltOnFailure = True
        ))
@@ -491,7 +502,7 @@ for target in targets:
        factory.addStep(ShellCommand(
                name = "pkgbuild",
                description = "Building packages",
-               command=["make", WithProperties("-j%(nproc:~4)s"), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
+               command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
                #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
                haltOnFailure = True
        ))
@@ -500,14 +511,14 @@ for target in targets:
        factory.addStep(ShellCommand(
                name = "pkginstall",
                description = "Installing packages",
-               command=["make", WithProperties("-j%(nproc:~4)s"), "package/install", "V=s"],
+               command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "package/install", "V=s"],
                haltOnFailure = True
        ))
 
        factory.addStep(ShellCommand(
                name = "pkgindex",
                description = "Indexing packages",
-               command=["make", WithProperties("-j%(nproc:~4)s"), "package/index", "V=s"],
+               command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "package/index", "V=s"],
                haltOnFailure = True
        ))
 
@@ -515,7 +526,7 @@ for target in targets:
        factory.addStep(ShellCommand(
                name = "images",
                description = "Building images",
-               command=["make", WithProperties("-j%(nproc:~4)s"), "target/install", "V=s"],
+               command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "target/install", "V=s"],
                haltOnFailure = True
        ))