Use the formula (n_cpus / n_builds) + 1 to calculate a suitable value for -j.
Also introduce a new general configuration option "other_builds" which is
added to n_builds. This is needed in cases where multiple build masters run
on the same machine.
Signed-off-by: Jo-Philipp Wich <jo@mein.io>
slave_port = ini.getint("general", "port")
c['slaves'] = []
slave_port = ini.getint("general", "port")
c['slaves'] = []
for section in ini.sections():
if section.startswith("slave "):
if ini.has_option(section, "name") and ini.has_option(section, "password"):
name = ini.get(section, "name")
password = ini.get(section, "password")
for section in ini.sections():
if section.startswith("slave "):
if ini.has_option(section, "name") and ini.has_option(section, "password"):
name = ini.get(section, "name")
password = ini.get(section, "password")
if ini.has_option(section, "builds"):
if ini.has_option(section, "builds"):
- max_builds = ini.getint(section, "builds")
- c['slaves'].append(BuildSlave(name, password, max_builds = max_builds))
+ max_builds[name] = ini.getint(section, "builds")
+ c['slaves'].append(BuildSlave(name, password, max_builds = max_builds[name]))
# 'slavePortnum' defines the TCP port to listen on for connections from slaves.
# This must match the value configured into the buildslaves (with their
# 'slavePortnum' defines the TCP port to listen on for connections from slaves.
# This must match the value configured into the buildslaves (with their
home_dir = os.path.abspath(ini.get("general", "homedir"))
tree_expire = 0
home_dir = os.path.abspath(ini.get("general", "homedir"))
tree_expire = 0
if ini.has_option("general", "expire"):
tree_expire = ini.getint("general", "expire")
if ini.has_option("general", "expire"):
tree_expire = ini.getint("general", "expire")
+if ini.has_option("general", "other_builds"):
+ other_builds = ini.getint("general", "other_builds")
+
repo_url = ini.get("repo", "url")
repo_branch = "master"
repo_url = ini.get("repo", "url")
repo_branch = "master"
+def GetNumJobs(props):
+ if props.hasProperty("slavename") and props.hasProperty("nproc"):
+ return ((int(props["nproc"]) / (max_builds[props["slavename"]] + other_builds)) + 1)
+ else:
+ return 1
+
factory.addStep(ShellCommand(
name = "dltar",
description = "Building GNU tar",
factory.addStep(ShellCommand(
name = "dltar",
description = "Building GNU tar",
- command = ["make", WithProperties("-j%(nproc:~4)s"), "tools/tar/install", "V=s"],
+ command = ["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "tools/tar/install", "V=s"],
factory.addStep(ShellCommand(
name = "dlrun",
description = "Populating dl/",
factory.addStep(ShellCommand(
name = "dlrun",
description = "Populating dl/",
- command = ["make", WithProperties("-j%(nproc:~4)s"), "download", "V=s"],
+ command = ["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "download", "V=s"],
logEnviron = False,
locks = [dlLock.access('exclusive')]
))
logEnviron = False,
locks = [dlLock.access('exclusive')]
))
factory.addStep(ShellCommand(
name = "tools",
description = "Building tools",
factory.addStep(ShellCommand(
name = "tools",
description = "Building tools",
- command = ["make", WithProperties("-j%(nproc:~4)s"), "tools/install", "V=s"],
+ command = ["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "tools/install", "V=s"],
haltOnFailure = True
))
factory.addStep(ShellCommand(
name = "toolchain",
description = "Building toolchain",
haltOnFailure = True
))
factory.addStep(ShellCommand(
name = "toolchain",
description = "Building toolchain",
- command=["make", WithProperties("-j%(nproc:~4)s"), "toolchain/install", "V=s"],
+ command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "toolchain/install", "V=s"],
haltOnFailure = True
))
factory.addStep(ShellCommand(
name = "kmods",
description = "Building kmods",
haltOnFailure = True
))
factory.addStep(ShellCommand(
name = "kmods",
description = "Building kmods",
- command=["make", WithProperties("-j%(nproc:~4)s"), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
+ command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
#env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
haltOnFailure = True
))
#env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
haltOnFailure = True
))
factory.addStep(ShellCommand(
name = "pkgbuild",
description = "Building packages",
factory.addStep(ShellCommand(
name = "pkgbuild",
description = "Building packages",
- command=["make", WithProperties("-j%(nproc:~4)s"), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
+ command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
#env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
haltOnFailure = True
))
#env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
haltOnFailure = True
))
factory.addStep(ShellCommand(
name = "pkginstall",
description = "Installing packages",
factory.addStep(ShellCommand(
name = "pkginstall",
description = "Installing packages",
- command=["make", WithProperties("-j%(nproc:~4)s"), "package/install", "V=s"],
+ command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "package/install", "V=s"],
haltOnFailure = True
))
factory.addStep(ShellCommand(
name = "pkgindex",
description = "Indexing packages",
haltOnFailure = True
))
factory.addStep(ShellCommand(
name = "pkgindex",
description = "Indexing packages",
- command=["make", WithProperties("-j%(nproc:~4)s"), "package/index", "V=s"],
+ command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "package/index", "V=s"],
factory.addStep(ShellCommand(
name = "images",
description = "Building images",
factory.addStep(ShellCommand(
name = "images",
description = "Building images",
- command=["make", WithProperties("-j%(nproc:~4)s"), "target/install", "V=s"],
+ command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "target/install", "V=s"],
slave_port = 9990
persistent = False
slave_port = 9990
persistent = False
if ini.has_option("general", "port"):
slave_port = ini.getint("general", "port")
if ini.has_option("general", "port"):
slave_port = ini.getint("general", "port")
if ini.has_option("general", "persistent"):
persistent = ini.getboolean("general", "persistent")
if ini.has_option("general", "persistent"):
persistent = ini.getboolean("general", "persistent")
+if ini.has_option("general", "other_builds"):
+ other_builds = ini.getint("general", "other_builds")
+
for section in ini.sections():
if section.startswith("slave "):
if ini.has_option(section, "name") and ini.has_option(section, "password"):
name = ini.get(section, "name")
password = ini.get(section, "password")
for section in ini.sections():
if section.startswith("slave "):
if ini.has_option(section, "name") and ini.has_option(section, "password"):
name = ini.get(section, "name")
password = ini.get(section, "password")
if ini.has_option(section, "builds"):
if ini.has_option(section, "builds"):
- max_builds = ini.getint(section, "builds")
- c['slaves'].append(BuildSlave(name, password, max_builds = max_builds))
+ max_builds[name] = ini.getint(section, "builds")
+ c['slaves'].append(BuildSlave(name, password, max_builds = max_builds[name]))
# 'slavePortnum' defines the TCP port to listen on for connections from slaves.
# This must match the value configured into the buildslaves (with their
# 'slavePortnum' defines the TCP port to listen on for connections from slaves.
# This must match the value configured into the buildslaves (with their
+def GetNumJobs(props):
+ if props.hasProperty("slavename") and props.hasProperty("nproc"):
+ return ((int(props["nproc"]) / (max_builds[props["slavename"]] + other_builds)) + 1)
+ else:
+ return 1
+
name = "compile",
description = "Building packages",
workdir = "build/sdk",
name = "compile",
description = "Building packages",
workdir = "build/sdk",
- command = ["make", WithProperties("-j%(nproc:~4)s"), "V=s", "IGNORE_ERRORS=n m y", "BUILD_LOG=1", "CONFIG_SIGNED_PACKAGES=y"]))
+ command = ["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "V=s", "IGNORE_ERRORS=n m y", "BUILD_LOG=1", "CONFIG_SIGNED_PACKAGES=y"]))
if gpg_keyid is not None:
factory.addStep(MasterShellCommand(
if gpg_keyid is not None:
factory.addStep(MasterShellCommand(