Revert "Use slave instead of buildername for partial dir to avoid slashes"

[buildbot.git] / phase2 / master.cfg

diff --git a/phase2/master.cfg b/phase2/master.cfg
index 41d50c892e6c8abcae708cfa781838b0f5448af0..faba9c9aeab8d57fab61de1a401a64e2036319ee 100644 (file)
--- a/phase2/master.cfg
+++ b/phase2/master.cfg
@@ -51,8 +51,15 @@ home_dir = os.path.abspath(ini.get("general", "homedir"))
 
 repo_url = ini.get("repo", "url")
 
-rsync_url = ini.get("rsync", "url")
-rsync_key = ini.get("rsync", "password")
+rsync_bin_url = ini.get("rsync", "binary_url")
+rsync_bin_key = ini.get("rsync", "binary_password")
+
+rsync_src_url = None
+rsync_src_key = None
+
+if ini.has_option("rsync", "source_url"):
+       rsync_src_url = ini.get("rsync", "source_url")
+       rsync_src_key = ini.get("rsync", "source_password")
 
 # find arches
 arches = [ ]
@@ -112,6 +119,7 @@ c['schedulers'].append(ForceScheduler(
 from buildbot.process.factory import BuildFactory
 from buildbot.steps.source import Git
 from buildbot.steps.shell import ShellCommand
+from buildbot.steps.shell import SetProperty
 from buildbot.steps.transfer import FileDownload
 from buildbot.process.properties import WithProperties
 
@@ -129,6 +137,13 @@ for arch in arches:
 
        factory = BuildFactory()
 
+       # find number of cores
+       factory.addStep(SetProperty(
+               name = "nproc",
+               property = "nproc",
+               description = "Finding number of CPUs",
+               command = ["nproc"]))
+
        # prepare workspace
        factory.addStep(FileDownload(mastersrc="cleanup.sh", slavedest="cleanup.sh", mode=0755))
 
@@ -155,7 +170,7 @@ for arch in arches:
        factory.addStep(ShellCommand(
                name = "downloadsdk",
                description = "Downloading SDK archive",
-               command = ["rsync", "-va", "downloads.lede-project.org::downloads/snapshots/targets/%s/%s/LEDE-SDK-*.tar.bz2" %(ts[0], ts[1]), "sdk.tar.bz2"],
+               command = ["rsync", "-va", "downloads.lede-project.org::downloads/snapshots/targets/%s/%s/[Ll][Ee][Dd][Ee]-[Ss][Dd][Kk]-*.tar.bz2" %(ts[0], ts[1]), "sdk.tar.bz2"],
                haltOnFailure = True))
 
        factory.addStep(ShellCommand(
@@ -164,14 +179,20 @@ for arch in arches:
                command = ["tar", "--strip-components=1", "-C", "sdk/", "-vxjf", "sdk.tar.bz2"],
                haltOnFailure = True))
 
-       factory.addStep(FileDownload(mastersrc=home_dir+'/key-build', slavedest="sdk/key-build", mode=0400))
-       factory.addStep(FileDownload(mastersrc=home_dir+'/key-build.pub', slavedest="sdk/key-build.pub", mode=0400))
+       factory.addStep(FileDownload(mastersrc=home_dir+'/key-build', slavedest="sdk/key-build", mode=0600))
+       factory.addStep(FileDownload(mastersrc=home_dir+'/key-build.pub', slavedest="sdk/key-build.pub", mode=0600))
 
        factory.addStep(ShellCommand(
                name = "mkdldir",
                description = "Preparing download directory",
                command = ["sh", "-c", "mkdir -p $HOME/dl && rmdir ./sdk/dl && ln -sf $HOME/dl ./sdk/dl"]))
 
+       factory.addStep(ShellCommand(
+               name = "mkconf",
+               description = "Preparing SDK configuration",
+               workdir = "build/sdk",
+               command = ["sh", "-c", "rm -f .config && make defconfig"]))
+
        factory.addStep(ShellCommand(
                name = "updatefeeds",
                description = "Updating feeds",
@@ -188,14 +209,14 @@ for arch in arches:
                name = "compile",
                description = "Building packages",
                workdir = "build/sdk",
-               command = ["make", "-j4", "V=s", "IGNORE_ERRORS=n m y", "BUILD_LOG=1", "CONFIG_SIGNED_PACKAGES=y"]))
+               command = ["make", WithProperties("-j%(nproc:~4)s"), "V=s", "IGNORE_ERRORS=n m y", "BUILD_LOG=1", "CONFIG_SIGNED_PACKAGES=y"]))
 
        factory.addStep(ShellCommand(
                name = "uploadprepare",
                description = "Preparing package directory",
                workdir = "build/sdk",
-               command = ["rsync", "-av", "--include", "/%s/" %(arch[0]), "--exclude", "/*", "--exclude", "/%s/*" %(arch[0]), "bin/packages/", "%s/packages/" %(rsync_url)],
-               env={'RSYNC_PASSWORD': rsync_key},
+               command = ["rsync", "-av", "--include", "/%s/" %(arch[0]), "--exclude", "/*", "--exclude", "/%s/*" %(arch[0]), "bin/packages/", "%s/packages/" %(rsync_bin_url)],
+               env={'RSYNC_PASSWORD': rsync_bin_key},
                haltOnFailure = True,
                logEnviron = False
        ))
@@ -204,8 +225,8 @@ for arch in arches:
                name = "packageupload",
                description = "Uploading package files",
                workdir = "build/sdk",
-               command = ["rsync", "--delete", "--delay-updates", "-avz", "bin/packages/%s/" %(arch[0]), "%s/packages/%s/" %(rsync_url, arch[0])],
-               env={'RSYNC_PASSWORD': rsync_key},
+               command = ["rsync", "--delete", "--delay-updates", WithProperties("--partial-dir=.~tmp~%(buildername)s"), "-avz", "bin/packages/%s/" %(arch[0]), "%s/packages/%s/" %(rsync_bin_url, arch[0])],
+               env={'RSYNC_PASSWORD': rsync_bin_key},
                haltOnFailure = True,
                logEnviron = False
        ))
@@ -214,8 +235,8 @@ for arch in arches:
                name = "logprepare",
                description = "Preparing log directory",
                workdir = "build/sdk",
-               command = ["rsync", "-av", "--include", "/%s/" %(arch[0]), "--exclude", "/*", "--exclude", "/%s/*" %(arch[0]), "bin/packages/", "%s/faillogs/" %(rsync_url)],
-               env={'RSYNC_PASSWORD': rsync_key},
+               command = ["rsync", "-av", "--include", "/%s/" %(arch[0]), "--exclude", "/*", "--exclude", "/%s/*" %(arch[0]), "bin/packages/", "%s/faillogs/" %(rsync_bin_url)],
+               env={'RSYNC_PASSWORD': rsync_bin_key},
                haltOnFailure = True,
                logEnviron = False
        ))
@@ -240,12 +261,23 @@ for arch in arches:
                name = "logupload",
                description = "Uploading failure logs",
                workdir = "build/sdk",
-               command = ["rsync", "--delete", "--delay-updates", "-avz", "faillogs/", "%s/faillogs/%s/" %(rsync_url, arch[0])],
-               env={'RSYNC_PASSWORD': rsync_key},
+               command = ["rsync", "--delete", "--delay-updates", WithProperties("--partial-dir=.~tmp~%(buildername)s"), "-avz", "faillogs/", "%s/faillogs/%s/" %(rsync_bin_url, arch[0])],
+               env={'RSYNC_PASSWORD': rsync_bin_key},
                haltOnFailure = False,
                logEnviron = False
        ))
 
+       if rsync_src_url is not None:
+               factory.addStep(ShellCommand(
+                       name = "sourceupload",
+                       description = "Uploading source archives",
+                       workdir = "build/sdk",
+                       command = ["rsync", "--delay-updates", WithProperties("--partial-dir=.~tmp~%(buildername)s"), "-avz", "dl/", "%s/" %(rsync_src_url)],
+                       env={'RSYNC_PASSWORD': rsync_src_key},
+                       haltOnFailure = False,
+                       logEnviron = False
+               ))
+
        from buildbot.config import BuilderConfig
 
        c['builders'].append(BuilderConfig(name=arch[0], slavenames=slaveNames, factory=factory))
@@ -304,4 +336,3 @@ c['db'] = {
        # this at its default for all but the largest installations.
        'db_url' : "sqlite:///state.sqlite",
 }
-