Use unique rsync partial directories per builder

[buildbot.git] / phase2 / master.cfg

diff --git a/phase2/master.cfg b/phase2/master.cfg
index 11e2173e95bf4485430087c7a701eadb7df0fbc9..faba9c9aeab8d57fab61de1a401a64e2036319ee 100644 (file)
--- a/phase2/master.cfg
+++ b/phase2/master.cfg
@@ -51,8 +51,15 @@ home_dir = os.path.abspath(ini.get("general", "homedir"))
 
 repo_url = ini.get("repo", "url")
 
-rsync_url = ini.get("rsync", "url")
-rsync_key = ini.get("rsync", "password")
+rsync_bin_url = ini.get("rsync", "binary_url")
+rsync_bin_key = ini.get("rsync", "binary_password")
+
+rsync_src_url = None
+rsync_src_key = None
+
+if ini.has_option("rsync", "source_url"):
+       rsync_src_url = ini.get("rsync", "source_url")
+       rsync_src_key = ini.get("rsync", "source_password")
 
 # find arches
 arches = [ ]
@@ -163,7 +170,7 @@ for arch in arches:
        factory.addStep(ShellCommand(
                name = "downloadsdk",
                description = "Downloading SDK archive",
-               command = ["rsync", "-va", "downloads.lede-project.org::downloads/snapshots/targets/%s/%s/LEDE-SDK-*.tar.bz2" %(ts[0], ts[1]), "sdk.tar.bz2"],
+               command = ["rsync", "-va", "downloads.lede-project.org::downloads/snapshots/targets/%s/%s/[Ll][Ee][Dd][Ee]-[Ss][Dd][Kk]-*.tar.bz2" %(ts[0], ts[1]), "sdk.tar.bz2"],
                haltOnFailure = True))
 
        factory.addStep(ShellCommand(
@@ -208,8 +215,8 @@ for arch in arches:
                name = "uploadprepare",
                description = "Preparing package directory",
                workdir = "build/sdk",
-               command = ["rsync", "-av", "--include", "/%s/" %(arch[0]), "--exclude", "/*", "--exclude", "/%s/*" %(arch[0]), "bin/packages/", "%s/packages/" %(rsync_url)],
-               env={'RSYNC_PASSWORD': rsync_key},
+               command = ["rsync", "-av", "--include", "/%s/" %(arch[0]), "--exclude", "/*", "--exclude", "/%s/*" %(arch[0]), "bin/packages/", "%s/packages/" %(rsync_bin_url)],
+               env={'RSYNC_PASSWORD': rsync_bin_key},
                haltOnFailure = True,
                logEnviron = False
        ))
@@ -218,8 +225,8 @@ for arch in arches:
                name = "packageupload",
                description = "Uploading package files",
                workdir = "build/sdk",
-               command = ["rsync", "--delete", "--delay-updates", "-avz", "bin/packages/%s/" %(arch[0]), "%s/packages/%s/" %(rsync_url, arch[0])],
-               env={'RSYNC_PASSWORD': rsync_key},
+               command = ["rsync", "--delete", "--delay-updates", WithProperties("--partial-dir=.~tmp~%(buildername)s"), "-avz", "bin/packages/%s/" %(arch[0]), "%s/packages/%s/" %(rsync_bin_url, arch[0])],
+               env={'RSYNC_PASSWORD': rsync_bin_key},
                haltOnFailure = True,
                logEnviron = False
        ))
@@ -228,8 +235,8 @@ for arch in arches:
                name = "logprepare",
                description = "Preparing log directory",
                workdir = "build/sdk",
-               command = ["rsync", "-av", "--include", "/%s/" %(arch[0]), "--exclude", "/*", "--exclude", "/%s/*" %(arch[0]), "bin/packages/", "%s/faillogs/" %(rsync_url)],
-               env={'RSYNC_PASSWORD': rsync_key},
+               command = ["rsync", "-av", "--include", "/%s/" %(arch[0]), "--exclude", "/*", "--exclude", "/%s/*" %(arch[0]), "bin/packages/", "%s/faillogs/" %(rsync_bin_url)],
+               env={'RSYNC_PASSWORD': rsync_bin_key},
                haltOnFailure = True,
                logEnviron = False
        ))
@@ -254,12 +261,23 @@ for arch in arches:
                name = "logupload",
                description = "Uploading failure logs",
                workdir = "build/sdk",
-               command = ["rsync", "--delete", "--delay-updates", "-avz", "faillogs/", "%s/faillogs/%s/" %(rsync_url, arch[0])],
-               env={'RSYNC_PASSWORD': rsync_key},
+               command = ["rsync", "--delete", "--delay-updates", WithProperties("--partial-dir=.~tmp~%(buildername)s"), "-avz", "faillogs/", "%s/faillogs/%s/" %(rsync_bin_url, arch[0])],
+               env={'RSYNC_PASSWORD': rsync_bin_key},
                haltOnFailure = False,
                logEnviron = False
        ))
 
+       if rsync_src_url is not None:
+               factory.addStep(ShellCommand(
+                       name = "sourceupload",
+                       description = "Uploading source archives",
+                       workdir = "build/sdk",
+                       command = ["rsync", "--delay-updates", WithProperties("--partial-dir=.~tmp~%(buildername)s"), "-avz", "dl/", "%s/" %(rsync_src_url)],
+                       env={'RSYNC_PASSWORD': rsync_src_key},
+                       haltOnFailure = False,
+                       logEnviron = False
+               ))
+
        from buildbot.config import BuilderConfig
 
        c['builders'].append(BuilderConfig(name=arch[0], slavenames=slaveNames, factory=factory))