phase1: rsync based on checksum, not mtime
[buildbot.git] / phase1 / master.cfg
index 6ae57d9004437e3a132734b4c98f14110a4a888f..89243ffb214550bcb56a742b61be601769300c95 100644 (file)
@@ -48,6 +48,10 @@ c['mergeRequests'] = True
 ####### CHANGESOURCES
 
 home_dir = os.path.abspath(ini.get("general", "homedir"))
+tree_expire = 0
+
+if ini.has_option("general", "expire"):
+       tree_expire = ini.getint("general", "expire")
 
 repo_url = ini.get("repo", "url")
 
@@ -61,6 +65,20 @@ if ini.has_option("rsync", "source_url"):
        rsync_src_url = ini.get("rsync", "source_url")
        rsync_src_key = ini.get("rsync", "source_password")
 
+gpg_keyid = None
+gpg_comment = "Unattended build signature"
+gpg_passfile = "/dev/null"
+
+if ini.has_option("gpg", "keyid"):
+       gpg_keyid = ini.get("gpg", "keyid")
+
+if ini.has_option("gpg", "comment"):
+       gpg_comment = ini.get("gpg", "comment")
+
+if ini.has_option("gpg", "passfile"):
+       gpg_passfile = ini.get("gpg", "passfile")
+
+
 # find targets
 targets = [ ]
 
@@ -114,7 +132,9 @@ from buildbot.process.factory import BuildFactory
 from buildbot.steps.source import Git
 from buildbot.steps.shell import ShellCommand
 from buildbot.steps.shell import SetProperty
+from buildbot.steps.transfer import FileUpload
 from buildbot.steps.transfer import FileDownload
+from buildbot.steps.master import MasterShellCommand
 from buildbot.process.properties import WithProperties
 
 
@@ -255,6 +275,21 @@ for target in targets:
                description = "Finding number of CPUs",
                command = ["nproc"]))
 
+       # expire tree if needed
+       if tree_expire > 0:
+               factory.addStep(FileDownload(
+                       mastersrc = "expire.sh",
+                       slavedest = "../expire.sh",
+                       mode = 0755))
+
+               factory.addStep(ShellCommand(
+                       name = "expire",
+                       description = "Checking for build tree expiry",
+                       command = ["./expire.sh", str(tree_expire)],
+                       workdir = ".",
+                       haltOnFailure = True,
+                       timeout = 2400))
+
        # check out the source
        factory.addStep(Git(repourl=repo_url, mode='update'))
 
@@ -269,6 +304,12 @@ for target in targets:
 #              description = "Copy the feeds.conf",
 #              command='''cp ~/feeds.conf ./feeds.conf''' ))
 
+       # feed
+       factory.addStep(ShellCommand(
+               name = "rmfeedlinks",
+               description = "Remove feed symlinks",
+               command=["rm", "-rf", "package/feeds/"]))
+
        # feed
        factory.addStep(ShellCommand(
                name = "updatefeeds",
@@ -298,6 +339,9 @@ CONFIG_SIGNED_PACKAGES=y
 # CONFIG_PER_FEED_REPO_ADD_COMMENTED is not set
 CONFIG_KERNEL_KALLSYMS=y
 CONFIG_COLLECT_KERNEL_DEBUG=y
+CONFIG_TARGET_ALL_PROFILES=y
+CONFIG_TARGET_MULTI_PROFILE=y
+CONFIG_TARGET_PER_DEVICE_ROOTFS=y
 EOT''' %(ts[0], ts[0], ts[1]) ))
 
        factory.addStep(ShellCommand(
@@ -417,11 +461,61 @@ EOT''' %(ts[0], ts[0], ts[1]) ))
        factory.addStep(ShellCommand(
                name = "images",
                description = "Building images",
-               command=["make", "-j1", "target/install", "V=s"],
+               command=["make", WithProperties("-j%(nproc:~4)s"), "target/install", "V=s"],
+               doStepIf = isChangeBuiltin,
+               haltOnFailure = True
+       ))
+
+       factory.addStep(ShellCommand(
+               name = "checksums",
+               description = "Calculating checksums",
+               command=["make", "-j1", "checksum", "V=s"],
                doStepIf = isChangeBuiltin,
                haltOnFailure = True
        ))
 
+       # sign
+       if gpg_keyid is not None:
+               factory.addStep(MasterShellCommand(
+                       name = "signprepare",
+                       description = "Preparing temporary signing directory",
+                       command = ["mkdir", "-p", "%s/signing" %(home_dir)],
+                       haltOnFailure = True
+               ))
+
+               factory.addStep(ShellCommand(
+                       name = "signpack",
+                       description = "Packing files to sign",
+                       command = WithProperties("find bin/targets/%s/%s%%(libc)s/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz" %(ts[0], ts[1])),
+                       haltOnFailure = True
+               ))
+
+               factory.addStep(FileUpload(
+                       slavesrc = "sign.tar.gz",
+                       masterdest = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
+                       haltOnFailure = True
+               ))
+
+               factory.addStep(MasterShellCommand(
+                       name = "signfiles",
+                       description = "Signing files",
+                       command = ["%s/signall.sh" %(home_dir), "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]), gpg_keyid, gpg_passfile, gpg_comment],
+                       haltOnFailure = True
+               ))
+
+               factory.addStep(FileDownload(
+                       mastersrc = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
+                       slavedest = "sign.tar.gz",
+                       haltOnFailure = True
+               ))
+
+               factory.addStep(ShellCommand(
+                       name = "signunpack",
+                       description = "Unpacking signed files",
+                       command = ["tar", "-xzf", "sign.tar.gz"],
+                       haltOnFailure = True
+               ))
+
        # upload
        factory.addStep(ShellCommand(
                name = "uploadprepare",
@@ -435,7 +529,7 @@ EOT''' %(ts[0], ts[0], ts[1]) ))
        factory.addStep(ShellCommand(
                name = "targetupload",
                description = "Uploading target files",
-               command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", WithProperties("bin/targets/%s/%s%%(libc)s/" %(ts[0], ts[1])), "%s/targets/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
+               command=["rsync", "--delete", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", WithProperties("bin/targets/%s/%s%%(libc)s/" %(ts[0], ts[1])), "%s/targets/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
                env={'RSYNC_PASSWORD': rsync_bin_key},
                haltOnFailure = True,
                logEnviron = False
@@ -445,7 +539,7 @@ EOT''' %(ts[0], ts[0], ts[1]) ))
                factory.addStep(ShellCommand(
                        name = "sourceupload",
                        description = "Uploading source archives",
-                       command=["rsync", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "dl/", "%s/" %(rsync_src_url)],
+                       command=["rsync", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "dl/", "%s/" %(rsync_src_url)],
                        env={'RSYNC_PASSWORD': rsync_src_key},
                        haltOnFailure = True,
                        logEnviron = False
@@ -524,12 +618,12 @@ if ini.has_option("irc", "host") and ini.has_option("irc", "nickname") and ini.h
                irc_pass = ini.get("irc", "password")
 
        irc = words.IRC(irc_host, irc_nick, port = irc_port, password = irc_pass,
-                       channels = [{ "channel": irc_chan }],
-                       notify_events = {
-                         'exception': 1,
-                         'successToFailure': 1,
-                         'failureToSuccess': 1
-                       }
+                       channels = [{ "channel": irc_chan }],
+                       notify_events = {
+                         'exception': 1,
+                         'successToFailure': 1,
+                         'failureToSuccess': 1
+                       }
        )
 
        c['status'].append(irc)