From 32466280f5842e78c4cd1d95990b9fdad4c25ae4 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Thibaut=20VAR=C3=88NE?= Date: Wed, 27 Jun 2018 18:19:12 +0200 Subject: [PATCH] phase1: use sha2rsync.pl for 'targetupload' MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit This patch uses sha2rsync.pl to build a list of files to upload to the remote server via rsync. Because --files-from is taken as a literal list of files to consider by rsync (the file list generation is skipped), it will not delete files on the receiver. To work around that without implementing a broken client-side upload queue management system, 'targetupload' is split in two: the first step uploads and updates new and existing files using rsync standard mechanism, and the second step does nothing but delete files that are present on the receiver but not on the sender. As a side effect, this implements a '--delete-after' deletion style where files are only deleted after uploads/updates finished successfully. Signed-off-by: Thibaut VARÈNE --- phase1/master.cfg | 42 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/phase1/master.cfg b/phase1/master.cfg index 5bf8d90..54a5761 100644 --- a/phase1/master.cfg +++ b/phase1/master.cfg @@ -907,6 +907,33 @@ for target in targets: logEnviron = False, )) + # download remote sha256sums to 'target-sha256sums' + factory.addStep(ShellCommand( + name = "target-sha256sums", + description = "Fetching remote sha256sums for target", + command = ["rsync", "-4", "-vz", Interpolate("%(kw:rsyncbinurl)s/targets/%(kw:target)s/%(kw:subtarget)s/sha256sums", rsyncbinurl=rsync_bin_url, target=ts[0], subtarget=ts[1]), "target-sha256sums"], + env={'RSYNC_PASSWORD': rsync_bin_key}, + logEnviron = False, + haltOnFailure = False, + flunkOnFailure = False, + warnOnFailure = False, + )) + + # build list of files to upload + factory.addStep(FileDownload( + name = "dlsha2rsyncpl", + mastersrc = "sha2rsync.pl", + slavedest = "../sha2rsync.pl", + mode = 0755, + )) + + factory.addStep(ShellCommand( + name = "buildlist", + description = "Building list of files to upload", + command = ["../sha2rsync.pl", "target-sha256sums", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/sha256sums", target=ts[0], subtarget=ts[1]), "rsynclist"], + haltOnFailure = True, + )) + factory.addStep(FileDownload( name = "dlrsync.sh", mastersrc = "rsync.sh", @@ -914,10 +941,23 @@ for target in targets: mode = 0755 )) + # upload new files and update existing ones factory.addStep(ShellCommand( name = "targetupload", description = "Uploading target files", - command=["../rsync.sh", "-4", "--exclude=/kmods/", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), + command=["../rsync.sh", "-4", "--exclude=/kmods/", "--files-from=rsynclist", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), + "-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]), + Interpolate("%(kw:rsyncbinurl)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", rsyncbinurl=rsync_bin_url, target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)], + env={'RSYNC_PASSWORD': rsync_bin_key}, + haltOnFailure = True, + logEnviron = False, + )) + + # delete files which don't exist locally + factory.addStep(ShellCommand( + name = "targetprune", + description = "Pruning target files", + command=["../rsync.sh", "-4", "--exclude=/kmods/", "--delete", "--existing", "--ignore-existing", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]), Interpolate("%(kw:rsyncbinurl)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", rsyncbinurl=rsync_bin_url, target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)], env={'RSYNC_PASSWORD': rsync_bin_key}, -- 2.30.2