We set '-4' in rsync_defopts, asking rsync to "prefer" ipv4 whenever
possible on _all_ workers. This was historically done because some
workers had flaky ipv6 connectivity, however in the future the reverse
may be true, with worker having worse networking over ipv4 than ipv6.
This change introduces an 'rsync_ipv4' worker configuration option:
when set to a true value, the old rsync behavior (adding '-4') is used,
when unset or set to false, the extra rsync argument is not used.
Signed-off-by: Thibaut VARÈNE <hacks@slashdirt.org>
dl_lock = host1
# tag_only workers will only build forced tag buildrequests (i.e. release builds)
tag_only = yes
dl_lock = host1
# tag_only workers will only build forced tag buildrequests (i.e. release builds)
tag_only = yes
+# if rsync operations must prefer ipv4 ('rsync -4'), set the following
+rsync_ipv4 = yes
repo_url = ini['repo'].get("url")
repo_url = ini['repo'].get("url")
-rsync_defopts = ["-v", "-4", "--timeout=120"]
+rsync_defopts = ["-v", "--timeout=120"]
#if rsync_bin_url.find("::") > 0 or rsync_bin_url.find("rsync://") == 0:
# rsync_bin_defopts += ["--contimeout=20"]
#if rsync_bin_url.find("::") > 0 or rsync_bin_url.find("rsync://") == 0:
# rsync_bin_defopts += ["--contimeout=20"]
password = section.get("password")
phase = section.getint("phase")
tagonly = section.getboolean("tag_only")
password = section.get("password")
phase = section.getint("phase")
tagonly = section.getboolean("tag_only")
+ rsyncipv4 = section.getboolean("rsync_ipv4")
if not name or not password or not phase == 1:
log.msg("invalid worker configuration ignored: {}".format(repr(section)))
if not name or not password or not phase == 1:
log.msg("invalid worker configuration ignored: {}".format(repr(section)))
sl_props['ul_lock'] = lockname
if lockname not in NetLocks:
NetLocks[lockname] = locks.MasterLock(lockname)
sl_props['ul_lock'] = lockname
if lockname not in NetLocks:
NetLocks[lockname] = locks.MasterLock(lockname)
+ if rsyncipv4:
+ sl_props['rsync_ipv4'] = True # only set prop if required, we use '+' Interpolate substitution
log.msg("Configured worker: {}".format(name))
# NB: phase1 build factory requires workers to be single-build only
log.msg("Configured worker: {}".format(name))
# NB: phase1 build factory requires workers to be single-build only
name = "dirupload",
description = "Uploading directory structure",
descriptionDone = "Directory structure uploaded",
name = "dirupload",
description = "Uploading directory structure",
descriptionDone = "Directory structure uploaded",
- command = ["rsync", "-az"] + rsync_defopts + ["tmp/upload/", Interpolate("%(kw:url)s/", url=GetRsyncParams.withArgs("bin", "url"))],
+ command = ["rsync", "-az"] + rsync_defopts + [Interpolate("%(prop:rsync_ipv4:+-4)s"), "tmp/upload/", Interpolate("%(kw:url)s/", url=GetRsyncParams.withArgs("bin", "url"))],
env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
haltOnFailure = True,
logEnviron = False,
env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
haltOnFailure = True,
logEnviron = False,
name = "target-sha256sums",
description = "Fetching remote sha256sums for target",
descriptionDone = "Remote sha256sums for target fetched",
name = "target-sha256sums",
description = "Fetching remote sha256sums for target",
descriptionDone = "Remote sha256sums for target fetched",
- command = ["rsync", "-z"] + rsync_defopts + [Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/sha256sums", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix), "target-sha256sums"],
+ command = ["rsync", "-z"] + rsync_defopts + [Interpolate("%(prop:rsync_ipv4:+-4)s"), Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/sha256sums", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix), "target-sha256sums"],
env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
logEnviron = False,
haltOnFailure = False,
env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
logEnviron = False,
haltOnFailure = False,
description = "Uploading target files",
descriptionDone = "Target files uploaded",
command=["../rsync.sh", "--exclude=/kmods/", "--files-from=rsynclist", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_defopts +
description = "Uploading target files",
descriptionDone = "Target files uploaded",
command=["../rsync.sh", "--exclude=/kmods/", "--files-from=rsynclist", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_defopts +
- ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]),
+ ["-a", Interpolate("%(prop:rsync_ipv4:+-4)s"), Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]),
Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
haltOnFailure = True,
Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
haltOnFailure = True,
description = "Pruning target files",
descriptionDone = "Target files pruned",
command=["../rsync.sh", "--exclude=/kmods/", "--delete", "--existing", "--ignore-existing", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_defopts +
description = "Pruning target files",
descriptionDone = "Target files pruned",
command=["../rsync.sh", "--exclude=/kmods/", "--delete", "--existing", "--ignore-existing", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_defopts +
- ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]),
+ ["-a", Interpolate("%(prop:rsync_ipv4:+-4)s"), Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]),
Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
haltOnFailure = True,
Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
haltOnFailure = True,
description = "Uploading kmod archive",
descriptionDone = "Kmod archive uploaded",
command=["../rsync.sh", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_defopts +
description = "Uploading kmod archive",
descriptionDone = "Kmod archive uploaded",
command=["../rsync.sh", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_defopts +
- ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1]),
+ ["-a", Interpolate("%(prop:rsync_ipv4:+-4)s"), Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1]),
Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
haltOnFailure = True,
Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
haltOnFailure = True,
name = "sourceupload",
description = "Uploading source archives",
descriptionDone = "Source archives uploaded",
name = "sourceupload",
description = "Uploading source archives",
descriptionDone = "Source archives uploaded",
- command=["../rsync.sh", "--files-from=sourcelist", "--size-only", "--delay-updates"] + rsync_defopts +
- [Interpolate("--partial-dir=.~tmp~%(kw:target)s~%(kw:subtarget)s~%(prop:workername)s", target=ts[0], subtarget=ts[1]), "-a", "dl/", Interpolate("%(kw:url)s/", url=GetRsyncParams.withArgs("src", "url"))],
+ command=["../rsync.sh", "--files-from=sourcelist", "--size-only", "--delay-updates"] + rsync_defopts + [Interpolate("%(prop:rsync_ipv4:+-4)s"),
+ Interpolate("--partial-dir=.~tmp~%(kw:target)s~%(kw:subtarget)s~%(prop:workername)s", target=ts[0], subtarget=ts[1]), "-a", "dl/", Interpolate("%(kw:url)s/", url=GetRsyncParams.withArgs("src", "url"))],
env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("src", "key")) },
haltOnFailure = True,
logEnviron = False,
env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("src", "key")) },
haltOnFailure = True,
logEnviron = False,