import os
import re
+import base64
import subprocess
import ConfigParser
persistent = False
other_builds = 0
tree_expire = 0
+git_ssh = False
+git_ssh_key = None
if ini.has_option("general", "port"):
slave_port = ini.getint("general", "port")
if ini.has_option("general", "expire"):
tree_expire = ini.getint("general", "expire")
+if ini.has_option("general", "git_ssh"):
+ git_ssh = ini.getboolean("general", "git_ssh")
+
+if ini.has_option("general", "git_ssh_key"):
+ git_ssh_key = ini.get("general", "git_ssh_key")
+else:
+ git_ssh = False
+
c['slaves'] = []
max_builds = dict()
####### CHANGESOURCES
-home_dir = os.path.abspath(ini.get("general", "homedir"))
+work_dir = os.path.abspath(ini.get("general", "workdir") or ".")
+scripts_dir = os.path.abspath("../scripts")
rsync_bin_url = ini.get("rsync", "binary_url")
rsync_bin_key = ini.get("rsync", "binary_password")
if ini.has_option("rsync", "sdk_pattern"):
rsync_sdk_pat = ini.get("rsync", "sdk_pattern")
-gpg_home = "~/.gnupg"
-gpg_keyid = None
-gpg_comment = "Unattended build signature"
-gpg_passfile = "/dev/null"
+repo_url = ini.get("repo", "url")
+repo_branch = "master"
+
+if ini.has_option("repo", "branch"):
+ repo_branch = ini.get("repo", "branch")
-if ini.has_option("gpg", "home"):
- gpg_home = ini.get("gpg", "home")
+gpg_key = None
+gpg_passphrase = None
+gpg_comment = repo_branch.replace("-", " ").title() + " key"
-if ini.has_option("gpg", "keyid"):
- gpg_keyid = ini.get("gpg", "keyid")
+if ini.has_option("gpg", "key"):
+ gpg_key = ini.get("gpg", "key")
+
+if ini.has_option("gpg", "passphrase"):
+ gpg_passphrase = ini.get("gpg", "passphrase")
if ini.has_option("gpg", "comment"):
gpg_comment = ini.get("gpg", "comment")
-if ini.has_option("gpg", "passfile"):
- gpg_passfile = ini.get("gpg", "passfile")
+usign_key = None
+usign_comment = "untrusted comment: " + repo_branch.replace("-", " ").title() + " key"
+
+if ini.has_option("usign", "key"):
+ usign_key = ini.get("usign", "key")
+
+if ini.has_option("usign", "comment"):
+ usign_comment = ini.get("usign", "comment")
# find arches
arches = [ ]
archnames = [ ]
-findarches = subprocess.Popen([home_dir+'/dumpinfo.pl', 'architectures'],
- stdout = subprocess.PIPE, cwd = home_dir+'/source.git')
+if not os.path.isdir(work_dir+'/source.git'):
+ subprocess.call(["git", "clone", "--depth=1", "--branch="+repo_branch, repo_url, work_dir+'/source.git'])
+else:
+ subprocess.call(["git", "pull"], cwd = work_dir+'/source.git')
+
+findarches = subprocess.Popen([scripts_dir + '/dumpinfo.pl', 'architectures'],
+ stdout = subprocess.PIPE, cwd = work_dir+'/source.git')
while True:
line = findarches.stdout.readline()
feedbranches[url[0]] = branch
c['change_source'].append(GitPoller(url[0], branch=branch, workdir='%s/%s.git' %(os.getcwd(), parts[1]), pollinterval=300))
-make = subprocess.Popen(['make', '--no-print-directory', '-C', home_dir+'/source.git/target/sdk/', 'val.BASE_FEED'],
- env = dict(os.environ, TOPDIR=home_dir+'/source.git'), stdout = subprocess.PIPE)
+make = subprocess.Popen(['make', '--no-print-directory', '-C', work_dir+'/source.git/target/sdk/', 'val.BASE_FEED'],
+ env = dict(os.environ, TOPDIR=work_dir+'/source.git'), stdout = subprocess.PIPE)
line = make.stdout.readline()
if line:
parse_feed_entry(line)
-with open(home_dir+'/source.git/feeds.conf.default', 'r') as f:
+with open(work_dir+'/source.git/feeds.conf.default', 'r') as f:
for line in f:
parse_feed_entry(line)
else:
return "/"
+def UsignSec2Pub(seckey, comment="untrusted comment: secret key"):
+ try:
+ seckey = base64.b64decode(seckey)
+ except:
+ return None
+
+ return "{}\n{}".format(re.sub(r"\bsecret key$", "public key", comment),
+ base64.b64encode(seckey[0:2] + seckey[32:40] + seckey[72:]))
+
c['builders'] = []
command = ["nproc"]))
# prepare workspace
- factory.addStep(FileDownload(mastersrc="cleanup.sh", slavedest="cleanup.sh", mode=0755))
+ factory.addStep(FileDownload(
+ mastersrc = scripts_dir + '/cleanup-phase2.sh',
+ slavedest = "cleanup.sh",
+ mode = 0755))
if not persistent:
factory.addStep(ShellCommand(
# expire tree if needed
elif tree_expire > 0:
factory.addStep(FileDownload(
- mastersrc = home_dir+"/expire.sh",
+ mastersrc = scripts_dir + '/expire.sh',
slavedest = "../expire.sh",
mode = 0755))
workdir = "build/sdk",
command = ["make", "-f", "getversion.mk"]))
- factory.addStep(FileDownload(mastersrc=home_dir+'/key-build', slavedest="sdk/key-build", mode=0600))
- factory.addStep(FileDownload(mastersrc=home_dir+'/key-build.pub', slavedest="sdk/key-build.pub", mode=0600))
+ # install build key
+ if usign_key is not None:
+ factory.addStep(StringDownload(
+ name = "dlkeybuildpub",
+ s = UsignSec2Pub(usign_key, usign_comment),
+ slavedest = "sdk/key-build.pub",
+ mode = 0600))
+
+ factory.addStep(StringDownload(
+ name = "dlkeybuild",
+ s = "# fake private key",
+ slavedest = "sdk/key-build",
+ mode = 0600))
+
+ factory.addStep(StringDownload(
+ name = "dlkeybuilducert",
+ s = "# fake certificate",
+ slavedest = "sdk/key-build.ucert",
+ mode = 0600))
factory.addStep(ShellCommand(
name = "mkdldir",
command = ["sh", "-c", "rm -f .config && make defconfig"]))
factory.addStep(FileDownload(
- mastersrc = home_dir+'/ccache.sh',
+ mastersrc = scripts_dir + '/ccache.sh',
slavedest = 'sdk/ccache.sh',
mode = 0755))
command = ["./ccache.sh"],
haltOnFailure = True))
+ if git_ssh:
+ factory.addStep(StringDownload(
+ name = "dlgitclonekey",
+ s = git_ssh_key,
+ slavedest = "../git-clone.key",
+ mode = 0600))
+
+ factory.addStep(ShellCommand(
+ name = "patchfeedsconf",
+ description = "Patching feeds.conf",
+ workdir = "build/sdk",
+ command = "sed -e 's#https://#ssh://git@#g' feeds.conf.default > feeds.conf",
+ haltOnFailure = True))
+
factory.addStep(ShellCommand(
name = "updatefeeds",
description = "Updating feeds",
workdir = "build/sdk",
command = ["./scripts/feeds", "update", "-f"],
+ env = {'GIT_SSH_COMMAND': WithProperties("ssh -o IdentitiesOnly=yes -o IdentityFile=%(cwd)s/git-clone.key -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no", cwd=GetCwd)} if git_ssh else {},
haltOnFailure = True))
+ if git_ssh:
+ factory.addStep(ShellCommand(
+ name = "rmfeedsconf",
+ description = "Removing feeds.conf",
+ workdir = "build/sdk",
+ command=["rm", "feeds.conf"],
+ haltOnFailure = True))
+
factory.addStep(ShellCommand(
name = "installfeeds",
description = "Installing feeds",
description = "Building packages",
workdir = "build/sdk",
timeout = 3600,
- command = ["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "IGNORE_ERRORS=n m y", "BUILD_LOG=1", "CONFIG_SIGNED_PACKAGES=y", "CONFIG_AUTOREMOVE=y"],
+ command = ["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "IGNORE_ERRORS=n m y", "BUILD_LOG=1", "CONFIG_AUTOREMOVE=y"],
env = {'CCACHE_BASEDIR': WithProperties("%(cwd)s", cwd=GetCwd)},
haltOnFailure = True))
workdir = "build/sdk",
command = "./scripts/feeds list -s -f > bin/packages/%s/feeds.conf" %(arch[0])))
- if gpg_keyid is not None:
+ if gpg_key is not None or usign_key is not None:
factory.addStep(MasterShellCommand(
name = "signprepare",
description = "Preparing temporary signing directory",
- command = ["mkdir", "-p", "%s/signing" %(home_dir)],
+ command = ["mkdir", "-p", "%s/signing" %(work_dir)],
haltOnFailure = True
))
factory.addStep(FileUpload(
slavesrc = "sdk/sign.tar.gz",
- masterdest = "%s/signing/%s.tar.gz" %(home_dir, arch[0]),
+ masterdest = "%s/signing/%s.tar.gz" %(work_dir, arch[0]),
haltOnFailure = True
))
factory.addStep(MasterShellCommand(
name = "signfiles",
description = "Signing files",
- command = ["%s/signall.sh" %(home_dir), "%s/signing/%s.tar.gz" %(home_dir, arch[0]), gpg_keyid, gpg_comment],
- env = {'GNUPGHOME': gpg_home, 'PASSFILE': gpg_passfile},
+ command = ["%s/signall.sh" %(scripts_dir), "%s/signing/%s.tar.gz" %(work_dir, arch[0])],
+ env = {
+ 'GPGKEY': gpg_key,
+ 'GPGPASS': gpg_passphrase,
+ 'GPGCOMMENT': gpg_comment,
+ 'USIGNKEY': usign_key,
+ 'USIGNCOMMENT': usign_comment
+ },
haltOnFailure = True
))
factory.addStep(FileDownload(
- mastersrc = "%s/signing/%s.tar.gz" %(home_dir, arch[0]),
+ mastersrc = "%s/signing/%s.tar.gz" %(work_dir, arch[0]),
slavedest = "sdk/sign.tar.gz",
haltOnFailure = True
))
name = "logprepare",
description = "Preparing log directory",
workdir = "build/sdk",
- command = ["rsync", "-4", "-av", "--include", "/%s/" %(arch[0]), "--exclude", "/*", "--exclude", "/%s/*" %(arch[0]), "bin/packages/", "%s/faillogs/" %(rsync_bin_url)],
+ command = ["rsync", "-4", "-av", "--include", "/%s/" %(arch[0]), "--exclude", "/*", "--exclude", "/%s/*" %(arch[0]), "bin/packages/", WithProperties("%s/faillogs%%(suffix)s/" %(rsync_bin_url), suffix=GetDirectorySuffix)],
env={'RSYNC_PASSWORD': rsync_bin_key},
haltOnFailure = True,
logEnviron = False
name = "logupload",
description = "Uploading failure logs",
workdir = "build/sdk",
- command = ["rsync", "-4", "--progress", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-avz", "faillogs/", "%s/faillogs/%s/" %(rsync_bin_url, arch[0])],
+ command = ["rsync", "-4", "--progress", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-avz", "faillogs/", WithProperties("%s/faillogs%%(suffix)s/%s/" %(rsync_bin_url, arch[0]), suffix=GetDirectorySuffix)],
env={'RSYNC_PASSWORD': rsync_bin_key},
haltOnFailure = False,
logEnviron = False
))
if rsync_src_url is not None:
+ factory.addStep(ShellCommand(
+ name = "sourcelist",
+ description = "Finding source archives to upload",
+ workdir = "build/sdk",
+ command = "find dl/ -maxdepth 1 -type f -not -size 0 -not -name '.*' -newer ../sdk.archive -printf '%f\\n' > sourcelist",
+ haltOnFailure = True
+ ))
+
factory.addStep(ShellCommand(
name = "sourceupload",
description = "Uploading source archives",
workdir = "build/sdk",
- command = ["rsync", "-4", "--progress", "--checksum", "--delay-updates",
+ command = ["rsync", "--files-from=sourcelist", "-4", "--progress", "--checksum", "--delay-updates",
WithProperties("--partial-dir=.~tmp~%s~%%(slavename)s" %(arch[0])), "-avz", "dl/", "%s/" %(rsync_src_url)],
env={'RSYNC_PASSWORD': rsync_src_key},
haltOnFailure = False,
projects / buildbot.git / blobdiff