# -*- python -*-
# ex: set syntax=python:

import os
import re
import base64
import subprocess
import ConfigParser

from buildbot import locks

ini = ConfigParser.ConfigParser()
ini.read(os.getenv("BUILDMASTER_CONFIG", "./config.ini"))

buildbot_url = ini.get("phase2", "buildbot_url")

# This is a sample buildmaster config file. It must be installed as
# 'master.cfg' in your buildmaster's base directory.

# This is the dictionary that the buildmaster pays attention to. We also use
# a shorter alias to save typing.
c = BuildmasterConfig = {}

####### BUILDSLAVES

# The 'slaves' list defines the set of recognized buildslaves. Each element is
# a BuildSlave object, specifying a unique slave name and password.  The same
# slave name and password must be configured on the slave.
from buildbot.buildslave import BuildSlave

slave_port = 9990
persistent = False
other_builds = 0
tree_expire = 0
git_ssh = False
git_ssh_key = None

if ini.has_option("phase2", "port"):
	slave_port = ini.getint("phase2", "port")

if ini.has_option("phase2", "persistent"):
	persistent = ini.getboolean("phase2", "persistent")

if ini.has_option("phase2", "other_builds"):
	other_builds = ini.getint("phase2", "other_builds")

if ini.has_option("phase2", "expire"):
	tree_expire = ini.getint("phase2", "expire")

if ini.has_option("general", "git_ssh"):
	git_ssh = ini.getboolean("general", "git_ssh")

if ini.has_option("general", "git_ssh_key"):
	git_ssh_key = ini.get("general", "git_ssh_key")
else:
	git_ssh = False

c['slaves'] = []
max_builds = dict()

for section in ini.sections():
	if section.startswith("slave "):
		if ini.has_option(section, "name") and ini.has_option(section, "password") and \
		   ini.has_option(section, "phase") and ini.getint(section, "phase") == 2:
			name = ini.get(section, "name")
			password = ini.get(section, "password")
			max_builds[name] = 1
			if ini.has_option(section, "builds"):
				max_builds[name] = ini.getint(section, "builds")
			c['slaves'].append(BuildSlave(name, password, max_builds = max_builds[name]))

# 'slavePortnum' defines the TCP port to listen on for connections from slaves.
# This must match the value configured into the buildslaves (with their
# --master option)
c['slavePortnum'] = slave_port

# coalesce builds
c['mergeRequests'] = True

# Reduce amount of backlog data
c['buildHorizon'] = 30
c['logHorizon'] = 20

####### CHANGESOURCES

work_dir = os.path.abspath(ini.get("general", "workdir") or ".")
scripts_dir = os.path.abspath("../scripts")

rsync_bin_url = ini.get("rsync", "binary_url")
rsync_bin_key = ini.get("rsync", "binary_password")

rsync_src_url = None
rsync_src_key = None

if ini.has_option("rsync", "source_url"):
	rsync_src_url = ini.get("rsync", "source_url")
	rsync_src_key = ini.get("rsync", "source_password")

rsync_sdk_url = None
rsync_sdk_key = None
rsync_sdk_pat = "openwrt-sdk-*.tar.xz"

if ini.has_option("rsync", "sdk_url"):
	rsync_sdk_url = ini.get("rsync", "sdk_url")

if ini.has_option("rsync", "sdk_password"):
	rsync_sdk_key = ini.get("rsync", "sdk_password")

if ini.has_option("rsync", "sdk_pattern"):
	rsync_sdk_pat = ini.get("rsync", "sdk_pattern")

repo_url = ini.get("repo", "url")
repo_branch = "master"

if ini.has_option("repo", "branch"):
	repo_branch = ini.get("repo", "branch")

usign_key = None
usign_comment = "untrusted comment: " + repo_branch.replace("-", " ").title() + " key"

if ini.has_option("usign", "key"):
	usign_key = ini.get("usign", "key")

if ini.has_option("usign", "comment"):
	usign_comment = ini.get("usign", "comment")


# find arches
arches = [ ]
archnames = [ ]

if not os.path.isdir(work_dir+'/source.git'):
	subprocess.call(["git", "clone", "--depth=1", "--branch="+repo_branch, repo_url, work_dir+'/source.git'])
else:
	subprocess.call(["git", "pull"], cwd = work_dir+'/source.git')

findarches = subprocess.Popen([scripts_dir + '/dumpinfo.pl', 'architectures'],
	stdout = subprocess.PIPE, cwd = work_dir+'/source.git')

while True:
	line = findarches.stdout.readline()
	if not line:
		break
	at = line.strip().split()
	arches.append(at)
	archnames.append(at[0])


# find feeds
feeds = []
feedbranches = dict()

from buildbot.changes.gitpoller import GitPoller
c['change_source'] = []

def parse_feed_entry(line):
	parts = line.strip().split()
	if parts[0] == "src-git":
		feeds.append(parts)
		url = parts[2].strip().split(';')
		branch = url[1] if len(url) > 1 else 'master'
		feedbranches[url[0]] = branch
		c['change_source'].append(GitPoller(url[0], branch=branch, workdir='%s/%s.git' %(os.getcwd(), parts[1]), pollinterval=300))

make = subprocess.Popen(['make', '--no-print-directory', '-C', work_dir+'/source.git/target/sdk/', 'val.BASE_FEED'],
	env = dict(os.environ, TOPDIR=work_dir+'/source.git'), stdout = subprocess.PIPE)

line = make.stdout.readline()
if line:
	parse_feed_entry(line)

with open(work_dir+'/source.git/feeds.conf.default', 'r') as f:
	for line in f:
		parse_feed_entry(line)


####### SCHEDULERS

# Configure the Schedulers, which decide how to react to incoming changes.  In this
# case, just kick off a 'basebuild' build

def branch_change_filter(change):
	return change.branch == feedbranches[change.repository]

from buildbot.schedulers.basic import SingleBranchScheduler
from buildbot.schedulers.forcesched import ForceScheduler
from buildbot.changes import filter
c['schedulers'] = []
c['schedulers'].append(SingleBranchScheduler(
	name="all",
	change_filter=filter.ChangeFilter(filter_fn=branch_change_filter),
	treeStableTimer=60,
	builderNames=archnames))

c['schedulers'].append(ForceScheduler(
	name="force",
	builderNames=archnames))

####### BUILDERS

# The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
# what steps, and which slaves can execute them.  Note that any particular build will
# only take place on one slave.

from buildbot.process.factory import BuildFactory
from buildbot.steps.source import Git
from buildbot.steps.shell import ShellCommand
from buildbot.steps.shell import SetProperty
from buildbot.steps.transfer import FileUpload
from buildbot.steps.transfer import FileDownload
from buildbot.steps.transfer import StringDownload
from buildbot.steps.master import MasterShellCommand
from buildbot.process.properties import WithProperties


def GetDirectorySuffix(props):
	verpat = re.compile('^([0-9]{2})\.([0-9]{2})(?:\.([0-9]+)(?:-rc([0-9]+))?|-(SNAPSHOT))$')
	if props.hasProperty("release_version"):
		m = verpat.match(props["release_version"])
		if m is not None:
			return "-%02d.%02d" %(int(m.group(1)), int(m.group(2)))
	return ""

def GetNumJobs(props):
	if props.hasProperty("slavename") and props.hasProperty("nproc"):
		return ((int(props["nproc"]) / (max_builds[props["slavename"]] + other_builds)) + 1)
	else:
		return 1

def GetCwd(props):
	if props.hasProperty("builddir"):
		return props["builddir"]
	elif props.hasProperty("workdir"):
		return props["workdir"]
	else:
		return "/"

def UsignSec2Pub(seckey, comment="untrusted comment: secret key"):
	try:
		seckey = base64.b64decode(seckey)
	except:
		return None

	return "{}\n{}".format(re.sub(r"\bsecret key$", "public key", comment),
		base64.b64encode(seckey[0:2] + seckey[32:40] + seckey[72:]))


c['builders'] = []

dlLock = locks.SlaveLock("slave_dl")

slaveNames = [ ]

for slave in c['slaves']:
	slaveNames.append(slave.slavename)

for arch in arches:
	ts = arch[1].split('/')

	factory = BuildFactory()

	# find number of cores
	factory.addStep(SetProperty(
		name = "nproc",
		property = "nproc",
		description = "Finding number of CPUs",
		command = ["nproc"]))

	# prepare workspace
	factory.addStep(FileDownload(
		mastersrc = scripts_dir + '/cleanup-phase2.sh',
		slavedest = "cleanup.sh",
		mode = 0755))

	if not persistent:
		factory.addStep(ShellCommand(
			name = "cleanold",
			description = "Cleaning previous builds",
			command = ["./cleanup.sh", buildbot_url, WithProperties("%(slavename)s"), WithProperties("%(buildername)s"), "full"],
			haltOnFailure = True,
			timeout = 2400))

		factory.addStep(ShellCommand(
			name = "cleanup",
			description = "Cleaning work area",
			command = ["./cleanup.sh", buildbot_url, WithProperties("%(slavename)s"), WithProperties("%(buildername)s"), "single"],
			haltOnFailure = True,
			timeout = 2400))

	# expire tree if needed
	elif tree_expire > 0:
		factory.addStep(FileDownload(
			mastersrc = scripts_dir + '/expire.sh',
			slavedest = "../expire.sh",
			mode = 0755))

		factory.addStep(ShellCommand(
			name = "expire",
			description = "Checking for build tree expiry",
			command = ["./expire.sh", str(tree_expire)],
			workdir = ".",
			haltOnFailure = True,
			timeout = 2400))

	factory.addStep(ShellCommand(
		name = "mksdkdir",
		description = "Preparing SDK directory",
		command = ["mkdir", "-p", "sdk"],
		haltOnFailure = True))

	factory.addStep(ShellCommand(
		name = "downloadsdk",
		description = "Downloading SDK archive",
		command = ["rsync", "-4", "-va", "%s/%s/%s/%s" %(rsync_sdk_url, ts[0], ts[1], rsync_sdk_pat), "sdk.archive"],
		env={'RSYNC_PASSWORD': rsync_sdk_key},
		haltOnFailure = True,
		logEnviron = False))

	factory.addStep(ShellCommand(
		name = "unpacksdk",
		description = "Unpacking SDK archive",
		command = "rm -rf sdk_update && mkdir sdk_update && tar --strip-components=1 -C sdk_update/ -vxf sdk.archive",
		haltOnFailure = True))

	factory.addStep(ShellCommand(
		name = "updatesdk",
		description = "Updating SDK",
		command = "rsync --checksum -av sdk_update/ sdk/ && rm -rf sdk_update",
		haltOnFailure = True))

	factory.addStep(StringDownload(
		name = "writeversionmk",
		s = 'TOPDIR:=${CURDIR}\n\ninclude $(TOPDIR)/include/version.mk\n\nversion:\n\t@echo $(VERSION_NUMBER)\n',
		slavedest = "sdk/getversion.mk",
		mode = 0755))

	factory.addStep(SetProperty(
		name = "getversion",
		property = "release_version",
		description = "Finding SDK release version",
		workdir = "build/sdk",
		command = ["make", "-f", "getversion.mk"]))

	# install build key
	if usign_key is not None:
		factory.addStep(StringDownload(
			name = "dlkeybuildpub",
			s = UsignSec2Pub(usign_key, usign_comment),
			slavedest = "sdk/key-build.pub",
			mode = 0600))

		factory.addStep(StringDownload(
			name = "dlkeybuild",
			s = "# fake private key",
			slavedest = "sdk/key-build",
			mode = 0600))

		factory.addStep(StringDownload(
			name = "dlkeybuilducert",
			s = "# fake certificate",
			slavedest = "sdk/key-build.ucert",
			mode = 0600))

	factory.addStep(ShellCommand(
		name = "mkdldir",
		description = "Preparing download directory",
		command = ["sh", "-c", "mkdir -p $HOME/dl && rm -rf ./sdk/dl && ln -sf $HOME/dl ./sdk/dl"],
		haltOnFailure = True))

	factory.addStep(ShellCommand(
		name = "mkconf",
		description = "Preparing SDK configuration",
		workdir = "build/sdk",
		command = ["sh", "-c", "rm -f .config && make defconfig"]))

	factory.addStep(FileDownload(
		mastersrc = scripts_dir + '/ccache.sh',
		slavedest = 'sdk/ccache.sh',
		mode = 0755))

	factory.addStep(ShellCommand(
		name = "prepccache",
		description = "Preparing ccache",
		workdir = "build/sdk",
		command = ["./ccache.sh"],
		haltOnFailure = True))

	if git_ssh:
		factory.addStep(StringDownload(
			name = "dlgitclonekey",
			s = git_ssh_key,
			slavedest = "../git-clone.key",
			mode = 0600))

		factory.addStep(ShellCommand(
			name = "patchfeedsconf",
			description = "Patching feeds.conf",
			workdir = "build/sdk",
			command = "sed -e 's#https://#ssh://git@#g' feeds.conf.default > feeds.conf",
			haltOnFailure = True))

	factory.addStep(ShellCommand(
		name = "updatefeeds",
		description = "Updating feeds",
		workdir = "build/sdk",
		command = ["./scripts/feeds", "update", "-f"],
		env = {'GIT_SSH_COMMAND': WithProperties("ssh -o IdentitiesOnly=yes -o IdentityFile=%(cwd)s/git-clone.key -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no", cwd=GetCwd)} if git_ssh else {},
		haltOnFailure = True))

	if git_ssh:
		factory.addStep(ShellCommand(
			name = "rmfeedsconf",
			description = "Removing feeds.conf",
			workdir = "build/sdk",
			command=["rm", "feeds.conf"],
			haltOnFailure = True))

	factory.addStep(ShellCommand(
		name = "installfeeds",
		description = "Installing feeds",
		workdir = "build/sdk",
		command = ["./scripts/feeds", "install", "-a"],
		haltOnFailure = True))

	factory.addStep(ShellCommand(
		name = "logclear",
		description = "Clearing failure logs",
		workdir = "build/sdk",
		command = ["rm", "-rf", "logs/package/error.txt", "faillogs/"],
		haltOnFailure = False
	))

	factory.addStep(ShellCommand(
		name = "compile",
		description = "Building packages",
		workdir = "build/sdk",
		timeout = 3600,
		command = ["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "IGNORE_ERRORS=n m y", "BUILD_LOG=1", "CONFIG_AUTOREMOVE=y", "CONFIG_SIGNED_PACKAGES="],
		env = {'CCACHE_BASEDIR': WithProperties("%(cwd)s", cwd=GetCwd)},
		haltOnFailure = True))

	factory.addStep(ShellCommand(
		name = "mkfeedsconf",
		description = "Generating pinned feeds.conf",
		workdir = "build/sdk",
		command = "./scripts/feeds list -s -f > bin/packages/%s/feeds.conf" %(arch[0])))

	if ini.has_option("gpg", "key") or usign_key is not None:
		factory.addStep(MasterShellCommand(
			name = "signprepare",
			description = "Preparing temporary signing directory",
			command = ["mkdir", "-p", "%s/signing" %(work_dir)],
			haltOnFailure = True
		))

		factory.addStep(ShellCommand(
			name = "signpack",
			description = "Packing files to sign",
			workdir = "build/sdk",
			command = "find bin/packages/%s/ -mindepth 2 -maxdepth 2 -type f -name Packages -print0 | xargs -0 tar -czf sign.tar.gz" %(arch[0]),
			haltOnFailure = True
		))

		factory.addStep(FileUpload(
			slavesrc = "sdk/sign.tar.gz",
			masterdest = "%s/signing/%s.tar.gz" %(work_dir, arch[0]),
			haltOnFailure = True
		))

		factory.addStep(MasterShellCommand(
			name = "signfiles",
			description = "Signing files",
			command = ["%s/signall.sh" %(scripts_dir), "%s/signing/%s.tar.gz" %(work_dir, arch[0])],
			env = { 'CONFIG_INI': os.getenv("BUILDMASTER_CONFIG", "./config.ini") },
			haltOnFailure = True
		))

		factory.addStep(FileDownload(
			mastersrc = "%s/signing/%s.tar.gz" %(work_dir, arch[0]),
			slavedest = "sdk/sign.tar.gz",
			haltOnFailure = True
		))

		factory.addStep(ShellCommand(
			name = "signunpack",
			description = "Unpacking signed files",
			workdir = "build/sdk",
			command = ["tar", "-xzf", "sign.tar.gz"],
			haltOnFailure = True
		))

	factory.addStep(ShellCommand(
		name = "uploadprepare",
		description = "Preparing package directory",
		workdir = "build/sdk",
		command = ["rsync", "-4", "-av", "--include", "/%s/" %(arch[0]), "--exclude", "/*", "--exclude", "/%s/*" %(arch[0]), "bin/packages/", WithProperties("%s/packages%%(suffix)s/" %(rsync_bin_url), suffix=GetDirectorySuffix)],
		env={'RSYNC_PASSWORD': rsync_bin_key},
		haltOnFailure = True,
		logEnviron = False
	))

	factory.addStep(ShellCommand(
		name = "packageupload",
		description = "Uploading package files",
		workdir = "build/sdk",
		command = ["rsync", "-4", "--progress", "--delete", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-avz", "bin/packages/%s/" %(arch[0]), WithProperties("%s/packages%%(suffix)s/%s/" %(rsync_bin_url, arch[0]), suffix=GetDirectorySuffix)],
		env={'RSYNC_PASSWORD': rsync_bin_key},
		haltOnFailure = True,
		logEnviron = False
	))

	factory.addStep(ShellCommand(
		name = "logprepare",
		description = "Preparing log directory",
		workdir = "build/sdk",
		command = ["rsync", "-4", "-av", "--include", "/%s/" %(arch[0]), "--exclude", "/*", "--exclude", "/%s/*" %(arch[0]), "bin/packages/", WithProperties("%s/faillogs%%(suffix)s/" %(rsync_bin_url), suffix=GetDirectorySuffix)],
		env={'RSYNC_PASSWORD': rsync_bin_key},
		haltOnFailure = True,
		logEnviron = False
	))

	factory.addStep(ShellCommand(
		name = "logfind",
		description = "Finding failure logs",
		workdir = "build/sdk/logs/package/feeds",
		command = ["sh", "-c", "sed -ne 's!^ *ERROR: package/feeds/\\([^ ]*\\) .*$!\\1!p' ../error.txt | sort -u | xargs -r find > ../../../logs.txt"],
		haltOnFailure = False
	))

	factory.addStep(ShellCommand(
		name = "logcollect",
		description = "Collecting failure logs",
		workdir = "build/sdk",
		command = ["rsync", "-av", "--files-from=logs.txt", "logs/package/feeds/", "faillogs/"],
		haltOnFailure = False
	))

	factory.addStep(ShellCommand(
		name = "logupload",
		description = "Uploading failure logs",
		workdir = "build/sdk",
		command = ["rsync", "-4", "--progress", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-avz", "faillogs/", WithProperties("%s/faillogs%%(suffix)s/%s/" %(rsync_bin_url, arch[0]), suffix=GetDirectorySuffix)],
		env={'RSYNC_PASSWORD': rsync_bin_key},
		haltOnFailure = False,
		logEnviron = False
	))

	if rsync_src_url is not None:
		factory.addStep(ShellCommand(
			name = "sourcelist",
			description = "Finding source archives to upload",
			workdir = "build/sdk",
			command = "find dl/ -maxdepth 1 -type f -not -size 0 -not -name '.*' -newer ../sdk.archive -printf '%f\\n' > sourcelist",
			haltOnFailure = True
		))

		factory.addStep(ShellCommand(
			name = "sourceupload",
			description = "Uploading source archives",
			workdir = "build/sdk",
			command = ["rsync", "--files-from=sourcelist", "-4", "--progress", "--checksum", "--delay-updates",
			           WithProperties("--partial-dir=.~tmp~%s~%%(slavename)s" %(arch[0])), "-avz", "dl/", "%s/" %(rsync_src_url)],
			env={'RSYNC_PASSWORD': rsync_src_key},
			haltOnFailure = False,
			logEnviron = False
		))

	factory.addStep(ShellCommand(
		name = "df",
		description = "Reporting disk usage",
		command=["df", "-h", "."],
		env={'LC_ALL': 'C'},
		haltOnFailure = False,
		alwaysRun = True
	))

	from buildbot.config import BuilderConfig

	c['builders'].append(BuilderConfig(name=arch[0], slavenames=slaveNames, factory=factory))


####### STATUS arches

# 'status' is a list of Status arches. The results of each build will be
# pushed to these arches. buildbot/status/*.py has a variety to choose from,
# including web pages, email senders, and IRC bots.

c['status'] = []

from buildbot.status import html
from buildbot.status.web import authz, auth

if ini.has_option("phase2", "status_bind"):
	if ini.has_option("phase2", "status_user") and ini.has_option("phase2", "status_password"):
		authz_cfg=authz.Authz(
			# change any of these to True to enable; see the manual for more
			# options
			auth=auth.BasicAuth([(ini.get("phase2", "status_user"), ini.get("phase2", "status_password"))]),
			gracefulShutdown = 'auth',
			forceBuild = 'auth', # use this to test your slave once it is set up
			forceAllBuilds = 'auth',
			pingBuilder = False,
			stopBuild = 'auth',
			stopAllBuilds = 'auth',
			cancelPendingBuild = 'auth',
		)
		c['status'].append(html.WebStatus(http_port=ini.get("phase2", "status_bind"), authz=authz_cfg))
	else:
		c['status'].append(html.WebStatus(http_port=ini.get("phase2", "status_bind")))

####### PROJECT IDENTITY

# the 'title' string will appear at the top of this buildbot
# installation's html.WebStatus home page (linked to the
# 'titleURL') and is embedded in the title of the waterfall HTML page.

c['title'] = ini.get("general", "title")
c['titleURL'] = ini.get("general", "title_url")

# the 'buildbotURL' string should point to the location where the buildbot's
# internal web server (usually the html.WebStatus page) is visible. This
# typically uses the port number set in the Waterfall 'status' entry, but
# with an externally-visible host name which the buildbot cannot figure out
# without some help.

c['buildbotURL'] = buildbot_url

####### DB URL

c['db'] = {
	# This specifies what database buildbot uses to store its state.  You can leave
	# this at its default for all but the largest installations.
	'db_url' : "sqlite:///state.sqlite",
}