# -*- python -*-
# ex: set syntax=python:

import os
import re
import base64
import subprocess
import configparser

from dateutil.tz import tzutc
from datetime import datetime, timedelta

from twisted.internet import defer
from twisted.python import log

from buildbot import locks
from buildbot.data import resultspec
from buildbot.changes.gitpoller import GitPoller
from buildbot.config import BuilderConfig
from buildbot.plugins import reporters
from buildbot.plugins import schedulers
from buildbot.plugins import steps
from buildbot.plugins import util
from buildbot.process import properties
from buildbot.process import results
from buildbot.process.factory import BuildFactory
from buildbot.process.properties import Interpolate
from buildbot.process.properties import Property
from buildbot.schedulers.basic import AnyBranchScheduler
from buildbot.schedulers.forcesched import BaseParameter
from buildbot.schedulers.forcesched import ForceScheduler
from buildbot.schedulers.forcesched import ValidationError
from buildbot.steps.master import MasterShellCommand
from buildbot.steps.shell import SetPropertyFromCommand
from buildbot.steps.shell import ShellCommand
from buildbot.steps.source.git import Git
from buildbot.steps.transfer import FileDownload
from buildbot.steps.transfer import FileUpload
from buildbot.steps.transfer import StringDownload
from buildbot.worker import Worker


if not os.path.exists("twistd.pid"):
	with open("twistd.pid", "w") as pidfile:
		pidfile.write("{}".format(os.getpid()))

# This is a sample buildmaster config file. It must be installed as
# 'master.cfg' in your buildmaster's base directory.

ini = configparser.ConfigParser()
ini.read(os.getenv("BUILDMASTER_CONFIG", "./config.ini"))

if "general" not in ini or "phase1" not in ini:
	raise ValueError("Fix your configuration")

inip1 = ini['phase1']

# Globals
work_dir = os.path.abspath(ini['general'].get("workdir", "."))
scripts_dir = os.path.abspath("../scripts")

repo_url = ini['repo'].get("url")

rsync_defopts = ["-v", "-4", "--timeout=120"]

#if rsync_bin_url.find("::") > 0 or rsync_bin_url.find("rsync://") == 0:
#	rsync_bin_defopts += ["--contimeout=20"]

branches = {}

def ini_parse_branch(section):
	b = {}
	name = section.get("name")

	if not name:
		raise ValueError("missing 'name' in " + repr(section))
	if name in branches:
		raise ValueError("duplicate branch name in " + repr(section))

	b["name"] = name
	b["bin_url"] = section.get("binary_url")
	b["bin_key"] = section.get("binary_password")

	b["src_url"] = section.get("source_url")
	b["src_key"] = section.get("source_password")

	b["gpg_key"] = section.get("gpg_key")

	b["usign_key"] = section.get("usign_key")
	usign_comment = "untrusted comment: " + name.replace("-", " ").title() + " key"
	b["usign_comment"] = section.get("usign_comment", usign_comment)

	b["config_seed"] = section.get("config_seed")

	b["kmod_archive"] = section.getboolean("kmod_archive", False)

	branches[name] = b
	log.msg("Configured branch: {}".format(name))

# PB port can be either a numeric port or a connection string
pb_port = inip1.get("port") or 9989

# This is the dictionary that the buildmaster pays attention to. We also use
# a shorter alias to save typing.
c = BuildmasterConfig = {}

####### PROJECT IDENTITY

# the 'title' string will appear at the top of this buildbot
# installation's html.WebStatus home page (linked to the
# 'titleURL') and is embedded in the title of the waterfall HTML page.

c['title'] = ini['general'].get("title")
c['titleURL'] = ini['general'].get("title_url")

# the 'buildbotURL' string should point to the location where the buildbot's
# internal web server (usually the html.WebStatus page) is visible. This
# typically uses the port number set in the Waterfall 'status' entry, but
# with an externally-visible host name which the buildbot cannot figure out
# without some help.

c['buildbotURL'] = inip1.get("buildbot_url")

####### BUILDWORKERS

# The 'workers' list defines the set of recognized buildworkers. Each element is
# a Worker object, specifying a unique worker name and password.  The same
# worker name and password must be configured on the worker.

c['workers'] = []
NetLocks = dict()

for section in ini.sections():
	if section.startswith("branch "):
		ini_parse_branch(ini[section])

	if section.startswith("worker "):
		if ini.has_option(section, "name") and ini.has_option(section, "password") and \
		   (not ini.has_option(section, "phase") or ini.getint(section, "phase") == 1):
			sl_props = { 'dl_lock':None, 'ul_lock':None }
			name = ini.get(section, "name")
			password = ini.get(section, "password")
			if ini.has_option(section, "dl_lock"):
				lockname = ini.get(section, "dl_lock")
				sl_props['dl_lock'] = lockname
				if lockname not in NetLocks:
					NetLocks[lockname] = locks.MasterLock(lockname)
			if ini.has_option(section, "ul_lock"):
				lockname = ini.get(section, "ul_lock")
				sl_props['ul_lock'] = lockname
				if lockname not in NetLocks:
					NetLocks[lockname] = locks.MasterLock(lockname)
			c['workers'].append(Worker(name, password, max_builds = 1, properties = sl_props))

c['protocols'] = {'pb': {'port': pb_port}}

# coalesce builds
c['collapseRequests'] = True

# Reduce amount of backlog data
c['configurators'] = [util.JanitorConfigurator(
	logHorizon=timedelta(days=3),
	hour=6,
)]

@defer.inlineCallbacks
def getNewestCompleteTime(bldr):
	"""Returns the complete_at of the latest completed and not SKIPPED
	build request for this builder, or None if there are no such build
	requests. We need to filter out SKIPPED requests because we're
	using collapseRequests=True which is unfortunately marking all
	previous requests as complete when new buildset is created.

	@returns: datetime instance or None, via Deferred
	"""

	bldrid = yield bldr.getBuilderId()
	completed = yield bldr.master.data.get(
			('builders', bldrid, 'buildrequests'),
			[
				resultspec.Filter('complete', 'eq', [True]),
				resultspec.Filter('results', 'ne', [results.SKIPPED]),
			],
			order=['-complete_at'], limit=1)
	if not completed:
		return

	complete_at = completed[0]['complete_at']

	last_build = yield bldr.master.data.get(
			('builds', ),
			[
				resultspec.Filter('builderid', 'eq', [bldrid]),
			],
			order=['-started_at'], limit=1)

	if last_build and last_build[0]:
		last_complete_at = last_build[0]['complete_at']
		if last_complete_at and (last_complete_at > complete_at):
			return last_complete_at

	return complete_at

@defer.inlineCallbacks
def prioritizeBuilders(master, builders):
	"""Returns sorted list of builders by their last timestamp of completed and
	not skipped build.

	@returns: list of sorted builders
	"""

	def is_building(bldr):
		return bool(bldr.building) or bool(bldr.old_building)

	def bldr_info(bldr):
		d = defer.maybeDeferred(getNewestCompleteTime, bldr)
		d.addCallback(lambda complete_at: (complete_at, bldr))
		return d

	def bldr_sort(item):
		(complete_at, bldr) = item

		if not complete_at:
			date = datetime.min
			complete_at = date.replace(tzinfo=tzutc())

		if is_building(bldr):
			date = datetime.max
			complete_at = date.replace(tzinfo=tzutc())

		return (complete_at, bldr.name)

	results = yield defer.gatherResults([bldr_info(bldr) for bldr in builders])
	results.sort(key=bldr_sort)

	for r in results:
		log.msg("prioritizeBuilders: {:>20} complete_at: {}".format(r[1].name, r[0]))

	return [r[1] for r in results]

c['prioritizeBuilders'] = prioritizeBuilders

####### CHANGESOURCES

branchNames = [branches[b]["name"] for b in branches]

# find targets
targets = set()

def populateTargets():
	log.msg("Populating targets, this will take time")
	sourcegit = work_dir + '/source.git'
	for branch in branchNames:
		if os.path.isdir(sourcegit):
			subprocess.call(["rm", "-rf", sourcegit])

		subprocess.call(["git", "clone", "-q", "--depth=1", "--branch="+branch, repo_url, sourcegit])

		os.makedirs(sourcegit + '/tmp', exist_ok=True)
		findtargets = subprocess.Popen(['./scripts/dump-target-info.pl', 'targets'],
			stdout = subprocess.PIPE, stderr = subprocess.DEVNULL, cwd = sourcegit)

		while True:
			line = findtargets.stdout.readline()
			if not line:
				break
			ta = line.decode().strip().split(' ')
			targets.add(ta[0])

		subprocess.call(["rm", "-rf", sourcegit])

populateTargets()

# the 'change_source' setting tells the buildmaster how it should find out
# about source code changes.  Here we point to the buildbot clone of pyflakes.

c['change_source'] = []
c['change_source'].append(GitPoller(
	repo_url,
	workdir=work_dir+'/work.git', branches=branchNames,
	pollAtLaunch=True, pollinterval=300))

####### SCHEDULERS

# Configure the Schedulers, which decide how to react to incoming changes.  In this
# case, just kick off a 'basebuild' build

class TagChoiceParameter(BaseParameter):
	spec_attributes = ["strict", "choices"]
	type = "list"
	strict = True

	def __init__(self, name, label=None, **kw):
		super().__init__(name, label, **kw)
		self._choice_list = []

	@property
	def choices(self):
		taglist = []
		basever = re.search(r'-([0-9]+\.[0-9]+)$', "master")	# XXX FIXME

		if basever:
			findtags = subprocess.Popen(
				['git', 'ls-remote', '--tags', repo_url],
				stdout = subprocess.PIPE)

			while True:
				line = findtags.stdout.readline()

				if not line:
					break

				tagver = re.search(r'\brefs/tags/v([0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?)$', line.decode().strip())

				if tagver and tagver[1].find(basever[1]) == 0:
					taglist.append(tagver[1])

		taglist.sort(reverse=True, key=lambda tag: tag if re.search(r'-rc[0-9]+$', tag) else tag + '-z')
		taglist.insert(0, '')

		self._choice_list = taglist

		return self._choice_list

	def parse_from_arg(self, s):
		if self.strict and s not in self._choice_list:
			raise ValidationError("'%s' does not belong to list of available choices '%s'" % (s, self._choice_list))
		return s

c['schedulers'] = []
c['schedulers'].append(AnyBranchScheduler(
	name            = "all",
	change_filter   = util.ChangeFilter(branch=branchNames),
	treeStableTimer = 15*60,
	builderNames    = list(targets)))

c['schedulers'].append(ForceScheduler(
	name         = "force",
	buttonName   = "Force builds",
	label        = "Force build details",
	builderNames = [ "00_force_build" ],

	codebases = [
		util.CodebaseParameter(
			"",
			label      = "Repository",
			branch     = util.FixedParameter(name = "branch",     default = ""),
			revision   = util.FixedParameter(name = "revision",   default = ""),
			repository = util.FixedParameter(name = "repository", default = ""),
			project    = util.FixedParameter(name = "project",    default = "")
		)
	],

	reason = util.StringParameter(
		name     = "reason",
		label    = "Reason",
		default  = "Trigger build",
		required = True,
		size     = 80
	),

	properties = [
		util.NestedParameter(
			name="options",
			label="Build Options",
			layout="vertical",
			fields=[
				util.ChoiceStringParameter(
					name    = "target",
					label   = "Build target",
					default = "all",
					choices = set( "all" ) | targets
				),
				TagChoiceParameter(
					name    = "tag",
					label   = "Build tag",
					default = ""
				)
			]
		)
	]
))

####### BUILDERS

# The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
# what steps, and which workers can execute them.  Note that any particular build will
# only take place on one worker.

def IsTaggingRequested(step):
	tag = step.getProperty("tag")
	return tag and re.match(r"^[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", tag)

def IsNoMasterBuild(step):
	return step.getProperty("branch") != "master"

def IsUsignEnabled(step):
	branch = step.getProperty("branch")
	return branch and branches[branch].get("usign_key")

def IsSignEnabled(step):
	branch = step.getProperty("branch")
	return IsUsignEnabled(step) or branch and branches[branch].get("gpg_key")

def IsKmodArchiveEnabled(step):
	branch = step.getProperty("branch")
	return branch and branches[branch].get("kmod_archive")

def GetBaseVersion(branch):
	if re.match(r"^[^-]+-[0-9]+\.[0-9]+$", branch):
		return branch.split('-')[1]
	else:
		return "master"

@properties.renderer
def GetVersionPrefix(props):
	branch = props.getProperty("branch")
	basever = GetBaseVersion(branch)
	if props.hasProperty("tag") and re.match(r"^[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", props["tag"]):
		return "%s/" % props["tag"]
	elif basever != "master":
		return "%s-SNAPSHOT/" % basever
	else:
		return ""

@util.renderer
def GetConfigSeed(props):
	branch = props.getProperty("branch")
	return branch and branches[branch].get("config_seed") or ""

@util.renderer
def GetRsyncParams(props, srcorbin, urlorkey):
	# srcorbin: 'bin' or 'src'; urlorkey: 'url' or 'key'
	branch = props.getProperty("branch")
	opt = srcorbin + "_" + urlorkey
	return branch and branches[branch].get(opt)

@util.renderer
def GetUsignKey(props):
	branch = props.getProperty("branch")
	return branch and branches[branch].get("usign_key")

def GetNextBuild(builder, requests):
	for r in requests:
		if r.properties:
			# order tagged build first
			if r.properties.hasProperty("tag"):
				return r
			# then order by branch order
			pbranch = r.properties.getProperty("branch")
			for name in branchNames:
				if pbranch == name:
					return r

	r = requests[0]
	log.msg("GetNextBuild: {:>20} id: {} bsid: {}".format(builder.name, r.id, r.bsid))
	return r

def MakeEnv(overrides=None, tryccache=False):
	env = {
		'CCC': Interpolate("%(prop:cc_command:-gcc)s"),
		'CCXX': Interpolate("%(prop:cxx_command:-g++)s"),
	}
	if tryccache:
		env['CC'] = Interpolate("%(prop:builddir)s/ccache_cc.sh")
		env['CXX'] = Interpolate("%(prop:builddir)s/ccache_cxx.sh")
		env['CCACHE'] = Interpolate("%(prop:ccache_command:-)s")
	else:
		env['CC'] = env['CCC']
		env['CXX'] = env['CCXX']
		env['CCACHE'] = ''
	if overrides is not None:
		env.update(overrides)
	return env

@properties.renderer
def NetLockDl(props, extralock=None):
	lock = None
	locks = []
	if props.hasProperty("dl_lock"):
		lock = NetLocks[props["dl_lock"]]
	if lock is not None:
		locks.append(lock.access('exclusive'))
	if extralock is not None:
		locks.append(extralock)
	return locks

@properties.renderer
def NetLockUl(props):
	lock = None
	if props.hasProperty("ul_lock"):
		lock = NetLocks[props["ul_lock"]]
	if lock is not None:
		return [lock.access('exclusive')]
	else:
		return []

@util.renderer
def TagPropertyValue(props):
	if props.hasProperty("options"):
		options = props.getProperty("options")
		if type(options) is dict:
			return options.get("tag")
	return None

def IsTargetSelected(target):
	def CheckTargetProperty(step):
		try:
			options = step.getProperty("options")
			if type(options) is dict:
				selected_target = options.get("target", "all")
				if selected_target != "all" and selected_target != target:
					return False
		except KeyError:
			pass

		return True

	return CheckTargetProperty

@util.renderer
def UsignSec2Pub(props):
	branch = props.getProperty("branch")
	try:
		comment = branches[branch].get("usign_comment") or "untrusted comment: secret key"
		seckey = branches[branch].get("usign_key")
		seckey = base64.b64decode(seckey)
	except:
		return None

	return "{}\n{}".format(re.sub(r"\bsecret key$", "public key", comment),
		base64.b64encode(seckey[0:2] + seckey[32:40] + seckey[72:]))


c['builders'] = []

dlLock = locks.WorkerLock("worker_dl")

workerNames = [ ]

for worker in c['workers']:
	workerNames.append(worker.workername)

force_factory = BuildFactory()

c['builders'].append(BuilderConfig(
	name        = "00_force_build",
	workernames = workerNames,
	factory     = force_factory))

for target in targets:
	ts = target.split('/')

	factory = BuildFactory()

	# setup shared work directory if required
	factory.addStep(ShellCommand(
		name = "sharedwd",
		description = "Setting up shared work directory",
		command = 'test -L "$PWD" || (mkdir -p ../shared-workdir && rm -rf "$PWD" && ln -s shared-workdir "$PWD")',
		workdir = ".",
		haltOnFailure = True))

	# find number of cores
	factory.addStep(SetPropertyFromCommand(
		name = "nproc",
		property = "nproc",
		description = "Finding number of CPUs",
		command = ["nproc"]))

	# find gcc and g++ compilers
	factory.addStep(FileDownload(
		name = "dlfindbinpl",
		mastersrc = scripts_dir + '/findbin.pl',
		workerdest = "../findbin.pl",
		mode = 0o755))

	factory.addStep(SetPropertyFromCommand(
		name = "gcc",
		property = "cc_command",
		description = "Finding gcc command",
		command = [
			"../findbin.pl", "gcc", "", "",
		],
		haltOnFailure = True))

	factory.addStep(SetPropertyFromCommand(
		name = "g++",
		property = "cxx_command",
		description = "Finding g++ command",
		command = [
			"../findbin.pl", "g++", "", "",
		],
		haltOnFailure = True))

	# see if ccache is available
	factory.addStep(SetPropertyFromCommand(
		property = "ccache_command",
		command = ["which", "ccache"],
		description = "Testing for ccache command",
		haltOnFailure = False,
		flunkOnFailure = False,
		warnOnFailure = False,
	))

	# Workaround bug when switching from a checked out tag back to a branch
	# Ref: http://lists.infradead.org/pipermail/openwrt-devel/2019-June/017809.html
	factory.addStep(ShellCommand(
		name = "gitcheckout",
		description = "Ensure that Git HEAD is sane",
		command = Interpolate("if [ -d .git ]; then git checkout -f %(prop:branch)s && git branch --set-upstream-to origin/%(prop:branch)s || rm -fr .git; else exit 0; fi"),
		haltOnFailure = True))

	# check out the source
	# Git() runs:
	  # if repo doesn't exist: 'git clone repourl'
	  # method 'clean' runs 'git clean -d -f', method fresh runs 'git clean -d -f x'. Only works with mode='full'
	  # 'git fetch -t repourl branch; git reset --hard revision'
	factory.addStep(Git(
		name = "git",
		repourl = repo_url,
		mode = 'full',
		method = 'fresh',
		locks = NetLockDl,
		haltOnFailure = True,
	))

	# update remote refs
	factory.addStep(ShellCommand(
		name = "fetchrefs",
		description = "Fetching Git remote refs",
		command = ["git", "fetch", "origin", Interpolate("+refs/heads/%(prop:branch)s:refs/remotes/origin/%(prop:branch)s")],
		haltOnFailure = True
	))

	# switch to tag
	factory.addStep(ShellCommand(
		name = "switchtag",
		description = "Checking out Git tag",
		command = ["git", "checkout", Interpolate("tags/v%(prop:tag:-)s")],
		haltOnFailure = True,
		doStepIf = IsTaggingRequested
	))

	# Verify that Git HEAD points to a tag or branch
	# Ref: http://lists.infradead.org/pipermail/openwrt-devel/2019-June/017809.html
	factory.addStep(ShellCommand(
		name = "gitverify",
		description = "Ensure that Git HEAD is pointing to a branch or tag",
		command = 'git rev-parse --abbrev-ref HEAD | grep -vxqF HEAD || git show-ref --tags --dereference 2>/dev/null | sed -ne "/^$(git rev-parse HEAD) / { s|^.*/||; s|\\^.*||; p }" | grep -qE "^v[0-9][0-9]\\."',
		haltOnFailure = True))

	factory.addStep(ShellCommand(
		name = "rmtmp",
		description = "Remove tmp folder",
		command=["rm", "-rf", "tmp/"]))

	# feed
	factory.addStep(ShellCommand(
		name = "rmfeedlinks",
		description = "Remove feed symlinks",
		command=["rm", "-rf", "package/feeds/"]))

	factory.addStep(StringDownload(
		name = "ccachecc",
		s = '#!/bin/sh\nexec ${CCACHE} ${CCC} "$@"\n',
		workerdest = "../ccache_cc.sh",
		mode = 0o755,
	))

	factory.addStep(StringDownload(
		name = "ccachecxx",
		s = '#!/bin/sh\nexec ${CCACHE} ${CCXX} "$@"\n',
		workerdest = "../ccache_cxx.sh",
		mode = 0o755,
	))

	# feed
	factory.addStep(ShellCommand(
		name = "updatefeeds",
		description = "Updating feeds",
		command=["./scripts/feeds", "update"],
		env = MakeEnv(tryccache=True),
		haltOnFailure = True,
		locks = NetLockDl,
	))

	# feed
	factory.addStep(ShellCommand(
		name = "installfeeds",
		description = "Installing feeds",
		command=["./scripts/feeds", "install", "-a"],
		env = MakeEnv(tryccache=True),
		haltOnFailure = True
	))

	# seed config
	factory.addStep(StringDownload(
		name = "dlconfigseed",
		s = Interpolate("%(kw:seed)s\n", seed=GetConfigSeed),
		workerdest = ".config",
		mode = 0o644
	))

	# configure
	factory.addStep(ShellCommand(
		name = "newconfig",
		description = "Seeding .config",
		command = Interpolate("printf 'CONFIG_TARGET_%(kw:target)s=y\\nCONFIG_TARGET_%(kw:target)s_%(kw:subtarget)s=y\\nCONFIG_SIGNED_PACKAGES=%(kw:usign:#?|y|n)s\\n' >> .config", target=ts[0], subtarget=ts[1], usign=GetUsignKey)
	))

	factory.addStep(ShellCommand(
		name = "delbin",
		description = "Removing output directory",
		command = ["rm", "-rf", "bin/"]
	))

	factory.addStep(ShellCommand(
		name = "defconfig",
		description = "Populating .config",
		command = ["make", "defconfig"],
		env = MakeEnv()
	))

	# check arch - exit early if does not exist - NB: some targets do not define CONFIG_TARGET_target_subtarget
	factory.addStep(ShellCommand(
		name = "checkarch",
		description = "Checking architecture",
		command = 'grep -sq CONFIG_TARGET_%s=y .config && grep -sq CONFIG_TARGET_SUBTARGET=\\"%s\\" .config' %(ts[0], ts[1]),
		logEnviron = False,
		want_stdout = False,
		want_stderr = False,
		haltOnFailure = True,
		flunkOnFailure = False,	# this is not a build FAILURE
	))

	# find libc suffix
	factory.addStep(SetPropertyFromCommand(
		name = "libc",
		property = "libc",
		description = "Finding libc suffix",
		command = ["sed", "-ne", '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }', ".config"]))

	# install build key
	factory.addStep(StringDownload(
		name = "dlkeybuildpub",
		s = Interpolate("%(kw:sec2pub)s", sec2pub=UsignSec2Pub),
		workerdest = "key-build.pub",
		mode = 0o600,
		doStepIf = IsUsignEnabled,
	))

	factory.addStep(StringDownload(
		name = "dlkeybuild",
		s = "# fake private key",
		workerdest = "key-build",
		mode = 0o600,
		doStepIf = IsUsignEnabled,
	))

	factory.addStep(StringDownload(
		name = "dlkeybuilducert",
		s = "# fake certificate",
		workerdest = "key-build.ucert",
		mode = 0o600,
		doStepIf = IsUsignEnabled,
	))

	# prepare dl
	factory.addStep(ShellCommand(
		name = "dldir",
		description = "Preparing dl/",
		command = "mkdir -p $HOME/dl && rm -rf ./dl && ln -sf $HOME/dl ./dl",
		logEnviron = False,
		want_stdout = False
	))

	# prepare tar
	factory.addStep(ShellCommand(
		name = "dltar",
		description = "Building and installing GNU tar",
		command = ["make", Interpolate("-j%(prop:nproc:-1)s"), "tools/tar/compile", "V=s"],
		env = MakeEnv(tryccache=True),
		haltOnFailure = True
	))

	# populate dl
	factory.addStep(ShellCommand(
		name = "dlrun",
		description = "Populating dl/",
		command = ["make", Interpolate("-j%(prop:nproc:-1)s"), "download", "V=s"],
		env = MakeEnv(),
		logEnviron = False,
		locks = NetLockDl.withArgs(dlLock.access('exclusive')),
	))

	factory.addStep(ShellCommand(
		name = "cleanbase",
		description = "Cleaning base-files",
		command=["make", "package/base-files/clean", "V=s"]
	))

	# build
	factory.addStep(ShellCommand(
		name = "tools",
		description = "Building and installing tools",
		command = ["make", Interpolate("-j%(prop:nproc:-1)s"), "tools/install", "V=s"],
		env = MakeEnv(tryccache=True),
		haltOnFailure = True
	))

	factory.addStep(ShellCommand(
		name = "toolchain",
		description = "Building and installing toolchain",
		command=["make", Interpolate("-j%(prop:nproc:-1)s"), "toolchain/install", "V=s"],
		env = MakeEnv(),
		haltOnFailure = True
	))

	factory.addStep(ShellCommand(
		name = "kmods",
		description = "Building kmods",
		command=["make", Interpolate("-j%(prop:nproc:-1)s"), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
		env = MakeEnv(),
		haltOnFailure = True
	))

	# find kernel version
	factory.addStep(SetPropertyFromCommand(
		name = "kernelversion",
		property = "kernelversion",
		description = "Finding the effective Kernel version",
		command = "make --no-print-directory -C target/linux/ val.LINUX_VERSION val.LINUX_RELEASE val.LINUX_VERMAGIC | xargs printf '%s-%s-%s\\n'",
		env = { 'TOPDIR': Interpolate("%(prop:builddir)s/build") }
	))

	factory.addStep(ShellCommand(
		name = "pkgclean",
		description = "Cleaning up package build",
		command=["make", "package/cleanup", "V=s"]
	))

	factory.addStep(ShellCommand(
		name = "pkgbuild",
		description = "Building packages",
		command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
		env = MakeEnv(),
		haltOnFailure = True
	))

	factory.addStep(ShellCommand(
		name = "pkginstall",
		description = "Installing packages",
		command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/install", "V=s"],
		env = MakeEnv(),
		haltOnFailure = True
	))

	factory.addStep(ShellCommand(
		name = "pkgindex",
		description = "Indexing packages",
		command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/index", "V=s", "CONFIG_SIGNED_PACKAGES="],
		env = MakeEnv(),
		haltOnFailure = True
	))

	factory.addStep(ShellCommand(
		name = "images",
		description = "Building and installing images",
		command=["make", Interpolate("-j%(prop:nproc:-1)s"), "target/install", "V=s"],
		env = MakeEnv(),
		haltOnFailure = True
	))

	factory.addStep(ShellCommand(
		name = "buildinfo",
		description = "Generating config.buildinfo, version.buildinfo and feeds.buildinfo",
		command = "make -j1 buildinfo V=s || true",
		env = MakeEnv(),
		haltOnFailure = True
	))

	factory.addStep(ShellCommand(
		name = "json_overview_image_info",
		description = "Generate profiles.json in target folder",
		command = "make -j1 json_overview_image_info V=s || true",
		env = MakeEnv(),
		haltOnFailure = True
	))

	factory.addStep(ShellCommand(
		name = "checksums",
		description = "Calculating checksums",
		command=["make", "-j1", "checksum", "V=s"],
		env = MakeEnv(),
		haltOnFailure = True
	))

	factory.addStep(ShellCommand(
		name = "kmoddir",
		description = "Creating kmod directory",
		command=["mkdir", "-p", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s", target=ts[0], subtarget=ts[1])],
		haltOnFailure = True,
		doStepIf = IsKmodArchiveEnabled,
	))

	factory.addStep(ShellCommand(
		name = "kmodprepare",
		description = "Preparing kmod archive",
		command=["rsync", "--include=/kmod-*.ipk", "--exclude=*", "-va",
			Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/packages/", target=ts[0], subtarget=ts[1]),
			Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1])],
		haltOnFailure = True,
		doStepIf = IsKmodArchiveEnabled,
	))

	factory.addStep(ShellCommand(
		name = "kmodindex",
		description = "Indexing kmod archive",
		command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/index", "V=s", "CONFIG_SIGNED_PACKAGES=",
			Interpolate("PACKAGE_SUBDIRS=bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1])],
		env = MakeEnv(),
		haltOnFailure = True,
		doStepIf = IsKmodArchiveEnabled,
	))

	# sign
	factory.addStep(MasterShellCommand(
		name = "signprepare",
		description = "Preparing temporary signing directory",
		command = ["mkdir", "-p", "%s/signing" %(work_dir)],
		haltOnFailure = True,
		doStepIf = IsSignEnabled,

	))

	factory.addStep(ShellCommand(
		name = "signpack",
		description = "Packing files to sign",
		command = Interpolate("find bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/ bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz", target=ts[0], subtarget=ts[1]),
		haltOnFailure = True,
		doStepIf = IsSignEnabled,
	))

	factory.addStep(FileUpload(
		workersrc = "sign.tar.gz",
		masterdest = "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1]),
		haltOnFailure = True,
		doStepIf = IsSignEnabled,
	))

	factory.addStep(MasterShellCommand(
		name = "signfiles",
		description = "Signing files",
		command = ["%s/signall.sh" %(scripts_dir), "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1]), Interpolate("%(prop:branch)s")],
		env = { 'CONFIG_INI': os.getenv("BUILDMASTER_CONFIG", "./config.ini") },
		haltOnFailure = True,
		doStepIf = IsSignEnabled,
	))

	factory.addStep(FileDownload(
		name = "dlsigntargz",
		mastersrc = "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1]),
		workerdest = "sign.tar.gz",
		haltOnFailure = True,
		doStepIf = IsSignEnabled,
	))

	factory.addStep(ShellCommand(
		name = "signunpack",
		description = "Unpacking signed files",
		command = ["tar", "-xzf", "sign.tar.gz"],
		haltOnFailure = True,
		doStepIf = IsSignEnabled,
	))

	# upload
	factory.addStep(ShellCommand(
		name = "dirprepare",
		description = "Preparing upload directory structure",
		command = ["mkdir", "-p", Interpolate("tmp/upload/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s", target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
		haltOnFailure = True
	))

	factory.addStep(ShellCommand(
		name = "linkprepare",
		description = "Preparing repository symlink",
		command = ["ln", "-s", "-f", Interpolate("../packages-%(kw:basever)s", basever=util.Transform(GetBaseVersion, Property("branch"))), Interpolate("tmp/upload/%(kw:prefix)spackages", prefix=GetVersionPrefix)],
		doStepIf = IsNoMasterBuild,
		haltOnFailure = True
	))

	factory.addStep(ShellCommand(
		name = "kmoddirprepare",
		description = "Preparing kmod archive upload directory",
		command = ["mkdir", "-p", Interpolate("tmp/upload/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s", target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
		haltOnFailure = True,
		doStepIf = IsKmodArchiveEnabled,
	))

	factory.addStep(ShellCommand(
		name = "dirupload",
		description = "Uploading directory structure",
		command = ["rsync", "-az"] + rsync_defopts + ["tmp/upload/", Interpolate("%(kw:url)s/", url=GetRsyncParams.withArgs("bin", "url"))],
		env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
		haltOnFailure = True,
		logEnviron = False,
		locks = NetLockUl,
	))

	# download remote sha256sums to 'target-sha256sums'
	factory.addStep(ShellCommand(
		name = "target-sha256sums",
		description = "Fetching remote sha256sums for target",
		command = ["rsync", "-z"] + rsync_defopts + [Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/sha256sums", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix), "target-sha256sums"],
		env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
		logEnviron = False,
		haltOnFailure = False,
		flunkOnFailure = False,
		warnOnFailure = False,
	))

	# build list of files to upload
	factory.addStep(FileDownload(
		name = "dlsha2rsyncpl",
		mastersrc = scripts_dir + '/sha2rsync.pl',
		workerdest = "../sha2rsync.pl",
		mode = 0o755,
	))

	factory.addStep(ShellCommand(
		name = "buildlist",
		description = "Building list of files to upload",
		command = ["../sha2rsync.pl", "target-sha256sums", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/sha256sums", target=ts[0], subtarget=ts[1]), "rsynclist"],
		haltOnFailure = True,
	))

	factory.addStep(FileDownload(
		name = "dlrsync.sh",
		mastersrc = scripts_dir + '/rsync.sh',
		workerdest = "../rsync.sh",
		mode = 0o755
	))

	# upload new files and update existing ones
	factory.addStep(ShellCommand(
		name = "targetupload",
		description = "Uploading target files",
		command=["../rsync.sh", "--exclude=/kmods/", "--files-from=rsynclist", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_defopts +
			["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]),
			Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
		env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
		haltOnFailure = True,
		logEnviron = False,
	))

	# delete files which don't exist locally
	factory.addStep(ShellCommand(
		name = "targetprune",
		description = "Pruning target files",
		command=["../rsync.sh", "--exclude=/kmods/", "--delete", "--existing", "--ignore-existing", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_defopts +
			["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]),
			Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
		env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
		haltOnFailure = True,
		logEnviron = False,
		locks = NetLockUl,
	))

	factory.addStep(ShellCommand(
		name = "kmodupload",
		description = "Uploading kmod archive",
		command=["../rsync.sh", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_defopts +
			["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1]),
			Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
		env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
		haltOnFailure = True,
		logEnviron = False,
		locks = NetLockUl,
		doStepIf = IsKmodArchiveEnabled,
	))

	factory.addStep(ShellCommand(
		name = "sourcelist",
		description = "Finding source archives to upload",
		command = "find dl/ -maxdepth 1 -type f -not -size 0 -not -name '.*' -not -name '*.hash' -not -name '*.dl' -newer .config -printf '%f\\n' > sourcelist",
		haltOnFailure = True
	))

	factory.addStep(ShellCommand(
		name = "sourceupload",
		description = "Uploading source archives",
		command=["../rsync.sh", "--files-from=sourcelist", "--size-only", "--delay-updates"] + rsync_defopts +
			[Interpolate("--partial-dir=.~tmp~%(kw:target)s~%(kw:subtarget)s~%(prop:workername)s", target=ts[0], subtarget=ts[1]), "-a", "dl/", Interpolate("%(kw:url)s/", url=GetRsyncParams.withArgs("src", "url"))],
		env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("src", "key")) },
		haltOnFailure = True,
		logEnviron = False,
		locks = NetLockUl,
	))

	factory.addStep(ShellCommand(
		name = "df",
		description = "Reporting disk usage",
		command=["df", "-h", "."],
		env={'LC_ALL': 'C'},
		haltOnFailure = False,
		flunkOnFailure = False,
		warnOnFailure = False,
		alwaysRun = True
	))

	factory.addStep(ShellCommand(
		name = "du",
		description = "Reporting estimated file space usage",
		command=["du", "-sh", "."],
		env={'LC_ALL': 'C'},
		haltOnFailure = False,
		flunkOnFailure = False,
		warnOnFailure = False,
		alwaysRun = True
	))

	factory.addStep(ShellCommand(
		name = "ccachestat",
		description = "Reporting ccache stats",
		command=["ccache", "-s"],
		env = MakeEnv(overrides={ 'PATH': ["${PATH}", "./staging_dir/host/bin"] }),
		want_stderr = False,
		haltOnFailure = False,
		flunkOnFailure = False,
		warnOnFailure = False,
		alwaysRun = True,
	))

	c['builders'].append(BuilderConfig(name=target, workernames=workerNames, factory=factory, nextBuild=GetNextBuild))

	c['schedulers'].append(schedulers.Triggerable(name="trigger_%s" % target, builderNames=[ target ]))
	force_factory.addStep(steps.Trigger(
		name = "trigger_%s" % target,
		description = "Triggering %s build" % target,
		schedulerNames = [ "trigger_%s" % target ],
		set_properties = { "reason": Property("reason"), "tag": TagPropertyValue },
		doStepIf = IsTargetSelected(target)
	))


####### STATUS TARGETS

# 'status' is a list of Status Targets. The results of each build will be
# pushed to these targets. buildbot/status/*.py has a variety to choose from,
# including web pages, email senders, and IRC bots.

if "status_bind" in inip1:
	c['www'] = {
		'port': inip1.get("status_bind"),
		'plugins': {
			'waterfall_view': True,
			'console_view': True,
			'grid_view': True
		}
	}

	if "status_user" in inip1 and "status_password" in inip1:
		c['www']['auth'] = util.UserPasswordAuth([
			(inip1.get("status_user"), inip1.get("status_password"))
		])
		c['www']['authz'] = util.Authz(
			allowRules=[ util.AnyControlEndpointMatcher(role="admins") ],
			roleMatchers=[ util.RolesFromUsername(roles=["admins"], usernames=[inip1.get("status_user")]) ]
		)

c['services'] = []
if ini.has_section("irc"):
	iniirc = ini['irc']
	irc_host = iniirc.get("host", None)
	irc_port = iniirc.getint("port", 6667)
	irc_chan = iniirc.get("channel", None)
	irc_nick = iniirc.get("nickname", None)
	irc_pass = iniirc.get("password", None)

	if irc_host and irc_nick and irc_chan:
		irc = reporters.IRC(irc_host, irc_nick,
			port = irc_port,
			password = irc_pass,
			channels = [ irc_chan ],
			notify_events = [ 'exception', 'problem', 'recovery' ]
		)

		c['services'].append(irc)

c['revlink'] = util.RevlinkMatch([
	r'https://git.openwrt.org/openwrt/(.*).git'
	],
	r'https://git.openwrt.org/?p=openwrt/\1.git;a=commit;h=%s')

####### DB URL

c['db'] = {
	# This specifies what database buildbot uses to store its state.  You can leave
	# this at its default for all but the largest installations.
	'db_url' : "sqlite:///state.sqlite",
}

c['buildbotNetUsageData'] = None