phase1/master.cfg

   1 # -*- python -*-
   2 # ex: set syntax=python:
   3
   4 import os
   5 import re
   6 import base64
   7 import subprocess
   8 import configparser
   9
  10 from dateutil.tz import tzutc
  11 from datetime import datetime, timedelta
  12
  13 from twisted.internet import defer
  14 from twisted.python import log
  15
  16 from buildbot import locks
  17 from buildbot.data import resultspec
  18 from buildbot.changes.gitpoller import GitPoller
  19 from buildbot.config import BuilderConfig
  20 from buildbot.plugins import reporters
  21 from buildbot.plugins import schedulers
  22 from buildbot.plugins import steps
  23 from buildbot.plugins import util
  24 from buildbot.process import properties
  25 from buildbot.process import results
  26 from buildbot.process.factory import BuildFactory
  27 from buildbot.process.properties import Interpolate
  28 from buildbot.process.properties import Property
  29 from buildbot.schedulers.basic import AnyBranchScheduler
  30 from buildbot.schedulers.forcesched import BaseParameter
  31 from buildbot.schedulers.forcesched import ForceScheduler
  32 from buildbot.schedulers.forcesched import ValidationError
  33 from buildbot.steps.master import MasterShellCommand
  34 from buildbot.steps.shell import SetPropertyFromCommand
  35 from buildbot.steps.shell import ShellCommand
  36 from buildbot.steps.source.git import Git
  37 from buildbot.steps.transfer import FileDownload
  38 from buildbot.steps.transfer import FileUpload
  39 from buildbot.steps.transfer import StringDownload
  40 from buildbot.worker import Worker
  41 from buildbot.worker.local import LocalWorker
  42
  43
  44 if not os.path.exists("twistd.pid"):
  45         with open("twistd.pid", "w") as pidfile:
  46                 pidfile.write("{}".format(os.getpid()))
  47
  48 # This is a sample buildmaster config file. It must be installed as
  49 # 'master.cfg' in your buildmaster's base directory.
  50
  51 ini = configparser.ConfigParser()
  52 ini.read(os.getenv("BUILDMASTER_CONFIG", "./config.ini"))
  53
  54 if "general" not in ini or "phase1" not in ini:
  55         raise ValueError("Fix your configuration")
  56
  57 inip1 = ini['phase1']
  58
  59 # Globals
  60 work_dir = os.path.abspath(ini['general'].get("workdir", "."))
  61 scripts_dir = os.path.abspath("../scripts")
  62
  63 repo_url = ini['repo'].get("url")
  64
  65 rsync_defopts = ["-v", "-4", "--timeout=120"]
  66
  67 #if rsync_bin_url.find("::") > 0 or rsync_bin_url.find("rsync://") == 0:
  68 #       rsync_bin_defopts += ["--contimeout=20"]
  69
  70 branches = {}
  71
  72 def ini_parse_branch(section):
  73         b = {}
  74         name = section.get("name")
  75
  76         if not name:
  77                 raise ValueError("missing 'name' in " + repr(section))
  78         if name in branches:
  79                 raise ValueError("duplicate branch name in " + repr(section))
  80
  81         b["name"] = name
  82         b["bin_url"] = section.get("binary_url")
  83         b["bin_key"] = section.get("binary_password")
  84
  85         b["src_url"] = section.get("source_url")
  86         b["src_key"] = section.get("source_password")
  87
  88         b["gpg_key"] = section.get("gpg_key")
  89
  90         b["usign_key"] = section.get("usign_key")
  91         usign_comment = "untrusted comment: " + name.replace("-", " ").title() + " key"
  92         b["usign_comment"] = section.get("usign_comment", usign_comment)
  93
  94         b["config_seed"] = section.get("config_seed")
  95
  96         b["kmod_archive"] = section.getboolean("kmod_archive", False)
  97
  98         branches[name] = b
  99         log.msg("Configured branch: {}".format(name))
 100
 101 # PB port can be either a numeric port or a connection string
 102 pb_port = inip1.get("port") or 9989
 103
 104 # This is the dictionary that the buildmaster pays attention to. We also use
 105 # a shorter alias to save typing.
 106 c = BuildmasterConfig = {}
 107
 108 ####### PROJECT IDENTITY
 109
 110 # the 'title' string will appear at the top of this buildbot
 111 # installation's html.WebStatus home page (linked to the
 112 # 'titleURL') and is embedded in the title of the waterfall HTML page.
 113
 114 c['title'] = ini['general'].get("title")
 115 c['titleURL'] = ini['general'].get("title_url")
 116
 117 # the 'buildbotURL' string should point to the location where the buildbot's
 118 # internal web server (usually the html.WebStatus page) is visible. This
 119 # typically uses the port number set in the Waterfall 'status' entry, but
 120 # with an externally-visible host name which the buildbot cannot figure out
 121 # without some help.
 122
 123 c['buildbotURL'] = inip1.get("buildbot_url")
 124
 125 ####### BUILDWORKERS
 126
 127 # The 'workers' list defines the set of recognized buildworkers. Each element is
 128 # a Worker object, specifying a unique worker name and password.  The same
 129 # worker name and password must be configured on the worker.
 130
 131 c['workers'] = []
 132 NetLocks = dict()
 133
 134 for section in ini.sections():
 135         if section.startswith("branch "):
 136                 ini_parse_branch(ini[section])
 137
 138         if section.startswith("worker "):
 139                 if ini.has_option(section, "name") and ini.has_option(section, "password") and \
 140                    (not ini.has_option(section, "phase") or ini.getint(section, "phase") == 1):
 141                         sl_props = { 'dl_lock':None, 'ul_lock':None }
 142                         name = ini.get(section, "name")
 143                         password = ini.get(section, "password")
 144                         if ini.has_option(section, "dl_lock"):
 145                                 lockname = ini.get(section, "dl_lock")
 146                                 sl_props['dl_lock'] = lockname
 147                                 if lockname not in NetLocks:
 148                                         NetLocks[lockname] = locks.MasterLock(lockname)
 149                         if ini.has_option(section, "ul_lock"):
 150                                 lockname = ini.get(section, "ul_lock")
 151                                 sl_props['ul_lock'] = lockname
 152                                 if lockname not in NetLocks:
 153                                         NetLocks[lockname] = locks.MasterLock(lockname)
 154                         c['workers'].append(Worker(name, password, max_builds = 1, properties = sl_props))
 155
 156 c['protocols'] = {'pb': {'port': pb_port}}
 157
 158 # coalesce builds
 159 c['collapseRequests'] = True
 160
 161 # Reduce amount of backlog data
 162 c['configurators'] = [util.JanitorConfigurator(
 163         logHorizon=timedelta(days=3),
 164         hour=6,
 165 )]
 166
 167 @defer.inlineCallbacks
 168 def getNewestCompleteTime(bldr):
 169         """Returns the complete_at of the latest completed and not SKIPPED
 170         build request for this builder, or None if there are no such build
 171         requests. We need to filter out SKIPPED requests because we're
 172         using collapseRequests=True which is unfortunately marking all
 173         previous requests as complete when new buildset is created.
 174
 175         @returns: datetime instance or None, via Deferred
 176         """
 177
 178         bldrid = yield bldr.getBuilderId()
 179         completed = yield bldr.master.data.get(
 180                         ('builders', bldrid, 'buildrequests'),
 181                         [
 182                                 resultspec.Filter('complete', 'eq', [True]),
 183                                 resultspec.Filter('results', 'ne', [results.SKIPPED]),
 184                         ],
 185                         order=['-complete_at'], limit=1)
 186         if not completed:
 187                 return
 188
 189         complete_at = completed[0]['complete_at']
 190
 191         last_build = yield bldr.master.data.get(
 192                         ('builds', ),
 193                         [
 194                                 resultspec.Filter('builderid', 'eq', [bldrid]),
 195                         ],
 196                         order=['-started_at'], limit=1)
 197
 198         if last_build and last_build[0]:
 199                 last_complete_at = last_build[0]['complete_at']
 200                 if last_complete_at and (last_complete_at > complete_at):
 201                         return last_complete_at
 202
 203         return complete_at
 204
 205 @defer.inlineCallbacks
 206 def prioritizeBuilders(master, builders):
 207         """Returns sorted list of builders by their last timestamp of completed and
 208         not skipped build.
 209
 210         @returns: list of sorted builders
 211         """
 212
 213         def is_building(bldr):
 214                 return bool(bldr.building) or bool(bldr.old_building)
 215
 216         def bldr_info(bldr):
 217                 d = defer.maybeDeferred(getNewestCompleteTime, bldr)
 218                 d.addCallback(lambda complete_at: (complete_at, bldr))
 219                 return d
 220
 221         def bldr_sort(item):
 222                 (complete_at, bldr) = item
 223
 224                 if bldr.name == "00_force_build":
 225                         date = datetime.min
 226                         complete_at = date.replace(tzinfo=tzutc())
 227                         return (complete_at, bldr.name)
 228
 229                 if not complete_at:
 230                         date = datetime.min
 231                         complete_at = date.replace(tzinfo=tzutc())
 232
 233                 if is_building(bldr):
 234                         date = datetime.max
 235                         complete_at = date.replace(tzinfo=tzutc())
 236
 237                 return (complete_at, bldr.name)
 238
 239         results = yield defer.gatherResults([bldr_info(bldr) for bldr in builders])
 240         results.sort(key=bldr_sort)
 241
 242         for r in results:
 243                 log.msg("prioritizeBuilders: {:>20} complete_at: {}".format(r[1].name, r[0]))
 244
 245         return [r[1] for r in results]
 246
 247 c['prioritizeBuilders'] = prioritizeBuilders
 248
 249 ####### CHANGESOURCES
 250
 251 branchNames = [branches[b]["name"] for b in branches]
 252
 253 # find targets
 254 targets = set()
 255
 256 def populateTargets():
 257         log.msg("Populating targets, this will take time")
 258         sourcegit = work_dir + '/source.git'
 259         for branch in branchNames:
 260                 if os.path.isdir(sourcegit):
 261                         subprocess.call(["rm", "-rf", sourcegit])
 262
 263                 subprocess.call(["git", "clone", "-q", "--depth=1", "--branch="+branch, repo_url, sourcegit])
 264
 265                 os.makedirs(sourcegit + '/tmp', exist_ok=True)
 266                 findtargets = subprocess.Popen(['./scripts/dump-target-info.pl', 'targets'],
 267                         stdout = subprocess.PIPE, stderr = subprocess.DEVNULL, cwd = sourcegit)
 268
 269                 while True:
 270                         line = findtargets.stdout.readline()
 271                         if not line:
 272                                 break
 273                         ta = line.decode().strip().split(' ')
 274                         targets.add(ta[0])
 275
 276                 subprocess.call(["rm", "-rf", sourcegit])
 277
 278 populateTargets()
 279
 280 # the 'change_source' setting tells the buildmaster how it should find out
 281 # about source code changes.  Here we point to the buildbot clone of pyflakes.
 282
 283 c['change_source'] = []
 284 c['change_source'].append(GitPoller(
 285         repo_url,
 286         workdir=work_dir+'/work.git', branches=branchNames,
 287         pollAtLaunch=True, pollinterval=300))
 288
 289 ####### SCHEDULERS
 290
 291 # Configure the Schedulers, which decide how to react to incoming changes.  In this
 292 # case, just kick off a 'basebuild' build
 293
 294 class TagChoiceParameter(BaseParameter):
 295         spec_attributes = ["strict", "choices"]
 296         type = "list"
 297         strict = True
 298
 299         def __init__(self, name, label=None, **kw):
 300                 super().__init__(name, label, **kw)
 301                 self._choice_list = []
 302
 303         @property
 304         def choices(self):
 305                 taglist = []
 306                 branchvers = []
 307
 308                 for b in branchNames:
 309                         basever = re.search(r'-([0-9]+\.[0-9]+)$', b)
 310                         if basever:
 311                                 branchvers.append(basever[1])
 312
 313                 alltags = subprocess.Popen(
 314                         ['git', 'ls-remote', '--tags', repo_url],
 315                         stdout = subprocess.PIPE)
 316
 317                 while True:
 318                         line = alltags.stdout.readline()
 319
 320                         if not line:
 321                                 break
 322
 323                         (ref, tag) = line.split()
 324
 325                         tagver = re.search(r'\brefs/tags/(v[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?)$', tag.decode().strip())
 326
 327                         # only list tags matching configured branches
 328                         if tagver and any(tagver[1][1:].startswith(b) for b in branchvers):
 329                                 taglist.append(tagver[1])
 330
 331                 taglist.sort(reverse=True, key=lambda tag: tag if re.search(r'-rc[0-9]+$', tag) else tag + '-z')
 332                 taglist.insert(0, '')
 333
 334                 self._choice_list = taglist
 335
 336                 return self._choice_list
 337
 338         def updateFromKwargs(self, properties, kwargs, **unused):
 339                 tag = self.getFromKwargs(kwargs)
 340                 properties[self.name] = tag
 341
 342                 # find the commit matching the tag
 343                 findrev = subprocess.Popen(['git', 'rev-parse', 'tags/'+tag], stdout=subprocess.PIPE, cwd=work_dir+'/work.git')
 344                 findrev.wait(timeout=10)
 345                 line = findrev.stdout.readline()
 346
 347                 if findrev.returncode!=0 or not line:
 348                         raise ValidationError("Couldn't find tag")
 349
 350                 properties['force_revision'] = line.decode().strip()
 351
 352                 # find the branch matching the tag
 353                 branch = None
 354                 branchver = re.search(r'v([0-9]+\.[0-9]+)', tag)
 355                 for b in branchNames:
 356                         if b.endswith(branchver[1]):
 357                                 branch = b
 358
 359                 if not branch:
 360                         raise ValidationError("Couldn't find branch")
 361
 362                 properties['force_branch'] = branch
 363
 364         def parse_from_arg(self, s):
 365                 if self.strict and s not in self._choice_list:
 366                         raise ValidationError("'%s' does not belong to list of available choices '%s'" % (s, self._choice_list))
 367                 return s
 368
 369 c['schedulers'] = []
 370 c['schedulers'].append(AnyBranchScheduler(
 371         name            = "all",
 372         change_filter   = util.ChangeFilter(branch=branchNames),
 373         treeStableTimer = 15*60,
 374         builderNames    = list(targets)))
 375
 376 c['schedulers'].append(ForceScheduler(
 377         name         = "force",
 378         buttonName   = "Force builds",
 379         label        = "Force build details",
 380         builderNames = [ "00_force_build" ],
 381
 382         codebases = [
 383                 util.CodebaseParameter(
 384                         "",
 385                         label      = "Repository",
 386                         branch     = util.FixedParameter(name = "branch",     default = ""),
 387                         revision   = util.FixedParameter(name = "revision",   default = ""),
 388                         repository = util.FixedParameter(name = "repository", default = ""),
 389                         project    = util.FixedParameter(name = "project",    default = "")
 390                 )
 391         ],
 392
 393         reason = util.StringParameter(
 394                 name     = "reason",
 395                 label    = "Reason",
 396                 default  = "Trigger build",
 397                 required = True,
 398                 size     = 80
 399         ),
 400
 401         properties = [
 402                 util.ChoiceStringParameter(
 403                         name    = "target",
 404                         label   = "Build target",
 405                         default = "all",
 406                         choices = [ "all" ] + list(targets)
 407                 ),
 408                 TagChoiceParameter(
 409                         name    = "tag",
 410                         label   = "Build tag",
 411                         default = ""
 412                 )
 413         ]
 414 ))
 415
 416 ####### BUILDERS
 417
 418 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
 419 # what steps, and which workers can execute them.  Note that any particular build will
 420 # only take place on one worker.
 421
 422 def IsNoMasterBuild(step):
 423         return step.getProperty("branch") != "master"
 424
 425 def IsUsignEnabled(step):
 426         branch = step.getProperty("branch")
 427         return branch and branches[branch].get("usign_key")
 428
 429 def IsSignEnabled(step):
 430         branch = step.getProperty("branch")
 431         return IsUsignEnabled(step) or branch and branches[branch].get("gpg_key")
 432
 433 def IsKmodArchiveEnabled(step):
 434         branch = step.getProperty("branch")
 435         return branch and branches[branch].get("kmod_archive")
 436
 437 def GetBaseVersion(branch):
 438         if re.match(r"^[^-]+-[0-9]+\.[0-9]+$", branch):
 439                 return branch.split('-')[1]
 440         else:
 441                 return "master"
 442
 443 @properties.renderer
 444 def GetVersionPrefix(props):
 445         branch = props.getProperty("branch")
 446         basever = GetBaseVersion(branch)
 447         if props.hasProperty("tag") and re.match(r"^v[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", props["tag"]):
 448                 return "%s/" % props["tag"][1:]
 449         elif basever != "master":
 450                 return "%s-SNAPSHOT/" % basever
 451         else:
 452                 return ""
 453
 454 @util.renderer
 455 def GetConfigSeed(props):
 456         branch = props.getProperty("branch")
 457         return branch and branches[branch].get("config_seed") or ""
 458
 459 @util.renderer
 460 def GetRsyncParams(props, srcorbin, urlorkey):
 461         # srcorbin: 'bin' or 'src'; urlorkey: 'url' or 'key'
 462         branch = props.getProperty("branch")
 463         opt = srcorbin + "_" + urlorkey
 464         return branch and branches[branch].get(opt)
 465
 466 @util.renderer
 467 def GetUsignKey(props):
 468         branch = props.getProperty("branch")
 469         return branch and branches[branch].get("usign_key")
 470
 471 def GetNextBuild(builder, requests):
 472         for r in requests:
 473                 if r.properties:
 474                         # order tagged build first
 475                         if r.properties.hasProperty("tag"):
 476                                 return r
 477                         # then order by branch order
 478                         pbranch = r.properties.getProperty("branch")
 479                         for name in branchNames:
 480                                 if pbranch == name:
 481                                         return r
 482
 483         r = requests[0]
 484         log.msg("GetNextBuild: {:>20} id: {} bsid: {}".format(builder.name, r.id, r.bsid))
 485         return r
 486
 487 def MakeEnv(overrides=None, tryccache=False):
 488         env = {
 489                 'CCC': Interpolate("%(prop:cc_command:-gcc)s"),
 490                 'CCXX': Interpolate("%(prop:cxx_command:-g++)s"),
 491         }
 492         if tryccache:
 493                 env['CC'] = Interpolate("%(prop:builddir)s/ccache_cc.sh")
 494                 env['CXX'] = Interpolate("%(prop:builddir)s/ccache_cxx.sh")
 495                 env['CCACHE'] = Interpolate("%(prop:ccache_command:-)s")
 496         else:
 497                 env['CC'] = env['CCC']
 498                 env['CXX'] = env['CCXX']
 499                 env['CCACHE'] = ''
 500         if overrides is not None:
 501                 env.update(overrides)
 502         return env
 503
 504 @properties.renderer
 505 def NetLockDl(props, extralock=None):
 506         lock = None
 507         if props.hasProperty("dl_lock"):
 508                 lock = NetLocks[props["dl_lock"]]
 509         if lock is not None:
 510                 return [lock.access('exclusive')]
 511         else:
 512                 return []
 513
 514 @properties.renderer
 515 def NetLockUl(props):
 516         lock = None
 517         if props.hasProperty("ul_lock"):
 518                 lock = NetLocks[props["ul_lock"]]
 519         if lock is not None:
 520                 return [lock.access('exclusive')]
 521         else:
 522                 return []
 523
 524 def IsTargetSelected(target):
 525         def CheckTargetProperty(step):
 526                 selected_target = step.getProperty("target", "all")
 527                 if selected_target != "all" and selected_target != target:
 528                         return False
 529                 return True
 530
 531         return CheckTargetProperty
 532
 533 @util.renderer
 534 def UsignSec2Pub(props):
 535         branch = props.getProperty("branch")
 536         try:
 537                 comment = branches[branch].get("usign_comment") or "untrusted comment: secret key"
 538                 seckey = branches[branch].get("usign_key")
 539                 seckey = base64.b64decode(seckey)
 540         except:
 541                 return None
 542
 543         return "{}\n{}".format(re.sub(r"\bsecret key$", "public key", comment),
 544                 base64.b64encode(seckey[0:2] + seckey[32:40] + seckey[72:]))
 545
 546
 547 c['builders'] = []
 548
 549 workerNames = [ ]
 550
 551 for worker in c['workers']:
 552         workerNames.append(worker.workername)
 553
 554 # add a single LocalWorker to handle the forcebuild builder
 555 c['workers'].append(LocalWorker("__local_force_build", max_builds=1))
 556
 557 force_factory = BuildFactory()
 558
 559 c['builders'].append(BuilderConfig(
 560         name        = "00_force_build",
 561         workername  = "__local_force_build",
 562         factory     = force_factory))
 563
 564 for target in targets:
 565         ts = target.split('/')
 566
 567         factory = BuildFactory()
 568
 569         # setup shared work directory if required
 570         factory.addStep(ShellCommand(
 571                 name = "sharedwd",
 572                 descriptionDone = "Shared work directory set up",
 573                 command = 'test -L "$PWD" || (mkdir -p ../shared-workdir && rm -rf "$PWD" && ln -s shared-workdir "$PWD")',
 574                 workdir = ".",
 575                 haltOnFailure = True,
 576         ))
 577
 578         # find number of cores
 579         factory.addStep(SetPropertyFromCommand(
 580                 name = "nproc",
 581                 property = "nproc",
 582                 description = "Finding number of CPUs",
 583                 command = ["nproc"],
 584         ))
 585
 586         # find gcc and g++ compilers
 587         factory.addStep(FileDownload(
 588                 name = "dlfindbinpl",
 589                 mastersrc = scripts_dir + '/findbin.pl',
 590                 workerdest = "../findbin.pl",
 591                 mode = 0o755,
 592         ))
 593
 594         factory.addStep(SetPropertyFromCommand(
 595                 name = "gcc",
 596                 property = "cc_command",
 597                 description = "Finding gcc command",
 598                 command = ["../findbin.pl", "gcc", "", ""],
 599                 haltOnFailure = True,
 600         ))
 601
 602         factory.addStep(SetPropertyFromCommand(
 603                 name = "g++",
 604                 property = "cxx_command",
 605                 description = "Finding g++ command",
 606                 command = ["../findbin.pl", "g++", "", ""],
 607                 haltOnFailure = True,
 608         ))
 609
 610         # see if ccache is available
 611         factory.addStep(SetPropertyFromCommand(
 612                 name = "ccache",
 613                 property = "ccache_command",
 614                 description = "Testing for ccache command",
 615                 command = ["which", "ccache"],
 616                 haltOnFailure = False,
 617                 flunkOnFailure = False,
 618                 warnOnFailure = False,
 619                 hideStepIf = lambda r, s: r==results.FAILURE,
 620         ))
 621
 622         # check out the source
 623         # Git() runs:
 624           # if repo doesn't exist: 'git clone repourl'
 625           # method 'clean' runs 'git clean -d -f', method fresh runs 'git clean -f -f -d -x'. Only works with mode='full'
 626           # git cat-file -e <commit>
 627           # git checkout -f <commit>
 628           # git checkout -B <branch>
 629           # git rev-parse HEAD
 630         factory.addStep(Git(
 631                 name = "git",
 632                 repourl = repo_url,
 633                 mode = 'full',
 634                 method = 'fresh',
 635                 locks = NetLockDl,
 636                 haltOnFailure = True,
 637         ))
 638
 639         # update remote refs
 640         factory.addStep(ShellCommand(
 641                 name = "fetchrefs",
 642                 description = "Fetching Git remote refs",
 643                 command = ["git", "fetch", "origin", Interpolate("+refs/heads/%(prop:branch)s:refs/remotes/origin/%(prop:branch)s")],
 644                 haltOnFailure = True,
 645         ))
 646
 647         # Verify that Git HEAD points to a tag or branch
 648         # Ref: https://web.archive.org/web/20190729224316/http://lists.infradead.org/pipermail/openwrt-devel/2019-June/017809.html
 649         factory.addStep(ShellCommand(
 650                 name = "gitverify",
 651                 description = "Ensure that Git HEAD is pointing to a branch or tag",
 652                 command = 'git rev-parse --abbrev-ref HEAD | grep -vxqF HEAD || git show-ref --tags --dereference 2>/dev/null | sed -ne "/^$(git rev-parse HEAD) / { s|^.*/||; s|\\^.*||; p }" | grep -qE "^v[0-9][0-9]\\."',
 653                 haltOnFailure = True,
 654         ))
 655
 656         factory.addStep(ShellCommand(
 657                 name = "rmtmp",
 658                 description = "Remove tmp folder",
 659                 command=["rm", "-rf", "tmp/"],
 660         ))
 661
 662         # feed
 663         factory.addStep(ShellCommand(
 664                 name = "rmfeedlinks",
 665                 description = "Remove feed symlinks",
 666                 command=["rm", "-rf", "package/feeds/"],
 667         ))
 668
 669         factory.addStep(StringDownload(
 670                 name = "ccachecc",
 671                 s = '#!/bin/sh\nexec ${CCACHE} ${CCC} "$@"\n',
 672                 workerdest = "../ccache_cc.sh",
 673                 mode = 0o755,
 674         ))
 675
 676         factory.addStep(StringDownload(
 677                 name = "ccachecxx",
 678                 s = '#!/bin/sh\nexec ${CCACHE} ${CCXX} "$@"\n',
 679                 workerdest = "../ccache_cxx.sh",
 680                 mode = 0o755,
 681         ))
 682
 683         # feed
 684         factory.addStep(ShellCommand(
 685                 name = "updatefeeds",
 686                 description = "Updating feeds",
 687                 command=["./scripts/feeds", "update"],
 688                 env = MakeEnv(tryccache=True),
 689                 haltOnFailure = True,
 690                 locks = NetLockDl,
 691         ))
 692
 693         # feed
 694         factory.addStep(ShellCommand(
 695                 name = "installfeeds",
 696                 description = "Installing feeds",
 697                 command=["./scripts/feeds", "install", "-a"],
 698                 env = MakeEnv(tryccache=True),
 699                 haltOnFailure = True,
 700         ))
 701
 702         # seed config
 703         factory.addStep(StringDownload(
 704                 name = "dlconfigseed",
 705                 s = Interpolate("%(kw:seed)s\n", seed=GetConfigSeed),
 706                 workerdest = ".config",
 707                 mode = 0o644,
 708         ))
 709
 710         # configure
 711         factory.addStep(ShellCommand(
 712                 name = "newconfig",
 713                 descriptionDone = ".config seeded",
 714                 command = Interpolate("printf 'CONFIG_TARGET_%(kw:target)s=y\\nCONFIG_TARGET_%(kw:target)s_%(kw:subtarget)s=y\\nCONFIG_SIGNED_PACKAGES=%(kw:usign:#?|y|n)s\\n' >> .config", target=ts[0], subtarget=ts[1], usign=GetUsignKey),
 715         ))
 716
 717         factory.addStep(ShellCommand(
 718                 name = "delbin",
 719                 description = "Removing output directory",
 720                 command = ["rm", "-rf", "bin/"],
 721         ))
 722
 723         factory.addStep(ShellCommand(
 724                 name = "defconfig",
 725                 description = "Populating .config",
 726                 command = ["make", "defconfig"],
 727                 env = MakeEnv(),
 728         ))
 729
 730         # check arch - exit early if does not exist - NB: some targets do not define CONFIG_TARGET_target_subtarget
 731         factory.addStep(ShellCommand(
 732                 name = "checkarch",
 733                 description = "Checking architecture",
 734                 descriptionDone = "Architecture validated",
 735                 command = 'grep -sq CONFIG_TARGET_%s=y .config && grep -sq CONFIG_TARGET_SUBTARGET=\\"%s\\" .config' %(ts[0], ts[1]),
 736                 logEnviron = False,
 737                 want_stdout = False,
 738                 want_stderr = False,
 739                 haltOnFailure = True,
 740                 flunkOnFailure = False, # this is not a build FAILURE
 741         ))
 742
 743         # find libc suffix
 744         factory.addStep(SetPropertyFromCommand(
 745                 name = "libc",
 746                 property = "libc",
 747                 description = "Finding libc suffix",
 748                 command = ["sed", "-ne", '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }', ".config"],
 749         ))
 750
 751         # install build key
 752         factory.addStep(StringDownload(
 753                 name = "dlkeybuildpub",
 754                 s = Interpolate("%(kw:sec2pub)s", sec2pub=UsignSec2Pub),
 755                 workerdest = "key-build.pub",
 756                 mode = 0o600,
 757                 doStepIf = IsUsignEnabled,
 758         ))
 759
 760         factory.addStep(StringDownload(
 761                 name = "dlkeybuild",
 762                 s = "# fake private key",
 763                 workerdest = "key-build",
 764                 mode = 0o600,
 765                 doStepIf = IsUsignEnabled,
 766         ))
 767
 768         factory.addStep(StringDownload(
 769                 name = "dlkeybuilducert",
 770                 s = "# fake certificate",
 771                 workerdest = "key-build.ucert",
 772                 mode = 0o600,
 773                 doStepIf = IsUsignEnabled,
 774         ))
 775
 776         # prepare dl
 777         factory.addStep(ShellCommand(
 778                 name = "dldir",
 779                 description = "Preparing dl/",
 780                 descriptionDone = "dl/ prepared",
 781                 command = 'mkdir -p ../dl && rm -rf "build/dl" && ln -s ../../dl "build/dl"',
 782                 workdir = Property("builddir"),
 783                 logEnviron = False,
 784                 want_stdout = False,
 785         ))
 786
 787         # cleanup dl
 788         factory.addStep(ShellCommand(
 789                 name = "dlprune",
 790                 description = "Pruning dl/",
 791                 descriptionDone = "dl/ pruned",
 792                 command = 'find dl/ -atime +15 -delete -print',
 793                 logEnviron = False,
 794         ))
 795
 796         # prepare tar
 797         factory.addStep(ShellCommand(
 798                 name = "dltar",
 799                 description = "Building and installing GNU tar",
 800                 descriptionDone = "GNU tar built and installed",
 801                 command = ["make", Interpolate("-j%(prop:nproc:-1)s"), "tools/tar/compile", "V=s"],
 802                 env = MakeEnv(tryccache=True),
 803                 haltOnFailure = True,
 804         ))
 805
 806         # populate dl
 807         factory.addStep(ShellCommand(
 808                 name = "dlrun",
 809                 description = "Populating dl/",
 810                 descriptionDone = "dl/ populated",
 811                 command = ["make", Interpolate("-j%(prop:nproc:-1)s"), "download", "V=s"],
 812                 env = MakeEnv(),
 813                 logEnviron = False,
 814                 locks = NetLockDl,
 815         ))
 816
 817         factory.addStep(ShellCommand(
 818                 name = "cleanbase",
 819                 description = "Cleaning base-files",
 820                 command=["make", "package/base-files/clean", "V=s"],
 821         ))
 822
 823         # build
 824         factory.addStep(ShellCommand(
 825                 name = "tools",
 826                 description = "Building and installing tools",
 827                 descriptionDone = "Tools built and installed",
 828                 command = ["make", Interpolate("-j%(prop:nproc:-1)s"), "tools/install", "V=s"],
 829                 env = MakeEnv(tryccache=True),
 830                 haltOnFailure = True,
 831         ))
 832
 833         factory.addStep(ShellCommand(
 834                 name = "toolchain",
 835                 description = "Building and installing toolchain",
 836                 descriptionDone = "Toolchain built and installed",
 837                 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "toolchain/install", "V=s"],
 838                 env = MakeEnv(),
 839                 haltOnFailure = True,
 840         ))
 841
 842         factory.addStep(ShellCommand(
 843                 name = "kmods",
 844                 description = "Building kmods",
 845                 descriptionDone = "Kmods built",
 846                 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
 847                 env = MakeEnv(),
 848                 haltOnFailure = True,
 849         ))
 850
 851         # find kernel version
 852         factory.addStep(SetPropertyFromCommand(
 853                 name = "kernelversion",
 854                 property = "kernelversion",
 855                 description = "Finding the effective Kernel version",
 856                 command = "make --no-print-directory -C target/linux/ val.LINUX_VERSION val.LINUX_RELEASE val.LINUX_VERMAGIC | xargs printf '%s-%s-%s\\n'",
 857                 env = { 'TOPDIR': Interpolate("%(prop:builddir)s/build") },
 858         ))
 859
 860         factory.addStep(ShellCommand(
 861                 name = "pkgclean",
 862                 description = "Cleaning up package build",
 863                 descriptionDone = "Package build cleaned up",
 864                 command=["make", "package/cleanup", "V=s"],
 865         ))
 866
 867         factory.addStep(ShellCommand(
 868                 name = "pkgbuild",
 869                 description = "Building packages",
 870                 descriptionDone = "Packages built",
 871                 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
 872                 env = MakeEnv(),
 873                 haltOnFailure = True,
 874         ))
 875
 876         factory.addStep(ShellCommand(
 877                 name = "pkginstall",
 878                 description = "Installing packages",
 879                 descriptionDone = "Packages installed",
 880                 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/install", "V=s"],
 881                 env = MakeEnv(),
 882                 haltOnFailure = True,
 883         ))
 884
 885         factory.addStep(ShellCommand(
 886                 name = "pkgindex",
 887                 description = "Indexing packages",
 888                 descriptionDone = "Packages indexed",
 889                 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/index", "V=s", "CONFIG_SIGNED_PACKAGES="],
 890                 env = MakeEnv(),
 891                 haltOnFailure = True,
 892         ))
 893
 894         factory.addStep(ShellCommand(
 895                 name = "images",
 896                 description = "Building and installing images",
 897                 descriptionDone = "Images built and installed",
 898                 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "target/install", "V=s"],
 899                 env = MakeEnv(),
 900                 haltOnFailure = True,
 901         ))
 902
 903         factory.addStep(ShellCommand(
 904                 name = "buildinfo",
 905                 description = "Generating config.buildinfo, version.buildinfo and feeds.buildinfo",
 906                 command = "make -j1 buildinfo V=s || true",
 907                 env = MakeEnv(),
 908                 haltOnFailure = True,
 909         ))
 910
 911         factory.addStep(ShellCommand(
 912                 name = "json_overview_image_info",
 913                 description = "Generating profiles.json in target folder",
 914                 command = "make -j1 json_overview_image_info V=s || true",
 915                 env = MakeEnv(),
 916                 haltOnFailure = True,
 917         ))
 918
 919         factory.addStep(ShellCommand(
 920                 name = "checksums",
 921                 description = "Calculating checksums",
 922                 descriptionDone = "Checksums calculated",
 923                 command=["make", "-j1", "checksum", "V=s"],
 924                 env = MakeEnv(),
 925                 haltOnFailure = True,
 926         ))
 927
 928         factory.addStep(ShellCommand(
 929                 name = "kmoddir",
 930                 descriptionDone = "Kmod directory created",
 931                 command=["mkdir", "-p", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s", target=ts[0], subtarget=ts[1])],
 932                 haltOnFailure = True,
 933                 doStepIf = IsKmodArchiveEnabled,
 934         ))
 935
 936         factory.addStep(ShellCommand(
 937                 name = "kmodprepare",
 938                 description = "Preparing kmod archive",
 939                 descriptionDone = "Kmod archive prepared",
 940                 command=["rsync", "--include=/kmod-*.ipk", "--exclude=*", "-va",
 941                         Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/packages/", target=ts[0], subtarget=ts[1]),
 942                         Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1])],
 943                 haltOnFailure = True,
 944                 doStepIf = IsKmodArchiveEnabled,
 945         ))
 946
 947         factory.addStep(ShellCommand(
 948                 name = "kmodindex",
 949                 description = "Indexing kmod archive",
 950                 descriptionDone = "Kmod archive indexed",
 951                 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/index", "V=s", "CONFIG_SIGNED_PACKAGES=",
 952                         Interpolate("PACKAGE_SUBDIRS=bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1])],
 953                 env = MakeEnv(),
 954                 haltOnFailure = True,
 955                 doStepIf = IsKmodArchiveEnabled,
 956         ))
 957
 958         # sign
 959         factory.addStep(MasterShellCommand(
 960                 name = "signprepare",
 961                 descriptionDone = "Temporary signing directory prepared",
 962                 command = ["mkdir", "-p", "%s/signing" %(work_dir)],
 963                 haltOnFailure = True,
 964                 doStepIf = IsSignEnabled,
 965
 966         ))
 967
 968         factory.addStep(ShellCommand(
 969                 name = "signpack",
 970                 description = "Packing files to sign",
 971                 descriptionDone = "Files to sign packed",
 972                 command = Interpolate("find bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/ bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz", target=ts[0], subtarget=ts[1]),
 973                 haltOnFailure = True,
 974                 doStepIf = IsSignEnabled,
 975         ))
 976
 977         factory.addStep(FileUpload(
 978                 workersrc = "sign.tar.gz",
 979                 masterdest = "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1]),
 980                 haltOnFailure = True,
 981                 doStepIf = IsSignEnabled,
 982         ))
 983
 984         factory.addStep(MasterShellCommand(
 985                 name = "signfiles",
 986                 description = "Signing files",
 987                 descriptionDone = "Files signed",
 988                 command = ["%s/signall.sh" %(scripts_dir), "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1]), Interpolate("%(prop:branch)s")],
 989                 env = { 'CONFIG_INI': os.getenv("BUILDMASTER_CONFIG", "./config.ini") },
 990                 haltOnFailure = True,
 991                 doStepIf = IsSignEnabled,
 992         ))
 993
 994         factory.addStep(FileDownload(
 995                 name = "dlsigntargz",
 996                 mastersrc = "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1]),
 997                 workerdest = "sign.tar.gz",
 998                 haltOnFailure = True,
 999                 doStepIf = IsSignEnabled,
1000         ))
1001
1002         factory.addStep(ShellCommand(
1003                 name = "signunpack",
1004                 description = "Unpacking signed files",
1005                 descriptionDone = "Signed files unpacked",
1006                 command = ["tar", "-xzf", "sign.tar.gz"],
1007                 haltOnFailure = True,
1008                 doStepIf = IsSignEnabled,
1009         ))
1010
1011         # upload
1012         factory.addStep(ShellCommand(
1013                 name = "dirprepare",
1014                 descriptionDone = "Upload directory structure prepared",
1015                 command = ["mkdir", "-p", Interpolate("tmp/upload/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s", target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1016                 haltOnFailure = True,
1017         ))
1018
1019         factory.addStep(ShellCommand(
1020                 name = "linkprepare",
1021                 descriptionDone = "Repository symlink prepared",
1022                 command = ["ln", "-s", "-f", Interpolate("../packages-%(kw:basever)s", basever=util.Transform(GetBaseVersion, Property("branch"))), Interpolate("tmp/upload/%(kw:prefix)spackages", prefix=GetVersionPrefix)],
1023                 doStepIf = IsNoMasterBuild,
1024                 haltOnFailure = True,
1025         ))
1026
1027         factory.addStep(ShellCommand(
1028                 name = "kmoddirprepare",
1029                 descriptionDone = "Kmod archive upload directory prepared",
1030                 command = ["mkdir", "-p", Interpolate("tmp/upload/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s", target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1031                 haltOnFailure = True,
1032                 doStepIf = IsKmodArchiveEnabled,
1033         ))
1034
1035         factory.addStep(ShellCommand(
1036                 name = "dirupload",
1037                 description = "Uploading directory structure",
1038                 descriptionDone = "Directory structure uploaded",
1039                 command = ["rsync", "-az"] + rsync_defopts + ["tmp/upload/", Interpolate("%(kw:url)s/", url=GetRsyncParams.withArgs("bin", "url"))],
1040                 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1041                 haltOnFailure = True,
1042                 logEnviron = False,
1043                 locks = NetLockUl,
1044                 doStepIf = util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
1045         ))
1046
1047         # download remote sha256sums to 'target-sha256sums'
1048         factory.addStep(ShellCommand(
1049                 name = "target-sha256sums",
1050                 description = "Fetching remote sha256sums for target",
1051                 descriptionDone = "Remote sha256sums for target fetched",
1052                 command = ["rsync", "-z"] + rsync_defopts + [Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/sha256sums", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix), "target-sha256sums"],
1053                 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1054                 logEnviron = False,
1055                 haltOnFailure = False,
1056                 flunkOnFailure = False,
1057                 warnOnFailure = False,
1058                 doStepIf = util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
1059         ))
1060
1061         # build list of files to upload
1062         factory.addStep(FileDownload(
1063                 name = "dlsha2rsyncpl",
1064                 mastersrc = scripts_dir + '/sha2rsync.pl',
1065                 workerdest = "../sha2rsync.pl",
1066                 mode = 0o755,
1067         ))
1068
1069         factory.addStep(ShellCommand(
1070                 name = "buildlist",
1071                 description = "Building list of files to upload",
1072                 descriptionDone = "List of files to upload built",
1073                 command = ["../sha2rsync.pl", "target-sha256sums", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/sha256sums", target=ts[0], subtarget=ts[1]), "rsynclist"],
1074                 haltOnFailure = True,
1075         ))
1076
1077         factory.addStep(FileDownload(
1078                 name = "dlrsync.sh",
1079                 mastersrc = scripts_dir + '/rsync.sh',
1080                 workerdest = "../rsync.sh",
1081                 mode = 0o755,
1082         ))
1083
1084         # upload new files and update existing ones
1085         factory.addStep(ShellCommand(
1086                 name = "targetupload",
1087                 description = "Uploading target files",
1088                 descriptionDone = "Target files uploaded",
1089                 command=["../rsync.sh", "--exclude=/kmods/", "--files-from=rsynclist", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_defopts +
1090                         ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]),
1091                         Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1092                 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1093                 haltOnFailure = True,
1094                 logEnviron = False,
1095                 doStepIf = util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
1096         ))
1097
1098         # delete files which don't exist locally
1099         factory.addStep(ShellCommand(
1100                 name = "targetprune",
1101                 description = "Pruning target files",
1102                 descriptionDone = "Target files pruned",
1103                 command=["../rsync.sh", "--exclude=/kmods/", "--delete", "--existing", "--ignore-existing", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_defopts +
1104                         ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]),
1105                         Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1106                 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1107                 haltOnFailure = True,
1108                 logEnviron = False,
1109                 locks = NetLockUl,
1110                 doStepIf = util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
1111         ))
1112
1113         factory.addStep(ShellCommand(
1114                 name = "kmodupload",
1115                 description = "Uploading kmod archive",
1116                 descriptionDone = "Kmod archive uploaded",
1117                 command=["../rsync.sh", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_defopts +
1118                         ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1]),
1119                         Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1120                 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1121                 haltOnFailure = True,
1122                 logEnviron = False,
1123                 locks = NetLockUl,
1124                 doStepIf = util.Transform(lambda a, b: bool(a and b), IsKmodArchiveEnabled, GetRsyncParams.withArgs("bin", "url")),
1125         ))
1126
1127         factory.addStep(ShellCommand(
1128                 name = "sourcelist",
1129                 description = "Finding source archives to upload",
1130                 descriptionDone = "Source archives to upload found",
1131                 command = "find dl/ -maxdepth 1 -type f -not -size 0 -not -name '.*' -not -name '*.hash' -not -name '*.dl' -newer .config -printf '%f\\n' > sourcelist",
1132                 haltOnFailure = True,
1133         ))
1134
1135         factory.addStep(ShellCommand(
1136                 name = "sourceupload",
1137                 description = "Uploading source archives",
1138                 descriptionDone = "Source archives uploaded",
1139                 command=["../rsync.sh", "--files-from=sourcelist", "--size-only", "--delay-updates"] + rsync_defopts +
1140                         [Interpolate("--partial-dir=.~tmp~%(kw:target)s~%(kw:subtarget)s~%(prop:workername)s", target=ts[0], subtarget=ts[1]), "-a", "dl/", Interpolate("%(kw:url)s/", url=GetRsyncParams.withArgs("src", "url"))],
1141                 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("src", "key")) },
1142                 haltOnFailure = True,
1143                 logEnviron = False,
1144                 locks = NetLockUl,
1145                 doStepIf = util.Transform(bool, GetRsyncParams.withArgs("src", "url")),
1146         ))
1147
1148         factory.addStep(ShellCommand(
1149                 name = "df",
1150                 description = "Reporting disk usage",
1151                 command=["df", "-h", "."],
1152                 env={'LC_ALL': 'C'},
1153                 logEnviron = False,
1154                 haltOnFailure = False,
1155                 flunkOnFailure = False,
1156                 warnOnFailure = False,
1157                 alwaysRun = True,
1158         ))
1159
1160         factory.addStep(ShellCommand(
1161                 name = "du",
1162                 description = "Reporting estimated file space usage",
1163                 command=["du", "-sh", "."],
1164                 env={'LC_ALL': 'C'},
1165                 logEnviron = False,
1166                 haltOnFailure = False,
1167                 flunkOnFailure = False,
1168                 warnOnFailure = False,
1169                 alwaysRun = True,
1170         ))
1171
1172         factory.addStep(ShellCommand(
1173                 name = "ccachestat",
1174                 description = "Reporting ccache stats",
1175                 command=["ccache", "-s"],
1176                 env = MakeEnv(overrides={ 'PATH': ["${PATH}", "./staging_dir/host/bin"] }),
1177                 logEnviron = False,
1178                 want_stderr = False,
1179                 haltOnFailure = False,
1180                 flunkOnFailure = False,
1181                 warnOnFailure = False,
1182                 hideStepIf = lambda r, s: r==results.FAILURE,
1183         ))
1184
1185         c['builders'].append(BuilderConfig(name=target, workernames=workerNames, factory=factory, nextBuild=GetNextBuild))
1186
1187         c['schedulers'].append(schedulers.Triggerable(name="trigger_%s" % target, builderNames=[ target ]))
1188         force_factory.addStep(steps.Trigger(
1189                 name = "trigger_%s" % target,
1190                 description = "Triggering %s build" % target,
1191                 schedulerNames = [ "trigger_%s" % target ],
1192                 sourceStamps = [{ "codebase": "", "branch": Property("force_branch"), "revision": Property("force_revision"), "repository": repo_url, "project": "" }],
1193                 set_properties = { "reason": Property("reason"), "tag": Property("tag"), },
1194                 doStepIf = IsTargetSelected(target),
1195         ))
1196
1197
1198 ####### STATUS TARGETS
1199
1200 # 'status' is a list of Status Targets. The results of each build will be
1201 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
1202 # including web pages, email senders, and IRC bots.
1203
1204 if "status_bind" in inip1:
1205         c['www'] = {
1206                 'port': inip1.get("status_bind"),
1207                 'plugins': {
1208                         'waterfall_view': True,
1209                         'console_view': True,
1210                         'grid_view': True
1211                 }
1212         }
1213
1214         if "status_user" in inip1 and "status_password" in inip1:
1215                 c['www']['auth'] = util.UserPasswordAuth([
1216                         (inip1.get("status_user"), inip1.get("status_password"))
1217                 ])
1218                 c['www']['authz'] = util.Authz(
1219                         allowRules=[ util.AnyControlEndpointMatcher(role="admins") ],
1220                         roleMatchers=[ util.RolesFromUsername(roles=["admins"], usernames=[inip1.get("status_user")]) ]
1221                 )
1222
1223 c['services'] = []
1224 if ini.has_section("irc"):
1225         iniirc = ini['irc']
1226         irc_host = iniirc.get("host", None)
1227         irc_port = iniirc.getint("port", 6667)
1228         irc_chan = iniirc.get("channel", None)
1229         irc_nick = iniirc.get("nickname", None)
1230         irc_pass = iniirc.get("password", None)
1231
1232         if irc_host and irc_nick and irc_chan:
1233                 irc = reporters.IRC(irc_host, irc_nick,
1234                         port = irc_port,
1235                         password = irc_pass,
1236                         channels = [ irc_chan ],
1237                         notify_events = [ 'exception', 'problem', 'recovery' ]
1238                 )
1239
1240                 c['services'].append(irc)
1241
1242 c['revlink'] = util.RevlinkMatch([
1243         r'https://git.openwrt.org/openwrt/(.*).git'
1244         ],
1245         r'https://git.openwrt.org/?p=openwrt/\1.git;a=commit;h=%s')
1246
1247 ####### DB URL
1248
1249 c['db'] = {
1250         # This specifies what database buildbot uses to store its state.  You can leave
1251         # this at its default for all but the largest installations.
1252         'db_url' : "sqlite:///state.sqlite",
1253 }
1254
1255 c['buildbotNetUsageData'] = None