phase1/master.cfg

   1 # -*- python -*-
   2 # ex: set syntax=python:
   3
   4 import os
   5 import re
   6 import base64
   7 import subprocess
   8 import configparser
   9
  10 from dateutil.tz import tzutc
  11 from datetime import datetime, timedelta
  12
  13 from twisted.internet import defer
  14 from twisted.python import log
  15
  16 from buildbot import locks
  17 from buildbot.data import resultspec
  18 from buildbot.changes.gitpoller import GitPoller
  19 from buildbot.config import BuilderConfig
  20 from buildbot.plugins import reporters
  21 from buildbot.plugins import schedulers
  22 from buildbot.plugins import steps
  23 from buildbot.plugins import util
  24 from buildbot.process import properties
  25 from buildbot.process import results
  26 from buildbot.process.factory import BuildFactory
  27 from buildbot.process.properties import Interpolate
  28 from buildbot.process.properties import Property
  29 from buildbot.schedulers.basic import AnyBranchScheduler
  30 from buildbot.schedulers.forcesched import BaseParameter
  31 from buildbot.schedulers.forcesched import ForceScheduler
  32 from buildbot.schedulers.forcesched import ValidationError
  33 from buildbot.steps.master import MasterShellCommand
  34 from buildbot.steps.shell import SetPropertyFromCommand
  35 from buildbot.steps.shell import ShellCommand
  36 from buildbot.steps.source.git import Git
  37 from buildbot.steps.transfer import FileDownload
  38 from buildbot.steps.transfer import FileUpload
  39 from buildbot.steps.transfer import StringDownload
  40 from buildbot.worker import Worker
  41 from buildbot.worker.local import LocalWorker
  42
  43
  44 if not os.path.exists("twistd.pid"):
  45         with open("twistd.pid", "w") as pidfile:
  46                 pidfile.write("{}".format(os.getpid()))
  47
  48 # This is a sample buildmaster config file. It must be installed as
  49 # 'master.cfg' in your buildmaster's base directory.
  50
  51 ini = configparser.ConfigParser()
  52 ini.read(os.getenv("BUILDMASTER_CONFIG", "./config.ini"))
  53
  54 if "general" not in ini or "phase1" not in ini:
  55         raise ValueError("Fix your configuration")
  56
  57 inip1 = ini['phase1']
  58
  59 # Globals
  60 work_dir = os.path.abspath(ini['general'].get("workdir", "."))
  61 scripts_dir = os.path.abspath("../scripts")
  62
  63 repo_url = ini['repo'].get("url")
  64
  65 rsync_defopts = ["-v", "-4", "--timeout=120"]
  66
  67 #if rsync_bin_url.find("::") > 0 or rsync_bin_url.find("rsync://") == 0:
  68 #       rsync_bin_defopts += ["--contimeout=20"]
  69
  70 branches = {}
  71
  72 def ini_parse_branch(section):
  73         b = {}
  74         name = section.get("name")
  75
  76         if not name:
  77                 raise ValueError("missing 'name' in " + repr(section))
  78         if name in branches:
  79                 raise ValueError("duplicate branch name in " + repr(section))
  80
  81         b["name"] = name
  82         b["bin_url"] = section.get("binary_url")
  83         b["bin_key"] = section.get("binary_password")
  84
  85         b["src_url"] = section.get("source_url")
  86         b["src_key"] = section.get("source_password")
  87
  88         b["gpg_key"] = section.get("gpg_key")
  89
  90         b["usign_key"] = section.get("usign_key")
  91         usign_comment = "untrusted comment: " + name.replace("-", " ").title() + " key"
  92         b["usign_comment"] = section.get("usign_comment", usign_comment)
  93
  94         b["config_seed"] = section.get("config_seed")
  95
  96         b["kmod_archive"] = section.getboolean("kmod_archive", False)
  97
  98         branches[name] = b
  99         log.msg("Configured branch: {}".format(name))
 100
 101 # PB port can be either a numeric port or a connection string
 102 pb_port = inip1.get("port") or 9989
 103
 104 # This is the dictionary that the buildmaster pays attention to. We also use
 105 # a shorter alias to save typing.
 106 c = BuildmasterConfig = {}
 107
 108 ####### PROJECT IDENTITY
 109
 110 # the 'title' string will appear at the top of this buildbot
 111 # installation's html.WebStatus home page (linked to the
 112 # 'titleURL') and is embedded in the title of the waterfall HTML page.
 113
 114 c['title'] = ini['general'].get("title")
 115 c['titleURL'] = ini['general'].get("title_url")
 116
 117 # the 'buildbotURL' string should point to the location where the buildbot's
 118 # internal web server (usually the html.WebStatus page) is visible. This
 119 # typically uses the port number set in the Waterfall 'status' entry, but
 120 # with an externally-visible host name which the buildbot cannot figure out
 121 # without some help.
 122
 123 c['buildbotURL'] = inip1.get("buildbot_url")
 124
 125 ####### BUILDWORKERS
 126
 127 # The 'workers' list defines the set of recognized buildworkers. Each element is
 128 # a Worker object, specifying a unique worker name and password.  The same
 129 # worker name and password must be configured on the worker.
 130
 131 c['workers'] = []
 132 NetLocks = dict()
 133
 134 for section in ini.sections():
 135         if section.startswith("branch "):
 136                 ini_parse_branch(ini[section])
 137
 138         if section.startswith("worker "):
 139                 if ini.has_option(section, "name") and ini.has_option(section, "password") and \
 140                    (not ini.has_option(section, "phase") or ini.getint(section, "phase") == 1):
 141                         sl_props = { 'dl_lock':None, 'ul_lock':None }
 142                         name = ini.get(section, "name")
 143                         password = ini.get(section, "password")
 144                         if ini.has_option(section, "dl_lock"):
 145                                 lockname = ini.get(section, "dl_lock")
 146                                 sl_props['dl_lock'] = lockname
 147                                 if lockname not in NetLocks:
 148                                         NetLocks[lockname] = locks.MasterLock(lockname)
 149                         if ini.has_option(section, "ul_lock"):
 150                                 lockname = ini.get(section, "ul_lock")
 151                                 sl_props['ul_lock'] = lockname
 152                                 if lockname not in NetLocks:
 153                                         NetLocks[lockname] = locks.MasterLock(lockname)
 154                         c['workers'].append(Worker(name, password, max_builds = 1, properties = sl_props))
 155
 156 c['protocols'] = {'pb': {'port': pb_port}}
 157
 158 # coalesce builds
 159 c['collapseRequests'] = True
 160
 161 # Reduce amount of backlog data
 162 c['configurators'] = [util.JanitorConfigurator(
 163         logHorizon=timedelta(days=3),
 164         hour=6,
 165 )]
 166
 167 @defer.inlineCallbacks
 168 def getNewestCompleteTime(bldr):
 169         """Returns the complete_at of the latest completed and not SKIPPED
 170         build request for this builder, or None if there are no such build
 171         requests. We need to filter out SKIPPED requests because we're
 172         using collapseRequests=True which is unfortunately marking all
 173         previous requests as complete when new buildset is created.
 174
 175         @returns: datetime instance or None, via Deferred
 176         """
 177
 178         bldrid = yield bldr.getBuilderId()
 179         completed = yield bldr.master.data.get(
 180                         ('builders', bldrid, 'buildrequests'),
 181                         [
 182                                 resultspec.Filter('complete', 'eq', [True]),
 183                                 resultspec.Filter('results', 'ne', [results.SKIPPED]),
 184                         ],
 185                         order=['-complete_at'], limit=1)
 186         if not completed:
 187                 return
 188
 189         complete_at = completed[0]['complete_at']
 190
 191         last_build = yield bldr.master.data.get(
 192                         ('builds', ),
 193                         [
 194                                 resultspec.Filter('builderid', 'eq', [bldrid]),
 195                         ],
 196                         order=['-started_at'], limit=1)
 197
 198         if last_build and last_build[0]:
 199                 last_complete_at = last_build[0]['complete_at']
 200                 if last_complete_at and (last_complete_at > complete_at):
 201                         return last_complete_at
 202
 203         return complete_at
 204
 205 @defer.inlineCallbacks
 206 def prioritizeBuilders(master, builders):
 207         """Returns sorted list of builders by their last timestamp of completed and
 208         not skipped build.
 209
 210         @returns: list of sorted builders
 211         """
 212
 213         def is_building(bldr):
 214                 return bool(bldr.building) or bool(bldr.old_building)
 215
 216         def bldr_info(bldr):
 217                 d = defer.maybeDeferred(getNewestCompleteTime, bldr)
 218                 d.addCallback(lambda complete_at: (complete_at, bldr))
 219                 return d
 220
 221         def bldr_sort(item):
 222                 (complete_at, bldr) = item
 223
 224                 if bldr.name == "00_force_build":
 225                         date = datetime.min
 226                         complete_at = date.replace(tzinfo=tzutc())
 227                         return (complete_at, bldr.name)
 228
 229                 if not complete_at:
 230                         date = datetime.min
 231                         complete_at = date.replace(tzinfo=tzutc())
 232
 233                 if is_building(bldr):
 234                         date = datetime.max
 235                         complete_at = date.replace(tzinfo=tzutc())
 236
 237                 return (complete_at, bldr.name)
 238
 239         results = yield defer.gatherResults([bldr_info(bldr) for bldr in builders])
 240         results.sort(key=bldr_sort)
 241
 242         for r in results:
 243                 log.msg("prioritizeBuilders: {:>20} complete_at: {}".format(r[1].name, r[0]))
 244
 245         return [r[1] for r in results]
 246
 247 c['prioritizeBuilders'] = prioritizeBuilders
 248
 249 ####### CHANGESOURCES
 250
 251 branchNames = [branches[b]["name"] for b in branches]
 252
 253 # find targets
 254 targets = set()
 255
 256 def populateTargets():
 257         log.msg("Populating targets, this will take time")
 258         sourcegit = work_dir + '/source.git'
 259         for branch in branchNames:
 260                 if os.path.isdir(sourcegit):
 261                         subprocess.call(["rm", "-rf", sourcegit])
 262
 263                 subprocess.call(["git", "clone", "-q", "--depth=1", "--branch="+branch, repo_url, sourcegit])
 264
 265                 os.makedirs(sourcegit + '/tmp', exist_ok=True)
 266                 findtargets = subprocess.Popen(['./scripts/dump-target-info.pl', 'targets'],
 267                         stdout = subprocess.PIPE, stderr = subprocess.DEVNULL, cwd = sourcegit)
 268
 269                 while True:
 270                         line = findtargets.stdout.readline()
 271                         if not line:
 272                                 break
 273                         ta = line.decode().strip().split(' ')
 274                         targets.add(ta[0])
 275
 276                 subprocess.call(["rm", "-rf", sourcegit])
 277
 278 populateTargets()
 279
 280 # the 'change_source' setting tells the buildmaster how it should find out
 281 # about source code changes.  Here we point to the buildbot clone of pyflakes.
 282
 283 c['change_source'] = []
 284 c['change_source'].append(GitPoller(
 285         repo_url,
 286         workdir=work_dir+'/work.git', branches=branchNames,
 287         pollAtLaunch=True, pollinterval=300))
 288
 289 ####### SCHEDULERS
 290
 291 # Configure the Schedulers, which decide how to react to incoming changes.  In this
 292 # case, just kick off a 'basebuild' build
 293
 294 class TagChoiceParameter(BaseParameter):
 295         spec_attributes = ["strict", "choices"]
 296         type = "list"
 297         strict = True
 298
 299         def __init__(self, name, label=None, **kw):
 300                 super().__init__(name, label, **kw)
 301                 self._choice_list = []
 302
 303         def getRevTags(self, findtag=None):
 304                 taglist = []
 305                 branchvers = []
 306
 307                 for b in branchNames:
 308                         basever = re.search(r'-([0-9]+\.[0-9]+)$', b)
 309                         if basever:
 310                                 branchvers.append(basever[1])
 311
 312                 alltags = subprocess.Popen(
 313                         ['git', 'ls-remote', '--tags', repo_url],
 314                         stdout = subprocess.PIPE)
 315
 316                 while True:
 317                         line = alltags.stdout.readline()
 318
 319                         if not line:
 320                                 break
 321
 322                         (rev, tag) = line.split()
 323
 324                         tagver = re.search(r'\brefs/tags/(v[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?)$', tag.decode().strip())
 325
 326                         # only list tags matching configured branches
 327                         if tagver and any(tagver[1][1:].startswith(b) for b in branchvers):
 328                                 if findtag and findtag != tagver[1]:
 329                                         continue
 330                                 taglist.append({'rev': rev.decode().strip(), 'tag': tagver[1]})
 331
 332                 return taglist
 333
 334         @property
 335         def choices(self):
 336                 taglist = [rt['tag'] for rt in self.getRevTags()]
 337                 taglist.sort(reverse=True, key=lambda tag: tag if re.search(r'-rc[0-9]+$', tag) else tag + '-z')
 338                 taglist.insert(0, '')
 339
 340                 self._choice_list = taglist
 341
 342                 return self._choice_list
 343
 344         def updateFromKwargs(self, properties, kwargs, **unused):
 345                 tag = self.getFromKwargs(kwargs)
 346                 properties[self.name] = tag
 347
 348                 # find the commit matching the tag
 349                 findtag = self.getRevTags(tag)
 350
 351                 if not findtag:
 352                         raise ValidationError("Couldn't find tag")
 353
 354                 properties['force_revision'] = findtag[0]['rev']
 355
 356                 # find the branch matching the tag
 357                 branch = None
 358                 branchver = re.search(r'v([0-9]+\.[0-9]+)', tag)
 359                 for b in branchNames:
 360                         if b.endswith(branchver[1]):
 361                                 branch = b
 362
 363                 if not branch:
 364                         raise ValidationError("Couldn't find branch")
 365
 366                 properties['force_branch'] = branch
 367
 368         def parse_from_arg(self, s):
 369                 if self.strict and s not in self._choice_list:
 370                         raise ValidationError("'%s' does not belong to list of available choices '%s'" % (s, self._choice_list))
 371                 return s
 372
 373 c['schedulers'] = []
 374 c['schedulers'].append(AnyBranchScheduler(
 375         name            = "all",
 376         change_filter   = util.ChangeFilter(branch=branchNames),
 377         treeStableTimer = 15*60,
 378         builderNames    = list(targets)))
 379
 380 c['schedulers'].append(ForceScheduler(
 381         name         = "force",
 382         buttonName   = "Force builds",
 383         label        = "Force build details",
 384         builderNames = [ "00_force_build" ],
 385
 386         codebases = [
 387                 util.CodebaseParameter(
 388                         "",
 389                         label      = "Repository",
 390                         branch     = util.FixedParameter(name = "branch",     default = ""),
 391                         revision   = util.FixedParameter(name = "revision",   default = ""),
 392                         repository = util.FixedParameter(name = "repository", default = ""),
 393                         project    = util.FixedParameter(name = "project",    default = "")
 394                 )
 395         ],
 396
 397         reason = util.StringParameter(
 398                 name     = "reason",
 399                 label    = "Reason",
 400                 default  = "Trigger build",
 401                 required = True,
 402                 size     = 80
 403         ),
 404
 405         properties = [
 406                 util.ChoiceStringParameter(
 407                         name    = "target",
 408                         label   = "Build target",
 409                         default = "all",
 410                         choices = [ "all" ] + list(targets)
 411                 ),
 412                 TagChoiceParameter(
 413                         name    = "tag",
 414                         label   = "Build tag",
 415                         default = ""
 416                 )
 417         ]
 418 ))
 419
 420 ####### BUILDERS
 421
 422 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
 423 # what steps, and which workers can execute them.  Note that any particular build will
 424 # only take place on one worker.
 425
 426 def IsNoMasterBuild(step):
 427         return step.getProperty("branch") != "master"
 428
 429 def IsUsignEnabled(step):
 430         branch = step.getProperty("branch")
 431         return branch and branches[branch].get("usign_key")
 432
 433 def IsSignEnabled(step):
 434         branch = step.getProperty("branch")
 435         return IsUsignEnabled(step) or branch and branches[branch].get("gpg_key")
 436
 437 def IsKmodArchiveEnabled(step):
 438         branch = step.getProperty("branch")
 439         return branch and branches[branch].get("kmod_archive")
 440
 441 def GetBaseVersion(branch):
 442         if re.match(r"^[^-]+-[0-9]+\.[0-9]+$", branch):
 443                 return branch.split('-')[1]
 444         else:
 445                 return "master"
 446
 447 @properties.renderer
 448 def GetVersionPrefix(props):
 449         branch = props.getProperty("branch")
 450         basever = GetBaseVersion(branch)
 451         if props.hasProperty("tag") and re.match(r"^v[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", props["tag"]):
 452                 return "%s/" % props["tag"][1:]
 453         elif basever != "master":
 454                 return "%s-SNAPSHOT/" % basever
 455         else:
 456                 return ""
 457
 458 @util.renderer
 459 def GetConfigSeed(props):
 460         branch = props.getProperty("branch")
 461         return branch and branches[branch].get("config_seed") or ""
 462
 463 @util.renderer
 464 def GetRsyncParams(props, srcorbin, urlorkey):
 465         # srcorbin: 'bin' or 'src'; urlorkey: 'url' or 'key'
 466         branch = props.getProperty("branch")
 467         opt = srcorbin + "_" + urlorkey
 468         return branch and branches[branch].get(opt)
 469
 470 @util.renderer
 471 def GetUsignKey(props):
 472         branch = props.getProperty("branch")
 473         return branch and branches[branch].get("usign_key")
 474
 475 def GetNextBuild(builder, requests):
 476         for r in requests:
 477                 if r.properties:
 478                         # order tagged build first
 479                         if r.properties.hasProperty("tag"):
 480                                 return r
 481                         # then order by branch order
 482                         pbranch = r.properties.getProperty("branch")
 483                         for name in branchNames:
 484                                 if pbranch == name:
 485                                         return r
 486
 487         r = requests[0]
 488         log.msg("GetNextBuild: {:>20} id: {} bsid: {}".format(builder.name, r.id, r.bsid))
 489         return r
 490
 491 def MakeEnv(overrides=None, tryccache=False):
 492         env = {
 493                 'CCC': Interpolate("%(prop:cc_command:-gcc)s"),
 494                 'CCXX': Interpolate("%(prop:cxx_command:-g++)s"),
 495         }
 496         if tryccache:
 497                 env['CC'] = Interpolate("%(prop:builddir)s/ccache_cc.sh")
 498                 env['CXX'] = Interpolate("%(prop:builddir)s/ccache_cxx.sh")
 499                 env['CCACHE'] = Interpolate("%(prop:ccache_command:-)s")
 500         else:
 501                 env['CC'] = env['CCC']
 502                 env['CXX'] = env['CCXX']
 503                 env['CCACHE'] = ''
 504         if overrides is not None:
 505                 env.update(overrides)
 506         return env
 507
 508 @properties.renderer
 509 def NetLockDl(props, extralock=None):
 510         lock = None
 511         if props.hasProperty("dl_lock"):
 512                 lock = NetLocks[props["dl_lock"]]
 513         if lock is not None:
 514                 return [lock.access('exclusive')]
 515         else:
 516                 return []
 517
 518 @properties.renderer
 519 def NetLockUl(props):
 520         lock = None
 521         if props.hasProperty("ul_lock"):
 522                 lock = NetLocks[props["ul_lock"]]
 523         if lock is not None:
 524                 return [lock.access('exclusive')]
 525         else:
 526                 return []
 527
 528 def IsTargetSelected(target):
 529         def CheckTargetProperty(step):
 530                 selected_target = step.getProperty("target", "all")
 531                 if selected_target != "all" and selected_target != target:
 532                         return False
 533                 return True
 534
 535         return CheckTargetProperty
 536
 537 @util.renderer
 538 def UsignSec2Pub(props):
 539         branch = props.getProperty("branch")
 540         try:
 541                 comment = branches[branch].get("usign_comment") or "untrusted comment: secret key"
 542                 seckey = branches[branch].get("usign_key")
 543                 seckey = base64.b64decode(seckey)
 544         except:
 545                 return None
 546
 547         return "{}\n{}".format(re.sub(r"\bsecret key$", "public key", comment),
 548                 base64.b64encode(seckey[0:2] + seckey[32:40] + seckey[72:]))
 549
 550
 551 c['builders'] = []
 552
 553 workerNames = [ ]
 554
 555 for worker in c['workers']:
 556         workerNames.append(worker.workername)
 557
 558 # add a single LocalWorker to handle the forcebuild builder
 559 c['workers'].append(LocalWorker("__local_force_build", max_builds=1))
 560
 561 force_factory = BuildFactory()
 562
 563 c['builders'].append(BuilderConfig(
 564         name        = "00_force_build",
 565         workername  = "__local_force_build",
 566         factory     = force_factory))
 567
 568 for target in targets:
 569         ts = target.split('/')
 570
 571         factory = BuildFactory()
 572
 573         # setup shared work directory if required
 574         factory.addStep(ShellCommand(
 575                 name = "sharedwd",
 576                 descriptionDone = "Shared work directory set up",
 577                 command = 'test -L "$PWD" || (mkdir -p ../shared-workdir && rm -rf "$PWD" && ln -s shared-workdir "$PWD")',
 578                 workdir = ".",
 579                 haltOnFailure = True,
 580         ))
 581
 582         # find number of cores
 583         factory.addStep(SetPropertyFromCommand(
 584                 name = "nproc",
 585                 property = "nproc",
 586                 description = "Finding number of CPUs",
 587                 command = ["nproc"],
 588         ))
 589
 590         # find gcc and g++ compilers
 591         factory.addStep(FileDownload(
 592                 name = "dlfindbinpl",
 593                 mastersrc = scripts_dir + '/findbin.pl',
 594                 workerdest = "../findbin.pl",
 595                 mode = 0o755,
 596         ))
 597
 598         factory.addStep(SetPropertyFromCommand(
 599                 name = "gcc",
 600                 property = "cc_command",
 601                 description = "Finding gcc command",
 602                 command = ["../findbin.pl", "gcc", "", ""],
 603                 haltOnFailure = True,
 604         ))
 605
 606         factory.addStep(SetPropertyFromCommand(
 607                 name = "g++",
 608                 property = "cxx_command",
 609                 description = "Finding g++ command",
 610                 command = ["../findbin.pl", "g++", "", ""],
 611                 haltOnFailure = True,
 612         ))
 613
 614         # see if ccache is available
 615         factory.addStep(SetPropertyFromCommand(
 616                 name = "ccache",
 617                 property = "ccache_command",
 618                 description = "Testing for ccache command",
 619                 command = ["which", "ccache"],
 620                 haltOnFailure = False,
 621                 flunkOnFailure = False,
 622                 warnOnFailure = False,
 623                 hideStepIf = lambda r, s: r==results.FAILURE,
 624         ))
 625
 626         # check out the source
 627         # Git() runs:
 628           # if repo doesn't exist: 'git clone repourl'
 629           # method 'clean' runs 'git clean -d -f', method fresh runs 'git clean -f -f -d -x'. Only works with mode='full'
 630           # git cat-file -e <commit>
 631           # git checkout -f <commit>
 632           # git checkout -B <branch>
 633           # git rev-parse HEAD
 634         factory.addStep(Git(
 635                 name = "git",
 636                 repourl = repo_url,
 637                 mode = 'full',
 638                 method = 'fresh',
 639                 locks = NetLockDl,
 640                 haltOnFailure = True,
 641         ))
 642
 643         # update remote refs
 644         factory.addStep(ShellCommand(
 645                 name = "fetchrefs",
 646                 description = "Fetching Git remote refs",
 647                 command = ["git", "fetch", "origin", Interpolate("+refs/heads/%(prop:branch)s:refs/remotes/origin/%(prop:branch)s")],
 648                 haltOnFailure = True,
 649         ))
 650
 651         # Verify that Git HEAD points to a tag or branch
 652         # Ref: https://web.archive.org/web/20190729224316/http://lists.infradead.org/pipermail/openwrt-devel/2019-June/017809.html
 653         factory.addStep(ShellCommand(
 654                 name = "gitverify",
 655                 description = "Ensure that Git HEAD is pointing to a branch or tag",
 656                 command = 'git rev-parse --abbrev-ref HEAD | grep -vxqF HEAD || git show-ref --tags --dereference 2>/dev/null | sed -ne "/^$(git rev-parse HEAD) / { s|^.*/||; s|\\^.*||; p }" | grep -qE "^v[0-9][0-9]\\."',
 657                 haltOnFailure = True,
 658         ))
 659
 660         factory.addStep(ShellCommand(
 661                 name = "rmtmp",
 662                 description = "Remove tmp folder",
 663                 command=["rm", "-rf", "tmp/"],
 664         ))
 665
 666         # feed
 667         factory.addStep(ShellCommand(
 668                 name = "rmfeedlinks",
 669                 description = "Remove feed symlinks",
 670                 command=["rm", "-rf", "package/feeds/"],
 671         ))
 672
 673         factory.addStep(StringDownload(
 674                 name = "ccachecc",
 675                 s = '#!/bin/sh\nexec ${CCACHE} ${CCC} "$@"\n',
 676                 workerdest = "../ccache_cc.sh",
 677                 mode = 0o755,
 678         ))
 679
 680         factory.addStep(StringDownload(
 681                 name = "ccachecxx",
 682                 s = '#!/bin/sh\nexec ${CCACHE} ${CCXX} "$@"\n',
 683                 workerdest = "../ccache_cxx.sh",
 684                 mode = 0o755,
 685         ))
 686
 687         # feed
 688         factory.addStep(ShellCommand(
 689                 name = "updatefeeds",
 690                 description = "Updating feeds",
 691                 command=["./scripts/feeds", "update"],
 692                 env = MakeEnv(tryccache=True),
 693                 haltOnFailure = True,
 694                 locks = NetLockDl,
 695         ))
 696
 697         # feed
 698         factory.addStep(ShellCommand(
 699                 name = "installfeeds",
 700                 description = "Installing feeds",
 701                 command=["./scripts/feeds", "install", "-a"],
 702                 env = MakeEnv(tryccache=True),
 703                 haltOnFailure = True,
 704         ))
 705
 706         # seed config
 707         factory.addStep(StringDownload(
 708                 name = "dlconfigseed",
 709                 s = Interpolate("%(kw:seed)s\n", seed=GetConfigSeed),
 710                 workerdest = ".config",
 711                 mode = 0o644,
 712         ))
 713
 714         # configure
 715         factory.addStep(ShellCommand(
 716                 name = "newconfig",
 717                 descriptionDone = ".config seeded",
 718                 command = Interpolate("printf 'CONFIG_TARGET_%(kw:target)s=y\\nCONFIG_TARGET_%(kw:target)s_%(kw:subtarget)s=y\\nCONFIG_SIGNED_PACKAGES=%(kw:usign:#?|y|n)s\\n' >> .config", target=ts[0], subtarget=ts[1], usign=GetUsignKey),
 719         ))
 720
 721         factory.addStep(ShellCommand(
 722                 name = "delbin",
 723                 description = "Removing output directory",
 724                 command = ["rm", "-rf", "bin/"],
 725         ))
 726
 727         factory.addStep(ShellCommand(
 728                 name = "defconfig",
 729                 description = "Populating .config",
 730                 command = ["make", "defconfig"],
 731                 env = MakeEnv(),
 732         ))
 733
 734         # check arch - exit early if does not exist - NB: some targets do not define CONFIG_TARGET_target_subtarget
 735         factory.addStep(ShellCommand(
 736                 name = "checkarch",
 737                 description = "Checking architecture",
 738                 descriptionDone = "Architecture validated",
 739                 command = 'grep -sq CONFIG_TARGET_%s=y .config && grep -sq CONFIG_TARGET_SUBTARGET=\\"%s\\" .config' %(ts[0], ts[1]),
 740                 logEnviron = False,
 741                 want_stdout = False,
 742                 want_stderr = False,
 743                 haltOnFailure = True,
 744                 flunkOnFailure = False, # this is not a build FAILURE
 745         ))
 746
 747         # find libc suffix
 748         factory.addStep(SetPropertyFromCommand(
 749                 name = "libc",
 750                 property = "libc",
 751                 description = "Finding libc suffix",
 752                 command = ["sed", "-ne", '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }', ".config"],
 753         ))
 754
 755         # install build key
 756         factory.addStep(StringDownload(
 757                 name = "dlkeybuildpub",
 758                 s = Interpolate("%(kw:sec2pub)s", sec2pub=UsignSec2Pub),
 759                 workerdest = "key-build.pub",
 760                 mode = 0o600,
 761                 doStepIf = IsUsignEnabled,
 762         ))
 763
 764         factory.addStep(StringDownload(
 765                 name = "dlkeybuild",
 766                 s = "# fake private key",
 767                 workerdest = "key-build",
 768                 mode = 0o600,
 769                 doStepIf = IsUsignEnabled,
 770         ))
 771
 772         factory.addStep(StringDownload(
 773                 name = "dlkeybuilducert",
 774                 s = "# fake certificate",
 775                 workerdest = "key-build.ucert",
 776                 mode = 0o600,
 777                 doStepIf = IsUsignEnabled,
 778         ))
 779
 780         # prepare dl
 781         factory.addStep(ShellCommand(
 782                 name = "dldir",
 783                 description = "Preparing dl/",
 784                 descriptionDone = "dl/ prepared",
 785                 command = 'mkdir -p ../dl && rm -rf "build/dl" && ln -s ../../dl "build/dl"',
 786                 workdir = Property("builddir"),
 787                 logEnviron = False,
 788                 want_stdout = False,
 789         ))
 790
 791         # cleanup dl
 792         factory.addStep(ShellCommand(
 793                 name = "dlprune",
 794                 description = "Pruning dl/",
 795                 descriptionDone = "dl/ pruned",
 796                 command = 'find dl/ -atime +15 -delete -print',
 797                 logEnviron = False,
 798         ))
 799
 800         # prepare tar
 801         factory.addStep(ShellCommand(
 802                 name = "dltar",
 803                 description = "Building and installing GNU tar",
 804                 descriptionDone = "GNU tar built and installed",
 805                 command = ["make", Interpolate("-j%(prop:nproc:-1)s"), "tools/tar/compile", "V=s"],
 806                 env = MakeEnv(tryccache=True),
 807                 haltOnFailure = True,
 808         ))
 809
 810         # populate dl
 811         factory.addStep(ShellCommand(
 812                 name = "dlrun",
 813                 description = "Populating dl/",
 814                 descriptionDone = "dl/ populated",
 815                 command = ["make", Interpolate("-j%(prop:nproc:-1)s"), "download", "V=s"],
 816                 env = MakeEnv(),
 817                 logEnviron = False,
 818                 locks = NetLockDl,
 819         ))
 820
 821         factory.addStep(ShellCommand(
 822                 name = "cleanbase",
 823                 description = "Cleaning base-files",
 824                 command=["make", "package/base-files/clean", "V=s"],
 825         ))
 826
 827         # build
 828         factory.addStep(ShellCommand(
 829                 name = "tools",
 830                 description = "Building and installing tools",
 831                 descriptionDone = "Tools built and installed",
 832                 command = ["make", Interpolate("-j%(prop:nproc:-1)s"), "tools/install", "V=s"],
 833                 env = MakeEnv(tryccache=True),
 834                 haltOnFailure = True,
 835         ))
 836
 837         factory.addStep(ShellCommand(
 838                 name = "toolchain",
 839                 description = "Building and installing toolchain",
 840                 descriptionDone = "Toolchain built and installed",
 841                 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "toolchain/install", "V=s"],
 842                 env = MakeEnv(),
 843                 haltOnFailure = True,
 844         ))
 845
 846         factory.addStep(ShellCommand(
 847                 name = "kmods",
 848                 description = "Building kmods",
 849                 descriptionDone = "Kmods built",
 850                 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
 851                 env = MakeEnv(),
 852                 haltOnFailure = True,
 853         ))
 854
 855         # find kernel version
 856         factory.addStep(SetPropertyFromCommand(
 857                 name = "kernelversion",
 858                 property = "kernelversion",
 859                 description = "Finding the effective Kernel version",
 860                 command = "make --no-print-directory -C target/linux/ val.LINUX_VERSION val.LINUX_RELEASE val.LINUX_VERMAGIC | xargs printf '%s-%s-%s\\n'",
 861                 env = { 'TOPDIR': Interpolate("%(prop:builddir)s/build") },
 862         ))
 863
 864         factory.addStep(ShellCommand(
 865                 name = "pkgclean",
 866                 description = "Cleaning up package build",
 867                 descriptionDone = "Package build cleaned up",
 868                 command=["make", "package/cleanup", "V=s"],
 869         ))
 870
 871         factory.addStep(ShellCommand(
 872                 name = "pkgbuild",
 873                 description = "Building packages",
 874                 descriptionDone = "Packages built",
 875                 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
 876                 env = MakeEnv(),
 877                 haltOnFailure = True,
 878         ))
 879
 880         factory.addStep(ShellCommand(
 881                 name = "pkginstall",
 882                 description = "Installing packages",
 883                 descriptionDone = "Packages installed",
 884                 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/install", "V=s"],
 885                 env = MakeEnv(),
 886                 haltOnFailure = True,
 887         ))
 888
 889         factory.addStep(ShellCommand(
 890                 name = "pkgindex",
 891                 description = "Indexing packages",
 892                 descriptionDone = "Packages indexed",
 893                 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/index", "V=s", "CONFIG_SIGNED_PACKAGES="],
 894                 env = MakeEnv(),
 895                 haltOnFailure = True,
 896         ))
 897
 898         factory.addStep(ShellCommand(
 899                 name = "images",
 900                 description = "Building and installing images",
 901                 descriptionDone = "Images built and installed",
 902                 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "target/install", "V=s"],
 903                 env = MakeEnv(),
 904                 haltOnFailure = True,
 905         ))
 906
 907         factory.addStep(ShellCommand(
 908                 name = "buildinfo",
 909                 description = "Generating config.buildinfo, version.buildinfo and feeds.buildinfo",
 910                 command = "make -j1 buildinfo V=s || true",
 911                 env = MakeEnv(),
 912                 haltOnFailure = True,
 913         ))
 914
 915         factory.addStep(ShellCommand(
 916                 name = "json_overview_image_info",
 917                 description = "Generating profiles.json in target folder",
 918                 command = "make -j1 json_overview_image_info V=s || true",
 919                 env = MakeEnv(),
 920                 haltOnFailure = True,
 921         ))
 922
 923         factory.addStep(ShellCommand(
 924                 name = "checksums",
 925                 description = "Calculating checksums",
 926                 descriptionDone = "Checksums calculated",
 927                 command=["make", "-j1", "checksum", "V=s"],
 928                 env = MakeEnv(),
 929                 haltOnFailure = True,
 930         ))
 931
 932         factory.addStep(ShellCommand(
 933                 name = "kmoddir",
 934                 descriptionDone = "Kmod directory created",
 935                 command=["mkdir", "-p", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s", target=ts[0], subtarget=ts[1])],
 936                 haltOnFailure = True,
 937                 doStepIf = IsKmodArchiveEnabled,
 938         ))
 939
 940         factory.addStep(ShellCommand(
 941                 name = "kmodprepare",
 942                 description = "Preparing kmod archive",
 943                 descriptionDone = "Kmod archive prepared",
 944                 command=["rsync", "--include=/kmod-*.ipk", "--exclude=*", "-va",
 945                         Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/packages/", target=ts[0], subtarget=ts[1]),
 946                         Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1])],
 947                 haltOnFailure = True,
 948                 doStepIf = IsKmodArchiveEnabled,
 949         ))
 950
 951         factory.addStep(ShellCommand(
 952                 name = "kmodindex",
 953                 description = "Indexing kmod archive",
 954                 descriptionDone = "Kmod archive indexed",
 955                 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/index", "V=s", "CONFIG_SIGNED_PACKAGES=",
 956                         Interpolate("PACKAGE_SUBDIRS=bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1])],
 957                 env = MakeEnv(),
 958                 haltOnFailure = True,
 959                 doStepIf = IsKmodArchiveEnabled,
 960         ))
 961
 962         # sign
 963         factory.addStep(MasterShellCommand(
 964                 name = "signprepare",
 965                 descriptionDone = "Temporary signing directory prepared",
 966                 command = ["mkdir", "-p", "%s/signing" %(work_dir)],
 967                 haltOnFailure = True,
 968                 doStepIf = IsSignEnabled,
 969
 970         ))
 971
 972         factory.addStep(ShellCommand(
 973                 name = "signpack",
 974                 description = "Packing files to sign",
 975                 descriptionDone = "Files to sign packed",
 976                 command = Interpolate("find bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/ bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz", target=ts[0], subtarget=ts[1]),
 977                 haltOnFailure = True,
 978                 doStepIf = IsSignEnabled,
 979         ))
 980
 981         factory.addStep(FileUpload(
 982                 workersrc = "sign.tar.gz",
 983                 masterdest = "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1]),
 984                 haltOnFailure = True,
 985                 doStepIf = IsSignEnabled,
 986         ))
 987
 988         factory.addStep(MasterShellCommand(
 989                 name = "signfiles",
 990                 description = "Signing files",
 991                 descriptionDone = "Files signed",
 992                 command = ["%s/signall.sh" %(scripts_dir), "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1]), Interpolate("%(prop:branch)s")],
 993                 env = { 'CONFIG_INI': os.getenv("BUILDMASTER_CONFIG", "./config.ini") },
 994                 haltOnFailure = True,
 995                 doStepIf = IsSignEnabled,
 996         ))
 997
 998         factory.addStep(FileDownload(
 999                 name = "dlsigntargz",
1000                 mastersrc = "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1]),
1001                 workerdest = "sign.tar.gz",
1002                 haltOnFailure = True,
1003                 doStepIf = IsSignEnabled,
1004         ))
1005
1006         factory.addStep(ShellCommand(
1007                 name = "signunpack",
1008                 description = "Unpacking signed files",
1009                 descriptionDone = "Signed files unpacked",
1010                 command = ["tar", "-xzf", "sign.tar.gz"],
1011                 haltOnFailure = True,
1012                 doStepIf = IsSignEnabled,
1013         ))
1014
1015         # upload
1016         factory.addStep(ShellCommand(
1017                 name = "dirprepare",
1018                 descriptionDone = "Upload directory structure prepared",
1019                 command = ["mkdir", "-p", Interpolate("tmp/upload/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s", target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1020                 haltOnFailure = True,
1021         ))
1022
1023         factory.addStep(ShellCommand(
1024                 name = "linkprepare",
1025                 descriptionDone = "Repository symlink prepared",
1026                 command = ["ln", "-s", "-f", Interpolate("../packages-%(kw:basever)s", basever=util.Transform(GetBaseVersion, Property("branch"))), Interpolate("tmp/upload/%(kw:prefix)spackages", prefix=GetVersionPrefix)],
1027                 doStepIf = IsNoMasterBuild,
1028                 haltOnFailure = True,
1029         ))
1030
1031         factory.addStep(ShellCommand(
1032                 name = "kmoddirprepare",
1033                 descriptionDone = "Kmod archive upload directory prepared",
1034                 command = ["mkdir", "-p", Interpolate("tmp/upload/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s", target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1035                 haltOnFailure = True,
1036                 doStepIf = IsKmodArchiveEnabled,
1037         ))
1038
1039         factory.addStep(ShellCommand(
1040                 name = "dirupload",
1041                 description = "Uploading directory structure",
1042                 descriptionDone = "Directory structure uploaded",
1043                 command = ["rsync", "-az"] + rsync_defopts + ["tmp/upload/", Interpolate("%(kw:url)s/", url=GetRsyncParams.withArgs("bin", "url"))],
1044                 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1045                 haltOnFailure = True,
1046                 logEnviron = False,
1047                 locks = NetLockUl,
1048                 doStepIf = util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
1049         ))
1050
1051         # download remote sha256sums to 'target-sha256sums'
1052         factory.addStep(ShellCommand(
1053                 name = "target-sha256sums",
1054                 description = "Fetching remote sha256sums for target",
1055                 descriptionDone = "Remote sha256sums for target fetched",
1056                 command = ["rsync", "-z"] + rsync_defopts + [Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/sha256sums", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix), "target-sha256sums"],
1057                 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1058                 logEnviron = False,
1059                 haltOnFailure = False,
1060                 flunkOnFailure = False,
1061                 warnOnFailure = False,
1062                 doStepIf = util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
1063         ))
1064
1065         # build list of files to upload
1066         factory.addStep(FileDownload(
1067                 name = "dlsha2rsyncpl",
1068                 mastersrc = scripts_dir + '/sha2rsync.pl',
1069                 workerdest = "../sha2rsync.pl",
1070                 mode = 0o755,
1071         ))
1072
1073         factory.addStep(ShellCommand(
1074                 name = "buildlist",
1075                 description = "Building list of files to upload",
1076                 descriptionDone = "List of files to upload built",
1077                 command = ["../sha2rsync.pl", "target-sha256sums", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/sha256sums", target=ts[0], subtarget=ts[1]), "rsynclist"],
1078                 haltOnFailure = True,
1079         ))
1080
1081         factory.addStep(FileDownload(
1082                 name = "dlrsync.sh",
1083                 mastersrc = scripts_dir + '/rsync.sh',
1084                 workerdest = "../rsync.sh",
1085                 mode = 0o755,
1086         ))
1087
1088         # upload new files and update existing ones
1089         factory.addStep(ShellCommand(
1090                 name = "targetupload",
1091                 description = "Uploading target files",
1092                 descriptionDone = "Target files uploaded",
1093                 command=["../rsync.sh", "--exclude=/kmods/", "--files-from=rsynclist", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_defopts +
1094                         ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]),
1095                         Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1096                 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1097                 haltOnFailure = True,
1098                 logEnviron = False,
1099                 doStepIf = util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
1100         ))
1101
1102         # delete files which don't exist locally
1103         factory.addStep(ShellCommand(
1104                 name = "targetprune",
1105                 description = "Pruning target files",
1106                 descriptionDone = "Target files pruned",
1107                 command=["../rsync.sh", "--exclude=/kmods/", "--delete", "--existing", "--ignore-existing", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_defopts +
1108                         ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]),
1109                         Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1110                 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1111                 haltOnFailure = True,
1112                 logEnviron = False,
1113                 locks = NetLockUl,
1114                 doStepIf = util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
1115         ))
1116
1117         factory.addStep(ShellCommand(
1118                 name = "kmodupload",
1119                 description = "Uploading kmod archive",
1120                 descriptionDone = "Kmod archive uploaded",
1121                 command=["../rsync.sh", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_defopts +
1122                         ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1]),
1123                         Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1124                 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1125                 haltOnFailure = True,
1126                 logEnviron = False,
1127                 locks = NetLockUl,
1128                 doStepIf = util.Transform(lambda a, b: bool(a and b), IsKmodArchiveEnabled, GetRsyncParams.withArgs("bin", "url")),
1129         ))
1130
1131         factory.addStep(ShellCommand(
1132                 name = "sourcelist",
1133                 description = "Finding source archives to upload",
1134                 descriptionDone = "Source archives to upload found",
1135                 command = "find dl/ -maxdepth 1 -type f -not -size 0 -not -name '.*' -not -name '*.hash' -not -name '*.dl' -newer .config -printf '%f\\n' > sourcelist",
1136                 haltOnFailure = True,
1137         ))
1138
1139         factory.addStep(ShellCommand(
1140                 name = "sourceupload",
1141                 description = "Uploading source archives",
1142                 descriptionDone = "Source archives uploaded",
1143                 command=["../rsync.sh", "--files-from=sourcelist", "--size-only", "--delay-updates"] + rsync_defopts +
1144                         [Interpolate("--partial-dir=.~tmp~%(kw:target)s~%(kw:subtarget)s~%(prop:workername)s", target=ts[0], subtarget=ts[1]), "-a", "dl/", Interpolate("%(kw:url)s/", url=GetRsyncParams.withArgs("src", "url"))],
1145                 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("src", "key")) },
1146                 haltOnFailure = True,
1147                 logEnviron = False,
1148                 locks = NetLockUl,
1149                 doStepIf = util.Transform(bool, GetRsyncParams.withArgs("src", "url")),
1150         ))
1151
1152         factory.addStep(ShellCommand(
1153                 name = "df",
1154                 description = "Reporting disk usage",
1155                 command=["df", "-h", "."],
1156                 env={'LC_ALL': 'C'},
1157                 logEnviron = False,
1158                 haltOnFailure = False,
1159                 flunkOnFailure = False,
1160                 warnOnFailure = False,
1161                 alwaysRun = True,
1162         ))
1163
1164         factory.addStep(ShellCommand(
1165                 name = "du",
1166                 description = "Reporting estimated file space usage",
1167                 command=["du", "-sh", "."],
1168                 env={'LC_ALL': 'C'},
1169                 logEnviron = False,
1170                 haltOnFailure = False,
1171                 flunkOnFailure = False,
1172                 warnOnFailure = False,
1173                 alwaysRun = True,
1174         ))
1175
1176         factory.addStep(ShellCommand(
1177                 name = "ccachestat",
1178                 description = "Reporting ccache stats",
1179                 command=["ccache", "-s"],
1180                 env = MakeEnv(overrides={ 'PATH': ["${PATH}", "./staging_dir/host/bin"] }),
1181                 logEnviron = False,
1182                 want_stderr = False,
1183                 haltOnFailure = False,
1184                 flunkOnFailure = False,
1185                 warnOnFailure = False,
1186                 hideStepIf = lambda r, s: r==results.FAILURE,
1187         ))
1188
1189         c['builders'].append(BuilderConfig(name=target, workernames=workerNames, factory=factory, nextBuild=GetNextBuild))
1190
1191         c['schedulers'].append(schedulers.Triggerable(name="trigger_%s" % target, builderNames=[ target ]))
1192         force_factory.addStep(steps.Trigger(
1193                 name = "trigger_%s" % target,
1194                 description = "Triggering %s build" % target,
1195                 schedulerNames = [ "trigger_%s" % target ],
1196                 sourceStamps = [{ "codebase": "", "branch": Property("force_branch"), "revision": Property("force_revision"), "repository": repo_url, "project": "" }],
1197                 set_properties = { "reason": Property("reason"), "tag": Property("tag"), },
1198                 doStepIf = IsTargetSelected(target),
1199         ))
1200
1201
1202 ####### STATUS TARGETS
1203
1204 # 'status' is a list of Status Targets. The results of each build will be
1205 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
1206 # including web pages, email senders, and IRC bots.
1207
1208 if "status_bind" in inip1:
1209         c['www'] = {
1210                 'port': inip1.get("status_bind"),
1211                 'plugins': {
1212                         'waterfall_view': True,
1213                         'console_view': True,
1214                         'grid_view': True
1215                 }
1216         }
1217
1218         if "status_user" in inip1 and "status_password" in inip1:
1219                 c['www']['auth'] = util.UserPasswordAuth([
1220                         (inip1.get("status_user"), inip1.get("status_password"))
1221                 ])
1222                 c['www']['authz'] = util.Authz(
1223                         allowRules=[ util.AnyControlEndpointMatcher(role="admins") ],
1224                         roleMatchers=[ util.RolesFromUsername(roles=["admins"], usernames=[inip1.get("status_user")]) ]
1225                 )
1226
1227 c['services'] = []
1228 if ini.has_section("irc"):
1229         iniirc = ini['irc']
1230         irc_host = iniirc.get("host", None)
1231         irc_port = iniirc.getint("port", 6667)
1232         irc_chan = iniirc.get("channel", None)
1233         irc_nick = iniirc.get("nickname", None)
1234         irc_pass = iniirc.get("password", None)
1235
1236         if irc_host and irc_nick and irc_chan:
1237                 irc = reporters.IRC(irc_host, irc_nick,
1238                         port = irc_port,
1239                         password = irc_pass,
1240                         channels = [ irc_chan ],
1241                         notify_events = [ 'exception', 'problem', 'recovery' ]
1242                 )
1243
1244                 c['services'].append(irc)
1245
1246 c['revlink'] = util.RevlinkMatch([
1247         r'https://git.openwrt.org/openwrt/(.*).git'
1248         ],
1249         r'https://git.openwrt.org/?p=openwrt/\1.git;a=commit;h=%s')
1250
1251 ####### DB URL
1252
1253 c['db'] = {
1254         # This specifies what database buildbot uses to store its state.  You can leave
1255         # this at its default for all but the largest installations.
1256         'db_url' : "sqlite:///state.sqlite",
1257 }
1258
1259 c['buildbotNetUsageData'] = None