phase1/master.cfg

   1 # -*- python -*-
   2 # ex: set syntax=python:
   3
   4 import os
   5 import re
   6 import base64
   7 import subprocess
   8 import configparser
   9
  10 from dateutil.tz import tzutc
  11 from datetime import datetime, timedelta
  12
  13 from twisted.internet import defer
  14 from twisted.python import log
  15
  16 from buildbot import locks
  17 from buildbot.data import resultspec
  18 from buildbot.changes.gitpoller import GitPoller
  19 from buildbot.config import BuilderConfig
  20 from buildbot.plugins import reporters
  21 from buildbot.plugins import schedulers
  22 from buildbot.plugins import steps
  23 from buildbot.plugins import util
  24 from buildbot.process import properties
  25 from buildbot.process import results
  26 from buildbot.process.factory import BuildFactory
  27 from buildbot.process.properties import Interpolate
  28 from buildbot.process.properties import Property
  29 from buildbot.schedulers.basic import AnyBranchScheduler
  30 from buildbot.schedulers.forcesched import BaseParameter
  31 from buildbot.schedulers.forcesched import ForceScheduler
  32 from buildbot.schedulers.forcesched import ValidationError
  33 from buildbot.steps.master import MasterShellCommand
  34 from buildbot.steps.shell import SetPropertyFromCommand
  35 from buildbot.steps.shell import ShellCommand
  36 from buildbot.steps.source.git import Git
  37 from buildbot.steps.transfer import FileDownload
  38 from buildbot.steps.transfer import FileUpload
  39 from buildbot.steps.transfer import StringDownload
  40 from buildbot.worker import Worker
  41 from buildbot.worker.local import LocalWorker
  42
  43
  44 if not os.path.exists("twistd.pid"):
  45         with open("twistd.pid", "w") as pidfile:
  46                 pidfile.write("{}".format(os.getpid()))
  47
  48 # This is a sample buildmaster config file. It must be installed as
  49 # 'master.cfg' in your buildmaster's base directory.
  50
  51 ini = configparser.ConfigParser()
  52 ini.read(os.getenv("BUILDMASTER_CONFIG", "./config.ini"))
  53
  54 if "general" not in ini or "phase1" not in ini:
  55         raise ValueError("Fix your configuration")
  56
  57 inip1 = ini['phase1']
  58
  59 # Globals
  60 work_dir = os.path.abspath(ini['general'].get("workdir", "."))
  61 scripts_dir = os.path.abspath("../scripts")
  62
  63 repo_url = ini['repo'].get("url")
  64
  65 rsync_defopts = ["-v", "-4", "--timeout=120"]
  66
  67 #if rsync_bin_url.find("::") > 0 or rsync_bin_url.find("rsync://") == 0:
  68 #       rsync_bin_defopts += ["--contimeout=20"]
  69
  70 branches = {}
  71
  72 def ini_parse_branch(section):
  73         b = {}
  74         name = section.get("name")
  75
  76         if not name:
  77                 raise ValueError("missing 'name' in " + repr(section))
  78         if name in branches:
  79                 raise ValueError("duplicate branch name in " + repr(section))
  80
  81         b["name"] = name
  82         b["bin_url"] = section.get("binary_url")
  83         b["bin_key"] = section.get("binary_password")
  84
  85         b["src_url"] = section.get("source_url")
  86         b["src_key"] = section.get("source_password")
  87
  88         b["gpg_key"] = section.get("gpg_key")
  89
  90         b["usign_key"] = section.get("usign_key")
  91         usign_comment = "untrusted comment: " + name.replace("-", " ").title() + " key"
  92         b["usign_comment"] = section.get("usign_comment", usign_comment)
  93
  94         b["config_seed"] = section.get("config_seed")
  95
  96         b["kmod_archive"] = section.getboolean("kmod_archive", False)
  97
  98         branches[name] = b
  99         log.msg("Configured branch: {}".format(name))
 100
 101 # PB port can be either a numeric port or a connection string
 102 pb_port = inip1.get("port") or 9989
 103
 104 # This is the dictionary that the buildmaster pays attention to. We also use
 105 # a shorter alias to save typing.
 106 c = BuildmasterConfig = {}
 107
 108 ####### PROJECT IDENTITY
 109
 110 # the 'title' string will appear at the top of this buildbot
 111 # installation's html.WebStatus home page (linked to the
 112 # 'titleURL') and is embedded in the title of the waterfall HTML page.
 113
 114 c['title'] = ini['general'].get("title")
 115 c['titleURL'] = ini['general'].get("title_url")
 116
 117 # the 'buildbotURL' string should point to the location where the buildbot's
 118 # internal web server (usually the html.WebStatus page) is visible. This
 119 # typically uses the port number set in the Waterfall 'status' entry, but
 120 # with an externally-visible host name which the buildbot cannot figure out
 121 # without some help.
 122
 123 c['buildbotURL'] = inip1.get("buildbot_url")
 124
 125 ####### BUILDWORKERS
 126
 127 # The 'workers' list defines the set of recognized buildworkers. Each element is
 128 # a Worker object, specifying a unique worker name and password.  The same
 129 # worker name and password must be configured on the worker.
 130
 131 c['workers'] = []
 132 NetLocks = dict()
 133
 134 def ini_parse_workers(section):
 135         name = section.get("name")
 136         password = section.get("password")
 137         phase = section.getint("phase")
 138         tagonly = section.getboolean("tag_only")
 139
 140         if not name or not password or not phase == 1:
 141                 log.msg("invalid worker configuration ignored: {}".format(repr(section)))
 142                 return
 143
 144         sl_props = { 'dl_lock':None, 'ul_lock':None, 'tag_only':tagonly }
 145         if "dl_lock" in section:
 146                 lockname = section.get("dl_lock")
 147                 sl_props['dl_lock'] = lockname
 148                 if lockname not in NetLocks:
 149                         NetLocks[lockname] = locks.MasterLock(lockname)
 150         if "ul_lock" in section:
 151                 lockname = section.get("ul_lock")
 152                 sl_props['ul_lock'] = lockname
 153                 if lockname not in NetLocks:
 154                         NetLocks[lockname] = locks.MasterLock(lockname)
 155
 156         log.msg("Configured worker: {}".format(name))
 157         # NB: phase1 build factory requires workers to be single-build only
 158         c['workers'].append(Worker(name, password, max_builds = 1, properties = sl_props))
 159
 160
 161 for section in ini.sections():
 162         if section.startswith("branch "):
 163                 ini_parse_branch(ini[section])
 164
 165         if section.startswith("worker "):
 166                 ini_parse_workers(ini[section])
 167
 168 # list of branches in build-priority order
 169 branchNames = [branches[b]["name"] for b in branches]
 170
 171 c['protocols'] = {'pb': {'port': pb_port}}
 172
 173 # coalesce builds
 174 c['collapseRequests'] = True
 175
 176 # Reduce amount of backlog data
 177 c['configurators'] = [util.JanitorConfigurator(
 178         logHorizon=timedelta(days=3),
 179         hour=6,
 180 )]
 181
 182 @defer.inlineCallbacks
 183 def getNewestCompleteTime(bldr):
 184         """Returns the complete_at of the latest completed and not SKIPPED
 185         build request for this builder, or None if there are no such build
 186         requests. We need to filter out SKIPPED requests because we're
 187         using collapseRequests=True which is unfortunately marking all
 188         previous requests as complete when new buildset is created.
 189
 190         @returns: datetime instance or None, via Deferred
 191         """
 192
 193         bldrid = yield bldr.getBuilderId()
 194         completed = yield bldr.master.data.get(
 195                         ('builders', bldrid, 'buildrequests'),
 196                         [
 197                                 resultspec.Filter('complete', 'eq', [True]),
 198                                 resultspec.Filter('results', 'ne', [results.SKIPPED]),
 199                         ],
 200                         order=['-complete_at'], limit=1)
 201         if not completed:
 202                 return
 203
 204         complete_at = completed[0]['complete_at']
 205
 206         last_build = yield bldr.master.data.get(
 207                         ('builds', ),
 208                         [
 209                                 resultspec.Filter('builderid', 'eq', [bldrid]),
 210                         ],
 211                         order=['-started_at'], limit=1)
 212
 213         if last_build and last_build[0]:
 214                 last_complete_at = last_build[0]['complete_at']
 215                 if last_complete_at and (last_complete_at > complete_at):
 216                         return last_complete_at
 217
 218         return complete_at
 219
 220 @defer.inlineCallbacks
 221 def prioritizeBuilders(master, builders):
 222         """Returns sorted list of builders by their last timestamp of completed and
 223         not skipped build, ordered first by branch name.
 224
 225         @returns: list of sorted builders
 226         """
 227
 228         bldrNamePrio = { "__Janitor": 0, "00_force_build": 0 }
 229         i = 1
 230         for bname in branchNames:
 231                 bldrNamePrio[bname] = i
 232                 i += 1
 233
 234         def is_building(bldr):
 235                 return bool(bldr.building) or bool(bldr.old_building)
 236
 237         def bldr_info(bldr):
 238                 d = defer.maybeDeferred(getNewestCompleteTime, bldr)
 239                 d.addCallback(lambda complete_at: (complete_at, bldr))
 240                 return d
 241
 242         def bldr_sort(item):
 243                 (complete_at, bldr) = item
 244
 245                 pos = 99
 246                 for (name, prio) in bldrNamePrio.items():
 247                         if bldr.name.startswith(name):
 248                                 pos = prio
 249                                 break
 250
 251                 if not complete_at:
 252                         date = datetime.min
 253                         complete_at = date.replace(tzinfo=tzutc())
 254
 255                 if is_building(bldr):
 256                         date = datetime.max
 257                         complete_at = date.replace(tzinfo=tzutc())
 258
 259                 return (pos, complete_at, bldr.name)
 260
 261         results = yield defer.gatherResults([bldr_info(bldr) for bldr in builders])
 262         results.sort(key=bldr_sort)
 263
 264         #for r in results:
 265         #       log.msg("prioritizeBuilders: {:>20} complete_at: {}".format(r[1].name, r[0]))
 266
 267         return [r[1] for r in results]
 268
 269 c['prioritizeBuilders'] = prioritizeBuilders
 270
 271 ####### CHANGESOURCES
 272
 273 # find targets
 274 targets = set()
 275
 276 def populateTargets():
 277         """ fetch a shallow clone of each configured branch in turn:
 278         execute dump-target-info.pl and collate the results to ensure
 279         targets that only exist in specific branches get built.
 280         This takes a while during master startup but is executed only once.
 281         """
 282         log.msg("Populating targets, this will take time")
 283         sourcegit = work_dir + '/source.git'
 284         for branch in branchNames:
 285                 if os.path.isdir(sourcegit):
 286                         subprocess.call(["rm", "-rf", sourcegit])
 287
 288                 subprocess.call(["git", "clone", "-q", "--depth=1", "--branch="+branch, repo_url, sourcegit])
 289
 290                 os.makedirs(sourcegit + '/tmp', exist_ok=True)
 291                 findtargets = subprocess.Popen(['./scripts/dump-target-info.pl', 'targets'],
 292                         stdout = subprocess.PIPE, stderr = subprocess.DEVNULL, cwd = sourcegit)
 293
 294                 while True:
 295                         line = findtargets.stdout.readline()
 296                         if not line:
 297                                 break
 298                         ta = line.decode().strip().split(' ')
 299                         targets.add(ta[0])
 300
 301                 subprocess.call(["rm", "-rf", sourcegit])
 302
 303 populateTargets()
 304
 305 # the 'change_source' setting tells the buildmaster how it should find out
 306 # about source code changes.
 307
 308 c['change_source'] = []
 309 c['change_source'].append(GitPoller(
 310         repo_url,
 311         workdir=work_dir+'/work.git', branches=branchNames,
 312         pollAtLaunch=True, pollinterval=300))
 313
 314 ####### SCHEDULERS
 315
 316 # Configure the Schedulers, which decide how to react to incoming changes.
 317
 318 # Selector for known valid tags
 319 class TagChoiceParameter(BaseParameter):
 320         spec_attributes = ["strict", "choices"]
 321         type = "list"
 322         strict = True
 323
 324         def __init__(self, name, label=None, **kw):
 325                 super().__init__(name, label, **kw)
 326                 self._choice_list = []
 327
 328         def getRevTags(self, findtag=None):
 329                 taglist = []
 330                 branchvers = []
 331
 332                 # we will filter out tags that do no match the configured branches
 333                 for b in branchNames:
 334                         basever = re.search(r'-([0-9]+\.[0-9]+)$', b)
 335                         if basever:
 336                                 branchvers.append(basever[1])
 337
 338                 # grab tags from remote repository
 339                 alltags = subprocess.Popen(
 340                         ['git', 'ls-remote', '--tags', repo_url],
 341                         stdout = subprocess.PIPE)
 342
 343                 while True:
 344                         line = alltags.stdout.readline()
 345
 346                         if not line:
 347                                 break
 348
 349                         (rev, tag) = line.split()
 350
 351                         # does it match known format? ('vNN.NN.NN(-rcN)')
 352                         tagver = re.search(r'\brefs/tags/(v[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?)$', tag.decode().strip())
 353
 354                         # only list valid tags matching configured branches
 355                         if tagver and any(tagver[1][1:].startswith(b) for b in branchvers):
 356                                 # if we want a specific tag, ignore all that don't match
 357                                 if findtag and findtag != tagver[1]:
 358                                         continue
 359                                 taglist.append({'rev': rev.decode().strip(), 'tag': tagver[1]})
 360
 361                 return taglist
 362
 363         @property
 364         def choices(self):
 365                 taglist = [rt['tag'] for rt in self.getRevTags()]
 366                 taglist.sort(reverse=True, key=lambda tag: tag if re.search(r'-rc[0-9]+$', tag) else tag + '-z')
 367                 taglist.insert(0, '')
 368
 369                 self._choice_list = taglist
 370
 371                 return self._choice_list
 372
 373         def updateFromKwargs(self, properties, kwargs, **unused):
 374                 tag = self.getFromKwargs(kwargs)
 375                 properties[self.name] = tag
 376
 377                 # find the commit matching the tag
 378                 findtag = self.getRevTags(tag)
 379
 380                 if not findtag:
 381                         raise ValidationError("Couldn't find tag")
 382
 383                 properties['force_revision'] = findtag[0]['rev']
 384
 385                 # find the branch matching the tag
 386                 branch = None
 387                 branchver = re.search(r'v([0-9]+\.[0-9]+)', tag)
 388                 for b in branchNames:
 389                         if b.endswith(branchver[1]):
 390                                 branch = b
 391
 392                 if not branch:
 393                         raise ValidationError("Couldn't find branch")
 394
 395                 properties['force_branch'] = branch
 396
 397         def parse_from_arg(self, s):
 398                 if self.strict and s not in self._choice_list:
 399                         raise ValidationError("'%s' does not belong to list of available choices '%s'" % (s, self._choice_list))
 400                 return s
 401
 402 @util.renderer
 403 @defer.inlineCallbacks
 404 def builderNames(props):
 405         """ since we have per branch and per target builders,
 406         address the relevant builder for each new buildrequest
 407         based on the request's desired branch and target.
 408         """
 409         branch = props.getProperty("branch")
 410         target = props.getProperty("target", "")
 411
 412         if target == "all":
 413                 target = ""
 414
 415         # if that didn't work, try sourcestamp to find a branch
 416         if not branch:
 417                 # match builders with target branch
 418                 ss = props.sourcestamps[0]
 419                 if ss:
 420                         branch = ss['branch']
 421                 else:
 422                         log.msg("couldn't find builder")
 423                         return []       # nothing works
 424
 425         bname = branch + "_" + target
 426         builders = []
 427
 428         for b in (yield props.master.data.get(('builders',))):
 429                 if not b['name'].startswith(bname):
 430                         continue
 431                 builders.append(b['name'])
 432
 433         return builders
 434
 435 c['schedulers'] = []
 436 c['schedulers'].append(AnyBranchScheduler(
 437         name            = "all",
 438         change_filter   = util.ChangeFilter(branch=branchNames),
 439         treeStableTimer = 15*60,
 440         builderNames    = builderNames))
 441
 442 c['schedulers'].append(ForceScheduler(
 443         name         = "force",
 444         buttonName   = "Force builds",
 445         label        = "Force build details",
 446         builderNames = [ "00_force_build" ],
 447
 448         codebases = [
 449                 util.CodebaseParameter(
 450                         "",
 451                         label      = "Repository",
 452                         branch     = util.FixedParameter(name = "branch",     default = ""),
 453                         revision   = util.FixedParameter(name = "revision",   default = ""),
 454                         repository = util.FixedParameter(name = "repository", default = ""),
 455                         project    = util.FixedParameter(name = "project",    default = "")
 456                 )
 457         ],
 458
 459         reason = util.StringParameter(
 460                 name     = "reason",
 461                 label    = "Reason",
 462                 default  = "Trigger build",
 463                 required = True,
 464                 size     = 80
 465         ),
 466
 467         properties = [
 468                 # NB: avoid nesting to simplify processing of properties
 469                 util.ChoiceStringParameter(
 470                         name    = "target",
 471                         label   = "Build target",
 472                         default = "all",
 473                         choices = [ "all" ] + list(targets)
 474                 ),
 475                 TagChoiceParameter(
 476                         name    = "tag",
 477                         label   = "Build tag",
 478                         default = ""
 479                 )
 480         ]
 481 ))
 482
 483 c['schedulers'].append(schedulers.Triggerable(name="trigger", builderNames=builderNames))
 484
 485 ####### BUILDERS
 486
 487 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
 488 # what steps, and which workers can execute them.  Note that any particular build will
 489 # only take place on one worker.
 490
 491 def IsNoMasterBuild(step):
 492         return step.getProperty("branch") != "master"
 493
 494 def IsUsignEnabled(step):
 495         branch = step.getProperty("branch")
 496         return branch and branches[branch].get("usign_key")
 497
 498 def IsSignEnabled(step):
 499         branch = step.getProperty("branch")
 500         return IsUsignEnabled(step) or branch and branches[branch].get("gpg_key")
 501
 502 def IsKmodArchiveEnabled(step):
 503         branch = step.getProperty("branch")
 504         return branch and branches[branch].get("kmod_archive")
 505
 506 def GetBaseVersion(branch):
 507         if re.match(r"^[^-]+-[0-9]+\.[0-9]+$", branch):
 508                 return branch.split('-')[1]
 509         else:
 510                 return "master"
 511
 512 @properties.renderer
 513 def GetVersionPrefix(props):
 514         branch = props.getProperty("branch")
 515         basever = GetBaseVersion(branch)
 516         if props.hasProperty("tag") and re.match(r"^v[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", props["tag"]):
 517                 return "%s/" % props["tag"][1:]
 518         elif basever != "master":
 519                 return "%s-SNAPSHOT/" % basever
 520         else:
 521                 return ""
 522
 523 @util.renderer
 524 def GetConfigSeed(props):
 525         branch = props.getProperty("branch")
 526         return branch and branches[branch].get("config_seed") or ""
 527
 528 @util.renderer
 529 def GetRsyncParams(props, srcorbin, urlorkey):
 530         # srcorbin: 'bin' or 'src'; urlorkey: 'url' or 'key'
 531         branch = props.getProperty("branch")
 532         opt = srcorbin + "_" + urlorkey
 533         return branch and branches[branch].get(opt)
 534
 535 @util.renderer
 536 def GetUsignKey(props):
 537         branch = props.getProperty("branch")
 538         return branch and branches[branch].get("usign_key")
 539
 540 def GetNextBuild(builder, requests):
 541         for r in requests:
 542                 if r.properties:
 543                         # order tagged build first
 544                         if r.properties.hasProperty("tag"):
 545                                 return r
 546
 547         r = requests[0]
 548         #log.msg("GetNextBuild: {:>20} id: {} bsid: {}".format(builder.name, r.id, r.bsid))
 549         return r
 550
 551 def MakeEnv(overrides=None, tryccache=False):
 552         env = {
 553                 'CCC': Interpolate("%(prop:cc_command:-gcc)s"),
 554                 'CCXX': Interpolate("%(prop:cxx_command:-g++)s"),
 555         }
 556         if tryccache:
 557                 env['CC'] = Interpolate("%(prop:builddir)s/ccache_cc.sh")
 558                 env['CXX'] = Interpolate("%(prop:builddir)s/ccache_cxx.sh")
 559                 env['CCACHE'] = Interpolate("%(prop:ccache_command:-)s")
 560         else:
 561                 env['CC'] = env['CCC']
 562                 env['CXX'] = env['CCXX']
 563                 env['CCACHE'] = ''
 564         if overrides is not None:
 565                 env.update(overrides)
 566         return env
 567
 568 @properties.renderer
 569 def NetLockDl(props, extralock=None):
 570         lock = None
 571         if props.hasProperty("dl_lock"):
 572                 lock = NetLocks[props["dl_lock"]]
 573         if lock is not None:
 574                 return [lock.access('exclusive')]
 575         else:
 576                 return []
 577
 578 @properties.renderer
 579 def NetLockUl(props):
 580         lock = None
 581         if props.hasProperty("ul_lock"):
 582                 lock = NetLocks[props["ul_lock"]]
 583         if lock is not None:
 584                 return [lock.access('exclusive')]
 585         else:
 586                 return []
 587
 588 def IsTargetSelected(target):
 589         def CheckTargetProperty(step):
 590                 selected_target = step.getProperty("target", "all")
 591                 if selected_target != "all" and selected_target != target:
 592                         return False
 593                 return True
 594
 595         return CheckTargetProperty
 596
 597 @util.renderer
 598 def UsignSec2Pub(props):
 599         branch = props.getProperty("branch")
 600         try:
 601                 comment = branches[branch].get("usign_comment") or "untrusted comment: secret key"
 602                 seckey = branches[branch].get("usign_key")
 603                 seckey = base64.b64decode(seckey)
 604         except:
 605                 return None
 606
 607         return "{}\n{}".format(re.sub(r"\bsecret key$", "public key", comment),
 608                 base64.b64encode(seckey[0:2] + seckey[32:40] + seckey[72:]))
 609
 610
 611 def canStartBuild(builder, wfb, request):
 612         """ filter out non tag requests for tag_only workers. """
 613         wtagonly = wfb.worker.properties.getProperty('tag_only')
 614         tag = request.properties.getProperty('tag')
 615
 616         if wtagonly and not tag:
 617                 return False
 618
 619         return True
 620
 621 c['builders'] = []
 622
 623 workerNames = [ ]
 624
 625 for worker in c['workers']:
 626         workerNames.append(worker.workername)
 627
 628 # add a single LocalWorker to handle the forcebuild builder
 629 c['workers'].append(LocalWorker("__local_force_build", max_builds=1))
 630
 631 force_factory = BuildFactory()
 632 force_factory.addStep(steps.Trigger(
 633         name = "trigger_build",
 634         schedulerNames = [ "trigger" ],
 635         sourceStamps = [{ "codebase": "", "branch": Property("force_branch"), "revision": Property("force_revision"), "repository": repo_url, "project": "" }],
 636         set_properties = { "reason": Property("reason"), "tag": Property("tag"), "target": Property("target") },
 637 ))
 638
 639 c['builders'].append(BuilderConfig(
 640         name        = "00_force_build",
 641         workername  = "__local_force_build",
 642         factory     = force_factory))
 643
 644
 645 # NB the phase1 build factory assumes workers are single-build only
 646 for target in targets:
 647         ts = target.split('/')
 648
 649         factory = BuildFactory()
 650
 651         # setup shared work directory if required
 652         factory.addStep(ShellCommand(
 653                 name = "sharedwd",
 654                 descriptionDone = "Shared work directory set up",
 655                 command = 'test -L "$PWD" || (mkdir -p ../shared-workdir && rm -rf "$PWD" && ln -s shared-workdir "$PWD")',
 656                 workdir = ".",
 657                 haltOnFailure = True,
 658         ))
 659
 660         # find number of cores
 661         factory.addStep(SetPropertyFromCommand(
 662                 name = "nproc",
 663                 property = "nproc",
 664                 description = "Finding number of CPUs",
 665                 command = ["nproc"],
 666         ))
 667
 668         # find gcc and g++ compilers
 669         factory.addStep(FileDownload(
 670                 name = "dlfindbinpl",
 671                 mastersrc = scripts_dir + '/findbin.pl',
 672                 workerdest = "../findbin.pl",
 673                 mode = 0o755,
 674         ))
 675
 676         factory.addStep(SetPropertyFromCommand(
 677                 name = "gcc",
 678                 property = "cc_command",
 679                 description = "Finding gcc command",
 680                 command = ["../findbin.pl", "gcc", "", ""],
 681                 haltOnFailure = True,
 682         ))
 683
 684         factory.addStep(SetPropertyFromCommand(
 685                 name = "g++",
 686                 property = "cxx_command",
 687                 description = "Finding g++ command",
 688                 command = ["../findbin.pl", "g++", "", ""],
 689                 haltOnFailure = True,
 690         ))
 691
 692         # see if ccache is available
 693         factory.addStep(SetPropertyFromCommand(
 694                 name = "ccache",
 695                 property = "ccache_command",
 696                 description = "Testing for ccache command",
 697                 command = ["which", "ccache"],
 698                 haltOnFailure = False,
 699                 flunkOnFailure = False,
 700                 warnOnFailure = False,
 701                 hideStepIf = lambda r, s: r==results.FAILURE,
 702         ))
 703
 704         # check out the source
 705         # Git() runs:
 706           # if repo doesn't exist: 'git clone repourl'
 707           # method 'clean' runs 'git clean -d -f', method fresh runs 'git clean -f -f -d -x'. Only works with mode='full'
 708           # git cat-file -e <commit>
 709           # git checkout -f <commit>
 710           # git checkout -B <branch>
 711           # git rev-parse HEAD
 712         factory.addStep(Git(
 713                 name = "git",
 714                 repourl = repo_url,
 715                 mode = 'full',
 716                 method = 'fresh',
 717                 locks = NetLockDl,
 718                 haltOnFailure = True,
 719         ))
 720
 721         # update remote refs
 722         factory.addStep(ShellCommand(
 723                 name = "fetchrefs",
 724                 description = "Fetching Git remote refs",
 725                 descriptionDone = "Git remote refs fetched",
 726                 command = ["git", "fetch", "origin", Interpolate("+refs/heads/%(prop:branch)s:refs/remotes/origin/%(prop:branch)s")],
 727                 haltOnFailure = True,
 728         ))
 729
 730         # Verify that Git HEAD points to a tag or branch
 731         # Ref: https://web.archive.org/web/20190729224316/http://lists.infradead.org/pipermail/openwrt-devel/2019-June/017809.html
 732         factory.addStep(ShellCommand(
 733                 name = "gitverify",
 734                 description = "Ensuring that Git HEAD is pointing to a branch or tag",
 735                 descriptionDone = "Git HEAD is sane",
 736                 command = 'git rev-parse --abbrev-ref HEAD | grep -vxqF HEAD || git show-ref --tags --dereference 2>/dev/null | sed -ne "/^$(git rev-parse HEAD) / { s|^.*/||; s|\\^.*||; p }" | grep -qE "^v[0-9][0-9]\\."',
 737                 haltOnFailure = True,
 738         ))
 739
 740         factory.addStep(ShellCommand(
 741                 name = "rmtmp",
 742                 description = "Remove tmp folder",
 743                 command=["rm", "-rf", "tmp/"],
 744         ))
 745
 746         # feed
 747         factory.addStep(ShellCommand(
 748                 name = "rmfeedlinks",
 749                 description = "Remove feed symlinks",
 750                 command=["rm", "-rf", "package/feeds/"],
 751         ))
 752
 753         factory.addStep(StringDownload(
 754                 name = "ccachecc",
 755                 s = '#!/bin/sh\nexec ${CCACHE} ${CCC} "$@"\n',
 756                 workerdest = "../ccache_cc.sh",
 757                 mode = 0o755,
 758         ))
 759
 760         factory.addStep(StringDownload(
 761                 name = "ccachecxx",
 762                 s = '#!/bin/sh\nexec ${CCACHE} ${CCXX} "$@"\n',
 763                 workerdest = "../ccache_cxx.sh",
 764                 mode = 0o755,
 765         ))
 766
 767         # feed
 768         factory.addStep(ShellCommand(
 769                 name = "updatefeeds",
 770                 description = "Updating feeds",
 771                 command=["./scripts/feeds", "update"],
 772                 env = MakeEnv(tryccache=True),
 773                 haltOnFailure = True,
 774                 locks = NetLockDl,
 775         ))
 776
 777         # feed
 778         factory.addStep(ShellCommand(
 779                 name = "installfeeds",
 780                 description = "Installing feeds",
 781                 command=["./scripts/feeds", "install", "-a"],
 782                 env = MakeEnv(tryccache=True),
 783                 haltOnFailure = True,
 784         ))
 785
 786         # seed config
 787         factory.addStep(StringDownload(
 788                 name = "dlconfigseed",
 789                 s = Interpolate("%(kw:seed)s\n", seed=GetConfigSeed),
 790                 workerdest = ".config",
 791                 mode = 0o644,
 792         ))
 793
 794         # configure
 795         factory.addStep(ShellCommand(
 796                 name = "newconfig",
 797                 descriptionDone = ".config seeded",
 798                 command = Interpolate("printf 'CONFIG_TARGET_%(kw:target)s=y\\nCONFIG_TARGET_%(kw:target)s_%(kw:subtarget)s=y\\nCONFIG_SIGNED_PACKAGES=%(kw:usign:#?|y|n)s\\n' >> .config", target=ts[0], subtarget=ts[1], usign=GetUsignKey),
 799         ))
 800
 801         factory.addStep(ShellCommand(
 802                 name = "delbin",
 803                 description = "Removing output directory",
 804                 command = ["rm", "-rf", "bin/"],
 805         ))
 806
 807         factory.addStep(ShellCommand(
 808                 name = "defconfig",
 809                 description = "Populating .config",
 810                 command = ["make", "defconfig"],
 811                 env = MakeEnv(),
 812         ))
 813
 814         # check arch - exit early if does not exist - NB: some targets do not define CONFIG_TARGET_target_subtarget
 815         factory.addStep(ShellCommand(
 816                 name = "checkarch",
 817                 description = "Checking architecture",
 818                 descriptionDone = "Architecture validated",
 819                 command = 'grep -sq CONFIG_TARGET_%s=y .config && grep -sq CONFIG_TARGET_SUBTARGET=\\"%s\\" .config' %(ts[0], ts[1]),
 820                 logEnviron = False,
 821                 want_stdout = False,
 822                 want_stderr = False,
 823                 haltOnFailure = True,
 824                 flunkOnFailure = False, # this is not a build FAILURE - TODO mark build as SKIPPED
 825         ))
 826
 827         # find libc suffix
 828         factory.addStep(SetPropertyFromCommand(
 829                 name = "libc",
 830                 property = "libc",
 831                 description = "Finding libc suffix",
 832                 command = ["sed", "-ne", '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }', ".config"],
 833         ))
 834
 835         # install build key
 836         factory.addStep(StringDownload(
 837                 name = "dlkeybuildpub",
 838                 s = Interpolate("%(kw:sec2pub)s", sec2pub=UsignSec2Pub),
 839                 workerdest = "key-build.pub",
 840                 mode = 0o600,
 841                 doStepIf = IsUsignEnabled,
 842         ))
 843
 844         factory.addStep(StringDownload(
 845                 name = "dlkeybuild",
 846                 s = "# fake private key",
 847                 workerdest = "key-build",
 848                 mode = 0o600,
 849                 doStepIf = IsUsignEnabled,
 850         ))
 851
 852         factory.addStep(StringDownload(
 853                 name = "dlkeybuilducert",
 854                 s = "# fake certificate",
 855                 workerdest = "key-build.ucert",
 856                 mode = 0o600,
 857                 doStepIf = IsUsignEnabled,
 858         ))
 859
 860         # prepare dl
 861         factory.addStep(ShellCommand(
 862                 name = "dldir",
 863                 description = "Preparing dl/",
 864                 descriptionDone = "dl/ prepared",
 865                 command = 'mkdir -p ../dl && rm -rf "build/dl" && ln -s ../../dl "build/dl"',
 866                 workdir = Property("builddir"),
 867                 logEnviron = False,
 868                 want_stdout = False,
 869         ))
 870
 871         # cleanup dl
 872         factory.addStep(ShellCommand(
 873                 name = "dlprune",
 874                 description = "Pruning dl/",
 875                 descriptionDone = "dl/ pruned",
 876                 command = 'find dl/ -atime +15 -delete -print',
 877                 logEnviron = False,
 878         ))
 879
 880         # prepare tar
 881         factory.addStep(ShellCommand(
 882                 name = "dltar",
 883                 description = "Building and installing GNU tar",
 884                 descriptionDone = "GNU tar built and installed",
 885                 command = ["make", Interpolate("-j%(prop:nproc:-1)s"), "tools/tar/compile", "V=s"],
 886                 env = MakeEnv(tryccache=True),
 887                 haltOnFailure = True,
 888         ))
 889
 890         # populate dl
 891         factory.addStep(ShellCommand(
 892                 name = "dlrun",
 893                 description = "Populating dl/",
 894                 descriptionDone = "dl/ populated",
 895                 command = ["make", Interpolate("-j%(prop:nproc:-1)s"), "download", "V=s"],
 896                 env = MakeEnv(),
 897                 logEnviron = False,
 898                 locks = NetLockDl,
 899         ))
 900
 901         factory.addStep(ShellCommand(
 902                 name = "cleanbase",
 903                 description = "Cleaning base-files",
 904                 command=["make", "package/base-files/clean", "V=s"],
 905         ))
 906
 907         # build
 908         factory.addStep(ShellCommand(
 909                 name = "tools",
 910                 description = "Building and installing tools",
 911                 descriptionDone = "Tools built and installed",
 912                 command = ["make", Interpolate("-j%(prop:nproc:-1)s"), "tools/install", "V=s"],
 913                 env = MakeEnv(tryccache=True),
 914                 haltOnFailure = True,
 915         ))
 916
 917         factory.addStep(ShellCommand(
 918                 name = "toolchain",
 919                 description = "Building and installing toolchain",
 920                 descriptionDone = "Toolchain built and installed",
 921                 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "toolchain/install", "V=s"],
 922                 env = MakeEnv(),
 923                 haltOnFailure = True,
 924         ))
 925
 926         factory.addStep(ShellCommand(
 927                 name = "kmods",
 928                 description = "Building kmods",
 929                 descriptionDone = "Kmods built",
 930                 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
 931                 env = MakeEnv(),
 932                 haltOnFailure = True,
 933         ))
 934
 935         # find kernel version
 936         factory.addStep(SetPropertyFromCommand(
 937                 name = "kernelversion",
 938                 property = "kernelversion",
 939                 description = "Finding the effective Kernel version",
 940                 command = "make --no-print-directory -C target/linux/ val.LINUX_VERSION val.LINUX_RELEASE val.LINUX_VERMAGIC | xargs printf '%s-%s-%s\\n'",
 941                 env = { 'TOPDIR': Interpolate("%(prop:builddir)s/build") },
 942         ))
 943
 944         factory.addStep(ShellCommand(
 945                 name = "pkgclean",
 946                 description = "Cleaning up package build",
 947                 descriptionDone = "Package build cleaned up",
 948                 command=["make", "package/cleanup", "V=s"],
 949         ))
 950
 951         factory.addStep(ShellCommand(
 952                 name = "pkgbuild",
 953                 description = "Building packages",
 954                 descriptionDone = "Packages built",
 955                 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
 956                 env = MakeEnv(),
 957                 haltOnFailure = True,
 958         ))
 959
 960         factory.addStep(ShellCommand(
 961                 name = "pkginstall",
 962                 description = "Installing packages",
 963                 descriptionDone = "Packages installed",
 964                 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/install", "V=s"],
 965                 env = MakeEnv(),
 966                 haltOnFailure = True,
 967         ))
 968
 969         factory.addStep(ShellCommand(
 970                 name = "pkgindex",
 971                 description = "Indexing packages",
 972                 descriptionDone = "Packages indexed",
 973                 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/index", "V=s", "CONFIG_SIGNED_PACKAGES="],
 974                 env = MakeEnv(),
 975                 haltOnFailure = True,
 976         ))
 977
 978         factory.addStep(ShellCommand(
 979                 name = "images",
 980                 description = "Building and installing images",
 981                 descriptionDone = "Images built and installed",
 982                 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "target/install", "V=s"],
 983                 env = MakeEnv(),
 984                 haltOnFailure = True,
 985         ))
 986
 987         factory.addStep(ShellCommand(
 988                 name = "buildinfo",
 989                 description = "Generating config.buildinfo, version.buildinfo and feeds.buildinfo",
 990                 command = "make -j1 buildinfo V=s || true",
 991                 env = MakeEnv(),
 992                 haltOnFailure = True,
 993         ))
 994
 995         factory.addStep(ShellCommand(
 996                 name = "json_overview_image_info",
 997                 description = "Generating profiles.json in target folder",
 998                 command = "make -j1 json_overview_image_info V=s || true",
 999                 env = MakeEnv(),
1000                 haltOnFailure = True,
1001         ))
1002
1003         factory.addStep(ShellCommand(
1004                 name = "checksums",
1005                 description = "Calculating checksums",
1006                 descriptionDone = "Checksums calculated",
1007                 command=["make", "-j1", "checksum", "V=s"],
1008                 env = MakeEnv(),
1009                 haltOnFailure = True,
1010         ))
1011
1012         factory.addStep(ShellCommand(
1013                 name = "kmoddir",
1014                 descriptionDone = "Kmod directory created",
1015                 command=["mkdir", "-p", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s", target=ts[0], subtarget=ts[1])],
1016                 haltOnFailure = True,
1017                 doStepIf = IsKmodArchiveEnabled,
1018         ))
1019
1020         factory.addStep(ShellCommand(
1021                 name = "kmodprepare",
1022                 description = "Preparing kmod archive",
1023                 descriptionDone = "Kmod archive prepared",
1024                 command=["rsync", "--include=/kmod-*.ipk", "--exclude=*", "-va",
1025                         Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/packages/", target=ts[0], subtarget=ts[1]),
1026                         Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1])],
1027                 haltOnFailure = True,
1028                 doStepIf = IsKmodArchiveEnabled,
1029         ))
1030
1031         factory.addStep(ShellCommand(
1032                 name = "kmodindex",
1033                 description = "Indexing kmod archive",
1034                 descriptionDone = "Kmod archive indexed",
1035                 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/index", "V=s", "CONFIG_SIGNED_PACKAGES=",
1036                         Interpolate("PACKAGE_SUBDIRS=bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1])],
1037                 env = MakeEnv(),
1038                 haltOnFailure = True,
1039                 doStepIf = IsKmodArchiveEnabled,
1040         ))
1041
1042         # sign
1043         factory.addStep(MasterShellCommand(
1044                 name = "signprepare",
1045                 descriptionDone = "Temporary signing directory prepared",
1046                 command = ["mkdir", "-p", "%s/signing" %(work_dir)],
1047                 haltOnFailure = True,
1048                 doStepIf = IsSignEnabled,
1049
1050         ))
1051
1052         factory.addStep(ShellCommand(
1053                 name = "signpack",
1054                 description = "Packing files to sign",
1055                 descriptionDone = "Files to sign packed",
1056                 command = Interpolate("find bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/ bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz", target=ts[0], subtarget=ts[1]),
1057                 haltOnFailure = True,
1058                 doStepIf = IsSignEnabled,
1059         ))
1060
1061         factory.addStep(FileUpload(
1062                 workersrc = "sign.tar.gz",
1063                 masterdest = "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1]),
1064                 haltOnFailure = True,
1065                 doStepIf = IsSignEnabled,
1066         ))
1067
1068         factory.addStep(MasterShellCommand(
1069                 name = "signfiles",
1070                 description = "Signing files",
1071                 descriptionDone = "Files signed",
1072                 command = ["%s/signall.sh" %(scripts_dir), "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1]), Interpolate("%(prop:branch)s")],
1073                 env = { 'CONFIG_INI': os.getenv("BUILDMASTER_CONFIG", "./config.ini") },
1074                 haltOnFailure = True,
1075                 doStepIf = IsSignEnabled,
1076         ))
1077
1078         factory.addStep(FileDownload(
1079                 name = "dlsigntargz",
1080                 mastersrc = "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1]),
1081                 workerdest = "sign.tar.gz",
1082                 haltOnFailure = True,
1083                 doStepIf = IsSignEnabled,
1084         ))
1085
1086         factory.addStep(ShellCommand(
1087                 name = "signunpack",
1088                 description = "Unpacking signed files",
1089                 descriptionDone = "Signed files unpacked",
1090                 command = ["tar", "-xzf", "sign.tar.gz"],
1091                 haltOnFailure = True,
1092                 doStepIf = IsSignEnabled,
1093         ))
1094
1095         # upload
1096         factory.addStep(ShellCommand(
1097                 name = "dirprepare",
1098                 descriptionDone = "Upload directory structure prepared",
1099                 command = ["mkdir", "-p", Interpolate("tmp/upload/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s", target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1100                 haltOnFailure = True,
1101         ))
1102
1103         factory.addStep(ShellCommand(
1104                 name = "linkprepare",
1105                 descriptionDone = "Repository symlink prepared",
1106                 command = ["ln", "-s", "-f", Interpolate("../packages-%(kw:basever)s", basever=util.Transform(GetBaseVersion, Property("branch"))), Interpolate("tmp/upload/%(kw:prefix)spackages", prefix=GetVersionPrefix)],
1107                 doStepIf = IsNoMasterBuild,
1108                 haltOnFailure = True,
1109         ))
1110
1111         factory.addStep(ShellCommand(
1112                 name = "kmoddirprepare",
1113                 descriptionDone = "Kmod archive upload directory prepared",
1114                 command = ["mkdir", "-p", Interpolate("tmp/upload/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s", target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1115                 haltOnFailure = True,
1116                 doStepIf = IsKmodArchiveEnabled,
1117         ))
1118
1119         factory.addStep(ShellCommand(
1120                 name = "dirupload",
1121                 description = "Uploading directory structure",
1122                 descriptionDone = "Directory structure uploaded",
1123                 command = ["rsync", "-az"] + rsync_defopts + ["tmp/upload/", Interpolate("%(kw:url)s/", url=GetRsyncParams.withArgs("bin", "url"))],
1124                 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1125                 haltOnFailure = True,
1126                 logEnviron = False,
1127                 locks = NetLockUl,
1128                 doStepIf = util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
1129         ))
1130
1131         # download remote sha256sums to 'target-sha256sums'
1132         factory.addStep(ShellCommand(
1133                 name = "target-sha256sums",
1134                 description = "Fetching remote sha256sums for target",
1135                 descriptionDone = "Remote sha256sums for target fetched",
1136                 command = ["rsync", "-z"] + rsync_defopts + [Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/sha256sums", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix), "target-sha256sums"],
1137                 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1138                 logEnviron = False,
1139                 haltOnFailure = False,
1140                 flunkOnFailure = False,
1141                 warnOnFailure = False,
1142                 doStepIf = util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
1143         ))
1144
1145         # build list of files to upload
1146         factory.addStep(FileDownload(
1147                 name = "dlsha2rsyncpl",
1148                 mastersrc = scripts_dir + '/sha2rsync.pl',
1149                 workerdest = "../sha2rsync.pl",
1150                 mode = 0o755,
1151         ))
1152
1153         factory.addStep(ShellCommand(
1154                 name = "buildlist",
1155                 description = "Building list of files to upload",
1156                 descriptionDone = "List of files to upload built",
1157                 command = ["../sha2rsync.pl", "target-sha256sums", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/sha256sums", target=ts[0], subtarget=ts[1]), "rsynclist"],
1158                 haltOnFailure = True,
1159         ))
1160
1161         factory.addStep(FileDownload(
1162                 name = "dlrsync.sh",
1163                 mastersrc = scripts_dir + '/rsync.sh',
1164                 workerdest = "../rsync.sh",
1165                 mode = 0o755,
1166         ))
1167
1168         # upload new files and update existing ones
1169         factory.addStep(ShellCommand(
1170                 name = "targetupload",
1171                 description = "Uploading target files",
1172                 descriptionDone = "Target files uploaded",
1173                 command=["../rsync.sh", "--exclude=/kmods/", "--files-from=rsynclist", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_defopts +
1174                         ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]),
1175                         Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1176                 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1177                 haltOnFailure = True,
1178                 logEnviron = False,
1179                 doStepIf = util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
1180         ))
1181
1182         # delete files which don't exist locally
1183         factory.addStep(ShellCommand(
1184                 name = "targetprune",
1185                 description = "Pruning target files",
1186                 descriptionDone = "Target files pruned",
1187                 command=["../rsync.sh", "--exclude=/kmods/", "--delete", "--existing", "--ignore-existing", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_defopts +
1188                         ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]),
1189                         Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1190                 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1191                 haltOnFailure = True,
1192                 logEnviron = False,
1193                 locks = NetLockUl,
1194                 doStepIf = util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
1195         ))
1196
1197         factory.addStep(ShellCommand(
1198                 name = "kmodupload",
1199                 description = "Uploading kmod archive",
1200                 descriptionDone = "Kmod archive uploaded",
1201                 command=["../rsync.sh", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_defopts +
1202                         ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1]),
1203                         Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1204                 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1205                 haltOnFailure = True,
1206                 logEnviron = False,
1207                 locks = NetLockUl,
1208                 doStepIf = util.Transform(lambda a, b: bool(a and b), IsKmodArchiveEnabled, GetRsyncParams.withArgs("bin", "url")),
1209         ))
1210
1211         factory.addStep(ShellCommand(
1212                 name = "sourcelist",
1213                 description = "Finding source archives to upload",
1214                 descriptionDone = "Source archives to upload found",
1215                 command = "find dl/ -maxdepth 1 -type f -not -size 0 -not -name '.*' -not -name '*.hash' -not -name '*.dl' -newer .config -printf '%f\\n' > sourcelist",
1216                 haltOnFailure = True,
1217         ))
1218
1219         factory.addStep(ShellCommand(
1220                 name = "sourceupload",
1221                 description = "Uploading source archives",
1222                 descriptionDone = "Source archives uploaded",
1223                 command=["../rsync.sh", "--files-from=sourcelist", "--size-only", "--delay-updates"] + rsync_defopts +
1224                         [Interpolate("--partial-dir=.~tmp~%(kw:target)s~%(kw:subtarget)s~%(prop:workername)s", target=ts[0], subtarget=ts[1]), "-a", "dl/", Interpolate("%(kw:url)s/", url=GetRsyncParams.withArgs("src", "url"))],
1225                 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("src", "key")) },
1226                 haltOnFailure = True,
1227                 logEnviron = False,
1228                 locks = NetLockUl,
1229                 doStepIf = util.Transform(bool, GetRsyncParams.withArgs("src", "url")),
1230         ))
1231
1232         factory.addStep(ShellCommand(
1233                 name = "df",
1234                 description = "Reporting disk usage",
1235                 command=["df", "-h", "."],
1236                 env={'LC_ALL': 'C'},
1237                 logEnviron = False,
1238                 haltOnFailure = False,
1239                 flunkOnFailure = False,
1240                 warnOnFailure = False,
1241                 alwaysRun = True,
1242         ))
1243
1244         factory.addStep(ShellCommand(
1245                 name = "du",
1246                 description = "Reporting estimated file space usage",
1247                 command=["du", "-sh", "."],
1248                 env={'LC_ALL': 'C'},
1249                 logEnviron = False,
1250                 haltOnFailure = False,
1251                 flunkOnFailure = False,
1252                 warnOnFailure = False,
1253                 alwaysRun = True,
1254         ))
1255
1256         factory.addStep(ShellCommand(
1257                 name = "ccachestat",
1258                 description = "Reporting ccache stats",
1259                 command=["ccache", "-s"],
1260                 logEnviron = False,
1261                 want_stderr = False,
1262                 haltOnFailure = False,
1263                 flunkOnFailure = False,
1264                 warnOnFailure = False,
1265                 doStepIf = util.Transform(bool, Property("ccache_command")),
1266         ))
1267
1268         for brname in branchNames:
1269                 bldrname = brname + "_" + target
1270                 c['builders'].append(BuilderConfig(name=bldrname, workernames=workerNames, factory=factory, tags=[brname,], nextBuild=GetNextBuild, canStartBuild=canStartBuild))
1271
1272
1273 ####### STATUS TARGETS
1274
1275 # 'status' is a list of Status Targets. The results of each build will be
1276 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
1277 # including web pages, email senders, and IRC bots.
1278
1279 if "status_bind" in inip1:
1280         c['www'] = {
1281                 'port': inip1.get("status_bind"),
1282                 'plugins': {
1283                         'waterfall_view': True,
1284                         'console_view': True,
1285                         'grid_view': True
1286                 }
1287         }
1288
1289         if "status_user" in inip1 and "status_password" in inip1:
1290                 c['www']['auth'] = util.UserPasswordAuth([
1291                         (inip1.get("status_user"), inip1.get("status_password"))
1292                 ])
1293                 c['www']['authz'] = util.Authz(
1294                         allowRules=[ util.AnyControlEndpointMatcher(role="admins") ],
1295                         roleMatchers=[ util.RolesFromUsername(roles=["admins"], usernames=[inip1.get("status_user")]) ]
1296                 )
1297
1298 c['services'] = []
1299 if ini.has_section("irc"):
1300         iniirc = ini['irc']
1301         irc_host = iniirc.get("host", None)
1302         irc_port = iniirc.getint("port", 6667)
1303         irc_chan = iniirc.get("channel", None)
1304         irc_nick = iniirc.get("nickname", None)
1305         irc_pass = iniirc.get("password", None)
1306
1307         if irc_host and irc_nick and irc_chan:
1308                 irc = reporters.IRC(irc_host, irc_nick,
1309                         port = irc_port,
1310                         password = irc_pass,
1311                         channels = [ irc_chan ],
1312                         notify_events = [ 'exception', 'problem', 'recovery' ]
1313                 )
1314
1315                 c['services'].append(irc)
1316
1317 c['revlink'] = util.RevlinkMatch([
1318         r'https://git.openwrt.org/openwrt/(.*).git'
1319         ],
1320         r'https://git.openwrt.org/?p=openwrt/\1.git;a=commit;h=%s')
1321
1322 ####### DB URL
1323
1324 c['db'] = {
1325         # This specifies what database buildbot uses to store its state.  You can leave
1326         # this at its default for all but the largest installations.
1327         'db_url' : "sqlite:///state.sqlite",
1328 }
1329
1330 c['buildbotNetUsageData'] = None