phase1/master.cfg

   1 # -*- python -*-
   2 # ex: set syntax=python:
   3
   4 import os
   5 import re
   6 import base64
   7 import subprocess
   8 import configparser
   9
  10 from dateutil.tz import tzutc
  11 from datetime import datetime, timedelta
  12
  13 from twisted.internet import defer
  14 from twisted.python import log
  15
  16 from buildbot import locks
  17 from buildbot.data import resultspec
  18 from buildbot.changes.gitpoller import GitPoller
  19 from buildbot.config import BuilderConfig
  20 from buildbot.plugins import reporters
  21 from buildbot.plugins import schedulers
  22 from buildbot.plugins import steps
  23 from buildbot.plugins import util
  24 from buildbot.process import properties
  25 from buildbot.process import results
  26 from buildbot.process.factory import BuildFactory
  27 from buildbot.process.properties import Interpolate
  28 from buildbot.process.properties import Property
  29 from buildbot.schedulers.basic import AnyBranchScheduler
  30 from buildbot.schedulers.forcesched import BaseParameter
  31 from buildbot.schedulers.forcesched import ForceScheduler
  32 from buildbot.schedulers.forcesched import ValidationError
  33 from buildbot.steps.master import MasterShellCommand
  34 from buildbot.steps.shell import SetPropertyFromCommand
  35 from buildbot.steps.shell import ShellCommand
  36 from buildbot.steps.source.git import Git
  37 from buildbot.steps.transfer import FileDownload
  38 from buildbot.steps.transfer import FileUpload
  39 from buildbot.steps.transfer import StringDownload
  40 from buildbot.worker import Worker
  41 from buildbot.worker.local import LocalWorker
  42
  43
  44 if not os.path.exists("twistd.pid"):
  45         with open("twistd.pid", "w") as pidfile:
  46                 pidfile.write("{}".format(os.getpid()))
  47
  48 # This is a sample buildmaster config file. It must be installed as
  49 # 'master.cfg' in your buildmaster's base directory.
  50
  51 ini = configparser.ConfigParser()
  52 ini.read(os.getenv("BUILDMASTER_CONFIG", "./config.ini"))
  53
  54 if "general" not in ini or "phase1" not in ini:
  55         raise ValueError("Fix your configuration")
  56
  57 inip1 = ini['phase1']
  58
  59 # Globals
  60 work_dir = os.path.abspath(ini['general'].get("workdir", "."))
  61 scripts_dir = os.path.abspath("../scripts")
  62
  63 repo_url = ini['repo'].get("url")
  64
  65 rsync_defopts = ["-v", "-4", "--timeout=120"]
  66
  67 #if rsync_bin_url.find("::") > 0 or rsync_bin_url.find("rsync://") == 0:
  68 #       rsync_bin_defopts += ["--contimeout=20"]
  69
  70 branches = {}
  71
  72 def ini_parse_branch(section):
  73         b = {}
  74         name = section.get("name")
  75
  76         if not name:
  77                 raise ValueError("missing 'name' in " + repr(section))
  78         if name in branches:
  79                 raise ValueError("duplicate branch name in " + repr(section))
  80
  81         b["name"] = name
  82         b["bin_url"] = section.get("binary_url")
  83         b["bin_key"] = section.get("binary_password")
  84
  85         b["src_url"] = section.get("source_url")
  86         b["src_key"] = section.get("source_password")
  87
  88         b["gpg_key"] = section.get("gpg_key")
  89
  90         b["usign_key"] = section.get("usign_key")
  91         usign_comment = "untrusted comment: " + name.replace("-", " ").title() + " key"
  92         b["usign_comment"] = section.get("usign_comment", usign_comment)
  93
  94         b["config_seed"] = section.get("config_seed")
  95
  96         b["kmod_archive"] = section.getboolean("kmod_archive", False)
  97
  98         branches[name] = b
  99         log.msg("Configured branch: {}".format(name))
 100
 101 # PB port can be either a numeric port or a connection string
 102 pb_port = inip1.get("port") or 9989
 103
 104 # This is the dictionary that the buildmaster pays attention to. We also use
 105 # a shorter alias to save typing.
 106 c = BuildmasterConfig = {}
 107
 108 ####### PROJECT IDENTITY
 109
 110 # the 'title' string will appear at the top of this buildbot
 111 # installation's html.WebStatus home page (linked to the
 112 # 'titleURL') and is embedded in the title of the waterfall HTML page.
 113
 114 c['title'] = ini['general'].get("title")
 115 c['titleURL'] = ini['general'].get("title_url")
 116
 117 # the 'buildbotURL' string should point to the location where the buildbot's
 118 # internal web server (usually the html.WebStatus page) is visible. This
 119 # typically uses the port number set in the Waterfall 'status' entry, but
 120 # with an externally-visible host name which the buildbot cannot figure out
 121 # without some help.
 122
 123 c['buildbotURL'] = inip1.get("buildbot_url")
 124
 125 ####### BUILDWORKERS
 126
 127 # The 'workers' list defines the set of recognized buildworkers. Each element is
 128 # a Worker object, specifying a unique worker name and password.  The same
 129 # worker name and password must be configured on the worker.
 130
 131 c['workers'] = []
 132 NetLocks = dict()
 133
 134 for section in ini.sections():
 135         if section.startswith("branch "):
 136                 ini_parse_branch(ini[section])
 137
 138         if section.startswith("worker "):
 139                 if ini.has_option(section, "name") and ini.has_option(section, "password") and \
 140                    (not ini.has_option(section, "phase") or ini.getint(section, "phase") == 1):
 141                         sl_props = { 'dl_lock':None, 'ul_lock':None }
 142                         name = ini.get(section, "name")
 143                         password = ini.get(section, "password")
 144                         if ini.has_option(section, "dl_lock"):
 145                                 lockname = ini.get(section, "dl_lock")
 146                                 sl_props['dl_lock'] = lockname
 147                                 if lockname not in NetLocks:
 148                                         NetLocks[lockname] = locks.MasterLock(lockname)
 149                         if ini.has_option(section, "ul_lock"):
 150                                 lockname = ini.get(section, "ul_lock")
 151                                 sl_props['ul_lock'] = lockname
 152                                 if lockname not in NetLocks:
 153                                         NetLocks[lockname] = locks.MasterLock(lockname)
 154                         c['workers'].append(Worker(name, password, max_builds = 1, properties = sl_props))
 155
 156 c['protocols'] = {'pb': {'port': pb_port}}
 157
 158 # coalesce builds
 159 c['collapseRequests'] = True
 160
 161 # Reduce amount of backlog data
 162 c['configurators'] = [util.JanitorConfigurator(
 163         logHorizon=timedelta(days=3),
 164         hour=6,
 165 )]
 166
 167 @defer.inlineCallbacks
 168 def getNewestCompleteTime(bldr):
 169         """Returns the complete_at of the latest completed and not SKIPPED
 170         build request for this builder, or None if there are no such build
 171         requests. We need to filter out SKIPPED requests because we're
 172         using collapseRequests=True which is unfortunately marking all
 173         previous requests as complete when new buildset is created.
 174
 175         @returns: datetime instance or None, via Deferred
 176         """
 177
 178         bldrid = yield bldr.getBuilderId()
 179         completed = yield bldr.master.data.get(
 180                         ('builders', bldrid, 'buildrequests'),
 181                         [
 182                                 resultspec.Filter('complete', 'eq', [True]),
 183                                 resultspec.Filter('results', 'ne', [results.SKIPPED]),
 184                         ],
 185                         order=['-complete_at'], limit=1)
 186         if not completed:
 187                 return
 188
 189         complete_at = completed[0]['complete_at']
 190
 191         last_build = yield bldr.master.data.get(
 192                         ('builds', ),
 193                         [
 194                                 resultspec.Filter('builderid', 'eq', [bldrid]),
 195                         ],
 196                         order=['-started_at'], limit=1)
 197
 198         if last_build and last_build[0]:
 199                 last_complete_at = last_build[0]['complete_at']
 200                 if last_complete_at and (last_complete_at > complete_at):
 201                         return last_complete_at
 202
 203         return complete_at
 204
 205 @defer.inlineCallbacks
 206 def prioritizeBuilders(master, builders):
 207         """Returns sorted list of builders by their last timestamp of completed and
 208         not skipped build, ordered first by branch name.
 209
 210         @returns: list of sorted builders
 211         """
 212
 213         bldrNamePrio = { "__Janitor": 0, "00_force_build": 0 }
 214         i = 1
 215         for bname in branchNames:
 216                 bldrNamePrio[bname] = i
 217                 i += 1
 218
 219         def is_building(bldr):
 220                 return bool(bldr.building) or bool(bldr.old_building)
 221
 222         def bldr_info(bldr):
 223                 d = defer.maybeDeferred(getNewestCompleteTime, bldr)
 224                 d.addCallback(lambda complete_at: (complete_at, bldr))
 225                 return d
 226
 227         def bldr_sort(item):
 228                 (complete_at, bldr) = item
 229
 230                 pos = 99
 231                 for (name, prio) in bldrNamePrio.items():
 232                         if bldr.name.startswith(name):
 233                                 pos = prio
 234                                 break
 235
 236                 if not complete_at:
 237                         date = datetime.min
 238                         complete_at = date.replace(tzinfo=tzutc())
 239
 240                 if is_building(bldr):
 241                         date = datetime.max
 242                         complete_at = date.replace(tzinfo=tzutc())
 243
 244                 return (pos, complete_at, bldr.name)
 245
 246         results = yield defer.gatherResults([bldr_info(bldr) for bldr in builders])
 247         results.sort(key=bldr_sort)
 248
 249         for r in results:
 250                 log.msg("prioritizeBuilders: {:>20} complete_at: {}".format(r[1].name, r[0]))
 251
 252         return [r[1] for r in results]
 253
 254 c['prioritizeBuilders'] = prioritizeBuilders
 255
 256 ####### CHANGESOURCES
 257
 258 branchNames = [branches[b]["name"] for b in branches]
 259
 260 # find targets
 261 targets = set()
 262
 263 def populateTargets():
 264         log.msg("Populating targets, this will take time")
 265         sourcegit = work_dir + '/source.git'
 266         for branch in branchNames:
 267                 if os.path.isdir(sourcegit):
 268                         subprocess.call(["rm", "-rf", sourcegit])
 269
 270                 subprocess.call(["git", "clone", "-q", "--depth=1", "--branch="+branch, repo_url, sourcegit])
 271
 272                 os.makedirs(sourcegit + '/tmp', exist_ok=True)
 273                 findtargets = subprocess.Popen(['./scripts/dump-target-info.pl', 'targets'],
 274                         stdout = subprocess.PIPE, stderr = subprocess.DEVNULL, cwd = sourcegit)
 275
 276                 while True:
 277                         line = findtargets.stdout.readline()
 278                         if not line:
 279                                 break
 280                         ta = line.decode().strip().split(' ')
 281                         targets.add(ta[0])
 282
 283                 subprocess.call(["rm", "-rf", sourcegit])
 284
 285 populateTargets()
 286
 287 # the 'change_source' setting tells the buildmaster how it should find out
 288 # about source code changes.  Here we point to the buildbot clone of pyflakes.
 289
 290 c['change_source'] = []
 291 c['change_source'].append(GitPoller(
 292         repo_url,
 293         workdir=work_dir+'/work.git', branches=branchNames,
 294         pollAtLaunch=True, pollinterval=300))
 295
 296 ####### SCHEDULERS
 297
 298 # Configure the Schedulers, which decide how to react to incoming changes.  In this
 299 # case, just kick off a 'basebuild' build
 300
 301 class TagChoiceParameter(BaseParameter):
 302         spec_attributes = ["strict", "choices"]
 303         type = "list"
 304         strict = True
 305
 306         def __init__(self, name, label=None, **kw):
 307                 super().__init__(name, label, **kw)
 308                 self._choice_list = []
 309
 310         def getRevTags(self, findtag=None):
 311                 taglist = []
 312                 branchvers = []
 313
 314                 for b in branchNames:
 315                         basever = re.search(r'-([0-9]+\.[0-9]+)$', b)
 316                         if basever:
 317                                 branchvers.append(basever[1])
 318
 319                 alltags = subprocess.Popen(
 320                         ['git', 'ls-remote', '--tags', repo_url],
 321                         stdout = subprocess.PIPE)
 322
 323                 while True:
 324                         line = alltags.stdout.readline()
 325
 326                         if not line:
 327                                 break
 328
 329                         (rev, tag) = line.split()
 330
 331                         tagver = re.search(r'\brefs/tags/(v[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?)$', tag.decode().strip())
 332
 333                         # only list tags matching configured branches
 334                         if tagver and any(tagver[1][1:].startswith(b) for b in branchvers):
 335                                 if findtag and findtag != tagver[1]:
 336                                         continue
 337                                 taglist.append({'rev': rev.decode().strip(), 'tag': tagver[1]})
 338
 339                 return taglist
 340
 341         @property
 342         def choices(self):
 343                 taglist = [rt['tag'] for rt in self.getRevTags()]
 344                 taglist.sort(reverse=True, key=lambda tag: tag if re.search(r'-rc[0-9]+$', tag) else tag + '-z')
 345                 taglist.insert(0, '')
 346
 347                 self._choice_list = taglist
 348
 349                 return self._choice_list
 350
 351         def updateFromKwargs(self, properties, kwargs, **unused):
 352                 tag = self.getFromKwargs(kwargs)
 353                 properties[self.name] = tag
 354
 355                 # find the commit matching the tag
 356                 findtag = self.getRevTags(tag)
 357
 358                 if not findtag:
 359                         raise ValidationError("Couldn't find tag")
 360
 361                 properties['force_revision'] = findtag[0]['rev']
 362
 363                 # find the branch matching the tag
 364                 branch = None
 365                 branchver = re.search(r'v([0-9]+\.[0-9]+)', tag)
 366                 for b in branchNames:
 367                         if b.endswith(branchver[1]):
 368                                 branch = b
 369
 370                 if not branch:
 371                         raise ValidationError("Couldn't find branch")
 372
 373                 properties['force_branch'] = branch
 374
 375         def parse_from_arg(self, s):
 376                 if self.strict and s not in self._choice_list:
 377                         raise ValidationError("'%s' does not belong to list of available choices '%s'" % (s, self._choice_list))
 378                 return s
 379
 380 @util.renderer
 381 @defer.inlineCallbacks
 382 def builderNames(props):
 383         """ since we have per branch and per target builders,
 384         address the relevant builder for each new buildrequest
 385         based on the request's desired branch and target.
 386         """
 387         branch = props.getProperty("branch")
 388         target = props.getProperty("target", "")
 389
 390         if target == "all":
 391                 target = ""
 392
 393         # if that didn't work, try sourcestamp to find a branch
 394         if not branch:
 395                 # match builders with target branch
 396                 ss = props.sourcestamps[0]
 397                 if ss:
 398                         branch = ss['branch']
 399                 else:
 400                         log.msg("couldn't find builder")
 401                         return []       # nothing works
 402
 403         bname = branch + "_" + target
 404         builders = []
 405
 406         for b in (yield props.master.data.get(('builders',))):
 407                 if not b['name'].startswith(bname):
 408                         continue
 409                 builders.append(b['name'])
 410
 411         return builders
 412
 413 c['schedulers'] = []
 414 c['schedulers'].append(AnyBranchScheduler(
 415         name            = "all",
 416         change_filter   = util.ChangeFilter(branch=branchNames),
 417         treeStableTimer = 15*60,
 418         builderNames    = builderNames))
 419
 420 c['schedulers'].append(ForceScheduler(
 421         name         = "force",
 422         buttonName   = "Force builds",
 423         label        = "Force build details",
 424         builderNames = [ "00_force_build" ],
 425
 426         codebases = [
 427                 util.CodebaseParameter(
 428                         "",
 429                         label      = "Repository",
 430                         branch     = util.FixedParameter(name = "branch",     default = ""),
 431                         revision   = util.FixedParameter(name = "revision",   default = ""),
 432                         repository = util.FixedParameter(name = "repository", default = ""),
 433                         project    = util.FixedParameter(name = "project",    default = "")
 434                 )
 435         ],
 436
 437         reason = util.StringParameter(
 438                 name     = "reason",
 439                 label    = "Reason",
 440                 default  = "Trigger build",
 441                 required = True,
 442                 size     = 80
 443         ),
 444
 445         properties = [
 446                 util.ChoiceStringParameter(
 447                         name    = "target",
 448                         label   = "Build target",
 449                         default = "all",
 450                         choices = [ "all" ] + list(targets)
 451                 ),
 452                 TagChoiceParameter(
 453                         name    = "tag",
 454                         label   = "Build tag",
 455                         default = ""
 456                 )
 457         ]
 458 ))
 459
 460 c['schedulers'].append(schedulers.Triggerable(name="trigger", builderNames=builderNames))
 461
 462 ####### BUILDERS
 463
 464 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
 465 # what steps, and which workers can execute them.  Note that any particular build will
 466 # only take place on one worker.
 467
 468 def IsNoMasterBuild(step):
 469         return step.getProperty("branch") != "master"
 470
 471 def IsUsignEnabled(step):
 472         branch = step.getProperty("branch")
 473         return branch and branches[branch].get("usign_key")
 474
 475 def IsSignEnabled(step):
 476         branch = step.getProperty("branch")
 477         return IsUsignEnabled(step) or branch and branches[branch].get("gpg_key")
 478
 479 def IsKmodArchiveEnabled(step):
 480         branch = step.getProperty("branch")
 481         return branch and branches[branch].get("kmod_archive")
 482
 483 def GetBaseVersion(branch):
 484         if re.match(r"^[^-]+-[0-9]+\.[0-9]+$", branch):
 485                 return branch.split('-')[1]
 486         else:
 487                 return "master"
 488
 489 @properties.renderer
 490 def GetVersionPrefix(props):
 491         branch = props.getProperty("branch")
 492         basever = GetBaseVersion(branch)
 493         if props.hasProperty("tag") and re.match(r"^v[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", props["tag"]):
 494                 return "%s/" % props["tag"][1:]
 495         elif basever != "master":
 496                 return "%s-SNAPSHOT/" % basever
 497         else:
 498                 return ""
 499
 500 @util.renderer
 501 def GetConfigSeed(props):
 502         branch = props.getProperty("branch")
 503         return branch and branches[branch].get("config_seed") or ""
 504
 505 @util.renderer
 506 def GetRsyncParams(props, srcorbin, urlorkey):
 507         # srcorbin: 'bin' or 'src'; urlorkey: 'url' or 'key'
 508         branch = props.getProperty("branch")
 509         opt = srcorbin + "_" + urlorkey
 510         return branch and branches[branch].get(opt)
 511
 512 @util.renderer
 513 def GetUsignKey(props):
 514         branch = props.getProperty("branch")
 515         return branch and branches[branch].get("usign_key")
 516
 517 def GetNextBuild(builder, requests):
 518         for r in requests:
 519                 if r.properties:
 520                         # order tagged build first
 521                         if r.properties.hasProperty("tag"):
 522                                 return r
 523
 524         r = requests[0]
 525         log.msg("GetNextBuild: {:>20} id: {} bsid: {}".format(builder.name, r.id, r.bsid))
 526         return r
 527
 528 def MakeEnv(overrides=None, tryccache=False):
 529         env = {
 530                 'CCC': Interpolate("%(prop:cc_command:-gcc)s"),
 531                 'CCXX': Interpolate("%(prop:cxx_command:-g++)s"),
 532         }
 533         if tryccache:
 534                 env['CC'] = Interpolate("%(prop:builddir)s/ccache_cc.sh")
 535                 env['CXX'] = Interpolate("%(prop:builddir)s/ccache_cxx.sh")
 536                 env['CCACHE'] = Interpolate("%(prop:ccache_command:-)s")
 537         else:
 538                 env['CC'] = env['CCC']
 539                 env['CXX'] = env['CCXX']
 540                 env['CCACHE'] = ''
 541         if overrides is not None:
 542                 env.update(overrides)
 543         return env
 544
 545 @properties.renderer
 546 def NetLockDl(props, extralock=None):
 547         lock = None
 548         if props.hasProperty("dl_lock"):
 549                 lock = NetLocks[props["dl_lock"]]
 550         if lock is not None:
 551                 return [lock.access('exclusive')]
 552         else:
 553                 return []
 554
 555 @properties.renderer
 556 def NetLockUl(props):
 557         lock = None
 558         if props.hasProperty("ul_lock"):
 559                 lock = NetLocks[props["ul_lock"]]
 560         if lock is not None:
 561                 return [lock.access('exclusive')]
 562         else:
 563                 return []
 564
 565 def IsTargetSelected(target):
 566         def CheckTargetProperty(step):
 567                 selected_target = step.getProperty("target", "all")
 568                 if selected_target != "all" and selected_target != target:
 569                         return False
 570                 return True
 571
 572         return CheckTargetProperty
 573
 574 @util.renderer
 575 def UsignSec2Pub(props):
 576         branch = props.getProperty("branch")
 577         try:
 578                 comment = branches[branch].get("usign_comment") or "untrusted comment: secret key"
 579                 seckey = branches[branch].get("usign_key")
 580                 seckey = base64.b64decode(seckey)
 581         except:
 582                 return None
 583
 584         return "{}\n{}".format(re.sub(r"\bsecret key$", "public key", comment),
 585                 base64.b64encode(seckey[0:2] + seckey[32:40] + seckey[72:]))
 586
 587
 588 c['builders'] = []
 589
 590 workerNames = [ ]
 591
 592 for worker in c['workers']:
 593         workerNames.append(worker.workername)
 594
 595 # add a single LocalWorker to handle the forcebuild builder
 596 c['workers'].append(LocalWorker("__local_force_build", max_builds=1))
 597
 598 force_factory = BuildFactory()
 599 force_factory.addStep(steps.Trigger(
 600         name = "trigger_build",
 601         schedulerNames = [ "trigger" ],
 602         sourceStamps = [{ "codebase": "", "branch": Property("force_branch"), "revision": Property("force_revision"), "repository": repo_url, "project": "" }],
 603         set_properties = { "reason": Property("reason"), "tag": Property("tag"), "target": Property("target") },
 604 ))
 605
 606 c['builders'].append(BuilderConfig(
 607         name        = "00_force_build",
 608         workername  = "__local_force_build",
 609         factory     = force_factory))
 610
 611 for target in targets:
 612         ts = target.split('/')
 613
 614         factory = BuildFactory()
 615
 616         # setup shared work directory if required
 617         factory.addStep(ShellCommand(
 618                 name = "sharedwd",
 619                 descriptionDone = "Shared work directory set up",
 620                 command = 'test -L "$PWD" || (mkdir -p ../shared-workdir && rm -rf "$PWD" && ln -s shared-workdir "$PWD")',
 621                 workdir = ".",
 622                 haltOnFailure = True,
 623         ))
 624
 625         # find number of cores
 626         factory.addStep(SetPropertyFromCommand(
 627                 name = "nproc",
 628                 property = "nproc",
 629                 description = "Finding number of CPUs",
 630                 command = ["nproc"],
 631         ))
 632
 633         # find gcc and g++ compilers
 634         factory.addStep(FileDownload(
 635                 name = "dlfindbinpl",
 636                 mastersrc = scripts_dir + '/findbin.pl',
 637                 workerdest = "../findbin.pl",
 638                 mode = 0o755,
 639         ))
 640
 641         factory.addStep(SetPropertyFromCommand(
 642                 name = "gcc",
 643                 property = "cc_command",
 644                 description = "Finding gcc command",
 645                 command = ["../findbin.pl", "gcc", "", ""],
 646                 haltOnFailure = True,
 647         ))
 648
 649         factory.addStep(SetPropertyFromCommand(
 650                 name = "g++",
 651                 property = "cxx_command",
 652                 description = "Finding g++ command",
 653                 command = ["../findbin.pl", "g++", "", ""],
 654                 haltOnFailure = True,
 655         ))
 656
 657         # see if ccache is available
 658         factory.addStep(SetPropertyFromCommand(
 659                 name = "ccache",
 660                 property = "ccache_command",
 661                 description = "Testing for ccache command",
 662                 command = ["which", "ccache"],
 663                 haltOnFailure = False,
 664                 flunkOnFailure = False,
 665                 warnOnFailure = False,
 666                 hideStepIf = lambda r, s: r==results.FAILURE,
 667         ))
 668
 669         # check out the source
 670         # Git() runs:
 671           # if repo doesn't exist: 'git clone repourl'
 672           # method 'clean' runs 'git clean -d -f', method fresh runs 'git clean -f -f -d -x'. Only works with mode='full'
 673           # git cat-file -e <commit>
 674           # git checkout -f <commit>
 675           # git checkout -B <branch>
 676           # git rev-parse HEAD
 677         factory.addStep(Git(
 678                 name = "git",
 679                 repourl = repo_url,
 680                 mode = 'full',
 681                 method = 'fresh',
 682                 locks = NetLockDl,
 683                 haltOnFailure = True,
 684         ))
 685
 686         # update remote refs
 687         factory.addStep(ShellCommand(
 688                 name = "fetchrefs",
 689                 description = "Fetching Git remote refs",
 690                 command = ["git", "fetch", "origin", Interpolate("+refs/heads/%(prop:branch)s:refs/remotes/origin/%(prop:branch)s")],
 691                 haltOnFailure = True,
 692         ))
 693
 694         # Verify that Git HEAD points to a tag or branch
 695         # Ref: https://web.archive.org/web/20190729224316/http://lists.infradead.org/pipermail/openwrt-devel/2019-June/017809.html
 696         factory.addStep(ShellCommand(
 697                 name = "gitverify",
 698                 description = "Ensure that Git HEAD is pointing to a branch or tag",
 699                 command = 'git rev-parse --abbrev-ref HEAD | grep -vxqF HEAD || git show-ref --tags --dereference 2>/dev/null | sed -ne "/^$(git rev-parse HEAD) / { s|^.*/||; s|\\^.*||; p }" | grep -qE "^v[0-9][0-9]\\."',
 700                 haltOnFailure = True,
 701         ))
 702
 703         factory.addStep(ShellCommand(
 704                 name = "rmtmp",
 705                 description = "Remove tmp folder",
 706                 command=["rm", "-rf", "tmp/"],
 707         ))
 708
 709         # feed
 710         factory.addStep(ShellCommand(
 711                 name = "rmfeedlinks",
 712                 description = "Remove feed symlinks",
 713                 command=["rm", "-rf", "package/feeds/"],
 714         ))
 715
 716         factory.addStep(StringDownload(
 717                 name = "ccachecc",
 718                 s = '#!/bin/sh\nexec ${CCACHE} ${CCC} "$@"\n',
 719                 workerdest = "../ccache_cc.sh",
 720                 mode = 0o755,
 721         ))
 722
 723         factory.addStep(StringDownload(
 724                 name = "ccachecxx",
 725                 s = '#!/bin/sh\nexec ${CCACHE} ${CCXX} "$@"\n',
 726                 workerdest = "../ccache_cxx.sh",
 727                 mode = 0o755,
 728         ))
 729
 730         # feed
 731         factory.addStep(ShellCommand(
 732                 name = "updatefeeds",
 733                 description = "Updating feeds",
 734                 command=["./scripts/feeds", "update"],
 735                 env = MakeEnv(tryccache=True),
 736                 haltOnFailure = True,
 737                 locks = NetLockDl,
 738         ))
 739
 740         # feed
 741         factory.addStep(ShellCommand(
 742                 name = "installfeeds",
 743                 description = "Installing feeds",
 744                 command=["./scripts/feeds", "install", "-a"],
 745                 env = MakeEnv(tryccache=True),
 746                 haltOnFailure = True,
 747         ))
 748
 749         # seed config
 750         factory.addStep(StringDownload(
 751                 name = "dlconfigseed",
 752                 s = Interpolate("%(kw:seed)s\n", seed=GetConfigSeed),
 753                 workerdest = ".config",
 754                 mode = 0o644,
 755         ))
 756
 757         # configure
 758         factory.addStep(ShellCommand(
 759                 name = "newconfig",
 760                 descriptionDone = ".config seeded",
 761                 command = Interpolate("printf 'CONFIG_TARGET_%(kw:target)s=y\\nCONFIG_TARGET_%(kw:target)s_%(kw:subtarget)s=y\\nCONFIG_SIGNED_PACKAGES=%(kw:usign:#?|y|n)s\\n' >> .config", target=ts[0], subtarget=ts[1], usign=GetUsignKey),
 762         ))
 763
 764         factory.addStep(ShellCommand(
 765                 name = "delbin",
 766                 description = "Removing output directory",
 767                 command = ["rm", "-rf", "bin/"],
 768         ))
 769
 770         factory.addStep(ShellCommand(
 771                 name = "defconfig",
 772                 description = "Populating .config",
 773                 command = ["make", "defconfig"],
 774                 env = MakeEnv(),
 775         ))
 776
 777         # check arch - exit early if does not exist - NB: some targets do not define CONFIG_TARGET_target_subtarget
 778         factory.addStep(ShellCommand(
 779                 name = "checkarch",
 780                 description = "Checking architecture",
 781                 descriptionDone = "Architecture validated",
 782                 command = 'grep -sq CONFIG_TARGET_%s=y .config && grep -sq CONFIG_TARGET_SUBTARGET=\\"%s\\" .config' %(ts[0], ts[1]),
 783                 logEnviron = False,
 784                 want_stdout = False,
 785                 want_stderr = False,
 786                 haltOnFailure = True,
 787                 flunkOnFailure = False, # this is not a build FAILURE
 788         ))
 789
 790         # find libc suffix
 791         factory.addStep(SetPropertyFromCommand(
 792                 name = "libc",
 793                 property = "libc",
 794                 description = "Finding libc suffix",
 795                 command = ["sed", "-ne", '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }', ".config"],
 796         ))
 797
 798         # install build key
 799         factory.addStep(StringDownload(
 800                 name = "dlkeybuildpub",
 801                 s = Interpolate("%(kw:sec2pub)s", sec2pub=UsignSec2Pub),
 802                 workerdest = "key-build.pub",
 803                 mode = 0o600,
 804                 doStepIf = IsUsignEnabled,
 805         ))
 806
 807         factory.addStep(StringDownload(
 808                 name = "dlkeybuild",
 809                 s = "# fake private key",
 810                 workerdest = "key-build",
 811                 mode = 0o600,
 812                 doStepIf = IsUsignEnabled,
 813         ))
 814
 815         factory.addStep(StringDownload(
 816                 name = "dlkeybuilducert",
 817                 s = "# fake certificate",
 818                 workerdest = "key-build.ucert",
 819                 mode = 0o600,
 820                 doStepIf = IsUsignEnabled,
 821         ))
 822
 823         # prepare dl
 824         factory.addStep(ShellCommand(
 825                 name = "dldir",
 826                 description = "Preparing dl/",
 827                 descriptionDone = "dl/ prepared",
 828                 command = 'mkdir -p ../dl && rm -rf "build/dl" && ln -s ../../dl "build/dl"',
 829                 workdir = Property("builddir"),
 830                 logEnviron = False,
 831                 want_stdout = False,
 832         ))
 833
 834         # cleanup dl
 835         factory.addStep(ShellCommand(
 836                 name = "dlprune",
 837                 description = "Pruning dl/",
 838                 descriptionDone = "dl/ pruned",
 839                 command = 'find dl/ -atime +15 -delete -print',
 840                 logEnviron = False,
 841         ))
 842
 843         # prepare tar
 844         factory.addStep(ShellCommand(
 845                 name = "dltar",
 846                 description = "Building and installing GNU tar",
 847                 descriptionDone = "GNU tar built and installed",
 848                 command = ["make", Interpolate("-j%(prop:nproc:-1)s"), "tools/tar/compile", "V=s"],
 849                 env = MakeEnv(tryccache=True),
 850                 haltOnFailure = True,
 851         ))
 852
 853         # populate dl
 854         factory.addStep(ShellCommand(
 855                 name = "dlrun",
 856                 description = "Populating dl/",
 857                 descriptionDone = "dl/ populated",
 858                 command = ["make", Interpolate("-j%(prop:nproc:-1)s"), "download", "V=s"],
 859                 env = MakeEnv(),
 860                 logEnviron = False,
 861                 locks = NetLockDl,
 862         ))
 863
 864         factory.addStep(ShellCommand(
 865                 name = "cleanbase",
 866                 description = "Cleaning base-files",
 867                 command=["make", "package/base-files/clean", "V=s"],
 868         ))
 869
 870         # build
 871         factory.addStep(ShellCommand(
 872                 name = "tools",
 873                 description = "Building and installing tools",
 874                 descriptionDone = "Tools built and installed",
 875                 command = ["make", Interpolate("-j%(prop:nproc:-1)s"), "tools/install", "V=s"],
 876                 env = MakeEnv(tryccache=True),
 877                 haltOnFailure = True,
 878         ))
 879
 880         factory.addStep(ShellCommand(
 881                 name = "toolchain",
 882                 description = "Building and installing toolchain",
 883                 descriptionDone = "Toolchain built and installed",
 884                 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "toolchain/install", "V=s"],
 885                 env = MakeEnv(),
 886                 haltOnFailure = True,
 887         ))
 888
 889         factory.addStep(ShellCommand(
 890                 name = "kmods",
 891                 description = "Building kmods",
 892                 descriptionDone = "Kmods built",
 893                 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
 894                 env = MakeEnv(),
 895                 haltOnFailure = True,
 896         ))
 897
 898         # find kernel version
 899         factory.addStep(SetPropertyFromCommand(
 900                 name = "kernelversion",
 901                 property = "kernelversion",
 902                 description = "Finding the effective Kernel version",
 903                 command = "make --no-print-directory -C target/linux/ val.LINUX_VERSION val.LINUX_RELEASE val.LINUX_VERMAGIC | xargs printf '%s-%s-%s\\n'",
 904                 env = { 'TOPDIR': Interpolate("%(prop:builddir)s/build") },
 905         ))
 906
 907         factory.addStep(ShellCommand(
 908                 name = "pkgclean",
 909                 description = "Cleaning up package build",
 910                 descriptionDone = "Package build cleaned up",
 911                 command=["make", "package/cleanup", "V=s"],
 912         ))
 913
 914         factory.addStep(ShellCommand(
 915                 name = "pkgbuild",
 916                 description = "Building packages",
 917                 descriptionDone = "Packages built",
 918                 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
 919                 env = MakeEnv(),
 920                 haltOnFailure = True,
 921         ))
 922
 923         factory.addStep(ShellCommand(
 924                 name = "pkginstall",
 925                 description = "Installing packages",
 926                 descriptionDone = "Packages installed",
 927                 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/install", "V=s"],
 928                 env = MakeEnv(),
 929                 haltOnFailure = True,
 930         ))
 931
 932         factory.addStep(ShellCommand(
 933                 name = "pkgindex",
 934                 description = "Indexing packages",
 935                 descriptionDone = "Packages indexed",
 936                 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/index", "V=s", "CONFIG_SIGNED_PACKAGES="],
 937                 env = MakeEnv(),
 938                 haltOnFailure = True,
 939         ))
 940
 941         factory.addStep(ShellCommand(
 942                 name = "images",
 943                 description = "Building and installing images",
 944                 descriptionDone = "Images built and installed",
 945                 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "target/install", "V=s"],
 946                 env = MakeEnv(),
 947                 haltOnFailure = True,
 948         ))
 949
 950         factory.addStep(ShellCommand(
 951                 name = "buildinfo",
 952                 description = "Generating config.buildinfo, version.buildinfo and feeds.buildinfo",
 953                 command = "make -j1 buildinfo V=s || true",
 954                 env = MakeEnv(),
 955                 haltOnFailure = True,
 956         ))
 957
 958         factory.addStep(ShellCommand(
 959                 name = "json_overview_image_info",
 960                 description = "Generating profiles.json in target folder",
 961                 command = "make -j1 json_overview_image_info V=s || true",
 962                 env = MakeEnv(),
 963                 haltOnFailure = True,
 964         ))
 965
 966         factory.addStep(ShellCommand(
 967                 name = "checksums",
 968                 description = "Calculating checksums",
 969                 descriptionDone = "Checksums calculated",
 970                 command=["make", "-j1", "checksum", "V=s"],
 971                 env = MakeEnv(),
 972                 haltOnFailure = True,
 973         ))
 974
 975         factory.addStep(ShellCommand(
 976                 name = "kmoddir",
 977                 descriptionDone = "Kmod directory created",
 978                 command=["mkdir", "-p", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s", target=ts[0], subtarget=ts[1])],
 979                 haltOnFailure = True,
 980                 doStepIf = IsKmodArchiveEnabled,
 981         ))
 982
 983         factory.addStep(ShellCommand(
 984                 name = "kmodprepare",
 985                 description = "Preparing kmod archive",
 986                 descriptionDone = "Kmod archive prepared",
 987                 command=["rsync", "--include=/kmod-*.ipk", "--exclude=*", "-va",
 988                         Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/packages/", target=ts[0], subtarget=ts[1]),
 989                         Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1])],
 990                 haltOnFailure = True,
 991                 doStepIf = IsKmodArchiveEnabled,
 992         ))
 993
 994         factory.addStep(ShellCommand(
 995                 name = "kmodindex",
 996                 description = "Indexing kmod archive",
 997                 descriptionDone = "Kmod archive indexed",
 998                 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/index", "V=s", "CONFIG_SIGNED_PACKAGES=",
 999                         Interpolate("PACKAGE_SUBDIRS=bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1])],
1000                 env = MakeEnv(),
1001                 haltOnFailure = True,
1002                 doStepIf = IsKmodArchiveEnabled,
1003         ))
1004
1005         # sign
1006         factory.addStep(MasterShellCommand(
1007                 name = "signprepare",
1008                 descriptionDone = "Temporary signing directory prepared",
1009                 command = ["mkdir", "-p", "%s/signing" %(work_dir)],
1010                 haltOnFailure = True,
1011                 doStepIf = IsSignEnabled,
1012
1013         ))
1014
1015         factory.addStep(ShellCommand(
1016                 name = "signpack",
1017                 description = "Packing files to sign",
1018                 descriptionDone = "Files to sign packed",
1019                 command = Interpolate("find bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/ bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz", target=ts[0], subtarget=ts[1]),
1020                 haltOnFailure = True,
1021                 doStepIf = IsSignEnabled,
1022         ))
1023
1024         factory.addStep(FileUpload(
1025                 workersrc = "sign.tar.gz",
1026                 masterdest = "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1]),
1027                 haltOnFailure = True,
1028                 doStepIf = IsSignEnabled,
1029         ))
1030
1031         factory.addStep(MasterShellCommand(
1032                 name = "signfiles",
1033                 description = "Signing files",
1034                 descriptionDone = "Files signed",
1035                 command = ["%s/signall.sh" %(scripts_dir), "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1]), Interpolate("%(prop:branch)s")],
1036                 env = { 'CONFIG_INI': os.getenv("BUILDMASTER_CONFIG", "./config.ini") },
1037                 haltOnFailure = True,
1038                 doStepIf = IsSignEnabled,
1039         ))
1040
1041         factory.addStep(FileDownload(
1042                 name = "dlsigntargz",
1043                 mastersrc = "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1]),
1044                 workerdest = "sign.tar.gz",
1045                 haltOnFailure = True,
1046                 doStepIf = IsSignEnabled,
1047         ))
1048
1049         factory.addStep(ShellCommand(
1050                 name = "signunpack",
1051                 description = "Unpacking signed files",
1052                 descriptionDone = "Signed files unpacked",
1053                 command = ["tar", "-xzf", "sign.tar.gz"],
1054                 haltOnFailure = True,
1055                 doStepIf = IsSignEnabled,
1056         ))
1057
1058         # upload
1059         factory.addStep(ShellCommand(
1060                 name = "dirprepare",
1061                 descriptionDone = "Upload directory structure prepared",
1062                 command = ["mkdir", "-p", Interpolate("tmp/upload/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s", target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1063                 haltOnFailure = True,
1064         ))
1065
1066         factory.addStep(ShellCommand(
1067                 name = "linkprepare",
1068                 descriptionDone = "Repository symlink prepared",
1069                 command = ["ln", "-s", "-f", Interpolate("../packages-%(kw:basever)s", basever=util.Transform(GetBaseVersion, Property("branch"))), Interpolate("tmp/upload/%(kw:prefix)spackages", prefix=GetVersionPrefix)],
1070                 doStepIf = IsNoMasterBuild,
1071                 haltOnFailure = True,
1072         ))
1073
1074         factory.addStep(ShellCommand(
1075                 name = "kmoddirprepare",
1076                 descriptionDone = "Kmod archive upload directory prepared",
1077                 command = ["mkdir", "-p", Interpolate("tmp/upload/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s", target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1078                 haltOnFailure = True,
1079                 doStepIf = IsKmodArchiveEnabled,
1080         ))
1081
1082         factory.addStep(ShellCommand(
1083                 name = "dirupload",
1084                 description = "Uploading directory structure",
1085                 descriptionDone = "Directory structure uploaded",
1086                 command = ["rsync", "-az"] + rsync_defopts + ["tmp/upload/", Interpolate("%(kw:url)s/", url=GetRsyncParams.withArgs("bin", "url"))],
1087                 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1088                 haltOnFailure = True,
1089                 logEnviron = False,
1090                 locks = NetLockUl,
1091                 doStepIf = util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
1092         ))
1093
1094         # download remote sha256sums to 'target-sha256sums'
1095         factory.addStep(ShellCommand(
1096                 name = "target-sha256sums",
1097                 description = "Fetching remote sha256sums for target",
1098                 descriptionDone = "Remote sha256sums for target fetched",
1099                 command = ["rsync", "-z"] + rsync_defopts + [Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/sha256sums", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix), "target-sha256sums"],
1100                 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1101                 logEnviron = False,
1102                 haltOnFailure = False,
1103                 flunkOnFailure = False,
1104                 warnOnFailure = False,
1105                 doStepIf = util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
1106         ))
1107
1108         # build list of files to upload
1109         factory.addStep(FileDownload(
1110                 name = "dlsha2rsyncpl",
1111                 mastersrc = scripts_dir + '/sha2rsync.pl',
1112                 workerdest = "../sha2rsync.pl",
1113                 mode = 0o755,
1114         ))
1115
1116         factory.addStep(ShellCommand(
1117                 name = "buildlist",
1118                 description = "Building list of files to upload",
1119                 descriptionDone = "List of files to upload built",
1120                 command = ["../sha2rsync.pl", "target-sha256sums", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/sha256sums", target=ts[0], subtarget=ts[1]), "rsynclist"],
1121                 haltOnFailure = True,
1122         ))
1123
1124         factory.addStep(FileDownload(
1125                 name = "dlrsync.sh",
1126                 mastersrc = scripts_dir + '/rsync.sh',
1127                 workerdest = "../rsync.sh",
1128                 mode = 0o755,
1129         ))
1130
1131         # upload new files and update existing ones
1132         factory.addStep(ShellCommand(
1133                 name = "targetupload",
1134                 description = "Uploading target files",
1135                 descriptionDone = "Target files uploaded",
1136                 command=["../rsync.sh", "--exclude=/kmods/", "--files-from=rsynclist", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_defopts +
1137                         ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]),
1138                         Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1139                 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1140                 haltOnFailure = True,
1141                 logEnviron = False,
1142                 doStepIf = util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
1143         ))
1144
1145         # delete files which don't exist locally
1146         factory.addStep(ShellCommand(
1147                 name = "targetprune",
1148                 description = "Pruning target files",
1149                 descriptionDone = "Target files pruned",
1150                 command=["../rsync.sh", "--exclude=/kmods/", "--delete", "--existing", "--ignore-existing", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_defopts +
1151                         ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]),
1152                         Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1153                 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1154                 haltOnFailure = True,
1155                 logEnviron = False,
1156                 locks = NetLockUl,
1157                 doStepIf = util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
1158         ))
1159
1160         factory.addStep(ShellCommand(
1161                 name = "kmodupload",
1162                 description = "Uploading kmod archive",
1163                 descriptionDone = "Kmod archive uploaded",
1164                 command=["../rsync.sh", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_defopts +
1165                         ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1]),
1166                         Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1167                 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1168                 haltOnFailure = True,
1169                 logEnviron = False,
1170                 locks = NetLockUl,
1171                 doStepIf = util.Transform(lambda a, b: bool(a and b), IsKmodArchiveEnabled, GetRsyncParams.withArgs("bin", "url")),
1172         ))
1173
1174         factory.addStep(ShellCommand(
1175                 name = "sourcelist",
1176                 description = "Finding source archives to upload",
1177                 descriptionDone = "Source archives to upload found",
1178                 command = "find dl/ -maxdepth 1 -type f -not -size 0 -not -name '.*' -not -name '*.hash' -not -name '*.dl' -newer .config -printf '%f\\n' > sourcelist",
1179                 haltOnFailure = True,
1180         ))
1181
1182         factory.addStep(ShellCommand(
1183                 name = "sourceupload",
1184                 description = "Uploading source archives",
1185                 descriptionDone = "Source archives uploaded",
1186                 command=["../rsync.sh", "--files-from=sourcelist", "--size-only", "--delay-updates"] + rsync_defopts +
1187                         [Interpolate("--partial-dir=.~tmp~%(kw:target)s~%(kw:subtarget)s~%(prop:workername)s", target=ts[0], subtarget=ts[1]), "-a", "dl/", Interpolate("%(kw:url)s/", url=GetRsyncParams.withArgs("src", "url"))],
1188                 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("src", "key")) },
1189                 haltOnFailure = True,
1190                 logEnviron = False,
1191                 locks = NetLockUl,
1192                 doStepIf = util.Transform(bool, GetRsyncParams.withArgs("src", "url")),
1193         ))
1194
1195         factory.addStep(ShellCommand(
1196                 name = "df",
1197                 description = "Reporting disk usage",
1198                 command=["df", "-h", "."],
1199                 env={'LC_ALL': 'C'},
1200                 logEnviron = False,
1201                 haltOnFailure = False,
1202                 flunkOnFailure = False,
1203                 warnOnFailure = False,
1204                 alwaysRun = True,
1205         ))
1206
1207         factory.addStep(ShellCommand(
1208                 name = "du",
1209                 description = "Reporting estimated file space usage",
1210                 command=["du", "-sh", "."],
1211                 env={'LC_ALL': 'C'},
1212                 logEnviron = False,
1213                 haltOnFailure = False,
1214                 flunkOnFailure = False,
1215                 warnOnFailure = False,
1216                 alwaysRun = True,
1217         ))
1218
1219         factory.addStep(ShellCommand(
1220                 name = "ccachestat",
1221                 description = "Reporting ccache stats",
1222                 command=["ccache", "-s"],
1223                 env = MakeEnv(overrides={ 'PATH': ["${PATH}", "./staging_dir/host/bin"] }),
1224                 logEnviron = False,
1225                 want_stderr = False,
1226                 haltOnFailure = False,
1227                 flunkOnFailure = False,
1228                 warnOnFailure = False,
1229                 hideStepIf = lambda r, s: r==results.FAILURE,
1230         ))
1231
1232         for brname in branchNames:
1233                 bldrname = brname + "_" + target
1234                 c['builders'].append(BuilderConfig(name=bldrname, workernames=workerNames, factory=factory, nextBuild=GetNextBuild))
1235
1236
1237 ####### STATUS TARGETS
1238
1239 # 'status' is a list of Status Targets. The results of each build will be
1240 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
1241 # including web pages, email senders, and IRC bots.
1242
1243 if "status_bind" in inip1:
1244         c['www'] = {
1245                 'port': inip1.get("status_bind"),
1246                 'plugins': {
1247                         'waterfall_view': True,
1248                         'console_view': True,
1249                         'grid_view': True
1250                 }
1251         }
1252
1253         if "status_user" in inip1 and "status_password" in inip1:
1254                 c['www']['auth'] = util.UserPasswordAuth([
1255                         (inip1.get("status_user"), inip1.get("status_password"))
1256                 ])
1257                 c['www']['authz'] = util.Authz(
1258                         allowRules=[ util.AnyControlEndpointMatcher(role="admins") ],
1259                         roleMatchers=[ util.RolesFromUsername(roles=["admins"], usernames=[inip1.get("status_user")]) ]
1260                 )
1261
1262 c['services'] = []
1263 if ini.has_section("irc"):
1264         iniirc = ini['irc']
1265         irc_host = iniirc.get("host", None)
1266         irc_port = iniirc.getint("port", 6667)
1267         irc_chan = iniirc.get("channel", None)
1268         irc_nick = iniirc.get("nickname", None)
1269         irc_pass = iniirc.get("password", None)
1270
1271         if irc_host and irc_nick and irc_chan:
1272                 irc = reporters.IRC(irc_host, irc_nick,
1273                         port = irc_port,
1274                         password = irc_pass,
1275                         channels = [ irc_chan ],
1276                         notify_events = [ 'exception', 'problem', 'recovery' ]
1277                 )
1278
1279                 c['services'].append(irc)
1280
1281 c['revlink'] = util.RevlinkMatch([
1282         r'https://git.openwrt.org/openwrt/(.*).git'
1283         ],
1284         r'https://git.openwrt.org/?p=openwrt/\1.git;a=commit;h=%s')
1285
1286 ####### DB URL
1287
1288 c['db'] = {
1289         # This specifies what database buildbot uses to store its state.  You can leave
1290         # this at its default for all but the largest installations.
1291         'db_url' : "sqlite:///state.sqlite",
1292 }
1293
1294 c['buildbotNetUsageData'] = None