phase1/master.cfg

   1 # -*- python -*-
   2 # ex: set syntax=python:
   3
   4 import os
   5 import re
   6 import base64
   7 import subprocess
   8 import configparser
   9
  10 from dateutil.tz import tzutc
  11 from datetime import datetime, timedelta
  12
  13 from twisted.internet import defer
  14 from twisted.python import log
  15
  16 from buildbot import locks
  17 from buildbot.data import resultspec
  18 from buildbot.changes.gitpoller import GitPoller
  19 from buildbot.config import BuilderConfig
  20 from buildbot.plugins import reporters
  21 from buildbot.plugins import schedulers
  22 from buildbot.plugins import steps
  23 from buildbot.plugins import util
  24 from buildbot.process import properties
  25 from buildbot.process import results
  26 from buildbot.process.factory import BuildFactory
  27 from buildbot.process.properties import Interpolate
  28 from buildbot.process.properties import Property
  29 from buildbot.schedulers.basic import AnyBranchScheduler
  30 from buildbot.schedulers.forcesched import BaseParameter
  31 from buildbot.schedulers.forcesched import ForceScheduler
  32 from buildbot.schedulers.forcesched import ValidationError
  33 from buildbot.steps.master import MasterShellCommand
  34 from buildbot.steps.shell import SetPropertyFromCommand
  35 from buildbot.steps.shell import ShellCommand
  36 from buildbot.steps.source.git import Git
  37 from buildbot.steps.transfer import FileDownload
  38 from buildbot.steps.transfer import FileUpload
  39 from buildbot.steps.transfer import StringDownload
  40 from buildbot.worker import Worker
  41 from buildbot.worker.local import LocalWorker
  42
  43
  44 if not os.path.exists("twistd.pid"):
  45         with open("twistd.pid", "w") as pidfile:
  46                 pidfile.write("{}".format(os.getpid()))
  47
  48 # This is a sample buildmaster config file. It must be installed as
  49 # 'master.cfg' in your buildmaster's base directory.
  50
  51 ini = configparser.ConfigParser()
  52 ini.read(os.getenv("BUILDMASTER_CONFIG", "./config.ini"))
  53
  54 if "general" not in ini or "phase1" not in ini:
  55         raise ValueError("Fix your configuration")
  56
  57 inip1 = ini['phase1']
  58
  59 # Globals
  60 work_dir = os.path.abspath(ini['general'].get("workdir", "."))
  61 scripts_dir = os.path.abspath("../scripts")
  62
  63 repo_url = ini['repo'].get("url")
  64
  65 rsync_defopts = ["-v", "-4", "--timeout=120"]
  66
  67 #if rsync_bin_url.find("::") > 0 or rsync_bin_url.find("rsync://") == 0:
  68 #       rsync_bin_defopts += ["--contimeout=20"]
  69
  70 branches = {}
  71
  72 def ini_parse_branch(section):
  73         b = {}
  74         name = section.get("name")
  75
  76         if not name:
  77                 raise ValueError("missing 'name' in " + repr(section))
  78         if name in branches:
  79                 raise ValueError("duplicate branch name in " + repr(section))
  80
  81         b["name"] = name
  82         b["bin_url"] = section.get("binary_url")
  83         b["bin_key"] = section.get("binary_password")
  84
  85         b["src_url"] = section.get("source_url")
  86         b["src_key"] = section.get("source_password")
  87
  88         b["gpg_key"] = section.get("gpg_key")
  89
  90         b["usign_key"] = section.get("usign_key")
  91         usign_comment = "untrusted comment: " + name.replace("-", " ").title() + " key"
  92         b["usign_comment"] = section.get("usign_comment", usign_comment)
  93
  94         b["config_seed"] = section.get("config_seed")
  95
  96         b["kmod_archive"] = section.getboolean("kmod_archive", False)
  97
  98         branches[name] = b
  99         log.msg("Configured branch: {}".format(name))
 100
 101 # PB port can be either a numeric port or a connection string
 102 pb_port = inip1.get("port") or 9989
 103
 104 # This is the dictionary that the buildmaster pays attention to. We also use
 105 # a shorter alias to save typing.
 106 c = BuildmasterConfig = {}
 107
 108 ####### PROJECT IDENTITY
 109
 110 # the 'title' string will appear at the top of this buildbot
 111 # installation's html.WebStatus home page (linked to the
 112 # 'titleURL') and is embedded in the title of the waterfall HTML page.
 113
 114 c['title'] = ini['general'].get("title")
 115 c['titleURL'] = ini['general'].get("title_url")
 116
 117 # the 'buildbotURL' string should point to the location where the buildbot's
 118 # internal web server (usually the html.WebStatus page) is visible. This
 119 # typically uses the port number set in the Waterfall 'status' entry, but
 120 # with an externally-visible host name which the buildbot cannot figure out
 121 # without some help.
 122
 123 c['buildbotURL'] = inip1.get("buildbot_url")
 124
 125 ####### BUILDWORKERS
 126
 127 # The 'workers' list defines the set of recognized buildworkers. Each element is
 128 # a Worker object, specifying a unique worker name and password.  The same
 129 # worker name and password must be configured on the worker.
 130
 131 c['workers'] = []
 132 NetLocks = dict()
 133
 134 def ini_parse_workers(section):
 135         name = section.get("name")
 136         password = section.get("password")
 137         phase = section.getint("phase")
 138
 139         if not name or not password or not phase == 1:
 140                 log.msg("invalid worker configuration ignored: {}".format(repr(section)))
 141                 return
 142
 143         sl_props = { 'dl_lock':None, 'ul_lock':None }
 144         if "dl_lock" in section:
 145                 lockname = section.get("dl_lock")
 146                 sl_props['dl_lock'] = lockname
 147                 if lockname not in NetLocks:
 148                         NetLocks[lockname] = locks.MasterLock(lockname)
 149         if "ul_lock" in section:
 150                 lockname = section.get("ul_lock")
 151                 sl_props['ul_lock'] = lockname
 152                 if lockname not in NetLocks:
 153                         NetLocks[lockname] = locks.MasterLock(lockname)
 154
 155         log.msg("Configured worker: {}".format(name))
 156         c['workers'].append(Worker(name, password, max_builds = 1, properties = sl_props))
 157
 158
 159 for section in ini.sections():
 160         if section.startswith("branch "):
 161                 ini_parse_branch(ini[section])
 162
 163         if section.startswith("worker "):
 164                 ini_parse_workers(ini[section])
 165
 166 branchNames = [branches[b]["name"] for b in branches]
 167
 168 c['protocols'] = {'pb': {'port': pb_port}}
 169
 170 # coalesce builds
 171 c['collapseRequests'] = True
 172
 173 # Reduce amount of backlog data
 174 c['configurators'] = [util.JanitorConfigurator(
 175         logHorizon=timedelta(days=3),
 176         hour=6,
 177 )]
 178
 179 @defer.inlineCallbacks
 180 def getNewestCompleteTime(bldr):
 181         """Returns the complete_at of the latest completed and not SKIPPED
 182         build request for this builder, or None if there are no such build
 183         requests. We need to filter out SKIPPED requests because we're
 184         using collapseRequests=True which is unfortunately marking all
 185         previous requests as complete when new buildset is created.
 186
 187         @returns: datetime instance or None, via Deferred
 188         """
 189
 190         bldrid = yield bldr.getBuilderId()
 191         completed = yield bldr.master.data.get(
 192                         ('builders', bldrid, 'buildrequests'),
 193                         [
 194                                 resultspec.Filter('complete', 'eq', [True]),
 195                                 resultspec.Filter('results', 'ne', [results.SKIPPED]),
 196                         ],
 197                         order=['-complete_at'], limit=1)
 198         if not completed:
 199                 return
 200
 201         complete_at = completed[0]['complete_at']
 202
 203         last_build = yield bldr.master.data.get(
 204                         ('builds', ),
 205                         [
 206                                 resultspec.Filter('builderid', 'eq', [bldrid]),
 207                         ],
 208                         order=['-started_at'], limit=1)
 209
 210         if last_build and last_build[0]:
 211                 last_complete_at = last_build[0]['complete_at']
 212                 if last_complete_at and (last_complete_at > complete_at):
 213                         return last_complete_at
 214
 215         return complete_at
 216
 217 @defer.inlineCallbacks
 218 def prioritizeBuilders(master, builders):
 219         """Returns sorted list of builders by their last timestamp of completed and
 220         not skipped build, ordered first by branch name.
 221
 222         @returns: list of sorted builders
 223         """
 224
 225         bldrNamePrio = { "__Janitor": 0, "00_force_build": 0 }
 226         i = 1
 227         for bname in branchNames:
 228                 bldrNamePrio[bname] = i
 229                 i += 1
 230
 231         def is_building(bldr):
 232                 return bool(bldr.building) or bool(bldr.old_building)
 233
 234         def bldr_info(bldr):
 235                 d = defer.maybeDeferred(getNewestCompleteTime, bldr)
 236                 d.addCallback(lambda complete_at: (complete_at, bldr))
 237                 return d
 238
 239         def bldr_sort(item):
 240                 (complete_at, bldr) = item
 241
 242                 pos = 99
 243                 for (name, prio) in bldrNamePrio.items():
 244                         if bldr.name.startswith(name):
 245                                 pos = prio
 246                                 break
 247
 248                 if not complete_at:
 249                         date = datetime.min
 250                         complete_at = date.replace(tzinfo=tzutc())
 251
 252                 if is_building(bldr):
 253                         date = datetime.max
 254                         complete_at = date.replace(tzinfo=tzutc())
 255
 256                 return (pos, complete_at, bldr.name)
 257
 258         results = yield defer.gatherResults([bldr_info(bldr) for bldr in builders])
 259         results.sort(key=bldr_sort)
 260
 261         #for r in results:
 262         #       log.msg("prioritizeBuilders: {:>20} complete_at: {}".format(r[1].name, r[0]))
 263
 264         return [r[1] for r in results]
 265
 266 c['prioritizeBuilders'] = prioritizeBuilders
 267
 268 ####### CHANGESOURCES
 269
 270 # find targets
 271 targets = set()
 272
 273 def populateTargets():
 274         log.msg("Populating targets, this will take time")
 275         sourcegit = work_dir + '/source.git'
 276         for branch in branchNames:
 277                 if os.path.isdir(sourcegit):
 278                         subprocess.call(["rm", "-rf", sourcegit])
 279
 280                 subprocess.call(["git", "clone", "-q", "--depth=1", "--branch="+branch, repo_url, sourcegit])
 281
 282                 os.makedirs(sourcegit + '/tmp', exist_ok=True)
 283                 findtargets = subprocess.Popen(['./scripts/dump-target-info.pl', 'targets'],
 284                         stdout = subprocess.PIPE, stderr = subprocess.DEVNULL, cwd = sourcegit)
 285
 286                 while True:
 287                         line = findtargets.stdout.readline()
 288                         if not line:
 289                                 break
 290                         ta = line.decode().strip().split(' ')
 291                         targets.add(ta[0])
 292
 293                 subprocess.call(["rm", "-rf", sourcegit])
 294
 295 populateTargets()
 296
 297 # the 'change_source' setting tells the buildmaster how it should find out
 298 # about source code changes.  Here we point to the buildbot clone of pyflakes.
 299
 300 c['change_source'] = []
 301 c['change_source'].append(GitPoller(
 302         repo_url,
 303         workdir=work_dir+'/work.git', branches=branchNames,
 304         pollAtLaunch=True, pollinterval=300))
 305
 306 ####### SCHEDULERS
 307
 308 # Configure the Schedulers, which decide how to react to incoming changes.  In this
 309 # case, just kick off a 'basebuild' build
 310
 311 class TagChoiceParameter(BaseParameter):
 312         spec_attributes = ["strict", "choices"]
 313         type = "list"
 314         strict = True
 315
 316         def __init__(self, name, label=None, **kw):
 317                 super().__init__(name, label, **kw)
 318                 self._choice_list = []
 319
 320         def getRevTags(self, findtag=None):
 321                 taglist = []
 322                 branchvers = []
 323
 324                 for b in branchNames:
 325                         basever = re.search(r'-([0-9]+\.[0-9]+)$', b)
 326                         if basever:
 327                                 branchvers.append(basever[1])
 328
 329                 alltags = subprocess.Popen(
 330                         ['git', 'ls-remote', '--tags', repo_url],
 331                         stdout = subprocess.PIPE)
 332
 333                 while True:
 334                         line = alltags.stdout.readline()
 335
 336                         if not line:
 337                                 break
 338
 339                         (rev, tag) = line.split()
 340
 341                         tagver = re.search(r'\brefs/tags/(v[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?)$', tag.decode().strip())
 342
 343                         # only list tags matching configured branches
 344                         if tagver and any(tagver[1][1:].startswith(b) for b in branchvers):
 345                                 if findtag and findtag != tagver[1]:
 346                                         continue
 347                                 taglist.append({'rev': rev.decode().strip(), 'tag': tagver[1]})
 348
 349                 return taglist
 350
 351         @property
 352         def choices(self):
 353                 taglist = [rt['tag'] for rt in self.getRevTags()]
 354                 taglist.sort(reverse=True, key=lambda tag: tag if re.search(r'-rc[0-9]+$', tag) else tag + '-z')
 355                 taglist.insert(0, '')
 356
 357                 self._choice_list = taglist
 358
 359                 return self._choice_list
 360
 361         def updateFromKwargs(self, properties, kwargs, **unused):
 362                 tag = self.getFromKwargs(kwargs)
 363                 properties[self.name] = tag
 364
 365                 # find the commit matching the tag
 366                 findtag = self.getRevTags(tag)
 367
 368                 if not findtag:
 369                         raise ValidationError("Couldn't find tag")
 370
 371                 properties['force_revision'] = findtag[0]['rev']
 372
 373                 # find the branch matching the tag
 374                 branch = None
 375                 branchver = re.search(r'v([0-9]+\.[0-9]+)', tag)
 376                 for b in branchNames:
 377                         if b.endswith(branchver[1]):
 378                                 branch = b
 379
 380                 if not branch:
 381                         raise ValidationError("Couldn't find branch")
 382
 383                 properties['force_branch'] = branch
 384
 385         def parse_from_arg(self, s):
 386                 if self.strict and s not in self._choice_list:
 387                         raise ValidationError("'%s' does not belong to list of available choices '%s'" % (s, self._choice_list))
 388                 return s
 389
 390 @util.renderer
 391 @defer.inlineCallbacks
 392 def builderNames(props):
 393         """ since we have per branch and per target builders,
 394         address the relevant builder for each new buildrequest
 395         based on the request's desired branch and target.
 396         """
 397         branch = props.getProperty("branch")
 398         target = props.getProperty("target", "")
 399
 400         if target == "all":
 401                 target = ""
 402
 403         # if that didn't work, try sourcestamp to find a branch
 404         if not branch:
 405                 # match builders with target branch
 406                 ss = props.sourcestamps[0]
 407                 if ss:
 408                         branch = ss['branch']
 409                 else:
 410                         log.msg("couldn't find builder")
 411                         return []       # nothing works
 412
 413         bname = branch + "_" + target
 414         builders = []
 415
 416         for b in (yield props.master.data.get(('builders',))):
 417                 if not b['name'].startswith(bname):
 418                         continue
 419                 builders.append(b['name'])
 420
 421         return builders
 422
 423 c['schedulers'] = []
 424 c['schedulers'].append(AnyBranchScheduler(
 425         name            = "all",
 426         change_filter   = util.ChangeFilter(branch=branchNames),
 427         treeStableTimer = 15*60,
 428         builderNames    = builderNames))
 429
 430 c['schedulers'].append(ForceScheduler(
 431         name         = "force",
 432         buttonName   = "Force builds",
 433         label        = "Force build details",
 434         builderNames = [ "00_force_build" ],
 435
 436         codebases = [
 437                 util.CodebaseParameter(
 438                         "",
 439                         label      = "Repository",
 440                         branch     = util.FixedParameter(name = "branch",     default = ""),
 441                         revision   = util.FixedParameter(name = "revision",   default = ""),
 442                         repository = util.FixedParameter(name = "repository", default = ""),
 443                         project    = util.FixedParameter(name = "project",    default = "")
 444                 )
 445         ],
 446
 447         reason = util.StringParameter(
 448                 name     = "reason",
 449                 label    = "Reason",
 450                 default  = "Trigger build",
 451                 required = True,
 452                 size     = 80
 453         ),
 454
 455         properties = [
 456                 util.ChoiceStringParameter(
 457                         name    = "target",
 458                         label   = "Build target",
 459                         default = "all",
 460                         choices = [ "all" ] + list(targets)
 461                 ),
 462                 TagChoiceParameter(
 463                         name    = "tag",
 464                         label   = "Build tag",
 465                         default = ""
 466                 )
 467         ]
 468 ))
 469
 470 c['schedulers'].append(schedulers.Triggerable(name="trigger", builderNames=builderNames))
 471
 472 ####### BUILDERS
 473
 474 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
 475 # what steps, and which workers can execute them.  Note that any particular build will
 476 # only take place on one worker.
 477
 478 def IsNoMasterBuild(step):
 479         return step.getProperty("branch") != "master"
 480
 481 def IsUsignEnabled(step):
 482         branch = step.getProperty("branch")
 483         return branch and branches[branch].get("usign_key")
 484
 485 def IsSignEnabled(step):
 486         branch = step.getProperty("branch")
 487         return IsUsignEnabled(step) or branch and branches[branch].get("gpg_key")
 488
 489 def IsKmodArchiveEnabled(step):
 490         branch = step.getProperty("branch")
 491         return branch and branches[branch].get("kmod_archive")
 492
 493 def GetBaseVersion(branch):
 494         if re.match(r"^[^-]+-[0-9]+\.[0-9]+$", branch):
 495                 return branch.split('-')[1]
 496         else:
 497                 return "master"
 498
 499 @properties.renderer
 500 def GetVersionPrefix(props):
 501         branch = props.getProperty("branch")
 502         basever = GetBaseVersion(branch)
 503         if props.hasProperty("tag") and re.match(r"^v[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", props["tag"]):
 504                 return "%s/" % props["tag"][1:]
 505         elif basever != "master":
 506                 return "%s-SNAPSHOT/" % basever
 507         else:
 508                 return ""
 509
 510 @util.renderer
 511 def GetConfigSeed(props):
 512         branch = props.getProperty("branch")
 513         return branch and branches[branch].get("config_seed") or ""
 514
 515 @util.renderer
 516 def GetRsyncParams(props, srcorbin, urlorkey):
 517         # srcorbin: 'bin' or 'src'; urlorkey: 'url' or 'key'
 518         branch = props.getProperty("branch")
 519         opt = srcorbin + "_" + urlorkey
 520         return branch and branches[branch].get(opt)
 521
 522 @util.renderer
 523 def GetUsignKey(props):
 524         branch = props.getProperty("branch")
 525         return branch and branches[branch].get("usign_key")
 526
 527 def GetNextBuild(builder, requests):
 528         for r in requests:
 529                 if r.properties:
 530                         # order tagged build first
 531                         if r.properties.hasProperty("tag"):
 532                                 return r
 533
 534         r = requests[0]
 535         #log.msg("GetNextBuild: {:>20} id: {} bsid: {}".format(builder.name, r.id, r.bsid))
 536         return r
 537
 538 def MakeEnv(overrides=None, tryccache=False):
 539         env = {
 540                 'CCC': Interpolate("%(prop:cc_command:-gcc)s"),
 541                 'CCXX': Interpolate("%(prop:cxx_command:-g++)s"),
 542         }
 543         if tryccache:
 544                 env['CC'] = Interpolate("%(prop:builddir)s/ccache_cc.sh")
 545                 env['CXX'] = Interpolate("%(prop:builddir)s/ccache_cxx.sh")
 546                 env['CCACHE'] = Interpolate("%(prop:ccache_command:-)s")
 547         else:
 548                 env['CC'] = env['CCC']
 549                 env['CXX'] = env['CCXX']
 550                 env['CCACHE'] = ''
 551         if overrides is not None:
 552                 env.update(overrides)
 553         return env
 554
 555 @properties.renderer
 556 def NetLockDl(props, extralock=None):
 557         lock = None
 558         if props.hasProperty("dl_lock"):
 559                 lock = NetLocks[props["dl_lock"]]
 560         if lock is not None:
 561                 return [lock.access('exclusive')]
 562         else:
 563                 return []
 564
 565 @properties.renderer
 566 def NetLockUl(props):
 567         lock = None
 568         if props.hasProperty("ul_lock"):
 569                 lock = NetLocks[props["ul_lock"]]
 570         if lock is not None:
 571                 return [lock.access('exclusive')]
 572         else:
 573                 return []
 574
 575 def IsTargetSelected(target):
 576         def CheckTargetProperty(step):
 577                 selected_target = step.getProperty("target", "all")
 578                 if selected_target != "all" and selected_target != target:
 579                         return False
 580                 return True
 581
 582         return CheckTargetProperty
 583
 584 @util.renderer
 585 def UsignSec2Pub(props):
 586         branch = props.getProperty("branch")
 587         try:
 588                 comment = branches[branch].get("usign_comment") or "untrusted comment: secret key"
 589                 seckey = branches[branch].get("usign_key")
 590                 seckey = base64.b64decode(seckey)
 591         except:
 592                 return None
 593
 594         return "{}\n{}".format(re.sub(r"\bsecret key$", "public key", comment),
 595                 base64.b64encode(seckey[0:2] + seckey[32:40] + seckey[72:]))
 596
 597
 598 c['builders'] = []
 599
 600 workerNames = [ ]
 601
 602 for worker in c['workers']:
 603         workerNames.append(worker.workername)
 604
 605 # add a single LocalWorker to handle the forcebuild builder
 606 c['workers'].append(LocalWorker("__local_force_build", max_builds=1))
 607
 608 force_factory = BuildFactory()
 609 force_factory.addStep(steps.Trigger(
 610         name = "trigger_build",
 611         schedulerNames = [ "trigger" ],
 612         sourceStamps = [{ "codebase": "", "branch": Property("force_branch"), "revision": Property("force_revision"), "repository": repo_url, "project": "" }],
 613         set_properties = { "reason": Property("reason"), "tag": Property("tag"), "target": Property("target") },
 614 ))
 615
 616 c['builders'].append(BuilderConfig(
 617         name        = "00_force_build",
 618         workername  = "__local_force_build",
 619         factory     = force_factory))
 620
 621 for target in targets:
 622         ts = target.split('/')
 623
 624         factory = BuildFactory()
 625
 626         # setup shared work directory if required
 627         factory.addStep(ShellCommand(
 628                 name = "sharedwd",
 629                 descriptionDone = "Shared work directory set up",
 630                 command = 'test -L "$PWD" || (mkdir -p ../shared-workdir && rm -rf "$PWD" && ln -s shared-workdir "$PWD")',
 631                 workdir = ".",
 632                 haltOnFailure = True,
 633         ))
 634
 635         # find number of cores
 636         factory.addStep(SetPropertyFromCommand(
 637                 name = "nproc",
 638                 property = "nproc",
 639                 description = "Finding number of CPUs",
 640                 command = ["nproc"],
 641         ))
 642
 643         # find gcc and g++ compilers
 644         factory.addStep(FileDownload(
 645                 name = "dlfindbinpl",
 646                 mastersrc = scripts_dir + '/findbin.pl',
 647                 workerdest = "../findbin.pl",
 648                 mode = 0o755,
 649         ))
 650
 651         factory.addStep(SetPropertyFromCommand(
 652                 name = "gcc",
 653                 property = "cc_command",
 654                 description = "Finding gcc command",
 655                 command = ["../findbin.pl", "gcc", "", ""],
 656                 haltOnFailure = True,
 657         ))
 658
 659         factory.addStep(SetPropertyFromCommand(
 660                 name = "g++",
 661                 property = "cxx_command",
 662                 description = "Finding g++ command",
 663                 command = ["../findbin.pl", "g++", "", ""],
 664                 haltOnFailure = True,
 665         ))
 666
 667         # see if ccache is available
 668         factory.addStep(SetPropertyFromCommand(
 669                 name = "ccache",
 670                 property = "ccache_command",
 671                 description = "Testing for ccache command",
 672                 command = ["which", "ccache"],
 673                 haltOnFailure = False,
 674                 flunkOnFailure = False,
 675                 warnOnFailure = False,
 676                 hideStepIf = lambda r, s: r==results.FAILURE,
 677         ))
 678
 679         # check out the source
 680         # Git() runs:
 681           # if repo doesn't exist: 'git clone repourl'
 682           # method 'clean' runs 'git clean -d -f', method fresh runs 'git clean -f -f -d -x'. Only works with mode='full'
 683           # git cat-file -e <commit>
 684           # git checkout -f <commit>
 685           # git checkout -B <branch>
 686           # git rev-parse HEAD
 687         factory.addStep(Git(
 688                 name = "git",
 689                 repourl = repo_url,
 690                 mode = 'full',
 691                 method = 'fresh',
 692                 locks = NetLockDl,
 693                 haltOnFailure = True,
 694         ))
 695
 696         # update remote refs
 697         factory.addStep(ShellCommand(
 698                 name = "fetchrefs",
 699                 description = "Fetching Git remote refs",
 700                 command = ["git", "fetch", "origin", Interpolate("+refs/heads/%(prop:branch)s:refs/remotes/origin/%(prop:branch)s")],
 701                 haltOnFailure = True,
 702         ))
 703
 704         # Verify that Git HEAD points to a tag or branch
 705         # Ref: https://web.archive.org/web/20190729224316/http://lists.infradead.org/pipermail/openwrt-devel/2019-June/017809.html
 706         factory.addStep(ShellCommand(
 707                 name = "gitverify",
 708                 description = "Ensure that Git HEAD is pointing to a branch or tag",
 709                 command = 'git rev-parse --abbrev-ref HEAD | grep -vxqF HEAD || git show-ref --tags --dereference 2>/dev/null | sed -ne "/^$(git rev-parse HEAD) / { s|^.*/||; s|\\^.*||; p }" | grep -qE "^v[0-9][0-9]\\."',
 710                 haltOnFailure = True,
 711         ))
 712
 713         factory.addStep(ShellCommand(
 714                 name = "rmtmp",
 715                 description = "Remove tmp folder",
 716                 command=["rm", "-rf", "tmp/"],
 717         ))
 718
 719         # feed
 720         factory.addStep(ShellCommand(
 721                 name = "rmfeedlinks",
 722                 description = "Remove feed symlinks",
 723                 command=["rm", "-rf", "package/feeds/"],
 724         ))
 725
 726         factory.addStep(StringDownload(
 727                 name = "ccachecc",
 728                 s = '#!/bin/sh\nexec ${CCACHE} ${CCC} "$@"\n',
 729                 workerdest = "../ccache_cc.sh",
 730                 mode = 0o755,
 731         ))
 732
 733         factory.addStep(StringDownload(
 734                 name = "ccachecxx",
 735                 s = '#!/bin/sh\nexec ${CCACHE} ${CCXX} "$@"\n',
 736                 workerdest = "../ccache_cxx.sh",
 737                 mode = 0o755,
 738         ))
 739
 740         # feed
 741         factory.addStep(ShellCommand(
 742                 name = "updatefeeds",
 743                 description = "Updating feeds",
 744                 command=["./scripts/feeds", "update"],
 745                 env = MakeEnv(tryccache=True),
 746                 haltOnFailure = True,
 747                 locks = NetLockDl,
 748         ))
 749
 750         # feed
 751         factory.addStep(ShellCommand(
 752                 name = "installfeeds",
 753                 description = "Installing feeds",
 754                 command=["./scripts/feeds", "install", "-a"],
 755                 env = MakeEnv(tryccache=True),
 756                 haltOnFailure = True,
 757         ))
 758
 759         # seed config
 760         factory.addStep(StringDownload(
 761                 name = "dlconfigseed",
 762                 s = Interpolate("%(kw:seed)s\n", seed=GetConfigSeed),
 763                 workerdest = ".config",
 764                 mode = 0o644,
 765         ))
 766
 767         # configure
 768         factory.addStep(ShellCommand(
 769                 name = "newconfig",
 770                 descriptionDone = ".config seeded",
 771                 command = Interpolate("printf 'CONFIG_TARGET_%(kw:target)s=y\\nCONFIG_TARGET_%(kw:target)s_%(kw:subtarget)s=y\\nCONFIG_SIGNED_PACKAGES=%(kw:usign:#?|y|n)s\\n' >> .config", target=ts[0], subtarget=ts[1], usign=GetUsignKey),
 772         ))
 773
 774         factory.addStep(ShellCommand(
 775                 name = "delbin",
 776                 description = "Removing output directory",
 777                 command = ["rm", "-rf", "bin/"],
 778         ))
 779
 780         factory.addStep(ShellCommand(
 781                 name = "defconfig",
 782                 description = "Populating .config",
 783                 command = ["make", "defconfig"],
 784                 env = MakeEnv(),
 785         ))
 786
 787         # check arch - exit early if does not exist - NB: some targets do not define CONFIG_TARGET_target_subtarget
 788         factory.addStep(ShellCommand(
 789                 name = "checkarch",
 790                 description = "Checking architecture",
 791                 descriptionDone = "Architecture validated",
 792                 command = 'grep -sq CONFIG_TARGET_%s=y .config && grep -sq CONFIG_TARGET_SUBTARGET=\\"%s\\" .config' %(ts[0], ts[1]),
 793                 logEnviron = False,
 794                 want_stdout = False,
 795                 want_stderr = False,
 796                 haltOnFailure = True,
 797                 flunkOnFailure = False, # this is not a build FAILURE
 798         ))
 799
 800         # find libc suffix
 801         factory.addStep(SetPropertyFromCommand(
 802                 name = "libc",
 803                 property = "libc",
 804                 description = "Finding libc suffix",
 805                 command = ["sed", "-ne", '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }', ".config"],
 806         ))
 807
 808         # install build key
 809         factory.addStep(StringDownload(
 810                 name = "dlkeybuildpub",
 811                 s = Interpolate("%(kw:sec2pub)s", sec2pub=UsignSec2Pub),
 812                 workerdest = "key-build.pub",
 813                 mode = 0o600,
 814                 doStepIf = IsUsignEnabled,
 815         ))
 816
 817         factory.addStep(StringDownload(
 818                 name = "dlkeybuild",
 819                 s = "# fake private key",
 820                 workerdest = "key-build",
 821                 mode = 0o600,
 822                 doStepIf = IsUsignEnabled,
 823         ))
 824
 825         factory.addStep(StringDownload(
 826                 name = "dlkeybuilducert",
 827                 s = "# fake certificate",
 828                 workerdest = "key-build.ucert",
 829                 mode = 0o600,
 830                 doStepIf = IsUsignEnabled,
 831         ))
 832
 833         # prepare dl
 834         factory.addStep(ShellCommand(
 835                 name = "dldir",
 836                 description = "Preparing dl/",
 837                 descriptionDone = "dl/ prepared",
 838                 command = 'mkdir -p ../dl && rm -rf "build/dl" && ln -s ../../dl "build/dl"',
 839                 workdir = Property("builddir"),
 840                 logEnviron = False,
 841                 want_stdout = False,
 842         ))
 843
 844         # cleanup dl
 845         factory.addStep(ShellCommand(
 846                 name = "dlprune",
 847                 description = "Pruning dl/",
 848                 descriptionDone = "dl/ pruned",
 849                 command = 'find dl/ -atime +15 -delete -print',
 850                 logEnviron = False,
 851         ))
 852
 853         # prepare tar
 854         factory.addStep(ShellCommand(
 855                 name = "dltar",
 856                 description = "Building and installing GNU tar",
 857                 descriptionDone = "GNU tar built and installed",
 858                 command = ["make", Interpolate("-j%(prop:nproc:-1)s"), "tools/tar/compile", "V=s"],
 859                 env = MakeEnv(tryccache=True),
 860                 haltOnFailure = True,
 861         ))
 862
 863         # populate dl
 864         factory.addStep(ShellCommand(
 865                 name = "dlrun",
 866                 description = "Populating dl/",
 867                 descriptionDone = "dl/ populated",
 868                 command = ["make", Interpolate("-j%(prop:nproc:-1)s"), "download", "V=s"],
 869                 env = MakeEnv(),
 870                 logEnviron = False,
 871                 locks = NetLockDl,
 872         ))
 873
 874         factory.addStep(ShellCommand(
 875                 name = "cleanbase",
 876                 description = "Cleaning base-files",
 877                 command=["make", "package/base-files/clean", "V=s"],
 878         ))
 879
 880         # build
 881         factory.addStep(ShellCommand(
 882                 name = "tools",
 883                 description = "Building and installing tools",
 884                 descriptionDone = "Tools built and installed",
 885                 command = ["make", Interpolate("-j%(prop:nproc:-1)s"), "tools/install", "V=s"],
 886                 env = MakeEnv(tryccache=True),
 887                 haltOnFailure = True,
 888         ))
 889
 890         factory.addStep(ShellCommand(
 891                 name = "toolchain",
 892                 description = "Building and installing toolchain",
 893                 descriptionDone = "Toolchain built and installed",
 894                 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "toolchain/install", "V=s"],
 895                 env = MakeEnv(),
 896                 haltOnFailure = True,
 897         ))
 898
 899         factory.addStep(ShellCommand(
 900                 name = "kmods",
 901                 description = "Building kmods",
 902                 descriptionDone = "Kmods built",
 903                 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
 904                 env = MakeEnv(),
 905                 haltOnFailure = True,
 906         ))
 907
 908         # find kernel version
 909         factory.addStep(SetPropertyFromCommand(
 910                 name = "kernelversion",
 911                 property = "kernelversion",
 912                 description = "Finding the effective Kernel version",
 913                 command = "make --no-print-directory -C target/linux/ val.LINUX_VERSION val.LINUX_RELEASE val.LINUX_VERMAGIC | xargs printf '%s-%s-%s\\n'",
 914                 env = { 'TOPDIR': Interpolate("%(prop:builddir)s/build") },
 915         ))
 916
 917         factory.addStep(ShellCommand(
 918                 name = "pkgclean",
 919                 description = "Cleaning up package build",
 920                 descriptionDone = "Package build cleaned up",
 921                 command=["make", "package/cleanup", "V=s"],
 922         ))
 923
 924         factory.addStep(ShellCommand(
 925                 name = "pkgbuild",
 926                 description = "Building packages",
 927                 descriptionDone = "Packages built",
 928                 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
 929                 env = MakeEnv(),
 930                 haltOnFailure = True,
 931         ))
 932
 933         factory.addStep(ShellCommand(
 934                 name = "pkginstall",
 935                 description = "Installing packages",
 936                 descriptionDone = "Packages installed",
 937                 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/install", "V=s"],
 938                 env = MakeEnv(),
 939                 haltOnFailure = True,
 940         ))
 941
 942         factory.addStep(ShellCommand(
 943                 name = "pkgindex",
 944                 description = "Indexing packages",
 945                 descriptionDone = "Packages indexed",
 946                 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/index", "V=s", "CONFIG_SIGNED_PACKAGES="],
 947                 env = MakeEnv(),
 948                 haltOnFailure = True,
 949         ))
 950
 951         factory.addStep(ShellCommand(
 952                 name = "images",
 953                 description = "Building and installing images",
 954                 descriptionDone = "Images built and installed",
 955                 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "target/install", "V=s"],
 956                 env = MakeEnv(),
 957                 haltOnFailure = True,
 958         ))
 959
 960         factory.addStep(ShellCommand(
 961                 name = "buildinfo",
 962                 description = "Generating config.buildinfo, version.buildinfo and feeds.buildinfo",
 963                 command = "make -j1 buildinfo V=s || true",
 964                 env = MakeEnv(),
 965                 haltOnFailure = True,
 966         ))
 967
 968         factory.addStep(ShellCommand(
 969                 name = "json_overview_image_info",
 970                 description = "Generating profiles.json in target folder",
 971                 command = "make -j1 json_overview_image_info V=s || true",
 972                 env = MakeEnv(),
 973                 haltOnFailure = True,
 974         ))
 975
 976         factory.addStep(ShellCommand(
 977                 name = "checksums",
 978                 description = "Calculating checksums",
 979                 descriptionDone = "Checksums calculated",
 980                 command=["make", "-j1", "checksum", "V=s"],
 981                 env = MakeEnv(),
 982                 haltOnFailure = True,
 983         ))
 984
 985         factory.addStep(ShellCommand(
 986                 name = "kmoddir",
 987                 descriptionDone = "Kmod directory created",
 988                 command=["mkdir", "-p", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s", target=ts[0], subtarget=ts[1])],
 989                 haltOnFailure = True,
 990                 doStepIf = IsKmodArchiveEnabled,
 991         ))
 992
 993         factory.addStep(ShellCommand(
 994                 name = "kmodprepare",
 995                 description = "Preparing kmod archive",
 996                 descriptionDone = "Kmod archive prepared",
 997                 command=["rsync", "--include=/kmod-*.ipk", "--exclude=*", "-va",
 998                         Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/packages/", target=ts[0], subtarget=ts[1]),
 999                         Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1])],
1000                 haltOnFailure = True,
1001                 doStepIf = IsKmodArchiveEnabled,
1002         ))
1003
1004         factory.addStep(ShellCommand(
1005                 name = "kmodindex",
1006                 description = "Indexing kmod archive",
1007                 descriptionDone = "Kmod archive indexed",
1008                 command=["make", Interpolate("-j%(prop:nproc:-1)s"), "package/index", "V=s", "CONFIG_SIGNED_PACKAGES=",
1009                         Interpolate("PACKAGE_SUBDIRS=bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1])],
1010                 env = MakeEnv(),
1011                 haltOnFailure = True,
1012                 doStepIf = IsKmodArchiveEnabled,
1013         ))
1014
1015         # sign
1016         factory.addStep(MasterShellCommand(
1017                 name = "signprepare",
1018                 descriptionDone = "Temporary signing directory prepared",
1019                 command = ["mkdir", "-p", "%s/signing" %(work_dir)],
1020                 haltOnFailure = True,
1021                 doStepIf = IsSignEnabled,
1022
1023         ))
1024
1025         factory.addStep(ShellCommand(
1026                 name = "signpack",
1027                 description = "Packing files to sign",
1028                 descriptionDone = "Files to sign packed",
1029                 command = Interpolate("find bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/ bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz", target=ts[0], subtarget=ts[1]),
1030                 haltOnFailure = True,
1031                 doStepIf = IsSignEnabled,
1032         ))
1033
1034         factory.addStep(FileUpload(
1035                 workersrc = "sign.tar.gz",
1036                 masterdest = "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1]),
1037                 haltOnFailure = True,
1038                 doStepIf = IsSignEnabled,
1039         ))
1040
1041         factory.addStep(MasterShellCommand(
1042                 name = "signfiles",
1043                 description = "Signing files",
1044                 descriptionDone = "Files signed",
1045                 command = ["%s/signall.sh" %(scripts_dir), "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1]), Interpolate("%(prop:branch)s")],
1046                 env = { 'CONFIG_INI': os.getenv("BUILDMASTER_CONFIG", "./config.ini") },
1047                 haltOnFailure = True,
1048                 doStepIf = IsSignEnabled,
1049         ))
1050
1051         factory.addStep(FileDownload(
1052                 name = "dlsigntargz",
1053                 mastersrc = "%s/signing/%s.%s.tar.gz" %(work_dir, ts[0], ts[1]),
1054                 workerdest = "sign.tar.gz",
1055                 haltOnFailure = True,
1056                 doStepIf = IsSignEnabled,
1057         ))
1058
1059         factory.addStep(ShellCommand(
1060                 name = "signunpack",
1061                 description = "Unpacking signed files",
1062                 descriptionDone = "Signed files unpacked",
1063                 command = ["tar", "-xzf", "sign.tar.gz"],
1064                 haltOnFailure = True,
1065                 doStepIf = IsSignEnabled,
1066         ))
1067
1068         # upload
1069         factory.addStep(ShellCommand(
1070                 name = "dirprepare",
1071                 descriptionDone = "Upload directory structure prepared",
1072                 command = ["mkdir", "-p", Interpolate("tmp/upload/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s", target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1073                 haltOnFailure = True,
1074         ))
1075
1076         factory.addStep(ShellCommand(
1077                 name = "linkprepare",
1078                 descriptionDone = "Repository symlink prepared",
1079                 command = ["ln", "-s", "-f", Interpolate("../packages-%(kw:basever)s", basever=util.Transform(GetBaseVersion, Property("branch"))), Interpolate("tmp/upload/%(kw:prefix)spackages", prefix=GetVersionPrefix)],
1080                 doStepIf = IsNoMasterBuild,
1081                 haltOnFailure = True,
1082         ))
1083
1084         factory.addStep(ShellCommand(
1085                 name = "kmoddirprepare",
1086                 descriptionDone = "Kmod archive upload directory prepared",
1087                 command = ["mkdir", "-p", Interpolate("tmp/upload/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s", target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1088                 haltOnFailure = True,
1089                 doStepIf = IsKmodArchiveEnabled,
1090         ))
1091
1092         factory.addStep(ShellCommand(
1093                 name = "dirupload",
1094                 description = "Uploading directory structure",
1095                 descriptionDone = "Directory structure uploaded",
1096                 command = ["rsync", "-az"] + rsync_defopts + ["tmp/upload/", Interpolate("%(kw:url)s/", url=GetRsyncParams.withArgs("bin", "url"))],
1097                 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1098                 haltOnFailure = True,
1099                 logEnviron = False,
1100                 locks = NetLockUl,
1101                 doStepIf = util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
1102         ))
1103
1104         # download remote sha256sums to 'target-sha256sums'
1105         factory.addStep(ShellCommand(
1106                 name = "target-sha256sums",
1107                 description = "Fetching remote sha256sums for target",
1108                 descriptionDone = "Remote sha256sums for target fetched",
1109                 command = ["rsync", "-z"] + rsync_defopts + [Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/sha256sums", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix), "target-sha256sums"],
1110                 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1111                 logEnviron = False,
1112                 haltOnFailure = False,
1113                 flunkOnFailure = False,
1114                 warnOnFailure = False,
1115                 doStepIf = util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
1116         ))
1117
1118         # build list of files to upload
1119         factory.addStep(FileDownload(
1120                 name = "dlsha2rsyncpl",
1121                 mastersrc = scripts_dir + '/sha2rsync.pl',
1122                 workerdest = "../sha2rsync.pl",
1123                 mode = 0o755,
1124         ))
1125
1126         factory.addStep(ShellCommand(
1127                 name = "buildlist",
1128                 description = "Building list of files to upload",
1129                 descriptionDone = "List of files to upload built",
1130                 command = ["../sha2rsync.pl", "target-sha256sums", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/sha256sums", target=ts[0], subtarget=ts[1]), "rsynclist"],
1131                 haltOnFailure = True,
1132         ))
1133
1134         factory.addStep(FileDownload(
1135                 name = "dlrsync.sh",
1136                 mastersrc = scripts_dir + '/rsync.sh',
1137                 workerdest = "../rsync.sh",
1138                 mode = 0o755,
1139         ))
1140
1141         # upload new files and update existing ones
1142         factory.addStep(ShellCommand(
1143                 name = "targetupload",
1144                 description = "Uploading target files",
1145                 descriptionDone = "Target files uploaded",
1146                 command=["../rsync.sh", "--exclude=/kmods/", "--files-from=rsynclist", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_defopts +
1147                         ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]),
1148                         Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1149                 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1150                 haltOnFailure = True,
1151                 logEnviron = False,
1152                 doStepIf = util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
1153         ))
1154
1155         # delete files which don't exist locally
1156         factory.addStep(ShellCommand(
1157                 name = "targetprune",
1158                 description = "Pruning target files",
1159                 descriptionDone = "Target files pruned",
1160                 command=["../rsync.sh", "--exclude=/kmods/", "--delete", "--existing", "--ignore-existing", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_defopts +
1161                         ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]),
1162                         Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1163                 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1164                 haltOnFailure = True,
1165                 logEnviron = False,
1166                 locks = NetLockUl,
1167                 doStepIf = util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
1168         ))
1169
1170         factory.addStep(ShellCommand(
1171                 name = "kmodupload",
1172                 description = "Uploading kmod archive",
1173                 descriptionDone = "Kmod archive uploaded",
1174                 command=["../rsync.sh", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1])] + rsync_defopts +
1175                         ["-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1]),
1176                         Interpolate("%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s/", url=GetRsyncParams.withArgs("bin", "url"), target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
1177                 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")) },
1178                 haltOnFailure = True,
1179                 logEnviron = False,
1180                 locks = NetLockUl,
1181                 doStepIf = util.Transform(lambda a, b: bool(a and b), IsKmodArchiveEnabled, GetRsyncParams.withArgs("bin", "url")),
1182         ))
1183
1184         factory.addStep(ShellCommand(
1185                 name = "sourcelist",
1186                 description = "Finding source archives to upload",
1187                 descriptionDone = "Source archives to upload found",
1188                 command = "find dl/ -maxdepth 1 -type f -not -size 0 -not -name '.*' -not -name '*.hash' -not -name '*.dl' -newer .config -printf '%f\\n' > sourcelist",
1189                 haltOnFailure = True,
1190         ))
1191
1192         factory.addStep(ShellCommand(
1193                 name = "sourceupload",
1194                 description = "Uploading source archives",
1195                 descriptionDone = "Source archives uploaded",
1196                 command=["../rsync.sh", "--files-from=sourcelist", "--size-only", "--delay-updates"] + rsync_defopts +
1197                         [Interpolate("--partial-dir=.~tmp~%(kw:target)s~%(kw:subtarget)s~%(prop:workername)s", target=ts[0], subtarget=ts[1]), "-a", "dl/", Interpolate("%(kw:url)s/", url=GetRsyncParams.withArgs("src", "url"))],
1198                 env={ 'RSYNC_PASSWORD': Interpolate("%(kw:key)s", key=GetRsyncParams.withArgs("src", "key")) },
1199                 haltOnFailure = True,
1200                 logEnviron = False,
1201                 locks = NetLockUl,
1202                 doStepIf = util.Transform(bool, GetRsyncParams.withArgs("src", "url")),
1203         ))
1204
1205         factory.addStep(ShellCommand(
1206                 name = "df",
1207                 description = "Reporting disk usage",
1208                 command=["df", "-h", "."],
1209                 env={'LC_ALL': 'C'},
1210                 logEnviron = False,
1211                 haltOnFailure = False,
1212                 flunkOnFailure = False,
1213                 warnOnFailure = False,
1214                 alwaysRun = True,
1215         ))
1216
1217         factory.addStep(ShellCommand(
1218                 name = "du",
1219                 description = "Reporting estimated file space usage",
1220                 command=["du", "-sh", "."],
1221                 env={'LC_ALL': 'C'},
1222                 logEnviron = False,
1223                 haltOnFailure = False,
1224                 flunkOnFailure = False,
1225                 warnOnFailure = False,
1226                 alwaysRun = True,
1227         ))
1228
1229         factory.addStep(ShellCommand(
1230                 name = "ccachestat",
1231                 description = "Reporting ccache stats",
1232                 command=["ccache", "-s"],
1233                 env = MakeEnv(overrides={ 'PATH': ["${PATH}", "./staging_dir/host/bin"] }),
1234                 logEnviron = False,
1235                 want_stderr = False,
1236                 haltOnFailure = False,
1237                 flunkOnFailure = False,
1238                 warnOnFailure = False,
1239                 hideStepIf = lambda r, s: r==results.FAILURE,
1240         ))
1241
1242         for brname in branchNames:
1243                 bldrname = brname + "_" + target
1244                 c['builders'].append(BuilderConfig(name=bldrname, workernames=workerNames, factory=factory, nextBuild=GetNextBuild))
1245
1246
1247 ####### STATUS TARGETS
1248
1249 # 'status' is a list of Status Targets. The results of each build will be
1250 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
1251 # including web pages, email senders, and IRC bots.
1252
1253 if "status_bind" in inip1:
1254         c['www'] = {
1255                 'port': inip1.get("status_bind"),
1256                 'plugins': {
1257                         'waterfall_view': True,
1258                         'console_view': True,
1259                         'grid_view': True
1260                 }
1261         }
1262
1263         if "status_user" in inip1 and "status_password" in inip1:
1264                 c['www']['auth'] = util.UserPasswordAuth([
1265                         (inip1.get("status_user"), inip1.get("status_password"))
1266                 ])
1267                 c['www']['authz'] = util.Authz(
1268                         allowRules=[ util.AnyControlEndpointMatcher(role="admins") ],
1269                         roleMatchers=[ util.RolesFromUsername(roles=["admins"], usernames=[inip1.get("status_user")]) ]
1270                 )
1271
1272 c['services'] = []
1273 if ini.has_section("irc"):
1274         iniirc = ini['irc']
1275         irc_host = iniirc.get("host", None)
1276         irc_port = iniirc.getint("port", 6667)
1277         irc_chan = iniirc.get("channel", None)
1278         irc_nick = iniirc.get("nickname", None)
1279         irc_pass = iniirc.get("password", None)
1280
1281         if irc_host and irc_nick and irc_chan:
1282                 irc = reporters.IRC(irc_host, irc_nick,
1283                         port = irc_port,
1284                         password = irc_pass,
1285                         channels = [ irc_chan ],
1286                         notify_events = [ 'exception', 'problem', 'recovery' ]
1287                 )
1288
1289                 c['services'].append(irc)
1290
1291 c['revlink'] = util.RevlinkMatch([
1292         r'https://git.openwrt.org/openwrt/(.*).git'
1293         ],
1294         r'https://git.openwrt.org/?p=openwrt/\1.git;a=commit;h=%s')
1295
1296 ####### DB URL
1297
1298 c['db'] = {
1299         # This specifies what database buildbot uses to store its state.  You can leave
1300         # this at its default for all but the largest installations.
1301         'db_url' : "sqlite:///state.sqlite",
1302 }
1303
1304 c['buildbotNetUsageData'] = None