phase1/master.cfg

   1 # -*- python -*-
   2 # ex: set syntax=python:
   3
   4 import os
   5 import re
   6 import base64
   7 import subprocess
   8 import configparser
   9
  10 from dateutil.tz import tzutc
  11 from datetime import datetime, timedelta
  12
  13 from twisted.internet import defer
  14 from twisted.python import log
  15
  16 from buildbot import locks
  17 from buildbot.data import resultspec
  18 from buildbot.changes.gitpoller import GitPoller
  19 from buildbot.config import BuilderConfig
  20 from buildbot.plugins import reporters
  21 from buildbot.plugins import schedulers
  22 from buildbot.plugins import steps
  23 from buildbot.plugins import util
  24 from buildbot.process import properties
  25 from buildbot.process import results
  26 from buildbot.process.factory import BuildFactory
  27 from buildbot.process.properties import Interpolate
  28 from buildbot.process.properties import Property
  29 from buildbot.schedulers.basic import AnyBranchScheduler
  30 from buildbot.schedulers.forcesched import BaseParameter
  31 from buildbot.schedulers.forcesched import ForceScheduler
  32 from buildbot.schedulers.forcesched import ValidationError
  33 from buildbot.steps.master import MasterShellCommand
  34 from buildbot.steps.shell import SetPropertyFromCommand
  35 from buildbot.steps.shell import ShellCommand
  36 from buildbot.steps.source.git import Git
  37 from buildbot.steps.transfer import FileDownload
  38 from buildbot.steps.transfer import FileUpload
  39 from buildbot.steps.transfer import StringDownload
  40 from buildbot.worker import Worker
  41 from buildbot.worker.local import LocalWorker
  42
  43
  44 if not os.path.exists("twistd.pid"):
  45     with open("twistd.pid", "w") as pidfile:
  46         pidfile.write("{}".format(os.getpid()))
  47
  48 # This is a sample buildmaster config file. It must be installed as
  49 # 'master.cfg' in your buildmaster's base directory.
  50
  51 ini = configparser.ConfigParser()
  52 ini.read(os.getenv("BUILDMASTER_CONFIG", "./config.ini"))
  53
  54 if "general" not in ini or "phase1" not in ini:
  55     raise ValueError("Fix your configuration")
  56
  57 inip1 = ini["phase1"]
  58
  59 # Globals
  60 work_dir = os.path.abspath(ini["general"].get("workdir", "."))
  61 scripts_dir = os.path.abspath("../scripts")
  62
  63 repo_url = ini["repo"].get("url")
  64
  65 rsync_defopts = ["-v", "--timeout=120"]
  66
  67 # if rsync_bin_url.find("::") > 0 or rsync_bin_url.find("rsync://") == 0:
  68 #       rsync_bin_defopts += ["--contimeout=20"]
  69
  70 branches = {}
  71
  72
  73 def ini_parse_branch(section):
  74     b = {}
  75     name = section.get("name")
  76
  77     if not name:
  78         raise ValueError("missing 'name' in " + repr(section))
  79     if name in branches:
  80         raise ValueError("duplicate branch name in " + repr(section))
  81
  82     b["name"] = name
  83     b["bin_url"] = section.get("binary_url")
  84     b["bin_key"] = section.get("binary_password")
  85
  86     b["src_url"] = section.get("source_url")
  87     b["src_key"] = section.get("source_password")
  88
  89     b["gpg_key"] = section.get("gpg_key")
  90
  91     b["usign_key"] = section.get("usign_key")
  92     usign_comment = "untrusted comment: " + name.replace("-", " ").title() + " key"
  93     b["usign_comment"] = section.get("usign_comment", usign_comment)
  94
  95     b["config_seed"] = section.get("config_seed")
  96
  97     b["kmod_archive"] = section.getboolean("kmod_archive", False)
  98
  99     branches[name] = b
 100     log.msg("Configured branch: {}".format(name))
 101
 102
 103 # PB port can be either a numeric port or a connection string
 104 pb_port = inip1.get("port") or 9989
 105
 106 # This is the dictionary that the buildmaster pays attention to. We also use
 107 # a shorter alias to save typing.
 108 c = BuildmasterConfig = {}
 109
 110 ####### PROJECT IDENTITY
 111
 112 # the 'title' string will appear at the top of this buildbot
 113 # installation's html.WebStatus home page (linked to the
 114 # 'titleURL') and is embedded in the title of the waterfall HTML page.
 115
 116 c["title"] = ini["general"].get("title")
 117 c["titleURL"] = ini["general"].get("title_url")
 118
 119 # the 'buildbotURL' string should point to the location where the buildbot's
 120 # internal web server (usually the html.WebStatus page) is visible. This
 121 # typically uses the port number set in the Waterfall 'status' entry, but
 122 # with an externally-visible host name which the buildbot cannot figure out
 123 # without some help.
 124
 125 c["buildbotURL"] = inip1.get("buildbot_url")
 126
 127 ####### BUILDWORKERS
 128
 129 # The 'workers' list defines the set of recognized buildworkers. Each element is
 130 # a Worker object, specifying a unique worker name and password.  The same
 131 # worker name and password must be configured on the worker.
 132
 133 c["workers"] = []
 134 NetLocks = dict()
 135
 136
 137 def ini_parse_workers(section):
 138     name = section.get("name")
 139     password = section.get("password")
 140     phase = section.getint("phase")
 141     tagonly = section.getboolean("tag_only")
 142     rsyncipv4 = section.getboolean("rsync_ipv4")
 143
 144     if not name or not password or not phase == 1:
 145         log.msg("invalid worker configuration ignored: {}".format(repr(section)))
 146         return
 147
 148     sl_props = {"tag_only": tagonly}
 149     if "dl_lock" in section:
 150         lockname = section.get("dl_lock")
 151         sl_props["dl_lock"] = lockname
 152         if lockname not in NetLocks:
 153             NetLocks[lockname] = locks.MasterLock(lockname)
 154     if "ul_lock" in section:
 155         lockname = section.get("ul_lock")
 156         sl_props["ul_lock"] = lockname
 157         if lockname not in NetLocks:
 158             NetLocks[lockname] = locks.MasterLock(lockname)
 159     if rsyncipv4:
 160         sl_props[
 161             "rsync_ipv4"
 162         ] = True  # only set prop if required, we use '+' Interpolate substitution
 163
 164     log.msg("Configured worker: {}".format(name))
 165     # NB: phase1 build factory requires workers to be single-build only
 166     c["workers"].append(Worker(name, password, max_builds=1, properties=sl_props))
 167
 168
 169 for section in ini.sections():
 170     if section.startswith("branch "):
 171         ini_parse_branch(ini[section])
 172
 173     if section.startswith("worker "):
 174         ini_parse_workers(ini[section])
 175
 176 # list of branches in build-priority order
 177 branchNames = [branches[b]["name"] for b in branches]
 178
 179 c["protocols"] = {"pb": {"port": pb_port}}
 180
 181 # coalesce builds
 182 c["collapseRequests"] = True
 183
 184 # Reduce amount of backlog data
 185 c["configurators"] = [
 186     util.JanitorConfigurator(
 187         logHorizon=timedelta(days=3),
 188         hour=6,
 189     )
 190 ]
 191
 192
 193 @defer.inlineCallbacks
 194 def getNewestCompleteTimePrio(bldr):
 195     """Returns the priority and the complete_at of the latest completed and not SKIPPED
 196     build request for this builder, or None if there are no such build
 197     requests. We need to filter out SKIPPED requests because we're
 198     using collapseRequests=True which is unfortunately marking all
 199     previous requests as complete when new buildset is created.
 200
 201     @returns: (priority, datetime instance or None), via Deferred
 202     """
 203
 204     prio = yield bldr.get_highest_priority()
 205     if prio is None:
 206         prio = 0
 207
 208     bldrid = yield bldr.getBuilderId()
 209     completed = yield bldr.master.data.get(
 210         ("builders", bldrid, "buildrequests"),
 211         [
 212             resultspec.Filter("complete", "eq", [True]),
 213             resultspec.Filter("results", "ne", [results.SKIPPED]),
 214         ],
 215         order=["-complete_at"],
 216         limit=1,
 217     )
 218     if not completed:
 219         return (prio, None)
 220
 221     complete_at = completed[0]["complete_at"]
 222
 223     last_build = yield bldr.master.data.get(
 224         ("builds",),
 225         [
 226             resultspec.Filter("builderid", "eq", [bldrid]),
 227         ],
 228         order=["-started_at"],
 229         limit=1,
 230     )
 231
 232     if last_build and last_build[0]:
 233         last_complete_at = last_build[0]["complete_at"]
 234         if last_complete_at and (last_complete_at > complete_at):
 235             return (prio, last_complete_at)
 236
 237     return (prio, complete_at)
 238
 239
 240 @defer.inlineCallbacks
 241 def prioritizeBuilders(master, builders):
 242     """Returns sorted list of builders by their last timestamp of completed and
 243     not skipped build, ordered first by branch name.
 244
 245     @returns: list of sorted builders
 246     """
 247
 248     bldrNamePrio = {"__Janitor": 0, "00_force_build": 0}
 249     i = 1
 250     for bname in branchNames:
 251         bldrNamePrio[bname] = i
 252         i += 1
 253
 254     def is_building(bldr):
 255         return bool(bldr.building) or bool(bldr.old_building)
 256
 257     def bldr_info(bldr):
 258         d = defer.maybeDeferred(getNewestCompleteTimePrio, bldr)
 259         d.addCallback(lambda retval: (retval, bldr))
 260         return d
 261
 262     def bldr_sort(item):
 263         ((hiprio, complete_at), bldr) = item
 264
 265         # check if we have some high prio build requests pending (i.e. tag builds),
 266         # if so, front-run these builders, while preserving the per-branch static priority
 267         pos = 99
 268         for name, prio in bldrNamePrio.items():
 269             if bldr.name.startswith(name):
 270                 pos = prio + 50 - min(hiprio, 50)    # higher priority (larger positive number) raises position
 271                 break
 272
 273         # pos order: janitor/local (0), tag builds per branch order if any [1..50], !tag builds per branch order [51...]
 274
 275         if not complete_at:
 276             date = datetime.min
 277             complete_at = date.replace(tzinfo=tzutc())
 278
 279         if is_building(bldr):
 280             date = datetime.max
 281             complete_at = date.replace(tzinfo=tzutc())
 282
 283         return (pos, complete_at, bldr.name)
 284
 285     results = yield defer.gatherResults([bldr_info(bldr) for bldr in builders])
 286     results.sort(key=bldr_sort)
 287
 288     # for r in results:
 289     #   log.msg("prioritizeBuilders: {:>20} complete_at: {}".format(r[1].name, r[0]))
 290
 291     return [r[1] for r in results]
 292
 293
 294 c["prioritizeBuilders"] = prioritizeBuilders
 295
 296 ####### CHANGESOURCES
 297
 298 # find targets
 299 targets = dict()
 300
 301
 302 def populateTargets():
 303     """fetch a shallow clone of each configured branch in turn:
 304     execute dump-target-info.pl and collate the results to ensure
 305     targets that only exist in specific branches get built.
 306     This takes a while during master startup but is executed only once.
 307     """
 308     sourcegit = work_dir + "/source.git"
 309     for branch in branchNames:
 310         log.msg(f"Populating targets for {branch}, this will take time")
 311
 312         if os.path.isdir(sourcegit):
 313             subprocess.call(["rm", "-rf", sourcegit])
 314
 315         subprocess.call(
 316             [
 317                 "git",
 318                 "clone",
 319                 "-q",
 320                 "--depth=1",
 321                 "--branch=" + branch,
 322                 repo_url,
 323                 sourcegit,
 324             ]
 325         )
 326
 327         os.makedirs(sourcegit + "/tmp", exist_ok=True)
 328         findtargets = subprocess.Popen(
 329             ["./scripts/dump-target-info.pl", "targets"],
 330             stdout=subprocess.PIPE,
 331             stderr=subprocess.DEVNULL,
 332             cwd=sourcegit,
 333         )
 334
 335         targets[branch] = set()
 336         while True:
 337             line = findtargets.stdout.readline()
 338             if not line:
 339                 break
 340             ta = line.decode().strip().split(" ")
 341             targets[branch].add(ta[0])
 342
 343         subprocess.call(["rm", "-rf", sourcegit])
 344
 345
 346 populateTargets()
 347
 348 # the 'change_source' setting tells the buildmaster how it should find out
 349 # about source code changes.
 350
 351 c["change_source"] = []
 352 c["change_source"].append(
 353     GitPoller(
 354         repo_url,
 355         workdir=work_dir + "/work.git",
 356         branches=branchNames,
 357         pollAtLaunch=True,
 358         pollinterval=300,
 359     )
 360 )
 361
 362 ####### SCHEDULERS
 363
 364 # Configure the Schedulers, which decide how to react to incoming changes.
 365
 366
 367 # Selector for known valid tags
 368 class TagChoiceParameter(BaseParameter):
 369     spec_attributes = ["strict", "choices"]
 370     type = "list"
 371     strict = True
 372
 373     def __init__(self, name, label=None, **kw):
 374         super().__init__(name, label, **kw)
 375         self._choice_list = []
 376
 377     def getRevTags(self, findtag=None):
 378         taglist = []
 379         branchvers = []
 380
 381         # we will filter out tags that do no match the configured branches
 382         for b in branchNames:
 383             basever = re.search(r"-([0-9]+\.[0-9]+)$", b)
 384             if basever:
 385                 branchvers.append(basever[1])
 386
 387         # grab tags from remote repository
 388         alltags = subprocess.Popen(
 389             ["git", "ls-remote", "--tags", repo_url], stdout=subprocess.PIPE
 390         )
 391
 392         while True:
 393             line = alltags.stdout.readline()
 394
 395             if not line:
 396                 break
 397
 398             (rev, tag) = line.split()
 399
 400             # does it match known format? ('vNN.NN.NN(-rcN)')
 401             tagver = re.search(
 402                 r"\brefs/tags/(v[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?)$",
 403                 tag.decode().strip(),
 404             )
 405
 406             # only list valid tags matching configured branches
 407             if tagver and any(tagver[1][1:].startswith(b) for b in branchvers):
 408                 # if we want a specific tag, ignore all that don't match
 409                 if findtag and findtag != tagver[1]:
 410                     continue
 411                 taglist.append({"rev": rev.decode().strip(), "tag": tagver[1]})
 412
 413         return taglist
 414
 415     @property
 416     def choices(self):
 417         taglist = [rt["tag"] for rt in self.getRevTags()]
 418         taglist.sort(
 419             reverse=True,
 420             key=lambda tag: tag if re.search(r"-rc[0-9]+$", tag) else tag + "-z",
 421         )
 422         taglist.insert(0, "")
 423
 424         self._choice_list = taglist
 425
 426         return self._choice_list
 427
 428     def updateFromKwargs(self, properties, kwargs, **unused):
 429         tag = self.getFromKwargs(kwargs)
 430         properties[self.name] = tag
 431
 432         # find the commit matching the tag
 433         findtag = self.getRevTags(tag)
 434
 435         if not findtag:
 436             raise ValidationError("Couldn't find tag")
 437
 438         properties["force_revision"] = findtag[0]["rev"]
 439
 440         # find the branch matching the tag
 441         branch = None
 442         branchver = re.search(r"v([0-9]+\.[0-9]+)", tag)
 443         for b in branchNames:
 444             if b.endswith(branchver[1]):
 445                 branch = b
 446
 447         if not branch:
 448             raise ValidationError("Couldn't find branch")
 449
 450         properties["force_branch"] = branch
 451
 452     def parse_from_arg(self, s):
 453         if self.strict and s not in self._choice_list:
 454             raise ValidationError(
 455                 "'%s' does not belong to list of available choices '%s'"
 456                 % (s, self._choice_list)
 457             )
 458         return s
 459
 460
 461 @util.renderer
 462 @defer.inlineCallbacks
 463 def builderNames(props):
 464     """since we have per branch and per target builders,
 465     address the relevant builder for each new buildrequest
 466     based on the request's desired branch and target.
 467     """
 468     branch = props.getProperty("branch")
 469     target = props.getProperty("target", "")
 470
 471     if target == "all":
 472         target = ""
 473
 474     # if that didn't work, try sourcestamp to find a branch
 475     if not branch:
 476         # match builders with target branch
 477         ss = props.sourcestamps[0]
 478         if ss:
 479             branch = ss["branch"]
 480         else:
 481             log.msg("couldn't find builder")
 482             return []  # nothing works
 483
 484     bname = branch + "_" + target
 485     builders = []
 486
 487     for b in (yield props.master.data.get(("builders",))):
 488         if not b["name"].startswith(bname):
 489             continue
 490         builders.append(b["name"])
 491
 492     return builders
 493
 494
 495 c["schedulers"] = []
 496 c["schedulers"].append(
 497     AnyBranchScheduler(
 498         name="all",
 499         change_filter=util.ChangeFilter(branch=branchNames),
 500         treeStableTimer=15 * 60,
 501         builderNames=builderNames,
 502     )
 503 )
 504
 505 c["schedulers"].append(
 506     ForceScheduler(
 507         name="force",
 508         buttonName="Force builds",
 509         label="Force build details",
 510         builderNames=["00_force_build"],
 511         codebases=[
 512             util.CodebaseParameter(
 513                 "",
 514                 label="Repository",
 515                 branch=util.FixedParameter(name="branch", default=""),
 516                 revision=util.FixedParameter(name="revision", default=""),
 517                 repository=util.FixedParameter(name="repository", default=""),
 518                 project=util.FixedParameter(name="project", default=""),
 519             )
 520         ],
 521         reason=util.StringParameter(
 522             name="reason",
 523             label="Reason",
 524             default="Trigger build",
 525             required=True,
 526             size=80,
 527         ),
 528         properties=[
 529             # NB: avoid nesting to simplify processing of properties
 530             util.ChoiceStringParameter(
 531                 name="target",
 532                 label="Build target",
 533                 default="all",
 534                 choices=["all"] + [t for b in branchNames for t in targets[b]],
 535             ),
 536             TagChoiceParameter(name="tag", label="Build tag", default=""),
 537         ],
 538     )
 539 )
 540
 541 c["schedulers"].append(
 542     schedulers.Triggerable(name="trigger", builderNames=builderNames, priority=20)
 543 )
 544
 545 ####### BUILDERS
 546
 547 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
 548 # what steps, and which workers can execute them.  Note that any particular build will
 549 # only take place on one worker.
 550
 551
 552 def IsNoMasterBuild(step):
 553     return step.getProperty("branch") != "master"
 554
 555
 556 def IsUsignEnabled(step):
 557     branch = step.getProperty("branch")
 558     return branch and branches[branch].get("usign_key")
 559
 560
 561 def IsSignEnabled(step):
 562     branch = step.getProperty("branch")
 563     return IsUsignEnabled(step) or branch and branches[branch].get("gpg_key")
 564
 565
 566 def IsKmodArchiveEnabled(step):
 567     branch = step.getProperty("branch")
 568     return branch and branches[branch].get("kmod_archive")
 569
 570
 571 def IsKmodArchiveAndRsyncEnabled(step):
 572     branch = step.getProperty("branch")
 573     return bool(IsKmodArchiveEnabled(step) and branches[branch].get("bin_url"))
 574
 575
 576 def GetBaseVersion(branch):
 577     if re.match(r"^[^-]+-[0-9]+\.[0-9]+$", branch):
 578         return branch.split("-")[1]
 579     else:
 580         return "master"
 581
 582
 583 @properties.renderer
 584 def GetVersionPrefix(props):
 585     branch = props.getProperty("branch")
 586     basever = GetBaseVersion(branch)
 587     if props.hasProperty("tag") and re.match(
 588         r"^v[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", props["tag"]
 589     ):
 590         return "%s/" % props["tag"][1:]
 591     elif basever != "master":
 592         return "%s-SNAPSHOT/" % basever
 593     else:
 594         return ""
 595
 596
 597 @util.renderer
 598 def GetConfigSeed(props):
 599     branch = props.getProperty("branch")
 600     return branch and branches[branch].get("config_seed") or ""
 601
 602
 603 @util.renderer
 604 def GetRsyncParams(props, srcorbin, urlorkey):
 605     # srcorbin: 'bin' or 'src'; urlorkey: 'url' or 'key'
 606     branch = props.getProperty("branch")
 607     opt = srcorbin + "_" + urlorkey
 608     return branch and branches[branch].get(opt)
 609
 610
 611 @util.renderer
 612 def GetUsignKey(props):
 613     branch = props.getProperty("branch")
 614     return branch and branches[branch].get("usign_key")
 615
 616
 617 def GetNextBuild(builder, requests):
 618     for r in requests:
 619         if r.properties:
 620             # order tagged build first
 621             if r.properties.hasProperty("tag"):
 622                 return r
 623
 624     r = requests[0]
 625     # log.msg("GetNextBuild: {:>20} id: {} bsid: {}".format(builder.name, r.id, r.bsid))
 626     return r
 627
 628
 629 def MakeEnv(overrides=None, tryccache=False):
 630     env = {
 631         "CCC": Interpolate("%(prop:cc_command:-gcc)s"),
 632         "CCXX": Interpolate("%(prop:cxx_command:-g++)s"),
 633     }
 634     if tryccache:
 635         env["CC"] = Interpolate("%(prop:builddir)s/ccache_cc.sh")
 636         env["CXX"] = Interpolate("%(prop:builddir)s/ccache_cxx.sh")
 637         env["CCACHE"] = Interpolate("%(prop:ccache_command:-)s")
 638     else:
 639         env["CC"] = env["CCC"]
 640         env["CXX"] = env["CCXX"]
 641         env["CCACHE"] = ""
 642     if overrides is not None:
 643         env.update(overrides)
 644     return env
 645
 646
 647 @properties.renderer
 648 def NetLockDl(props, extralock=None):
 649     lock = None
 650     if props.hasProperty("dl_lock"):
 651         lock = NetLocks[props["dl_lock"]]
 652     if lock is not None:
 653         return [lock.access("exclusive")]
 654     else:
 655         return []
 656
 657
 658 @properties.renderer
 659 def NetLockUl(props):
 660     lock = None
 661     if props.hasProperty("ul_lock"):
 662         lock = NetLocks[props["ul_lock"]]
 663     if lock is not None:
 664         return [lock.access("exclusive")]
 665     else:
 666         return []
 667
 668
 669 def IsTargetSelected(target):
 670     def CheckTargetProperty(step):
 671         selected_target = step.getProperty("target", "all")
 672         if selected_target != "all" and selected_target != target:
 673             return False
 674         return True
 675
 676     return CheckTargetProperty
 677
 678
 679 @util.renderer
 680 def UsignSec2Pub(props):
 681     branch = props.getProperty("branch")
 682     try:
 683         comment = (
 684             branches[branch].get("usign_comment") or "untrusted comment: secret key"
 685         )
 686         seckey = branches[branch].get("usign_key")
 687         seckey = base64.b64decode(seckey)
 688     except Exception:
 689         return None
 690
 691     return "{}\n{}".format(
 692         re.sub(r"\bsecret key$", "public key", comment),
 693         base64.b64encode(seckey[0:2] + seckey[32:40] + seckey[72:]),
 694     )
 695
 696
 697 def canStartBuild(builder, wfb, request):
 698     """filter out non tag requests for tag_only workers."""
 699     wtagonly = wfb.worker.properties.getProperty("tag_only")
 700     tag = request.properties.getProperty("tag")
 701
 702     if wtagonly and not tag:
 703         return False
 704
 705     return True
 706
 707
 708 c["builders"] = []
 709
 710 workerNames = []
 711
 712 for worker in c["workers"]:
 713     workerNames.append(worker.workername)
 714
 715 # add a single LocalWorker to handle the forcebuild builder
 716 c["workers"].append(LocalWorker("__local_force_build", max_builds=1))
 717
 718 force_factory = BuildFactory()
 719 force_factory.addStep(
 720     steps.Trigger(
 721         name="trigger_build",
 722         schedulerNames=["trigger"],
 723         sourceStamps=[
 724             {
 725                 "codebase": "",
 726                 "branch": Property("force_branch"),
 727                 "revision": Property("force_revision"),
 728                 "repository": repo_url,
 729                 "project": "",
 730             }
 731         ],
 732         set_properties={
 733             "reason": Property("reason"),
 734             "tag": Property("tag"),
 735             "target": Property("target"),
 736         },
 737     )
 738 )
 739
 740 c["builders"].append(
 741     BuilderConfig(
 742         name="00_force_build", workername="__local_force_build", factory=force_factory
 743     )
 744 )
 745
 746
 747 # NB the phase1 build factory assumes workers are single-build only
 748 def prepareFactory(target):
 749     (target, subtarget) = target.split("/")
 750
 751     factory = BuildFactory()
 752
 753     # setup shared work directory if required
 754     factory.addStep(
 755         ShellCommand(
 756             name="sharedwd",
 757             descriptionDone="Shared work directory set up",
 758             command='test -L "$PWD" || (mkdir -p ../shared-workdir && rm -rf "$PWD" && ln -s shared-workdir "$PWD")',
 759             workdir=".",
 760             haltOnFailure=True,
 761         )
 762     )
 763
 764     # find number of cores
 765     factory.addStep(
 766         SetPropertyFromCommand(
 767             name="nproc",
 768             property="nproc",
 769             description="Finding number of CPUs",
 770             command=["nproc"],
 771         )
 772     )
 773
 774     # find gcc and g++ compilers
 775     factory.addStep(
 776         FileDownload(
 777             name="dlfindbinpl",
 778             mastersrc=scripts_dir + "/findbin.pl",
 779             workerdest="../findbin.pl",
 780             mode=0o755,
 781         )
 782     )
 783
 784     factory.addStep(
 785         SetPropertyFromCommand(
 786             name="gcc",
 787             property="cc_command",
 788             description="Finding gcc command",
 789             command=["../findbin.pl", "gcc", "", ""],
 790             haltOnFailure=True,
 791         )
 792     )
 793
 794     factory.addStep(
 795         SetPropertyFromCommand(
 796             name="g++",
 797             property="cxx_command",
 798             description="Finding g++ command",
 799             command=["../findbin.pl", "g++", "", ""],
 800             haltOnFailure=True,
 801         )
 802     )
 803
 804     # see if ccache is available
 805     factory.addStep(
 806         SetPropertyFromCommand(
 807             name="ccache",
 808             property="ccache_command",
 809             description="Testing for ccache command",
 810             command=["which", "ccache"],
 811             haltOnFailure=False,
 812             flunkOnFailure=False,
 813             warnOnFailure=False,
 814             hideStepIf=lambda r, s: r == results.FAILURE,
 815         )
 816     )
 817
 818     # check out the source
 819     # Git() runs:
 820     # if repo doesn't exist: 'git clone repourl'
 821     # method 'clean' runs 'git clean -d -f', method fresh runs 'git clean -f -f -d -x'. Only works with mode='full'
 822     # git cat-file -e <commit>
 823     # git checkout -f <commit>
 824     # git checkout -B <branch>
 825     # git rev-parse HEAD
 826     factory.addStep(
 827         Git(
 828             name="git",
 829             repourl=repo_url,
 830             mode="full",
 831             method="fresh",
 832             locks=NetLockDl,
 833             haltOnFailure=True,
 834         )
 835     )
 836
 837     # workaround for https://github.com/openwrt/buildbot/issues/5
 838     factory.addStep(
 839         Git(
 840             name="git me once more please",
 841             repourl=repo_url,
 842             mode="full",
 843             method="fresh",
 844             locks=NetLockDl,
 845             haltOnFailure=True,
 846         )
 847     )
 848
 849     # update remote refs
 850     factory.addStep(
 851         ShellCommand(
 852             name="fetchrefs",
 853             description="Fetching Git remote refs",
 854             descriptionDone="Git remote refs fetched",
 855             command=[
 856                 "git",
 857                 "fetch",
 858                 "origin",
 859                 Interpolate(
 860                     "+refs/heads/%(prop:branch)s:refs/remotes/origin/%(prop:branch)s"
 861                 ),
 862             ],
 863             haltOnFailure=True,
 864         )
 865     )
 866
 867     # getver.sh requires local branches to track upstream otherwise version computation fails.
 868     # Git() does not set tracking branches when cloning or switching, so work around this here
 869     factory.addStep(
 870         ShellCommand(
 871             name="trackupstream",
 872             description="Setting upstream branch",
 873             descriptionDone="getver.sh is happy now",
 874             command=["git", "branch", "-u", Interpolate("origin/%(prop:branch)s")],
 875             haltOnFailure=True,
 876         )
 877     )
 878
 879     # Verify that Git HEAD points to a tag or branch
 880     # Ref: https://web.archive.org/web/20190729224316/http://lists.infradead.org/pipermail/openwrt-devel/2019-June/017809.html
 881     factory.addStep(
 882         ShellCommand(
 883             name="gitverify",
 884             description="Ensuring that Git HEAD is pointing to a branch or tag",
 885             descriptionDone="Git HEAD is sane",
 886             command='git rev-parse --abbrev-ref HEAD | grep -vxqF HEAD || git show-ref --tags --dereference 2>/dev/null | sed -ne "/^$(git rev-parse HEAD) / { s|^.*/||; s|\\^.*||; p }" | grep -qE "^v[0-9][0-9]\\."',
 887             haltOnFailure=True,
 888         )
 889     )
 890
 891     factory.addStep(
 892         StringDownload(
 893             name="ccachecc",
 894             s='#!/bin/sh\nexec ${CCACHE} ${CCC} "$@"\n',
 895             workerdest="../ccache_cc.sh",
 896             mode=0o755,
 897         )
 898     )
 899
 900     factory.addStep(
 901         StringDownload(
 902             name="ccachecxx",
 903             s='#!/bin/sh\nexec ${CCACHE} ${CCXX} "$@"\n',
 904             workerdest="../ccache_cxx.sh",
 905             mode=0o755,
 906         )
 907     )
 908
 909     # feed
 910     factory.addStep(
 911         ShellCommand(
 912             name="updatefeeds",
 913             description="Updating feeds",
 914             command=["./scripts/feeds", "update"],
 915             env=MakeEnv(tryccache=True),
 916             haltOnFailure=True,
 917             locks=NetLockDl,
 918         )
 919     )
 920
 921     # feed
 922     factory.addStep(
 923         ShellCommand(
 924             name="installfeeds",
 925             description="Installing feeds",
 926             command=["./scripts/feeds", "install", "-a"],
 927             env=MakeEnv(tryccache=True),
 928             haltOnFailure=True,
 929         )
 930     )
 931
 932     # seed config
 933     factory.addStep(
 934         StringDownload(
 935             name="dlconfigseed",
 936             s=Interpolate("%(kw:seed)s\n", seed=GetConfigSeed),
 937             workerdest=".config",
 938             mode=0o644,
 939         )
 940     )
 941
 942     # configure
 943     factory.addStep(
 944         ShellCommand(
 945             name="newconfig",
 946             descriptionDone=".config seeded",
 947             command=Interpolate(
 948                 "printf 'CONFIG_TARGET_%(kw:target)s=y\\nCONFIG_TARGET_%(kw:target)s_%(kw:subtarget)s=y\\nCONFIG_SIGNED_PACKAGES=%(kw:usign:#?|y|n)s\\n' >> .config",
 949                 target=target,
 950                 subtarget=subtarget,
 951                 usign=GetUsignKey,
 952             ),
 953         )
 954     )
 955
 956     factory.addStep(
 957         ShellCommand(
 958             name="defconfig",
 959             description="Populating .config",
 960             command=["make", "defconfig"],
 961             env=MakeEnv(),
 962         )
 963     )
 964
 965     # check arch - exit early if does not exist - NB: some targets do not define CONFIG_TARGET_target_subtarget
 966     factory.addStep(
 967         ShellCommand(
 968             name="checkarch",
 969             description="Checking architecture",
 970             descriptionDone="Architecture validated",
 971             command='grep -sq CONFIG_TARGET_%s=y .config && grep -sq CONFIG_TARGET_SUBTARGET=\\"%s\\" .config'
 972             % (target, subtarget),
 973             logEnviron=False,
 974             want_stdout=False,
 975             want_stderr=False,
 976             haltOnFailure=True,
 977             flunkOnFailure=False,  # this is not a build FAILURE - TODO mark build as SKIPPED
 978         )
 979     )
 980
 981     # find libc suffix
 982     factory.addStep(
 983         SetPropertyFromCommand(
 984             name="libc",
 985             property="libc",
 986             description="Finding libc suffix",
 987             command=[
 988                 "sed",
 989                 "-ne",
 990                 '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }',
 991                 ".config",
 992             ],
 993         )
 994     )
 995
 996     # install build key
 997     factory.addStep(
 998         StringDownload(
 999             name="dlkeybuildpub",
1000             s=Interpolate("%(kw:sec2pub)s", sec2pub=UsignSec2Pub),
1001             workerdest="key-build.pub",
1002             mode=0o600,
1003             doStepIf=IsUsignEnabled,
1004         )
1005     )
1006
1007     factory.addStep(
1008         StringDownload(
1009             name="dlkeybuild",
1010             s="# fake private key",
1011             workerdest="key-build",
1012             mode=0o600,
1013             doStepIf=IsUsignEnabled,
1014         )
1015     )
1016
1017     factory.addStep(
1018         StringDownload(
1019             name="dlkeybuilducert",
1020             s="# fake certificate",
1021             workerdest="key-build.ucert",
1022             mode=0o600,
1023             doStepIf=IsUsignEnabled,
1024         )
1025     )
1026
1027     # prepare dl
1028     factory.addStep(
1029         ShellCommand(
1030             name="dldir",
1031             description="Preparing dl/",
1032             descriptionDone="dl/ prepared",
1033             command='mkdir -p ../dl && rm -rf "build/dl" && ln -s ../../dl "build/dl"',
1034             workdir=Property("builddir"),
1035             logEnviron=False,
1036             want_stdout=False,
1037         )
1038     )
1039
1040     # cleanup dl
1041     factory.addStep(
1042         ShellCommand(
1043             name="dlprune",
1044             description="Pruning dl/",
1045             descriptionDone="dl/ pruned",
1046             command="find dl/ -mindepth 1 -atime +15 -delete -print",
1047             logEnviron=False,
1048             haltOnFailure=False,
1049             flunkOnFailure=False,
1050             warnOnFailure=False,
1051         )
1052     )
1053
1054     # prepare tar
1055     factory.addStep(
1056         ShellCommand(
1057             name="dltar",
1058             description="Building and installing GNU tar",
1059             descriptionDone="GNU tar built and installed",
1060             command=[
1061                 "make",
1062                 Interpolate("-j%(prop:nproc:-1)s"),
1063                 "tools/tar/compile",
1064                 "V=s",
1065             ],
1066             env=MakeEnv(tryccache=True),
1067             haltOnFailure=True,
1068         )
1069     )
1070
1071     # populate dl
1072     factory.addStep(
1073         ShellCommand(
1074             name="dlrun",
1075             description="Populating dl/",
1076             descriptionDone="dl/ populated",
1077             command=["make", Interpolate("-j%(prop:nproc:-1)s"), "download", "V=s"],
1078             env=MakeEnv(),
1079             logEnviron=False,
1080             locks=NetLockDl,
1081         )
1082     )
1083
1084     factory.addStep(
1085         ShellCommand(
1086             name="cleanbase",
1087             description="Cleaning base-files",
1088             command=["make", "package/base-files/clean", "V=s"],
1089         )
1090     )
1091
1092     # build
1093     factory.addStep(
1094         ShellCommand(
1095             name="tools",
1096             description="Building and installing tools",
1097             descriptionDone="Tools built and installed",
1098             command=[
1099                 "make",
1100                 Interpolate("-j%(prop:nproc:-1)s"),
1101                 "tools/install",
1102                 "V=s",
1103             ],
1104             env=MakeEnv(tryccache=True),
1105             haltOnFailure=True,
1106         )
1107     )
1108
1109     factory.addStep(
1110         ShellCommand(
1111             name="toolchain",
1112             description="Building and installing toolchain",
1113             descriptionDone="Toolchain built and installed",
1114             command=[
1115                 "make",
1116                 Interpolate("-j%(prop:nproc:-1)s"),
1117                 "toolchain/install",
1118                 "V=s",
1119             ],
1120             env=MakeEnv(),
1121             haltOnFailure=True,
1122         )
1123     )
1124
1125     factory.addStep(
1126         ShellCommand(
1127             name="kmods",
1128             description="Building kmods",
1129             descriptionDone="Kmods built",
1130             command=[
1131                 "make",
1132                 Interpolate("-j%(prop:nproc:-1)s"),
1133                 "target/compile",
1134                 "V=s",
1135                 "IGNORE_ERRORS=n m",
1136                 "BUILD_LOG=1",
1137             ],
1138             env=MakeEnv(),
1139             haltOnFailure=True,
1140         )
1141     )
1142
1143     # find kernel version
1144     factory.addStep(
1145         SetPropertyFromCommand(
1146             name="kernelversion",
1147             property="kernelversion",
1148             description="Finding the effective Kernel version",
1149             command="make --no-print-directory -C target/linux/ val.LINUX_VERSION val.LINUX_RELEASE val.LINUX_VERMAGIC | xargs printf '%s-%s-%s\\n'",
1150             env={"TOPDIR": Interpolate("%(prop:builddir)s/build")},
1151         )
1152     )
1153
1154     factory.addStep(
1155         ShellCommand(
1156             name="pkgclean",
1157             description="Cleaning up package build",
1158             descriptionDone="Package build cleaned up",
1159             command=["make", "package/cleanup", "V=s"],
1160         )
1161     )
1162
1163     factory.addStep(
1164         ShellCommand(
1165             name="pkgbuild",
1166             description="Building packages",
1167             descriptionDone="Packages built",
1168             command=[
1169                 "make",
1170                 Interpolate("-j%(prop:nproc:-1)s"),
1171                 "package/compile",
1172                 "V=s",
1173                 "IGNORE_ERRORS=n m",
1174                 "BUILD_LOG=1",
1175             ],
1176             env=MakeEnv(),
1177             haltOnFailure=True,
1178         )
1179     )
1180
1181     factory.addStep(
1182         ShellCommand(
1183             name="pkginstall",
1184             description="Installing packages",
1185             descriptionDone="Packages installed",
1186             command=[
1187                 "make",
1188                 Interpolate("-j%(prop:nproc:-1)s"),
1189                 "package/install",
1190                 "V=s",
1191             ],
1192             env=MakeEnv(),
1193             haltOnFailure=True,
1194         )
1195     )
1196
1197     factory.addStep(
1198         ShellCommand(
1199             name="pkgindex",
1200             description="Indexing packages",
1201             descriptionDone="Packages indexed",
1202             command=[
1203                 "make",
1204                 Interpolate("-j%(prop:nproc:-1)s"),
1205                 "package/index",
1206                 "V=s",
1207                 "CONFIG_SIGNED_PACKAGES=",
1208             ],
1209             env=MakeEnv(),
1210             haltOnFailure=True,
1211         )
1212     )
1213
1214     factory.addStep(
1215         ShellCommand(
1216             name="images",
1217             description="Building and installing images",
1218             descriptionDone="Images built and installed",
1219             command=[
1220                 "make",
1221                 Interpolate("-j%(prop:nproc:-1)s"),
1222                 "target/install",
1223                 "V=s",
1224             ],
1225             env=MakeEnv(),
1226             haltOnFailure=True,
1227         )
1228     )
1229
1230     factory.addStep(
1231         ShellCommand(
1232             name="buildinfo",
1233             description="Generating config.buildinfo, version.buildinfo and feeds.buildinfo",
1234             command="make -j1 buildinfo V=s || true",
1235             env=MakeEnv(),
1236             haltOnFailure=True,
1237         )
1238     )
1239
1240     factory.addStep(
1241         ShellCommand(
1242             name="json_overview_image_info",
1243             description="Generating profiles.json in target folder",
1244             command="make -j1 json_overview_image_info V=s || true",
1245             env=MakeEnv(),
1246             haltOnFailure=True,
1247         )
1248     )
1249
1250     factory.addStep(
1251         ShellCommand(
1252             name="checksums",
1253             description="Calculating checksums",
1254             descriptionDone="Checksums calculated",
1255             command=["make", "-j1", "checksum", "V=s"],
1256             env=MakeEnv(),
1257             haltOnFailure=True,
1258         )
1259     )
1260
1261     factory.addStep(
1262         ShellCommand(
1263             name="kmoddir",
1264             descriptionDone="Kmod directory created",
1265             command=[
1266                 "mkdir",
1267                 "-p",
1268                 Interpolate(
1269                     "bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s",
1270                     target=target,
1271                     subtarget=subtarget,
1272                 ),
1273             ],
1274             haltOnFailure=True,
1275             doStepIf=IsKmodArchiveEnabled,
1276         )
1277     )
1278
1279     factory.addStep(
1280         ShellCommand(
1281             name="kmodprepare",
1282             description="Preparing kmod archive",
1283             descriptionDone="Kmod archive prepared",
1284             command=[
1285                 "rsync",
1286                 "--include=/kmod-*.ipk",
1287                 "--exclude=*",
1288                 "-va",
1289                 Interpolate(
1290                     "bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/packages/",
1291                     target=target,
1292                     subtarget=subtarget,
1293                 ),
1294                 Interpolate(
1295                     "bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/",
1296                     target=target,
1297                     subtarget=subtarget,
1298                 ),
1299             ],
1300             haltOnFailure=True,
1301             doStepIf=IsKmodArchiveEnabled,
1302         )
1303     )
1304
1305     factory.addStep(
1306         ShellCommand(
1307             name="kmodindex",
1308             description="Indexing kmod archive",
1309             descriptionDone="Kmod archive indexed",
1310             command=[
1311                 "make",
1312                 Interpolate("-j%(prop:nproc:-1)s"),
1313                 "package/index",
1314                 "V=s",
1315                 "CONFIG_SIGNED_PACKAGES=",
1316                 Interpolate(
1317                     "PACKAGE_SUBDIRS=bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/",
1318                     target=target,
1319                     subtarget=subtarget,
1320                 ),
1321             ],
1322             env=MakeEnv(),
1323             haltOnFailure=True,
1324             doStepIf=IsKmodArchiveEnabled,
1325         )
1326     )
1327
1328     # sign
1329     factory.addStep(
1330         MasterShellCommand(
1331             name="signprepare",
1332             descriptionDone="Temporary signing directory prepared",
1333             command=["mkdir", "-p", "%s/signing" % (work_dir)],
1334             haltOnFailure=True,
1335             doStepIf=IsSignEnabled,
1336         )
1337     )
1338
1339     factory.addStep(
1340         ShellCommand(
1341             name="signpack",
1342             description="Packing files to sign",
1343             descriptionDone="Files to sign packed",
1344             command=Interpolate(
1345                 "find bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/ bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz",
1346                 target=target,
1347                 subtarget=subtarget,
1348             ),
1349             haltOnFailure=True,
1350             doStepIf=IsSignEnabled,
1351         )
1352     )
1353
1354     factory.addStep(
1355         FileUpload(
1356             workersrc="sign.tar.gz",
1357             masterdest="%s/signing/%s.%s.tar.gz" % (work_dir, target, subtarget),
1358             haltOnFailure=True,
1359             doStepIf=IsSignEnabled,
1360         )
1361     )
1362
1363     factory.addStep(
1364         MasterShellCommand(
1365             name="signfiles",
1366             description="Signing files",
1367             descriptionDone="Files signed",
1368             command=[
1369                 "%s/signall.sh" % (scripts_dir),
1370                 "%s/signing/%s.%s.tar.gz" % (work_dir, target, subtarget),
1371                 Interpolate("%(prop:branch)s"),
1372             ],
1373             env={"CONFIG_INI": os.getenv("BUILDMASTER_CONFIG", "./config.ini")},
1374             haltOnFailure=True,
1375             doStepIf=IsSignEnabled,
1376         )
1377     )
1378
1379     factory.addStep(
1380         FileDownload(
1381             name="dlsigntargz",
1382             mastersrc="%s/signing/%s.%s.tar.gz" % (work_dir, target, subtarget),
1383             workerdest="sign.tar.gz",
1384             haltOnFailure=True,
1385             doStepIf=IsSignEnabled,
1386         )
1387     )
1388
1389     factory.addStep(
1390         ShellCommand(
1391             name="signunpack",
1392             description="Unpacking signed files",
1393             descriptionDone="Signed files unpacked",
1394             command=["tar", "-xzf", "sign.tar.gz"],
1395             haltOnFailure=True,
1396             doStepIf=IsSignEnabled,
1397         )
1398     )
1399
1400     # upload
1401     factory.addStep(
1402         ShellCommand(
1403             name="dirprepare",
1404             descriptionDone="Upload directory structure prepared",
1405             command=[
1406                 "mkdir",
1407                 "-p",
1408                 Interpolate(
1409                     "tmp/upload/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s",
1410                     target=target,
1411                     subtarget=subtarget,
1412                     prefix=GetVersionPrefix,
1413                 ),
1414             ],
1415             haltOnFailure=True,
1416         )
1417     )
1418
1419     factory.addStep(
1420         ShellCommand(
1421             name="linkprepare",
1422             descriptionDone="Repository symlink prepared",
1423             command=[
1424                 "ln",
1425                 "-s",
1426                 "-f",
1427                 Interpolate(
1428                     "../packages-%(kw:basever)s",
1429                     basever=util.Transform(GetBaseVersion, Property("branch")),
1430                 ),
1431                 Interpolate(
1432                     "tmp/upload/%(kw:prefix)spackages", prefix=GetVersionPrefix
1433                 ),
1434             ],
1435             doStepIf=IsNoMasterBuild,
1436             haltOnFailure=True,
1437         )
1438     )
1439
1440     factory.addStep(
1441         ShellCommand(
1442             name="kmoddirprepare",
1443             descriptionDone="Kmod archive upload directory prepared",
1444             command=[
1445                 "mkdir",
1446                 "-p",
1447                 Interpolate(
1448                     "tmp/upload/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s",
1449                     target=target,
1450                     subtarget=subtarget,
1451                     prefix=GetVersionPrefix,
1452                 ),
1453             ],
1454             haltOnFailure=True,
1455             doStepIf=IsKmodArchiveEnabled,
1456         )
1457     )
1458
1459     factory.addStep(
1460         ShellCommand(
1461             name="dirupload",
1462             description="Uploading directory structure",
1463             descriptionDone="Directory structure uploaded",
1464             command=["rsync", Interpolate("-az%(prop:rsync_ipv4:+4)s")]
1465             + rsync_defopts
1466             + [
1467                 "tmp/upload/",
1468                 Interpolate("%(kw:url)s/", url=GetRsyncParams.withArgs("bin", "url")),
1469             ],
1470             env={
1471                 "RSYNC_PASSWORD": Interpolate(
1472                     "%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")
1473                 )
1474             },
1475             haltOnFailure=True,
1476             logEnviron=False,
1477             locks=NetLockUl,
1478             doStepIf=util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
1479         )
1480     )
1481
1482     # download remote sha256sums to 'target-sha256sums'
1483     factory.addStep(
1484         ShellCommand(
1485             name="target-sha256sums",
1486             description="Fetching remote sha256sums for target",
1487             descriptionDone="Remote sha256sums for target fetched",
1488             command=["rsync", Interpolate("-z%(prop:rsync_ipv4:+4)s")]
1489             + rsync_defopts
1490             + [
1491                 Interpolate(
1492                     "%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/sha256sums",
1493                     url=GetRsyncParams.withArgs("bin", "url"),
1494                     target=target,
1495                     subtarget=subtarget,
1496                     prefix=GetVersionPrefix,
1497                 ),
1498                 "target-sha256sums",
1499             ],
1500             env={
1501                 "RSYNC_PASSWORD": Interpolate(
1502                     "%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")
1503                 )
1504             },
1505             logEnviron=False,
1506             haltOnFailure=False,
1507             flunkOnFailure=False,
1508             warnOnFailure=False,
1509             doStepIf=util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
1510         )
1511     )
1512
1513     # build list of files to upload
1514     factory.addStep(
1515         FileDownload(
1516             name="dlsha2rsyncpl",
1517             mastersrc=scripts_dir + "/sha2rsync.pl",
1518             workerdest="../sha2rsync.pl",
1519             mode=0o755,
1520         )
1521     )
1522
1523     factory.addStep(
1524         ShellCommand(
1525             name="buildlist",
1526             description="Building list of files to upload",
1527             descriptionDone="List of files to upload built",
1528             command=[
1529                 "../sha2rsync.pl",
1530                 "target-sha256sums",
1531                 Interpolate(
1532                     "bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/sha256sums",
1533                     target=target,
1534                     subtarget=subtarget,
1535                 ),
1536                 "rsynclist",
1537             ],
1538             haltOnFailure=True,
1539         )
1540     )
1541
1542     factory.addStep(
1543         FileDownload(
1544             name="dlrsync.sh",
1545             mastersrc=scripts_dir + "/rsync.sh",
1546             workerdest="../rsync.sh",
1547             mode=0o755,
1548         )
1549     )
1550
1551     # upload new files and update existing ones
1552     factory.addStep(
1553         ShellCommand(
1554             name="targetupload",
1555             description="Uploading target files",
1556             descriptionDone="Target files uploaded",
1557             command=[
1558                 "../rsync.sh",
1559                 "--exclude=/kmods/",
1560                 "--files-from=rsynclist",
1561                 "--delay-updates",
1562                 "--partial-dir=.~tmp~%s~%s" % (target, subtarget),
1563             ]
1564             + rsync_defopts
1565             + [
1566                 Interpolate("-a%(prop:rsync_ipv4:+4)s"),
1567                 Interpolate(
1568                     "bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/",
1569                     target=target,
1570                     subtarget=subtarget,
1571                 ),
1572                 Interpolate(
1573                     "%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/",
1574                     url=GetRsyncParams.withArgs("bin", "url"),
1575                     target=target,
1576                     subtarget=subtarget,
1577                     prefix=GetVersionPrefix,
1578                 ),
1579             ],
1580             env={
1581                 "RSYNC_PASSWORD": Interpolate(
1582                     "%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")
1583                 )
1584             },
1585             haltOnFailure=True,
1586             logEnviron=False,
1587             doStepIf=util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
1588         )
1589     )
1590
1591     # delete files which don't exist locally
1592     factory.addStep(
1593         ShellCommand(
1594             name="targetprune",
1595             description="Pruning target files",
1596             descriptionDone="Target files pruned",
1597             command=[
1598                 "../rsync.sh",
1599                 "--exclude=/kmods/",
1600                 "--delete",
1601                 "--existing",
1602                 "--ignore-existing",
1603                 "--delay-updates",
1604                 "--partial-dir=.~tmp~%s~%s" % (target, subtarget),
1605             ]
1606             + rsync_defopts
1607             + [
1608                 Interpolate("-a%(prop:rsync_ipv4:+4)s"),
1609                 Interpolate(
1610                     "bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/",
1611                     target=target,
1612                     subtarget=subtarget,
1613                 ),
1614                 Interpolate(
1615                     "%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/",
1616                     url=GetRsyncParams.withArgs("bin", "url"),
1617                     target=target,
1618                     subtarget=subtarget,
1619                     prefix=GetVersionPrefix,
1620                 ),
1621             ],
1622             env={
1623                 "RSYNC_PASSWORD": Interpolate(
1624                     "%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")
1625                 )
1626             },
1627             haltOnFailure=True,
1628             logEnviron=False,
1629             locks=NetLockUl,
1630             doStepIf=util.Transform(bool, GetRsyncParams.withArgs("bin", "url")),
1631         )
1632     )
1633
1634     factory.addStep(
1635         ShellCommand(
1636             name="kmodupload",
1637             description="Uploading kmod archive",
1638             descriptionDone="Kmod archive uploaded",
1639             command=[
1640                 "../rsync.sh",
1641                 "--delete",
1642                 "--delay-updates",
1643                 "--partial-dir=.~tmp~%s~%s" % (target, subtarget),
1644             ]
1645             + rsync_defopts
1646             + [
1647                 Interpolate("-a%(prop:rsync_ipv4:+4)s"),
1648                 Interpolate(
1649                     "bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/",
1650                     target=target,
1651                     subtarget=subtarget,
1652                 ),
1653                 Interpolate(
1654                     "%(kw:url)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s/",
1655                     url=GetRsyncParams.withArgs("bin", "url"),
1656                     target=target,
1657                     subtarget=subtarget,
1658                     prefix=GetVersionPrefix,
1659                 ),
1660             ],
1661             env={
1662                 "RSYNC_PASSWORD": Interpolate(
1663                     "%(kw:key)s", key=GetRsyncParams.withArgs("bin", "key")
1664                 )
1665             },
1666             haltOnFailure=True,
1667             logEnviron=False,
1668             locks=NetLockUl,
1669             doStepIf=IsKmodArchiveAndRsyncEnabled,
1670         )
1671     )
1672
1673     factory.addStep(
1674         ShellCommand(
1675             name="sourcelist",
1676             description="Finding source archives to upload",
1677             descriptionDone="Source archives to upload found",
1678             command="find dl/ -maxdepth 1 -type f -not -size 0 -not -name '.*' -not -name '*.hash' -not -name '*.dl' -newer .config -printf '%f\\n' > sourcelist",
1679             haltOnFailure=True,
1680         )
1681     )
1682
1683     factory.addStep(
1684         ShellCommand(
1685             name="sourceupload",
1686             description="Uploading source archives",
1687             descriptionDone="Source archives uploaded",
1688             command=[
1689                 "../rsync.sh",
1690                 "--files-from=sourcelist",
1691                 "--size-only",
1692                 "--delay-updates",
1693             ]
1694             + rsync_defopts
1695             + [
1696                 Interpolate(
1697                     "--partial-dir=.~tmp~%(kw:target)s~%(kw:subtarget)s~%(prop:workername)s",
1698                     target=target,
1699                     subtarget=subtarget,
1700                 ),
1701                 Interpolate("-a%(prop:rsync_ipv4:+4)s"),
1702                 "dl/",
1703                 Interpolate("%(kw:url)s/", url=GetRsyncParams.withArgs("src", "url")),
1704             ],
1705             env={
1706                 "RSYNC_PASSWORD": Interpolate(
1707                     "%(kw:key)s", key=GetRsyncParams.withArgs("src", "key")
1708                 )
1709             },
1710             haltOnFailure=True,
1711             logEnviron=False,
1712             locks=NetLockUl,
1713             doStepIf=util.Transform(bool, GetRsyncParams.withArgs("src", "url")),
1714         )
1715     )
1716
1717     factory.addStep(
1718         ShellCommand(
1719             name="df",
1720             description="Reporting disk usage",
1721             command=["df", "-h", "."],
1722             env={"LC_ALL": "C"},
1723             logEnviron=False,
1724             haltOnFailure=False,
1725             flunkOnFailure=False,
1726             warnOnFailure=False,
1727             alwaysRun=True,
1728         )
1729     )
1730
1731     factory.addStep(
1732         ShellCommand(
1733             name="du",
1734             description="Reporting estimated file space usage",
1735             command=["du", "-sh", "."],
1736             env={"LC_ALL": "C"},
1737             logEnviron=False,
1738             haltOnFailure=False,
1739             flunkOnFailure=False,
1740             warnOnFailure=False,
1741             alwaysRun=True,
1742         )
1743     )
1744
1745     factory.addStep(
1746         ShellCommand(
1747             name="ccachestat",
1748             description="Reporting ccache stats",
1749             command=["ccache", "-s"],
1750             logEnviron=False,
1751             want_stderr=False,
1752             haltOnFailure=False,
1753             flunkOnFailure=False,
1754             warnOnFailure=False,
1755             doStepIf=util.Transform(bool, Property("ccache_command")),
1756         )
1757     )
1758
1759     return factory
1760
1761
1762 for brname in branchNames:
1763     for target in targets[brname]:
1764         bldrname = brname + "_" + target
1765         c["builders"].append(
1766             BuilderConfig(
1767                 name=bldrname,
1768                 workernames=workerNames,
1769                 factory=prepareFactory(target),
1770                 tags=[
1771                     brname,
1772                 ],
1773                 nextBuild=GetNextBuild,
1774                 canStartBuild=canStartBuild,
1775             )
1776         )
1777
1778
1779 ####### STATUS TARGETS
1780
1781 # 'status' is a list of Status Targets. The results of each build will be
1782 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
1783 # including web pages, email senders, and IRC bots.
1784
1785 if "status_bind" in inip1:
1786     c["www"] = {
1787         "port": inip1.get("status_bind"),
1788         "plugins": {"waterfall_view": True, "console_view": True, "grid_view": True},
1789     }
1790
1791     if "status_user" in inip1 and "status_password" in inip1:
1792         c["www"]["auth"] = util.UserPasswordAuth(
1793             [(inip1.get("status_user"), inip1.get("status_password"))]
1794         )
1795         c["www"]["authz"] = util.Authz(
1796             allowRules=[util.AnyControlEndpointMatcher(role="admins")],
1797             roleMatchers=[
1798                 util.RolesFromUsername(
1799                     roles=["admins"], usernames=[inip1.get("status_user")]
1800                 )
1801             ],
1802         )
1803
1804 c["services"] = []
1805 if ini.has_section("irc"):
1806     iniirc = ini["irc"]
1807     irc_host = iniirc.get("host", None)
1808     irc_port = iniirc.getint("port", 6667)
1809     irc_chan = iniirc.get("channel", None)
1810     irc_nick = iniirc.get("nickname", None)
1811     irc_pass = iniirc.get("password", None)
1812
1813     if irc_host and irc_nick and irc_chan:
1814         irc = reporters.IRC(
1815             irc_host,
1816             irc_nick,
1817             port=irc_port,
1818             password=irc_pass,
1819             channels=[irc_chan],
1820             notify_events=["exception", "problem", "recovery"],
1821         )
1822
1823         c["services"].append(irc)
1824
1825 c["revlink"] = util.RevlinkMatch(
1826     [r"https://git.openwrt.org/openwrt/(.*).git"],
1827     r"https://git.openwrt.org/?p=openwrt/\1.git;a=commit;h=%s",
1828 )
1829
1830 ####### DB URL
1831
1832 c["db"] = {
1833     # This specifies what database buildbot uses to store its state.  You can leave
1834     # this at its default for all but the largest installations.
1835     "db_url": "sqlite:///state.sqlite",
1836 }
1837
1838 c["buildbotNetUsageData"] = None