phase1/master.cfg

   1 # -*- python -*-
   2 # ex: set syntax=python:
   3
   4 import os
   5 import re
   6 import subprocess
   7 import ConfigParser
   8
   9 from buildbot import locks
  10
  11 # This is a sample buildmaster config file. It must be installed as
  12 # 'master.cfg' in your buildmaster's base directory.
  13
  14 ini = ConfigParser.ConfigParser()
  15 ini.read("./config.ini")
  16
  17 # This is the dictionary that the buildmaster pays attention to. We also use
  18 # a shorter alias to save typing.
  19 c = BuildmasterConfig = {}
  20
  21 ####### PROJECT IDENTITY
  22
  23 # the 'title' string will appear at the top of this buildbot
  24 # installation's html.WebStatus home page (linked to the
  25 # 'titleURL') and is embedded in the title of the waterfall HTML page.
  26
  27 c['title'] = ini.get("general", "title")
  28 c['titleURL'] = ini.get("general", "title_url")
  29
  30 # the 'buildbotURL' string should point to the location where the buildbot's
  31 # internal web server (usually the html.WebStatus page) is visible. This
  32 # typically uses the port number set in the Waterfall 'status' entry, but
  33 # with an externally-visible host name which the buildbot cannot figure out
  34 # without some help.
  35
  36 c['buildbotURL'] = ini.get("general", "buildbot_url")
  37
  38 ####### BUILDSLAVES
  39
  40 # The 'slaves' list defines the set of recognized buildslaves. Each element is
  41 # a BuildSlave object, specifying a unique slave name and password.  The same
  42 # slave name and password must be configured on the slave.
  43 from buildbot.buildslave import BuildSlave
  44
  45 slave_port = 9989
  46
  47 if ini.has_option("general", "port"):
  48         slave_port = ini.getint("general", "port")
  49
  50 c['slaves'] = []
  51 NetLocks = dict()
  52
  53 for section in ini.sections():
  54         if section.startswith("slave "):
  55                 if ini.has_option(section, "name") and ini.has_option(section, "password"):
  56                         sl_props = { 'dl_lock':None, 'ul_lock':None, 'do_cleanup':False, 'max_builds':1, 'shared_wd':False }
  57                         name = ini.get(section, "name")
  58                         password = ini.get(section, "password")
  59                         max_builds = 1
  60                         if ini.has_option(section, "builds"):
  61                                 max_builds = ini.getint(section, "builds")
  62                                 sl_props['max_builds'] = max_builds
  63                         if ini.has_option(section, "cleanup"):
  64                                 sl_props['do_cleanup'] = ini.getboolean(section, "cleanup")
  65                         if ini.has_option(section, "dl_lock"):
  66                                 lockname = ini.get(section, "dl_lock")
  67                                 sl_props['dl_lock'] = lockname
  68                                 if lockname not in NetLocks:
  69                                         NetLocks[lockname] = locks.MasterLock(lockname)
  70                         if ini.has_option(section, "ul_lock"):
  71                                 lockname = ini.get(section, "dl_lock")
  72                                 sl_props['ul_lock'] = lockname
  73                                 if lockname not in NetLocks:
  74                                         NetLocks[lockname] = locks.MasterLock(lockname)
  75                         if ini.has_option(section, "shared_wd"):
  76                                 shared_wd = ini.getboolean(section, "shared_wd")
  77                                 sl_props['shared_wd'] = shared_wd
  78                                 if shared_wd and (max_builds != 1):
  79                                         raise ValueError('max_builds must be 1 with shared workdir!')
  80                         c['slaves'].append(BuildSlave(name, password, max_builds = max_builds, properties = sl_props))
  81
  82 # 'slavePortnum' defines the TCP port to listen on for connections from slaves.
  83 # This must match the value configured into the buildslaves (with their
  84 # --master option)
  85 c['slavePortnum'] = slave_port
  86
  87 # coalesce builds
  88 c['mergeRequests'] = True
  89
  90 # Reduce amount of backlog data
  91 c['buildHorizon'] = 30
  92 c['logHorizon'] = 20
  93
  94 ####### CHANGESOURCES
  95
  96 home_dir = os.path.abspath(ini.get("general", "homedir"))
  97 tree_expire = 0
  98 other_builds = 0
  99 cc_version = None
 100
 101 cc_command = "gcc"
 102 cxx_command = "g++"
 103
 104 if ini.has_option("general", "expire"):
 105         tree_expire = ini.getint("general", "expire")
 106
 107 if ini.has_option("general", "other_builds"):
 108         other_builds = ini.getint("general", "other_builds")
 109
 110 if ini.has_option("general", "cc_version"):
 111         cc_version = ini.get("general", "cc_version").split()
 112         if len(cc_version) == 1:
 113                 cc_version = ["eq", cc_version[0]]
 114
 115 repo_url = ini.get("repo", "url")
 116 repo_branch = "master"
 117
 118 if ini.has_option("repo", "branch"):
 119         repo_branch = ini.get("repo", "branch")
 120
 121 rsync_bin_url = ini.get("rsync", "binary_url")
 122 rsync_bin_key = ini.get("rsync", "binary_password")
 123
 124 rsync_src_url = None
 125 rsync_src_key = None
 126
 127 if ini.has_option("rsync", "source_url"):
 128         rsync_src_url = ini.get("rsync", "source_url")
 129         rsync_src_key = ini.get("rsync", "source_password")
 130
 131 gpg_home = "~/.gnupg"
 132 gpg_keyid = None
 133 gpg_comment = "Unattended build signature"
 134 gpg_passfile = "/dev/null"
 135
 136 if ini.has_option("gpg", "home"):
 137         gpg_home = ini.get("gpg", "home")
 138
 139 if ini.has_option("gpg", "keyid"):
 140         gpg_keyid = ini.get("gpg", "keyid")
 141
 142 if ini.has_option("gpg", "comment"):
 143         gpg_comment = ini.get("gpg", "comment")
 144
 145 if ini.has_option("gpg", "passfile"):
 146         gpg_passfile = ini.get("gpg", "passfile")
 147
 148 enable_kmod_archive = True
 149
 150
 151 # find targets
 152 targets = [ ]
 153
 154 if not os.path.isdir(home_dir+'/source.git'):
 155         subprocess.call(["git", "clone", "--depth=1", "--branch="+repo_branch, repo_url, home_dir+'/source.git'])
 156 else:
 157         subprocess.call(["git", "pull"], cwd = home_dir+'/source.git')
 158
 159 findtargets = subprocess.Popen([home_dir+'/dumpinfo.pl', 'targets'],
 160         stdout = subprocess.PIPE, cwd = home_dir+'/source.git')
 161
 162 while True:
 163         line = findtargets.stdout.readline()
 164         if not line:
 165                 break
 166         ta = line.strip().split(' ')
 167         targets.append(ta[0])
 168
 169
 170 # the 'change_source' setting tells the buildmaster how it should find out
 171 # about source code changes.  Here we point to the buildbot clone of pyflakes.
 172
 173 from buildbot.changes.gitpoller import GitPoller
 174 c['change_source'] = []
 175 c['change_source'].append(GitPoller(
 176         repo_url,
 177         workdir=home_dir+'/work.git', branch=repo_branch,
 178         pollinterval=300))
 179
 180 ####### SCHEDULERS
 181
 182 # Configure the Schedulers, which decide how to react to incoming changes.  In this
 183 # case, just kick off a 'basebuild' build
 184
 185 from buildbot.schedulers.basic import SingleBranchScheduler
 186 from buildbot.schedulers.forcesched import ForceScheduler
 187 from buildbot.changes import filter
 188 c['schedulers'] = []
 189 c['schedulers'].append(SingleBranchScheduler(
 190         name="all",
 191         change_filter=filter.ChangeFilter(branch=repo_branch),
 192         treeStableTimer=60,
 193         builderNames=targets))
 194
 195 c['schedulers'].append(ForceScheduler(
 196         name="force",
 197         builderNames=targets))
 198
 199 ####### BUILDERS
 200
 201 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
 202 # what steps, and which slaves can execute them.  Note that any particular build will
 203 # only take place on one slave.
 204
 205 from buildbot.process.factory import BuildFactory
 206 from buildbot.steps.source.git import Git
 207 from buildbot.steps.shell import ShellCommand
 208 from buildbot.steps.shell import SetPropertyFromCommand
 209 from buildbot.steps.transfer import FileUpload
 210 from buildbot.steps.transfer import FileDownload
 211 from buildbot.steps.master import MasterShellCommand
 212 from buildbot.process.properties import Interpolate
 213 from buildbot.process import properties
 214
 215
 216 CleanTargetMap = [
 217         [ "tools",      "tools/clean"                   ],
 218         [ "chain",      "toolchain/clean"               ],
 219         [ "linux",      "target/linux/clean"    ],
 220         [ "dir",        "dirclean"                              ],
 221         [ "dist",       "distclean"                             ]
 222 ]
 223
 224 def IsMakeCleanRequested(pattern):
 225         def CheckCleanProperty(step):
 226                 val = step.getProperty("clean")
 227                 if val and re.match(pattern, val):
 228                         return True
 229                 else:
 230                         return False
 231
 232         return CheckCleanProperty
 233
 234 def IsCleanupRequested(step):
 235         shared_wd = step.getProperty("shared_wd")
 236         if shared_wd:
 237                 return False
 238         do_cleanup = step.getProperty("do_cleanup")
 239         if do_cleanup:
 240                 return True
 241         else:
 242                 return False
 243
 244 def IsExpireRequested(step):
 245         shared_wd = step.getProperty("shared_wd")
 246         if shared_wd:
 247                 return False
 248         else:
 249                 return not IsCleanupRequested(step)
 250
 251 def IsGitFreshRequested(step):
 252         do_cleanup = step.getProperty("do_cleanup")
 253         if do_cleanup:
 254                 return True
 255         else:
 256                 return False
 257
 258 def IsGitCleanRequested(step):
 259         return not IsGitFreshRequested(step)
 260
 261 def IsTaggingRequested(step):
 262         val = step.getProperty("tag")
 263         if val and re.match("^[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", val):
 264                 return True
 265         else:
 266                 return False
 267
 268 def IsNoTaggingRequested(step):
 269         return not IsTaggingRequested(step)
 270
 271 def IsNoMasterBuild(step):
 272         return repo_branch != "master"
 273
 274 def GetBaseVersion():
 275         if re.match("^[^-]+-[0-9]+\.[0-9]+$", repo_branch):
 276                 return repo_branch.split('-')[1]
 277         else:
 278                 return "master"
 279
 280 @properties.renderer
 281 def GetVersionPrefix(props):
 282         basever = GetBaseVersion()
 283         if props.hasProperty("tag") and re.match("^[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", props["tag"]):
 284                 return "%s/" % props["tag"]
 285         elif basever != "master":
 286                 return "%s-SNAPSHOT/" % basever
 287         else:
 288                 return ""
 289
 290 @properties.renderer
 291 def GetNumJobs(props):
 292         if props.hasProperty("max_builds") and props.hasProperty("nproc"):
 293                 return str(int(props["nproc"]) / (props["max_builds"] + other_builds))
 294         else:
 295                 return "1"
 296
 297 @properties.renderer
 298 def GetCC(props):
 299         if props.hasProperty("cc_command"):
 300                 return props["cc_command"]
 301         else:
 302                 return "gcc"
 303
 304 @properties.renderer
 305 def GetCXX(props):
 306         if props.hasProperty("cxx_command"):
 307                 return props["cxx_command"]
 308         else:
 309                 return "g++"
 310
 311 @properties.renderer
 312 def GetCwd(props):
 313         if props.hasProperty("builddir"):
 314                 return props["builddir"]
 315         elif props.hasProperty("workdir"):
 316                 return props["workdir"]
 317         else:
 318                 return "/"
 319
 320 @properties.renderer
 321 def GetCCache(props):
 322         if props.hasProperty("ccache_command") and "ccache" in props["ccache_command"]:
 323                 return props["ccache_command"] + " "
 324         else:
 325                 return ""
 326
 327 def GetNextBuild(builder, requests):
 328         for r in requests:
 329                 if r.properties and r.properties.hasProperty("tag"):
 330                         return r
 331         return requests[0]
 332
 333 def MakeEnv(overrides=None, tryccache=False):
 334         if tryccache:
 335                 envcc = Interpolate("%(kw:ccache)s%(kw:cc)s", ccache=GetCCache, cc=GetCC)
 336                 envcxx = Interpolate("%(kw:ccache)s%(kw:cxx)s", ccache=GetCCache, cxx=GetCXX)
 337         else:
 338                 envcc = Interpolate("%(kw:cc)s", cc=GetCC)
 339                 envcxx = Interpolate("%(kw:cxx)s", cxx=GetCXX)
 340         env = {
 341                 'CC': envcc,
 342                 'CXX': envcxx,
 343                 'CCACHE_BASEDIR': Interpolate("%(kw:cwd)s", cwd=GetCwd)
 344         }
 345         if overrides is not None:
 346                 env.update(overrides)
 347         return env
 348
 349 @properties.renderer
 350 def NetLockDl(props):
 351         lock = None
 352         if props.hasProperty("dl_lock"):
 353                 lock = NetLocks[props["dl_lock"]]
 354         if lock is not None:
 355                 return [lock.access('exclusive')]
 356         else:
 357                 return []
 358
 359 @properties.renderer
 360 def NetLockUl(props):
 361         lock = None
 362         if props.hasProperty("ul_lock"):
 363                 lock = NetLocks[props["ul_lock"]]
 364         if lock is not None:
 365                 return [lock.access('exclusive')]
 366         else:
 367                 return []
 368
 369 c['builders'] = []
 370
 371 dlLock = locks.SlaveLock("slave_dl")
 372
 373 checkBuiltin = re.sub('[\t\n ]+', ' ', """
 374         checkBuiltin() {
 375                 local symbol op path file;
 376                 for file in $CHANGED_FILES; do
 377                         case "$file" in
 378                                 package/*/*) : ;;
 379                                 *) return 0 ;;
 380                         esac;
 381                 done;
 382                 while read symbol op path; do
 383                         case "$symbol" in package-*)
 384                                 symbol="${symbol##*(}";
 385                                 symbol="${symbol%)}";
 386                                 for file in $CHANGED_FILES; do
 387                                         case "$file" in "package/$path/"*)
 388                                                 grep -qsx "$symbol=y" .config && return 0
 389                                         ;; esac;
 390                                 done;
 391                         esac;
 392                 done < tmp/.packagedeps;
 393                 return 1;
 394         }
 395 """).strip()
 396
 397
 398 class IfBuiltinShellCommand(ShellCommand):
 399         def _quote(self, str):
 400                 if re.search("[^a-zA-Z0-9/_.-]", str):
 401                         return "'%s'" %(re.sub("'", "'\"'\"'", str))
 402                 return str
 403
 404         def setCommand(self, command):
 405                 if not isinstance(command, (str, unicode)):
 406                         command = ' '.join(map(self._quote, command))
 407                 self.command = [
 408                         '/bin/sh', '-c',
 409                         '%s; if checkBuiltin; then %s; else exit 0; fi' %(checkBuiltin, command)
 410                 ]
 411
 412         def setupEnvironment(self, cmd):
 413                 slaveEnv = self.slaveEnvironment
 414                 if slaveEnv is None:
 415                         slaveEnv = { }
 416                 changedFiles = { }
 417                 for request in self.build.requests:
 418                         for source in request.sources:
 419                                 for change in source.changes:
 420                                         for file in change.files:
 421                                                 changedFiles[file] = True
 422                 fullSlaveEnv = slaveEnv.copy()
 423                 fullSlaveEnv['CHANGED_FILES'] = ' '.join(changedFiles.keys())
 424                 cmd.args['env'] = fullSlaveEnv
 425
 426 slaveNames = [ ]
 427
 428 for slave in c['slaves']:
 429         slaveNames.append(slave.slavename)
 430
 431 for target in targets:
 432         ts = target.split('/')
 433
 434         factory = BuildFactory()
 435
 436         # find number of cores
 437         factory.addStep(SetPropertyFromCommand(
 438                 name = "nproc",
 439                 property = "nproc",
 440                 description = "Finding number of CPUs",
 441                 command = ["nproc"]))
 442
 443         # find gcc and g++ compilers
 444         if cc_version is not None:
 445                 factory.addStep(FileDownload(
 446                         mastersrc = "findbin.pl",
 447                         slavedest = "../findbin.pl",
 448                         mode = 0755))
 449
 450                 factory.addStep(SetPropertyFromCommand(
 451                         name = "gcc",
 452                         property = "cc_command",
 453                         description = "Finding gcc command",
 454                         command = ["../findbin.pl", "gcc", cc_version[0], cc_version[1]],
 455                         haltOnFailure = True))
 456
 457                 factory.addStep(SetPropertyFromCommand(
 458                         name = "g++",
 459                         property = "cxx_command",
 460                         description = "Finding g++ command",
 461                         command = ["../findbin.pl", "g++", cc_version[0], cc_version[1]],
 462                         haltOnFailure = True))
 463
 464         # see if ccache is available
 465         factory.addStep(SetPropertyFromCommand(
 466                 property = "ccache_command",
 467                 command = ["which", "ccache"],
 468                 description = "Testing for ccache command",
 469                 haltOnFailure = False,
 470                 flunkOnFailure = False,
 471                 warnOnFailure = False,
 472         ))
 473
 474         # expire tree if needed
 475         if tree_expire > 0:
 476                 factory.addStep(FileDownload(
 477                         doStepIf = IsExpireRequested,
 478                         mastersrc = "expire.sh",
 479                         slavedest = "../expire.sh",
 480                         mode = 0755))
 481
 482                 factory.addStep(ShellCommand(
 483                         name = "expire",
 484                         description = "Checking for build tree expiry",
 485                         command = ["./expire.sh", str(tree_expire)],
 486                         workdir = ".",
 487                         haltOnFailure = True,
 488                         doStepIf = IsExpireRequested,
 489                         timeout = 2400))
 490
 491         # cleanup.sh if needed
 492         factory.addStep(FileDownload(
 493                 mastersrc = "cleanup.sh",
 494                 slavedest = "../cleanup.sh",
 495                 mode = 0755,
 496                 doStepIf = IsCleanupRequested))
 497
 498         factory.addStep(ShellCommand(
 499                 name = "cleanold",
 500                 description = "Cleaning previous builds",
 501                 command = ["./cleanup.sh", c['buildbotURL'], Interpolate("%(prop:slavename)s"), Interpolate("%(prop:buildername)s"), "full"],
 502                 workdir = ".",
 503                 haltOnFailure = True,
 504                 doStepIf = IsCleanupRequested,
 505                 timeout = 2400))
 506
 507         factory.addStep(ShellCommand(
 508                 name = "cleanup",
 509                 description = "Cleaning work area",
 510                 command = ["./cleanup.sh", c['buildbotURL'], Interpolate("%(prop:slavename)s"), Interpolate("%(prop:buildername)s"), "single"],
 511                 workdir = ".",
 512                 haltOnFailure = True,
 513                 doStepIf = IsCleanupRequested,
 514                 timeout = 2400))
 515
 516         # user-requested clean targets
 517         for tuple in CleanTargetMap:
 518                 factory.addStep(ShellCommand(
 519                         name = tuple[1],
 520                         description = 'User-requested "make %s"' % tuple[1],
 521                         command = ["make", tuple[1], "V=s"],
 522                         env = MakeEnv(),
 523                         doStepIf = IsMakeCleanRequested(tuple[0])
 524                 ))
 525
 526         # check out the source
 527         # Git() runs:
 528           # if repo doesn't exist: 'git clone repourl'
 529           # method 'clean' runs 'git clean -d -f', method fresh runs 'git clean -d -f x'. Only works with mode='full'
 530           # 'git fetch -t repourl branch; git reset --hard revision'
 531         # Git() parameters can't take a renderer until buildbot 0.8.10, so we have to split the fresh and clean cases
 532         # if buildbot is updated, one can use: method = Interpolate('%(prop:do_cleanup:#?|fresh|clean)s')
 533         factory.addStep(Git(
 534                 repourl = repo_url,
 535                 branch = repo_branch,
 536                 mode = 'full',
 537                 method = 'clean',
 538                 haltOnFailure = True,
 539                 doStepIf = IsGitCleanRequested,
 540         ))
 541
 542         factory.addStep(Git(
 543                 repourl = repo_url,
 544                 branch = repo_branch,
 545                 mode = 'full',
 546                 method = 'fresh',
 547                 haltOnFailure = True,
 548                 doStepIf = IsGitFreshRequested,
 549         ))
 550
 551         # update remote refs
 552         factory.addStep(ShellCommand(
 553                 name = "fetchrefs",
 554                 description = "Fetching Git remote refs",
 555                 command = ["git", "fetch", "origin", "+refs/heads/%s:refs/remotes/origin/%s" %(repo_branch, repo_branch)],
 556                 haltOnFailure = True
 557         ))
 558
 559         # switch to tag
 560         factory.addStep(ShellCommand(
 561                 name = "switchtag",
 562                 description = "Checking out Git tag",
 563                 command = ["git", "checkout", Interpolate("tags/v%(prop:tag:-)s")],
 564                 haltOnFailure = True,
 565                 doStepIf = IsTaggingRequested
 566         ))
 567
 568         factory.addStep(ShellCommand(
 569                 name = "rmtmp",
 570                 description = "Remove tmp folder",
 571                 command=["rm", "-rf", "tmp/"]))
 572
 573         # feed
 574 #       factory.addStep(ShellCommand(
 575 #               name = "feedsconf",
 576 #               description = "Copy the feeds.conf",
 577 #               command='''cp ~/feeds.conf ./feeds.conf''' ))
 578
 579         # feed
 580         factory.addStep(ShellCommand(
 581                 name = "rmfeedlinks",
 582                 description = "Remove feed symlinks",
 583                 command=["rm", "-rf", "package/feeds/"]))
 584
 585         # feed
 586         factory.addStep(ShellCommand(
 587                 name = "updatefeeds",
 588                 description = "Updating feeds",
 589                 command=["./scripts/feeds", "update"],
 590                 env = MakeEnv(),
 591         ))
 592
 593         # feed
 594         factory.addStep(ShellCommand(
 595                 name = "installfeeds",
 596                 description = "Installing feeds",
 597                 command=["./scripts/feeds", "install", "-a"],
 598                 env = MakeEnv()))
 599
 600         # seed config
 601         factory.addStep(FileDownload(
 602                 mastersrc = "config.seed",
 603                 slavedest = ".config",
 604                 mode = 0644
 605         ))
 606
 607         # configure
 608         factory.addStep(ShellCommand(
 609                 name = "newconfig",
 610                 description = "Seeding .config",
 611                 command = "printf 'CONFIG_TARGET_%s=y\\nCONFIG_TARGET_%s_%s=y\\n' >> .config" %(ts[0], ts[0], ts[1])
 612         ))
 613
 614         factory.addStep(ShellCommand(
 615                 name = "delbin",
 616                 description = "Removing output directory",
 617                 command = ["rm", "-rf", "bin/"]
 618         ))
 619
 620         factory.addStep(ShellCommand(
 621                 name = "defconfig",
 622                 description = "Populating .config",
 623                 command = ["make", "defconfig"],
 624                 env = MakeEnv()
 625         ))
 626
 627         # check arch
 628         factory.addStep(ShellCommand(
 629                 name = "checkarch",
 630                 description = "Checking architecture",
 631                 command = ["grep", "-sq", "CONFIG_TARGET_%s=y" %(ts[0]), ".config"],
 632                 logEnviron = False,
 633                 want_stdout = False,
 634                 want_stderr = False,
 635                 haltOnFailure = True
 636         ))
 637
 638         # find libc suffix
 639         factory.addStep(SetPropertyFromCommand(
 640                 name = "libc",
 641                 property = "libc",
 642                 description = "Finding libc suffix",
 643                 command = ["sed", "-ne", '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }', ".config"]))
 644
 645         # ccache helper
 646         factory.addStep(FileDownload(
 647                 mastersrc = "ccache.sh",
 648                 slavedest = "../ccache.sh",
 649                 mode = 0755
 650         ))
 651
 652         # ccache prepare
 653         factory.addStep(ShellCommand(
 654                 name = "prepccache",
 655                 description = "Preparing ccache",
 656                 command = ["../ccache.sh"]
 657         ))
 658
 659         # install build key
 660         factory.addStep(FileDownload(mastersrc=home_dir+'/key-build', slavedest="key-build", mode=0600))
 661         factory.addStep(FileDownload(mastersrc=home_dir+'/key-build.pub', slavedest="key-build.pub", mode=0600))
 662
 663         # prepare dl
 664         factory.addStep(ShellCommand(
 665                 name = "dldir",
 666                 description = "Preparing dl/",
 667                 command = "mkdir -p $HOME/dl && rm -rf ./dl && ln -sf $HOME/dl ./dl",
 668                 logEnviron = False,
 669                 want_stdout = False
 670         ))
 671
 672         # prepare tar
 673         factory.addStep(ShellCommand(
 674                 name = "dltar",
 675                 description = "Building and installing GNU tar",
 676                 command = ["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "tools/tar/compile", "V=s"],
 677                 env = MakeEnv(tryccache=True),
 678                 haltOnFailure = True
 679         ))
 680
 681         # populate dl
 682         factory.addStep(ShellCommand(
 683                 name = "dlrun",
 684                 description = "Populating dl/",
 685                 command = ["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "download", "V=s"],
 686                 env = MakeEnv(),
 687                 logEnviron = False,
 688                 locks = [dlLock.access('exclusive')],
 689         ))
 690
 691         factory.addStep(ShellCommand(
 692                 name = "cleanbase",
 693                 description = "Cleaning base-files",
 694                 command=["make", "package/base-files/clean", "V=s"]
 695         ))
 696
 697         # build
 698         factory.addStep(ShellCommand(
 699                 name = "tools",
 700                 description = "Building and installing tools",
 701                 command = ["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "tools/install", "V=s"],
 702                 env = MakeEnv(tryccache=True),
 703                 haltOnFailure = True
 704         ))
 705
 706         factory.addStep(ShellCommand(
 707                 name = "toolchain",
 708                 description = "Building and installing toolchain",
 709                 command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "toolchain/install", "V=s"],
 710                 env = MakeEnv(),
 711                 haltOnFailure = True
 712         ))
 713
 714         factory.addStep(ShellCommand(
 715                 name = "kmods",
 716                 description = "Building kmods",
 717                 command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
 718                 env = MakeEnv(),
 719                 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
 720                 haltOnFailure = True
 721         ))
 722
 723         # find kernel version
 724         factory.addStep(SetPropertyFromCommand(
 725                 name = "kernelversion",
 726                 property = "kernelversion",
 727                 description = "Finding the effective Kernel version",
 728                 command = "make --no-print-directory -C target/linux/ val.LINUX_VERSION val.LINUX_RELEASE val.LINUX_VERMAGIC | xargs printf '%s-%s-%s\\n'",
 729                 env = { 'TOPDIR': Interpolate("%(kw:cwd)s/build", cwd=GetCwd) }
 730         ))
 731
 732         factory.addStep(ShellCommand(
 733                 name = "pkgclean",
 734                 description = "Cleaning up package build",
 735                 command=["make", "package/cleanup", "V=s"]
 736         ))
 737
 738         factory.addStep(ShellCommand(
 739                 name = "pkgbuild",
 740                 description = "Building packages",
 741                 command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
 742                 env = MakeEnv(),
 743                 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
 744                 haltOnFailure = True
 745         ))
 746
 747         # factory.addStep(IfBuiltinShellCommand(
 748         factory.addStep(ShellCommand(
 749                 name = "pkginstall",
 750                 description = "Installing packages",
 751                 command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "package/install", "V=s"],
 752                 env = MakeEnv(),
 753                 haltOnFailure = True
 754         ))
 755
 756         factory.addStep(ShellCommand(
 757                 name = "pkgindex",
 758                 description = "Indexing packages",
 759                 command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "package/index", "V=s"],
 760                 env = MakeEnv(),
 761                 haltOnFailure = True
 762         ))
 763
 764         if enable_kmod_archive:
 765                 factory.addStep(ShellCommand(
 766                         name = "kmoddir",
 767                         description = "Creating kmod directory",
 768                         command=["mkdir", "-p", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s", target=ts[0], subtarget=ts[1])],
 769                         haltOnFailure = True
 770                 ))
 771
 772                 factory.addStep(ShellCommand(
 773                         name = "kmodprepare",
 774                         description = "Preparing kmod archive",
 775                         command=["rsync", "--include=/kmod-*.ipk", "--exclude=*", "-va",
 776                                  Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/packages/", target=ts[0], subtarget=ts[1]),
 777                                  Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1])],
 778                         haltOnFailure = True
 779                 ))
 780
 781                 factory.addStep(ShellCommand(
 782                         name = "kmodindex",
 783                         description = "Indexing kmod archive",
 784                         command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "package/index", "V=s",
 785                                  Interpolate("PACKAGE_SUBDIRS=bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1])],
 786                         env = MakeEnv(),
 787                         haltOnFailure = True
 788                 ))
 789
 790                 # find rootfs staging directory
 791                 factory.addStep(SetPropertyFromCommand(
 792                         name = "stageroot",
 793                         property = "stageroot",
 794                         description = "Finding the rootfs staging directory",
 795                         command=["make", "--no-print-directory", "val.STAGING_DIR_ROOT"],
 796                         env = { 'TOPDIR': Interpolate("%(kw:cwd)s/build", cwd=GetCwd) }
 797                 ))
 798
 799                 factory.addStep(ShellCommand(
 800                         name = "filesdir",
 801                         description = "Creating file overlay directory",
 802                         command=["mkdir", "-p", "files/etc/opkg"],
 803                         haltOnFailure = True
 804                 ))
 805
 806                 factory.addStep(ShellCommand(
 807                         name = "kmodconfig",
 808                         description = "Embedding kmod repository configuration",
 809                         command=Interpolate("sed -e 's#^\\(src/gz .*\\)_core \\(.*\\)/packages$#&\\n\\1_kmods \\2/kmods/%(prop:kernelversion)s#' " +
 810                                                "%(prop:stageroot)s/etc/opkg/distfeeds.conf > files/etc/opkg/distfeeds.conf"),
 811                         haltOnFailure = True
 812                 ))
 813
 814         #factory.addStep(IfBuiltinShellCommand(
 815         factory.addStep(ShellCommand(
 816                 name = "images",
 817                 description = "Building and installing images",
 818                 command=["make", Interpolate("-j%(kw:jobs)s", jobs=GetNumJobs), "target/install", "V=s"],
 819                 env = MakeEnv(),
 820                 haltOnFailure = True
 821         ))
 822
 823         factory.addStep(ShellCommand(
 824                 name = "diffconfig",
 825                 description = "Generating config.seed",
 826                 command=["make", "-j1", "diffconfig", "V=s"],
 827                 env = MakeEnv(),
 828                 haltOnFailure = True
 829         ))
 830
 831         factory.addStep(ShellCommand(
 832                 name = "checksums",
 833                 description = "Calculating checksums",
 834                 command=["make", "-j1", "checksum", "V=s"],
 835                 env = MakeEnv(),
 836                 haltOnFailure = True
 837         ))
 838
 839         # sign
 840         if gpg_keyid is not None:
 841                 factory.addStep(MasterShellCommand(
 842                         name = "signprepare",
 843                         description = "Preparing temporary signing directory",
 844                         command = ["mkdir", "-p", "%s/signing" %(home_dir)],
 845                         haltOnFailure = True
 846                 ))
 847
 848                 factory.addStep(ShellCommand(
 849                         name = "signpack",
 850                         description = "Packing files to sign",
 851                         command = Interpolate("find bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/ bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz", target=ts[0], subtarget=ts[1]),
 852                         haltOnFailure = True
 853                 ))
 854
 855                 factory.addStep(FileUpload(
 856                         slavesrc = "sign.tar.gz",
 857                         masterdest = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
 858                         haltOnFailure = True
 859                 ))
 860
 861                 factory.addStep(MasterShellCommand(
 862                         name = "signfiles",
 863                         description = "Signing files",
 864                         command = ["%s/signall.sh" %(home_dir), "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]), gpg_keyid, gpg_comment],
 865                         env = {'GNUPGHOME': gpg_home, 'PASSFILE': gpg_passfile},
 866                         haltOnFailure = True
 867                 ))
 868
 869                 factory.addStep(FileDownload(
 870                         mastersrc = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
 871                         slavedest = "sign.tar.gz",
 872                         haltOnFailure = True
 873                 ))
 874
 875                 factory.addStep(ShellCommand(
 876                         name = "signunpack",
 877                         description = "Unpacking signed files",
 878                         command = ["tar", "-xzf", "sign.tar.gz"],
 879                         haltOnFailure = True
 880                 ))
 881
 882         # upload
 883         factory.addStep(ShellCommand(
 884                 name = "dirprepare",
 885                 description = "Preparing upload directory structure",
 886                 command = ["mkdir", "-p", Interpolate("tmp/upload/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s", target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
 887                 haltOnFailure = True
 888         ))
 889
 890         factory.addStep(ShellCommand(
 891                 name = "linkprepare",
 892                 description = "Preparing repository symlink",
 893                 command = ["ln", "-s", "-f", Interpolate("../packages-%(kw:basever)s", basever=GetBaseVersion()), Interpolate("tmp/upload/%(kw:prefix)spackages", prefix=GetVersionPrefix)],
 894                 doStepIf = IsNoMasterBuild,
 895                 haltOnFailure = True
 896         ))
 897
 898         if enable_kmod_archive:
 899                 factory.addStep(ShellCommand(
 900                         name = "kmoddirprepare",
 901                         description = "Preparing kmod archive upload directory",
 902                         command = ["mkdir", "-p", Interpolate("tmp/upload/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s", target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
 903                         haltOnFailure = True
 904                 ))
 905
 906         factory.addStep(ShellCommand(
 907                 name = "dirupload",
 908                 description = "Uploading directory structure",
 909                 command = ["rsync", "-4", "-az", "tmp/upload/", "%s/" %(rsync_bin_url)],
 910                 env={'RSYNC_PASSWORD': rsync_bin_key},
 911                 haltOnFailure = True,
 912                 logEnviron = False,
 913         ))
 914
 915         factory.addStep(FileDownload(
 916                         mastersrc = "rsync.sh",
 917                         slavedest = "../rsync.sh",
 918                         mode = 0755))
 919
 920         factory.addStep(ShellCommand(
 921                 name = "targetupload",
 922                 description = "Uploading target files",
 923                 command=["../rsync.sh", "-4", "--exclude=/kmods/", "--delete", "--size-only", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]),
 924                          "-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/", target=ts[0], subtarget=ts[1]),
 925                          Interpolate("%(kw:rsyncbinurl)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/", rsyncbinurl=rsync_bin_url, target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
 926                 env={'RSYNC_PASSWORD': rsync_bin_key},
 927                 haltOnFailure = True,
 928                 logEnviron = False,
 929         ))
 930
 931         if enable_kmod_archive:
 932                 factory.addStep(ShellCommand(
 933                         name = "kmodupload",
 934                         description = "Uploading kmod archive",
 935                         command=["../rsync.sh", "-4", "--delete", "--size-only", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]),
 936                                  "-a", Interpolate("bin/targets/%(kw:target)s/%(kw:subtarget)s%(prop:libc)s/kmods/%(prop:kernelversion)s/", target=ts[0], subtarget=ts[1]),
 937                                  Interpolate("%(kw:rsyncbinurl)s/%(kw:prefix)stargets/%(kw:target)s/%(kw:subtarget)s/kmods/%(prop:kernelversion)s/", rsyncbinurl=rsync_bin_url, target=ts[0], subtarget=ts[1], prefix=GetVersionPrefix)],
 938                         env={'RSYNC_PASSWORD': rsync_bin_key},
 939                         haltOnFailure = True,
 940                         logEnviron = False,
 941                 ))
 942
 943         if rsync_src_url is not None:
 944                 factory.addStep(ShellCommand(
 945                         name = "sourceupload",
 946                         description = "Uploading source archives",
 947                         command=["../rsync.sh", "-4", "--size-only", "--delay-updates",
 948                                  Interpolate("--partial-dir=.~tmp~%(kw:target)s~%(kw:subtarget)s~%(prop:slavename)s", target=ts[0], subtarget=ts[1]), "-a", "dl/", "%s/" %(rsync_src_url)],
 949                         env={'RSYNC_PASSWORD': rsync_src_key},
 950                         haltOnFailure = True,
 951                         logEnviron = False,
 952                 ))
 953
 954         if False:
 955                 factory.addStep(ShellCommand(
 956                         name = "packageupload",
 957                         description = "Uploading package files",
 958                         command=["../rsync.sh", "-4", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-a", "bin/packages/", "%s/packages/" %(rsync_bin_url)],
 959                         env={'RSYNC_PASSWORD': rsync_bin_key},
 960                         haltOnFailure = False,
 961                         logEnviron = False,
 962                 ))
 963
 964         # logs
 965         if False:
 966                 factory.addStep(ShellCommand(
 967                         name = "upload",
 968                         description = "Uploading logs",
 969                         command=["../rsync.sh", "-4", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-az", "logs/", "%s/logs/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
 970                         env={'RSYNC_PASSWORD': rsync_bin_key},
 971                         haltOnFailure = False,
 972                         alwaysRun = True,
 973                         logEnviron = False,
 974                 ))
 975
 976         factory.addStep(ShellCommand(
 977                 name = "df",
 978                 description = "Reporting disk usage",
 979                 command=["df", "-h", "."],
 980                 env={'LC_ALL': 'C'},
 981                 haltOnFailure = False,
 982                 alwaysRun = True
 983         ))
 984
 985         factory.addStep(ShellCommand(
 986                 name = "ccachestat",
 987                 description = "Reporting ccache stats",
 988                 command=["ccache", "-s"],
 989                 env = MakeEnv(overrides={ 'PATH': ["./staging_dir/host/bin", "${PATH}"] }),
 990                 want_stderr = False,
 991                 haltOnFailure = False,
 992                 flunkOnFailure = False,
 993                 warnOnFailure = False,
 994                 alwaysRun = True,
 995         ))
 996
 997         from buildbot.config import BuilderConfig
 998
 999         c['builders'].append(BuilderConfig(name=target, slavenames=slaveNames, factory=factory, nextBuild=GetNextBuild))
1000
1001
1002 ####### STATUS TARGETS
1003
1004 # 'status' is a list of Status Targets. The results of each build will be
1005 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
1006 # including web pages, email senders, and IRC bots.
1007
1008 c['status'] = []
1009
1010 from buildbot.status import html
1011 from buildbot.status.web import authz, auth
1012
1013 if ini.has_option("status", "bind"):
1014         if ini.has_option("status", "user") and ini.has_option("status", "password"):
1015                 authz_cfg=authz.Authz(
1016                         # change any of these to True to enable; see the manual for more
1017                         # options
1018                         auth=auth.BasicAuth([(ini.get("status", "user"), ini.get("status", "password"))]),
1019                         gracefulShutdown = 'auth',
1020                         forceBuild = 'auth', # use this to test your slave once it is set up
1021                         forceAllBuilds = 'auth',
1022                         pingBuilder = False,
1023                         stopBuild = 'auth',
1024                         stopAllBuilds = 'auth',
1025                         cancelPendingBuild = 'auth',
1026                 )
1027                 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind"), authz=authz_cfg))
1028         else:
1029                 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind")))
1030
1031
1032 from buildbot.status import words
1033
1034 if ini.has_option("irc", "host") and ini.has_option("irc", "nickname") and ini.has_option("irc", "channel"):
1035         irc_host = ini.get("irc", "host")
1036         irc_port = 6667
1037         irc_chan = ini.get("irc", "channel")
1038         irc_nick = ini.get("irc", "nickname")
1039         irc_pass = None
1040
1041         if ini.has_option("irc", "port"):
1042                 irc_port = ini.getint("irc", "port")
1043
1044         if ini.has_option("irc", "password"):
1045                 irc_pass = ini.get("irc", "password")
1046
1047         irc = words.IRC(irc_host, irc_nick, port = irc_port, password = irc_pass,
1048                         channels = [{ "channel": irc_chan }],
1049                         notify_events = {
1050                           'exception': 1,
1051                           'successToFailure': 1,
1052                           'failureToSuccess': 1
1053                         }
1054         )
1055
1056         c['status'].append(irc)
1057
1058 ####### DB URL
1059
1060 c['db'] = {
1061         # This specifies what database buildbot uses to store its state.  You can leave
1062         # this at its default for all but the largest installations.
1063         'db_url' : "sqlite:///state.sqlite",
1064 }