phase1/master.cfg

   1 # -*- python -*-
   2 # ex: set syntax=python:
   3
   4 import os
   5 import re
   6 import subprocess
   7 import ConfigParser
   8
   9 from buildbot import locks
  10
  11 # This is a sample buildmaster config file. It must be installed as
  12 # 'master.cfg' in your buildmaster's base directory.
  13
  14 ini = ConfigParser.ConfigParser()
  15 ini.read("./config.ini")
  16
  17 # This is the dictionary that the buildmaster pays attention to. We also use
  18 # a shorter alias to save typing.
  19 c = BuildmasterConfig = {}
  20
  21 ####### BUILDSLAVES
  22
  23 # The 'slaves' list defines the set of recognized buildslaves. Each element is
  24 # a BuildSlave object, specifying a unique slave name and password.  The same
  25 # slave name and password must be configured on the slave.
  26 from buildbot.buildslave import BuildSlave
  27
  28 slave_port = 9989
  29
  30 if ini.has_option("general", "port"):
  31         slave_port = ini.getint("general", "port")
  32
  33 c['slaves'] = []
  34 max_builds = dict()
  35
  36 for section in ini.sections():
  37         if section.startswith("slave "):
  38                 if ini.has_option(section, "name") and ini.has_option(section, "password"):
  39                         name = ini.get(section, "name")
  40                         password = ini.get(section, "password")
  41                         max_builds[name] = 1
  42                         if ini.has_option(section, "builds"):
  43                                 max_builds[name] = ini.getint(section, "builds")
  44                         c['slaves'].append(BuildSlave(name, password, max_builds = max_builds[name]))
  45
  46 # 'slavePortnum' defines the TCP port to listen on for connections from slaves.
  47 # This must match the value configured into the buildslaves (with their
  48 # --master option)
  49 c['slavePortnum'] = slave_port
  50
  51 # coalesce builds
  52 c['mergeRequests'] = True
  53
  54 # Reduce amount of backlog data
  55 c['buildHorizon'] = 30
  56 c['logHorizon'] = 20
  57
  58 ####### CHANGESOURCES
  59
  60 home_dir = os.path.abspath(ini.get("general", "homedir"))
  61 tree_expire = 0
  62 other_builds = 0
  63 cc_version = None
  64
  65 cc_command = "gcc"
  66 cxx_command = "g++"
  67
  68 if ini.has_option("general", "expire"):
  69         tree_expire = ini.getint("general", "expire")
  70
  71 if ini.has_option("general", "other_builds"):
  72         other_builds = ini.getint("general", "other_builds")
  73
  74 if ini.has_option("general", "cc_version"):
  75         cc_version = ini.get("general", "cc_version").split()
  76         if len(cc_version) == 1:
  77                 cc_version = ["eq", cc_version[0]]
  78
  79 repo_url = ini.get("repo", "url")
  80 repo_branch = "master"
  81
  82 if ini.has_option("repo", "branch"):
  83         repo_branch = ini.get("repo", "branch")
  84
  85 rsync_bin_url = ini.get("rsync", "binary_url")
  86 rsync_bin_key = ini.get("rsync", "binary_password")
  87
  88 rsync_src_url = None
  89 rsync_src_key = None
  90
  91 if ini.has_option("rsync", "source_url"):
  92         rsync_src_url = ini.get("rsync", "source_url")
  93         rsync_src_key = ini.get("rsync", "source_password")
  94
  95 gpg_home = "~/.gnupg"
  96 gpg_keyid = None
  97 gpg_comment = "Unattended build signature"
  98 gpg_passfile = "/dev/null"
  99
 100 if ini.has_option("gpg", "home"):
 101         gpg_home = ini.get("gpg", "home")
 102
 103 if ini.has_option("gpg", "keyid"):
 104         gpg_keyid = ini.get("gpg", "keyid")
 105
 106 if ini.has_option("gpg", "comment"):
 107         gpg_comment = ini.get("gpg", "comment")
 108
 109 if ini.has_option("gpg", "passfile"):
 110         gpg_passfile = ini.get("gpg", "passfile")
 111
 112
 113 # find targets
 114 targets = [ ]
 115
 116 if not os.path.isdir(home_dir+'/source.git'):
 117         subprocess.call(["git", "clone", "--depth=1", "--branch="+repo_branch, repo_url, home_dir+'/source.git'])
 118 else:
 119         subprocess.call(["git", "pull"], cwd = home_dir+'/source.git')
 120
 121 findtargets = subprocess.Popen([home_dir+'/dumpinfo.pl', 'targets'],
 122         stdout = subprocess.PIPE, cwd = home_dir+'/source.git')
 123
 124 while True:
 125         line = findtargets.stdout.readline()
 126         if not line:
 127                 break
 128         ta = line.strip().split(' ')
 129         targets.append(ta[0])
 130
 131
 132 # the 'change_source' setting tells the buildmaster how it should find out
 133 # about source code changes.  Here we point to the buildbot clone of pyflakes.
 134
 135 from buildbot.changes.gitpoller import GitPoller
 136 c['change_source'] = []
 137 c['change_source'].append(GitPoller(
 138         repo_url,
 139         workdir=home_dir+'/work.git', branch=repo_branch,
 140         pollinterval=300))
 141
 142 ####### SCHEDULERS
 143
 144 # Configure the Schedulers, which decide how to react to incoming changes.  In this
 145 # case, just kick off a 'basebuild' build
 146
 147 from buildbot.schedulers.basic import SingleBranchScheduler
 148 from buildbot.schedulers.forcesched import ForceScheduler
 149 from buildbot.changes import filter
 150 c['schedulers'] = []
 151 c['schedulers'].append(SingleBranchScheduler(
 152         name="all",
 153         change_filter=filter.ChangeFilter(branch=repo_branch),
 154         treeStableTimer=60,
 155         builderNames=targets))
 156
 157 c['schedulers'].append(ForceScheduler(
 158         name="force",
 159         builderNames=targets))
 160
 161 ####### BUILDERS
 162
 163 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
 164 # what steps, and which slaves can execute them.  Note that any particular build will
 165 # only take place on one slave.
 166
 167 from buildbot.process.factory import BuildFactory
 168 from buildbot.steps.source.git import Git
 169 from buildbot.steps.shell import ShellCommand
 170 from buildbot.steps.shell import SetProperty
 171 from buildbot.steps.transfer import FileUpload
 172 from buildbot.steps.transfer import FileDownload
 173 from buildbot.steps.master import MasterShellCommand
 174 from buildbot.process.properties import WithProperties
 175
 176
 177 CleanTargetMap = [
 178         [ "tools",      "tools/clean"                   ],
 179         [ "chain",      "toolchain/clean"               ],
 180         [ "linux",      "target/linux/clean"    ],
 181         [ "dir",        "dirclean"                              ],
 182         [ "dist",       "distclean"                             ]
 183 ]
 184
 185 def IsCleanRequested(pattern):
 186         def CheckCleanProperty(step):
 187                 val = step.getProperty("clean")
 188                 if val and re.match(pattern, val):
 189                         return True
 190                 else:
 191                         return False
 192
 193         return CheckCleanProperty
 194
 195 def IsTaggingRequested(step):
 196         val = step.getProperty("tag")
 197         if val and re.match("^[0-9]+\.[0-9]+\.[0-9]+$", val):
 198                 return True
 199         else:
 200                 return False
 201
 202 def IsNoTaggingRequested(step):
 203         return not IsTaggingRequested(step)
 204
 205 def IsNoMasterBuild(step):
 206         return repo_branch != "master"
 207
 208 def GetBaseVersion(props):
 209         if re.match("^[^-]+-[0-9]+\.[0-9]+$", repo_branch):
 210                 return repo_branch.split('-')[1]
 211         else:
 212                 return "master"
 213
 214 def GetVersionPrefix(props):
 215         basever = GetBaseVersion(props)
 216         if props.hasProperty("tag") and re.match("^[0-9]+\.[0-9]+\.[0-9]+$", props["tag"]):
 217                 return "%s/" % props["tag"]
 218         elif basever != "master":
 219                 return "%s-SNAPSHOT/" % basever
 220         else:
 221                 return ""
 222
 223 def GetNumJobs(props):
 224         if props.hasProperty("slavename") and props.hasProperty("nproc"):
 225                 return ((int(props["nproc"]) / (max_builds[props["slavename"]] + other_builds)) + 1)
 226         else:
 227                 return 1
 228
 229 def GetCC(props):
 230         if props.hasProperty("cc_command"):
 231                 return props["cc_command"]
 232         else:
 233                 return "gcc"
 234
 235 def GetCXX(props):
 236         if props.hasProperty("cxx_command"):
 237                 return props["cxx_command"]
 238         else:
 239                 return "g++"
 240
 241 def MakeEnv(overrides=None):
 242         env = {
 243                 'CC': WithProperties("%(cc)s", cc=GetCC),
 244                 'CXX': WithProperties("%(cxx)s", cxx=GetCXX)
 245         }
 246         if overrides is not None:
 247                 env.update(overrides)
 248         return env
 249
 250
 251 c['builders'] = []
 252
 253 dlLock = locks.SlaveLock("slave_dl")
 254 tagLock = locks.MasterLock("make_tag")
 255
 256 checkBuiltin = re.sub('[\t\n ]+', ' ', """
 257         checkBuiltin() {
 258                 local symbol op path file;
 259                 for file in $CHANGED_FILES; do
 260                         case "$file" in
 261                                 package/*/*) : ;;
 262                                 *) return 0 ;;
 263                         esac;
 264                 done;
 265                 while read symbol op path; do
 266                         case "$symbol" in package-*)
 267                                 symbol="${symbol##*(}";
 268                                 symbol="${symbol%)}";
 269                                 for file in $CHANGED_FILES; do
 270                                         case "$file" in "package/$path/"*)
 271                                                 grep -qsx "$symbol=y" .config && return 0
 272                                         ;; esac;
 273                                 done;
 274                         esac;
 275                 done < tmp/.packagedeps;
 276                 return 1;
 277         }
 278 """).strip()
 279
 280
 281 class IfBuiltinShellCommand(ShellCommand):
 282         def _quote(self, str):
 283                 if re.search("[^a-zA-Z0-9/_.-]", str):
 284                         return "'%s'" %(re.sub("'", "'\"'\"'", str))
 285                 return str
 286
 287         def setCommand(self, command):
 288                 if not isinstance(command, (str, unicode)):
 289                         command = ' '.join(map(self._quote, command))
 290                 self.command = [
 291                         '/bin/sh', '-c',
 292                         '%s; if checkBuiltin; then %s; else exit 0; fi' %(checkBuiltin, command)
 293                 ]
 294
 295         def setupEnvironment(self, cmd):
 296                 slaveEnv = self.slaveEnvironment
 297                 if slaveEnv is None:
 298                         slaveEnv = { }
 299                 changedFiles = { }
 300                 for request in self.build.requests:
 301                         for source in request.sources:
 302                                 for change in source.changes:
 303                                         for file in change.files:
 304                                                 changedFiles[file] = True
 305                 fullSlaveEnv = slaveEnv.copy()
 306                 fullSlaveEnv['CHANGED_FILES'] = ' '.join(changedFiles.keys())
 307                 cmd.args['env'] = fullSlaveEnv
 308
 309 slaveNames = [ ]
 310
 311 for slave in c['slaves']:
 312         slaveNames.append(slave.slavename)
 313
 314 for target in targets:
 315         ts = target.split('/')
 316
 317         factory = BuildFactory()
 318
 319         # find number of cores
 320         factory.addStep(SetProperty(
 321                 name = "nproc",
 322                 property = "nproc",
 323                 description = "Finding number of CPUs",
 324                 command = ["nproc"]))
 325
 326         # find gcc and g++ compilers
 327         if cc_version is not None:
 328                 factory.addStep(FileDownload(
 329                         mastersrc = "findbin.pl",
 330                         slavedest = "../findbin.pl",
 331                         mode = 0755))
 332
 333                 factory.addStep(SetProperty(
 334                         name = "gcc",
 335                         property = "cc_command",
 336                         description = "Finding gcc command",
 337                         command = ["../findbin.pl", "gcc", cc_version[0], cc_version[1]],
 338                         haltOnFailure = True))
 339
 340                 factory.addStep(SetProperty(
 341                         name = "g++",
 342                         property = "cxx_command",
 343                         description = "Finding g++ command",
 344                         command = ["../findbin.pl", "g++", cc_version[0], cc_version[1]],
 345                         haltOnFailure = True))
 346
 347         # expire tree if needed
 348         if tree_expire > 0:
 349                 factory.addStep(FileDownload(
 350                         mastersrc = "expire.sh",
 351                         slavedest = "../expire.sh",
 352                         mode = 0755))
 353
 354                 factory.addStep(ShellCommand(
 355                         name = "expire",
 356                         description = "Checking for build tree expiry",
 357                         command = ["./expire.sh", str(tree_expire)],
 358                         workdir = ".",
 359                         haltOnFailure = True,
 360                         timeout = 2400))
 361
 362         # user-requested clean targets
 363         for tuple in CleanTargetMap:
 364                 factory.addStep(ShellCommand(
 365                         name = tuple[1],
 366                         description = 'User-requested "make %s"' % tuple[1],
 367                         command = ["make", tuple[1], "V=s"],
 368                         env = MakeEnv(),
 369                         doStepIf = IsCleanRequested(tuple[0])
 370                 ))
 371
 372         factory.addStep(MasterShellCommand(
 373                 name = "maketag",
 374                 description = "Tagging Git repository",
 375                 command = [home_dir+'/maketag.sh', '-i', '-k', str(gpg_keyid or ''),
 376                            '-p', str(gpg_passfile or ''), '-v', WithProperties("%(tag:-)s")],
 377                 path = home_dir+'/source.git',
 378                 env = {'GNUPGHOME': gpg_home},
 379                 haltOnFailure = True,
 380                 doStepIf = IsTaggingRequested,
 381                 locks = [tagLock.access('exclusive')]
 382         ))
 383
 384         # switch to branch
 385         factory.addStep(ShellCommand(
 386                 name = "switchbranch",
 387                 description = "Checking out Git branch",
 388                 command = "if [ -d .git ]; then git checkout '%s'; else exit 0; fi" % repo_branch,
 389                 haltOnFailure = True,
 390                 doStepIf = IsNoTaggingRequested
 391         ))
 392
 393         # check out the source
 394         factory.addStep(Git(
 395                 repourl = repo_url,
 396                 branch = repo_branch,
 397                 mode = 'incremental',
 398                 method = 'clean'))
 399
 400         # update remote refs
 401         factory.addStep(ShellCommand(
 402                 name = "fetchrefs",
 403                 description = "Fetching Git remote refs",
 404                 command = ["git", "fetch", "origin", "+refs/heads/%s:refs/remotes/origin/%s" %(repo_branch, repo_branch)],
 405                 haltOnFailure = True
 406         ))
 407
 408         # fetch tags
 409         factory.addStep(ShellCommand(
 410                 name = "fetchtag",
 411                 description = "Fetching Git tags",
 412                 command = ["git", "fetch", "--tags", "--", repo_url],
 413                 haltOnFailure = True,
 414                 doStepIf = IsTaggingRequested
 415         ))
 416
 417         # switch to tag
 418         factory.addStep(ShellCommand(
 419                 name = "switchtag",
 420                 description = "Checking out Git tag",
 421                 command = ["git", "checkout", WithProperties("tags/v%(tag:-)s")],
 422                 haltOnFailure = True,
 423                 doStepIf = IsTaggingRequested
 424         ))
 425
 426         factory.addStep(ShellCommand(
 427                 name = "rmtmp",
 428                 description = "Remove tmp folder",
 429                 command=["rm", "-rf", "tmp/"]))
 430
 431         # feed
 432 #       factory.addStep(ShellCommand(
 433 #               name = "feedsconf",
 434 #               description = "Copy the feeds.conf",
 435 #               command='''cp ~/feeds.conf ./feeds.conf''' ))
 436
 437         # feed
 438         factory.addStep(ShellCommand(
 439                 name = "rmfeedlinks",
 440                 description = "Remove feed symlinks",
 441                 command=["rm", "-rf", "package/feeds/"]))
 442
 443         # feed
 444         factory.addStep(ShellCommand(
 445                 name = "updatefeeds",
 446                 description = "Updating feeds",
 447                 command=["./scripts/feeds", "update"],
 448                 env = MakeEnv()))
 449
 450         # feed
 451         factory.addStep(ShellCommand(
 452                 name = "installfeeds",
 453                 description = "Installing feeds",
 454                 command=["./scripts/feeds", "install", "-a"],
 455                 env = MakeEnv()))
 456
 457         # seed config
 458         factory.addStep(FileDownload(
 459                 mastersrc = "config.seed",
 460                 slavedest = ".config",
 461                 mode = 0644
 462         ))
 463
 464         # configure
 465         factory.addStep(ShellCommand(
 466                 name = "newconfig",
 467                 description = "Seeding .config",
 468                 command = "printf 'CONFIG_TARGET_%s=y\\nCONFIG_TARGET_%s_%s=y\\n' >> .config" %(ts[0], ts[0], ts[1])
 469         ))
 470
 471         factory.addStep(ShellCommand(
 472                 name = "delbin",
 473                 description = "Removing output directory",
 474                 command = ["rm", "-rf", "bin/"]
 475         ))
 476
 477         factory.addStep(ShellCommand(
 478                 name = "defconfig",
 479                 description = "Populating .config",
 480                 command = ["make", "defconfig"],
 481                 env = MakeEnv()
 482         ))
 483
 484         # check arch
 485         factory.addStep(ShellCommand(
 486                 name = "checkarch",
 487                 description = "Checking architecture",
 488                 command = ["grep", "-sq", "CONFIG_TARGET_%s=y" %(ts[0]), ".config"],
 489                 logEnviron = False,
 490                 want_stdout = False,
 491                 want_stderr = False,
 492                 haltOnFailure = True
 493         ))
 494
 495         # find libc suffix
 496         factory.addStep(SetProperty(
 497                 name = "libc",
 498                 property = "libc",
 499                 description = "Finding libc suffix",
 500                 command = ["sed", "-ne", '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }', ".config"]))
 501
 502         # install build key
 503         factory.addStep(FileDownload(mastersrc=home_dir+'/key-build', slavedest="key-build", mode=0600))
 504         factory.addStep(FileDownload(mastersrc=home_dir+'/key-build.pub', slavedest="key-build.pub", mode=0600))
 505
 506         # prepare dl
 507         factory.addStep(ShellCommand(
 508                 name = "dldir",
 509                 description = "Preparing dl/",
 510                 command = "mkdir -p $HOME/dl && rm -rf ./dl && ln -sf $HOME/dl ./dl",
 511                 logEnviron = False,
 512                 want_stdout = False
 513         ))
 514
 515         # prepare tar
 516         factory.addStep(ShellCommand(
 517                 name = "dltar",
 518                 description = "Building GNU tar",
 519                 command = ["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "tools/tar/install", "V=s"],
 520                 env = MakeEnv(),
 521                 haltOnFailure = True
 522         ))
 523
 524         # populate dl
 525         factory.addStep(ShellCommand(
 526                 name = "dlrun",
 527                 description = "Populating dl/",
 528                 command = ["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "download", "V=s"],
 529                 env = MakeEnv(),
 530                 logEnviron = False,
 531                 locks = [dlLock.access('exclusive')]
 532         ))
 533
 534         factory.addStep(ShellCommand(
 535                 name = "cleanbase",
 536                 description = "Cleaning base-files",
 537                 command=["make", "package/base-files/clean", "V=s"]
 538         ))
 539
 540         # build
 541         factory.addStep(ShellCommand(
 542                 name = "tools",
 543                 description = "Building tools",
 544                 command = ["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "tools/install", "V=s"],
 545                 env = MakeEnv(),
 546                 haltOnFailure = True
 547         ))
 548
 549         factory.addStep(ShellCommand(
 550                 name = "toolchain",
 551                 description = "Building toolchain",
 552                 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "toolchain/install", "V=s"],
 553                 env = MakeEnv(),
 554                 haltOnFailure = True
 555         ))
 556
 557         factory.addStep(ShellCommand(
 558                 name = "kmods",
 559                 description = "Building kmods",
 560                 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
 561                 env = MakeEnv(),
 562                 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
 563                 haltOnFailure = True
 564         ))
 565
 566         factory.addStep(ShellCommand(
 567                 name = "pkgbuild",
 568                 description = "Building packages",
 569                 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
 570                 env = MakeEnv(),
 571                 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
 572                 haltOnFailure = True
 573         ))
 574
 575         # factory.addStep(IfBuiltinShellCommand(
 576         factory.addStep(ShellCommand(
 577                 name = "pkginstall",
 578                 description = "Installing packages",
 579                 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "package/install", "V=s"],
 580                 env = MakeEnv(),
 581                 haltOnFailure = True
 582         ))
 583
 584         factory.addStep(ShellCommand(
 585                 name = "pkgindex",
 586                 description = "Indexing packages",
 587                 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "package/index", "V=s"],
 588                 env = MakeEnv(),
 589                 haltOnFailure = True
 590         ))
 591
 592         #factory.addStep(IfBuiltinShellCommand(
 593         factory.addStep(ShellCommand(
 594                 name = "images",
 595                 description = "Building images",
 596                 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "target/install", "V=s"],
 597                 env = MakeEnv(),
 598                 haltOnFailure = True
 599         ))
 600
 601         factory.addStep(ShellCommand(
 602                 name = "diffconfig",
 603                 description = "Generating config.seed",
 604                 command=["make", "-j1", "diffconfig", "V=s"],
 605                 env = MakeEnv(),
 606                 haltOnFailure = True
 607         ))
 608
 609         factory.addStep(ShellCommand(
 610                 name = "checksums",
 611                 description = "Calculating checksums",
 612                 command=["make", "-j1", "checksum", "V=s"],
 613                 env = MakeEnv(),
 614                 haltOnFailure = True
 615         ))
 616
 617         # sign
 618         if gpg_keyid is not None:
 619                 factory.addStep(MasterShellCommand(
 620                         name = "signprepare",
 621                         description = "Preparing temporary signing directory",
 622                         command = ["mkdir", "-p", "%s/signing" %(home_dir)],
 623                         haltOnFailure = True
 624                 ))
 625
 626                 factory.addStep(ShellCommand(
 627                         name = "signpack",
 628                         description = "Packing files to sign",
 629                         command = WithProperties("find bin/targets/%s/%s%%(libc)s/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz" %(ts[0], ts[1])),
 630                         haltOnFailure = True
 631                 ))
 632
 633                 factory.addStep(FileUpload(
 634                         slavesrc = "sign.tar.gz",
 635                         masterdest = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
 636                         haltOnFailure = True
 637                 ))
 638
 639                 factory.addStep(MasterShellCommand(
 640                         name = "signfiles",
 641                         description = "Signing files",
 642                         command = ["%s/signall.sh" %(home_dir), "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]), gpg_keyid, gpg_comment],
 643                         env = {'GNUPGHOME': gpg_home, 'PASSFILE': gpg_passfile},
 644                         haltOnFailure = True
 645                 ))
 646
 647                 factory.addStep(FileDownload(
 648                         mastersrc = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
 649                         slavedest = "sign.tar.gz",
 650                         haltOnFailure = True
 651                 ))
 652
 653                 factory.addStep(ShellCommand(
 654                         name = "signunpack",
 655                         description = "Unpacking signed files",
 656                         command = ["tar", "-xzf", "sign.tar.gz"],
 657                         haltOnFailure = True
 658                 ))
 659
 660         # upload
 661         factory.addStep(ShellCommand(
 662                 name = "dirprepare",
 663                 description = "Preparing upload directory structure",
 664                 command = ["mkdir", "-p", WithProperties("tmp/upload/%%(prefix)stargets/%s/%s" %(ts[0], ts[1]), prefix=GetVersionPrefix)],
 665                 haltOnFailure = True
 666         ))
 667
 668         factory.addStep(ShellCommand(
 669                 name = "linkprepare",
 670                 description = "Preparing repository symlink",
 671                 command = ["ln", "-s", "-f", WithProperties("../packages-%(basever)s", basever=GetBaseVersion), WithProperties("tmp/upload/%(prefix)spackages", prefix=GetVersionPrefix)],
 672                 doStepIf = IsNoMasterBuild,
 673                 haltOnFailure = True
 674         ))
 675
 676         factory.addStep(ShellCommand(
 677                 name = "dirupload",
 678                 description = "Uploading directory structure",
 679                 command = ["rsync", "-avz", "tmp/upload/", "%s/" %(rsync_bin_url)],
 680                 env={'RSYNC_PASSWORD': rsync_bin_key},
 681                 haltOnFailure = True,
 682                 logEnviron = False
 683         ))
 684
 685         factory.addStep(ShellCommand(
 686                 name = "targetupload",
 687                 description = "Uploading target files",
 688                 command=["rsync", "--progress", "--delete", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]),
 689                          "-avz", WithProperties("bin/targets/%s/%s%%(libc)s/" %(ts[0], ts[1])),
 690                          WithProperties("%s/%%(prefix)stargets/%s/%s/" %(rsync_bin_url, ts[0], ts[1]), prefix=GetVersionPrefix)],
 691                 env={'RSYNC_PASSWORD': rsync_bin_key},
 692                 haltOnFailure = True,
 693                 logEnviron = False
 694         ))
 695
 696         if rsync_src_url is not None:
 697                 factory.addStep(ShellCommand(
 698                         name = "sourceupload",
 699                         description = "Uploading source archives",
 700                         command=["rsync", "--progress", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "dl/", "%s/" %(rsync_src_url)],
 701                         env={'RSYNC_PASSWORD': rsync_src_key},
 702                         haltOnFailure = True,
 703                         logEnviron = False
 704                 ))
 705
 706         if False:
 707                 factory.addStep(ShellCommand(
 708                         name = "packageupload",
 709                         description = "Uploading package files",
 710                         command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "bin/packages/", "%s/packages/" %(rsync_bin_url)],
 711                         env={'RSYNC_PASSWORD': rsync_bin_key},
 712                         haltOnFailure = False,
 713                         logEnviron = False
 714                 ))
 715
 716         # logs
 717         if False:
 718                 factory.addStep(ShellCommand(
 719                         name = "upload",
 720                         description = "Uploading logs",
 721                         command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "logs/", "%s/logs/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
 722                         env={'RSYNC_PASSWORD': rsync_bin_key},
 723                         haltOnFailure = False,
 724                         alwaysRun = True,
 725                         logEnviron = False
 726                 ))
 727
 728         from buildbot.config import BuilderConfig
 729
 730         c['builders'].append(BuilderConfig(name=target, slavenames=slaveNames, factory=factory))
 731
 732
 733 ####### STATUS TARGETS
 734
 735 # 'status' is a list of Status Targets. The results of each build will be
 736 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
 737 # including web pages, email senders, and IRC bots.
 738
 739 c['status'] = []
 740
 741 from buildbot.status import html
 742 from buildbot.status.web import authz, auth
 743
 744 if ini.has_option("status", "bind"):
 745         if ini.has_option("status", "user") and ini.has_option("status", "password"):
 746                 authz_cfg=authz.Authz(
 747                         # change any of these to True to enable; see the manual for more
 748                         # options
 749                         auth=auth.BasicAuth([(ini.get("status", "user"), ini.get("status", "password"))]),
 750                         gracefulShutdown = 'auth',
 751                         forceBuild = 'auth', # use this to test your slave once it is set up
 752                         forceAllBuilds = 'auth',
 753                         pingBuilder = False,
 754                         stopBuild = 'auth',
 755                         stopAllBuilds = 'auth',
 756                         cancelPendingBuild = 'auth',
 757                 )
 758                 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind"), authz=authz_cfg))
 759         else:
 760                 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind")))
 761
 762
 763 from buildbot.status import words
 764
 765 if ini.has_option("irc", "host") and ini.has_option("irc", "nickname") and ini.has_option("irc", "channel"):
 766         irc_host = ini.get("irc", "host")
 767         irc_port = 6667
 768         irc_chan = ini.get("irc", "channel")
 769         irc_nick = ini.get("irc", "nickname")
 770         irc_pass = None
 771
 772         if ini.has_option("irc", "port"):
 773                 irc_port = ini.getint("irc", "port")
 774
 775         if ini.has_option("irc", "password"):
 776                 irc_pass = ini.get("irc", "password")
 777
 778         irc = words.IRC(irc_host, irc_nick, port = irc_port, password = irc_pass,
 779                         channels = [{ "channel": irc_chan }],
 780                         notify_events = {
 781                           'exception': 1,
 782                           'successToFailure': 1,
 783                           'failureToSuccess': 1
 784                         }
 785         )
 786
 787         c['status'].append(irc)
 788
 789
 790 ####### PROJECT IDENTITY
 791
 792 # the 'title' string will appear at the top of this buildbot
 793 # installation's html.WebStatus home page (linked to the
 794 # 'titleURL') and is embedded in the title of the waterfall HTML page.
 795
 796 c['title'] = ini.get("general", "title")
 797 c['titleURL'] = ini.get("general", "title_url")
 798
 799 # the 'buildbotURL' string should point to the location where the buildbot's
 800 # internal web server (usually the html.WebStatus page) is visible. This
 801 # typically uses the port number set in the Waterfall 'status' entry, but
 802 # with an externally-visible host name which the buildbot cannot figure out
 803 # without some help.
 804
 805 c['buildbotURL'] = ini.get("general", "buildbot_url")
 806
 807 ####### DB URL
 808
 809 c['db'] = {
 810         # This specifies what database buildbot uses to store its state.  You can leave
 811         # this at its default for all but the largest installations.
 812         'db_url' : "sqlite:///state.sqlite",
 813 }