phase1/master.cfg

   1 # -*- python -*-
   2 # ex: set syntax=python:
   3
   4 import os
   5 import re
   6 import subprocess
   7 import ConfigParser
   8
   9 from buildbot import locks
  10
  11 # This is a sample buildmaster config file. It must be installed as
  12 # 'master.cfg' in your buildmaster's base directory.
  13
  14 ini = ConfigParser.ConfigParser()
  15 ini.read("./config.ini")
  16
  17 # This is the dictionary that the buildmaster pays attention to. We also use
  18 # a shorter alias to save typing.
  19 c = BuildmasterConfig = {}
  20
  21 ####### BUILDSLAVES
  22
  23 # The 'slaves' list defines the set of recognized buildslaves. Each element is
  24 # a BuildSlave object, specifying a unique slave name and password.  The same
  25 # slave name and password must be configured on the slave.
  26 from buildbot.buildslave import BuildSlave
  27
  28 slave_port = 9989
  29
  30 if ini.has_option("general", "port"):
  31         slave_port = ini.getint("general", "port")
  32
  33 c['slaves'] = []
  34 max_builds = dict()
  35
  36 for section in ini.sections():
  37         if section.startswith("slave "):
  38                 if ini.has_option(section, "name") and ini.has_option(section, "password"):
  39                         name = ini.get(section, "name")
  40                         password = ini.get(section, "password")
  41                         max_builds[name] = 1
  42                         if ini.has_option(section, "builds"):
  43                                 max_builds[name] = ini.getint(section, "builds")
  44                         c['slaves'].append(BuildSlave(name, password, max_builds = max_builds[name]))
  45
  46 # 'slavePortnum' defines the TCP port to listen on for connections from slaves.
  47 # This must match the value configured into the buildslaves (with their
  48 # --master option)
  49 c['slavePortnum'] = slave_port
  50
  51 # coalesce builds
  52 c['mergeRequests'] = True
  53
  54 # Reduce amount of backlog data
  55 c['buildHorizon'] = 30
  56 c['logHorizon'] = 20
  57
  58 ####### CHANGESOURCES
  59
  60 home_dir = os.path.abspath(ini.get("general", "homedir"))
  61 tree_expire = 0
  62 other_builds = 0
  63 cc_version = None
  64
  65 cc_command = "gcc"
  66 cxx_command = "g++"
  67
  68 if ini.has_option("general", "expire"):
  69         tree_expire = ini.getint("general", "expire")
  70
  71 if ini.has_option("general", "other_builds"):
  72         other_builds = ini.getint("general", "other_builds")
  73
  74 if ini.has_option("general", "cc_version"):
  75         cc_version = ini.get("general", "cc_version").split()
  76         if len(cc_version) == 1:
  77                 cc_version = ["eq", cc_version[0]]
  78
  79 repo_url = ini.get("repo", "url")
  80 repo_branch = "master"
  81
  82 if ini.has_option("repo", "branch"):
  83         repo_branch = ini.get("repo", "branch")
  84
  85 rsync_bin_url = ini.get("rsync", "binary_url")
  86 rsync_bin_key = ini.get("rsync", "binary_password")
  87
  88 rsync_src_url = None
  89 rsync_src_key = None
  90
  91 if ini.has_option("rsync", "source_url"):
  92         rsync_src_url = ini.get("rsync", "source_url")
  93         rsync_src_key = ini.get("rsync", "source_password")
  94
  95 gpg_home = "~/.gnupg"
  96 gpg_keyid = None
  97 gpg_comment = "Unattended build signature"
  98 gpg_passfile = "/dev/null"
  99
 100 if ini.has_option("gpg", "home"):
 101         gpg_home = ini.get("gpg", "home")
 102
 103 if ini.has_option("gpg", "keyid"):
 104         gpg_keyid = ini.get("gpg", "keyid")
 105
 106 if ini.has_option("gpg", "comment"):
 107         gpg_comment = ini.get("gpg", "comment")
 108
 109 if ini.has_option("gpg", "passfile"):
 110         gpg_passfile = ini.get("gpg", "passfile")
 111
 112
 113 # find targets
 114 targets = [ ]
 115
 116 if not os.path.isdir(home_dir+'/source.git'):
 117         subprocess.call(["git", "clone", "--depth=1", "--branch="+repo_branch, repo_url, home_dir+'/source.git'])
 118 else:
 119         subprocess.call(["git", "pull"], cwd = home_dir+'/source.git')
 120
 121 findtargets = subprocess.Popen([home_dir+'/dumpinfo.pl', 'targets'],
 122         stdout = subprocess.PIPE, cwd = home_dir+'/source.git')
 123
 124 while True:
 125         line = findtargets.stdout.readline()
 126         if not line:
 127                 break
 128         ta = line.strip().split(' ')
 129         targets.append(ta[0])
 130
 131
 132 # the 'change_source' setting tells the buildmaster how it should find out
 133 # about source code changes.  Here we point to the buildbot clone of pyflakes.
 134
 135 from buildbot.changes.gitpoller import GitPoller
 136 c['change_source'] = []
 137 c['change_source'].append(GitPoller(
 138         repo_url,
 139         workdir=home_dir+'/work.git', branch=repo_branch,
 140         pollinterval=300))
 141
 142 ####### SCHEDULERS
 143
 144 # Configure the Schedulers, which decide how to react to incoming changes.  In this
 145 # case, just kick off a 'basebuild' build
 146
 147 from buildbot.schedulers.basic import SingleBranchScheduler
 148 from buildbot.schedulers.forcesched import ForceScheduler
 149 from buildbot.changes import filter
 150 c['schedulers'] = []
 151 c['schedulers'].append(SingleBranchScheduler(
 152         name="all",
 153         change_filter=filter.ChangeFilter(branch=repo_branch),
 154         treeStableTimer=60,
 155         builderNames=targets))
 156
 157 c['schedulers'].append(ForceScheduler(
 158         name="force",
 159         builderNames=targets))
 160
 161 ####### BUILDERS
 162
 163 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
 164 # what steps, and which slaves can execute them.  Note that any particular build will
 165 # only take place on one slave.
 166
 167 from buildbot.process.factory import BuildFactory
 168 from buildbot.steps.source.git import Git
 169 from buildbot.steps.shell import ShellCommand
 170 from buildbot.steps.shell import SetProperty
 171 from buildbot.steps.transfer import FileUpload
 172 from buildbot.steps.transfer import FileDownload
 173 from buildbot.steps.master import MasterShellCommand
 174 from buildbot.process.properties import WithProperties
 175
 176
 177 CleanTargetMap = [
 178         [ "tools",      "tools/clean"                   ],
 179         [ "chain",      "toolchain/clean"               ],
 180         [ "linux",      "target/linux/clean"    ],
 181         [ "dir",        "dirclean"                              ],
 182         [ "dist",       "distclean"                             ]
 183 ]
 184
 185 def IsCleanRequested(pattern):
 186         def CheckCleanProperty(step):
 187                 val = step.getProperty("clean")
 188                 if val and re.match(pattern, val):
 189                         return True
 190                 else:
 191                         return False
 192
 193         return CheckCleanProperty
 194
 195 def IsTaggingRequested(step):
 196         val = step.getProperty("tag")
 197         if val and re.match("^[0-9]+\.[0-9]+\.[0-9]+$", val):
 198                 return True
 199         else:
 200                 return False
 201
 202 def IsNoTaggingRequested(step):
 203         return not IsTaggingRequested(step)
 204
 205 def IsNoMasterBuild(step):
 206         return repo_branch != "master"
 207
 208 def GetBaseVersion(props):
 209         if re.match("^[^-]+-[0-9]+\.[0-9]+$", repo_branch):
 210                 return repo_branch.split('-')[1]
 211         else:
 212                 return "master"
 213
 214 def GetVersionPrefix(props):
 215         basever = GetBaseVersion(props)
 216         if props.hasProperty("tag") and re.match("^[0-9]+\.[0-9]+\.[0-9]+$", props["tag"]):
 217                 return "%s/" % props["tag"]
 218         elif basever != "master":
 219                 return "%s-SNAPSHOT/" % basever
 220         else:
 221                 return ""
 222
 223 def GetNumJobs(props):
 224         if props.hasProperty("slavename") and props.hasProperty("nproc"):
 225                 return ((int(props["nproc"]) / (max_builds[props["slavename"]] + other_builds)) + 1)
 226         else:
 227                 return 1
 228
 229 def GetCC(props):
 230         if props.hasProperty("cc_command"):
 231                 return props["cc_command"]
 232         else:
 233                 return "gcc"
 234
 235 def GetCXX(props):
 236         if props.hasProperty("cxx_command"):
 237                 return props["cxx_command"]
 238         else:
 239                 return "g++"
 240
 241 def MakeEnv(overrides=None):
 242         env = {
 243                 'CC': WithProperties("%(cc)s", cc=GetCC),
 244                 'CXX': WithProperties("%(cxx)s", cxx=GetCXX)
 245         }
 246         if overrides is not None:
 247                 env.update(overrides)
 248         return env
 249
 250
 251 c['builders'] = []
 252
 253 dlLock = locks.SlaveLock("slave_dl")
 254 tagLock = locks.MasterLock("make_tag")
 255
 256 checkBuiltin = re.sub('[\t\n ]+', ' ', """
 257         checkBuiltin() {
 258                 local symbol op path file;
 259                 for file in $CHANGED_FILES; do
 260                         case "$file" in
 261                                 package/*/*) : ;;
 262                                 *) return 0 ;;
 263                         esac;
 264                 done;
 265                 while read symbol op path; do
 266                         case "$symbol" in package-*)
 267                                 symbol="${symbol##*(}";
 268                                 symbol="${symbol%)}";
 269                                 for file in $CHANGED_FILES; do
 270                                         case "$file" in "package/$path/"*)
 271                                                 grep -qsx "$symbol=y" .config && return 0
 272                                         ;; esac;
 273                                 done;
 274                         esac;
 275                 done < tmp/.packagedeps;
 276                 return 1;
 277         }
 278 """).strip()
 279
 280
 281 class IfBuiltinShellCommand(ShellCommand):
 282         def _quote(self, str):
 283                 if re.search("[^a-zA-Z0-9/_.-]", str):
 284                         return "'%s'" %(re.sub("'", "'\"'\"'", str))
 285                 return str
 286
 287         def setCommand(self, command):
 288                 if not isinstance(command, (str, unicode)):
 289                         command = ' '.join(map(self._quote, command))
 290                 self.command = [
 291                         '/bin/sh', '-c',
 292                         '%s; if checkBuiltin; then %s; else exit 0; fi' %(checkBuiltin, command)
 293                 ]
 294
 295         def setupEnvironment(self, cmd):
 296                 slaveEnv = self.slaveEnvironment
 297                 if slaveEnv is None:
 298                         slaveEnv = { }
 299                 changedFiles = { }
 300                 for request in self.build.requests:
 301                         for source in request.sources:
 302                                 for change in source.changes:
 303                                         for file in change.files:
 304                                                 changedFiles[file] = True
 305                 fullSlaveEnv = slaveEnv.copy()
 306                 fullSlaveEnv['CHANGED_FILES'] = ' '.join(changedFiles.keys())
 307                 cmd.args['env'] = fullSlaveEnv
 308
 309 slaveNames = [ ]
 310
 311 for slave in c['slaves']:
 312         slaveNames.append(slave.slavename)
 313
 314 for target in targets:
 315         ts = target.split('/')
 316
 317         factory = BuildFactory()
 318
 319         # find number of cores
 320         factory.addStep(SetProperty(
 321                 name = "nproc",
 322                 property = "nproc",
 323                 description = "Finding number of CPUs",
 324                 command = ["nproc"]))
 325
 326         # find gcc and g++ compilers
 327         if cc_version is not None:
 328                 factory.addStep(FileDownload(
 329                         mastersrc = "findbin.pl",
 330                         slavedest = "../findbin.pl",
 331                         mode = 0755))
 332
 333                 factory.addStep(SetProperty(
 334                         name = "gcc",
 335                         property = "cc_command",
 336                         description = "Finding gcc command",
 337                         command = ["../findbin.pl", "gcc", cc_version[0], cc_version[1]],
 338                         haltOnFailure = True))
 339
 340                 factory.addStep(SetProperty(
 341                         name = "g++",
 342                         property = "cxx_command",
 343                         description = "Finding g++ command",
 344                         command = ["../findbin.pl", "g++", cc_version[0], cc_version[1]],
 345                         haltOnFailure = True))
 346
 347         # expire tree if needed
 348         if tree_expire > 0:
 349                 factory.addStep(FileDownload(
 350                         mastersrc = "expire.sh",
 351                         slavedest = "../expire.sh",
 352                         mode = 0755))
 353
 354                 factory.addStep(ShellCommand(
 355                         name = "expire",
 356                         description = "Checking for build tree expiry",
 357                         command = ["./expire.sh", str(tree_expire)],
 358                         workdir = ".",
 359                         haltOnFailure = True,
 360                         timeout = 2400))
 361
 362         # user-requested clean targets
 363         for tuple in CleanTargetMap:
 364                 factory.addStep(ShellCommand(
 365                         name = tuple[1],
 366                         description = 'User-requested "make %s"' % tuple[1],
 367                         command = ["make", tuple[1], "V=s"],
 368                         env = MakeEnv(),
 369                         doStepIf = IsCleanRequested(tuple[0])
 370                 ))
 371
 372         factory.addStep(MasterShellCommand(
 373                 name = "maketag",
 374                 description = "Tagging Git repository",
 375                 command = [home_dir+'/maketag.sh', '-i', '-k', str(gpg_keyid or ''),
 376                            '-p', str(gpg_passfile or ''), '-v', WithProperties("%(tag:-)s")],
 377                 path = home_dir+'/source.git',
 378                 env = {'GNUPGHOME': gpg_home},
 379                 haltOnFailure = True,
 380                 doStepIf = IsTaggingRequested,
 381                 locks = [tagLock.access('exclusive')]
 382         ))
 383
 384         # switch to branch
 385         factory.addStep(ShellCommand(
 386                 name = "switchbranch",
 387                 description = "Checking out Git branch",
 388                 command = "if [ -d .git ]; then git checkout '%s'; else exit 0; fi" % repo_branch,
 389                 haltOnFailure = True,
 390                 doStepIf = IsNoTaggingRequested
 391         ))
 392
 393         # check out the source
 394         factory.addStep(Git(
 395                 repourl = repo_url,
 396                 branch = repo_branch,
 397                 mode = 'incremental',
 398                 method = 'clean'))
 399
 400         # update remote refs
 401         factory.addStep(ShellCommand(
 402                 name = "fetchrefs",
 403                 description = "Fetching Git remote refs",
 404                 command = ["git", "fetch", "origin", "+refs/heads/%s:refs/remotes/origin/%s" %(repo_branch, repo_branch)],
 405                 haltOnFailure = True
 406         ))
 407
 408         # fetch tags
 409         factory.addStep(ShellCommand(
 410                 name = "fetchtag",
 411                 description = "Fetching Git tags",
 412                 command = ["git", "fetch", "--tags", "--", repo_url],
 413                 haltOnFailure = True,
 414                 doStepIf = IsTaggingRequested
 415         ))
 416
 417         # switch to tag
 418         factory.addStep(ShellCommand(
 419                 name = "switchtag",
 420                 description = "Checking out Git tag",
 421                 command = ["git", "checkout", WithProperties("tags/v%(tag:-)s")],
 422                 haltOnFailure = True,
 423                 doStepIf = IsTaggingRequested
 424         ))
 425
 426         factory.addStep(ShellCommand(
 427                 name = "rmtmp",
 428                 description = "Remove tmp folder",
 429                 command=["rm", "-rf", "tmp/"]))
 430
 431         # feed
 432 #       factory.addStep(ShellCommand(
 433 #               name = "feedsconf",
 434 #               description = "Copy the feeds.conf",
 435 #               command='''cp ~/feeds.conf ./feeds.conf''' ))
 436
 437         # feed
 438         factory.addStep(ShellCommand(
 439                 name = "rmfeedlinks",
 440                 description = "Remove feed symlinks",
 441                 command=["rm", "-rf", "package/feeds/"]))
 442
 443         # feed
 444         factory.addStep(ShellCommand(
 445                 name = "updatefeeds",
 446                 description = "Updating feeds",
 447                 command=["./scripts/feeds", "update"],
 448                 env = MakeEnv()))
 449
 450         # feed
 451         factory.addStep(ShellCommand(
 452                 name = "installfeeds",
 453                 description = "Installing feeds",
 454                 command=["./scripts/feeds", "install", "-a"],
 455                 env = MakeEnv()))
 456
 457         # seed config
 458         factory.addStep(FileDownload(
 459                 mastersrc = "config.seed",
 460                 slavedest = ".config",
 461                 mode = 0644
 462         ))
 463
 464         # configure
 465         factory.addStep(ShellCommand(
 466                 name = "newconfig",
 467                 description = "Seeding .config",
 468                 command = "printf 'CONFIG_TARGET_%s=y\\nCONFIG_TARGET_%s_%s=y\\n' >> .config" %(ts[0], ts[0], ts[1])
 469         ))
 470
 471         factory.addStep(ShellCommand(
 472                 name = "delbin",
 473                 description = "Removing output directory",
 474                 command = ["rm", "-rf", "bin/"]
 475         ))
 476
 477         factory.addStep(ShellCommand(
 478                 name = "defconfig",
 479                 description = "Populating .config",
 480                 command = ["make", "defconfig"],
 481                 env = MakeEnv()
 482         ))
 483
 484         # check arch
 485         factory.addStep(ShellCommand(
 486                 name = "checkarch",
 487                 description = "Checking architecture",
 488                 command = ["grep", "-sq", "CONFIG_TARGET_%s=y" %(ts[0]), ".config"],
 489                 logEnviron = False,
 490                 want_stdout = False,
 491                 want_stderr = False,
 492                 haltOnFailure = True
 493         ))
 494
 495         # find libc suffix
 496         factory.addStep(SetProperty(
 497                 name = "libc",
 498                 property = "libc",
 499                 description = "Finding libc suffix",
 500                 command = ["sed", "-ne", '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }', ".config"]))
 501
 502         # install build key
 503         factory.addStep(FileDownload(mastersrc=home_dir+'/key-build', slavedest="key-build", mode=0600))
 504         factory.addStep(FileDownload(mastersrc=home_dir+'/key-build.pub', slavedest="key-build.pub", mode=0600))
 505
 506         # prepare dl
 507         factory.addStep(ShellCommand(
 508                 name = "dldir",
 509                 description = "Preparing dl/",
 510                 command = "mkdir -p $HOME/dl && rm -rf ./dl && ln -sf $HOME/dl ./dl",
 511                 logEnviron = False,
 512                 want_stdout = False
 513         ))
 514
 515         # prepare tar
 516         factory.addStep(ShellCommand(
 517                 name = "dltar",
 518                 description = "Building GNU tar",
 519                 command = ["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "tools/tar/install", "V=s"],
 520                 env = MakeEnv(),
 521                 haltOnFailure = True
 522         ))
 523
 524         # populate dl
 525         factory.addStep(ShellCommand(
 526                 name = "dlrun",
 527                 description = "Populating dl/",
 528                 command = ["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "download", "V=s"],
 529                 env = MakeEnv(),
 530                 logEnviron = False,
 531                 locks = [dlLock.access('exclusive')]
 532         ))
 533
 534         factory.addStep(ShellCommand(
 535                 name = "cleanbase",
 536                 description = "Cleaning base-files",
 537                 command=["make", "package/base-files/clean", "V=s"]
 538         ))
 539
 540         # build
 541         factory.addStep(ShellCommand(
 542                 name = "tools",
 543                 description = "Building tools",
 544                 command = ["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "tools/install", "V=s"],
 545                 env = MakeEnv(),
 546                 haltOnFailure = True
 547         ))
 548
 549         factory.addStep(ShellCommand(
 550                 name = "toolchain",
 551                 description = "Building toolchain",
 552                 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "toolchain/install", "V=s"],
 553                 env = MakeEnv(),
 554                 haltOnFailure = True
 555         ))
 556
 557         factory.addStep(ShellCommand(
 558                 name = "kmods",
 559                 description = "Building kmods",
 560                 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
 561                 env = MakeEnv(),
 562                 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
 563                 haltOnFailure = True
 564         ))
 565
 566         factory.addStep(ShellCommand(
 567                 name = "pkgclean",
 568                 description = "Cleaning up package build",
 569                 command=["make", "package/cleanup", "V=s"]
 570         ))
 571
 572         factory.addStep(ShellCommand(
 573                 name = "pkgbuild",
 574                 description = "Building packages",
 575                 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
 576                 env = MakeEnv(),
 577                 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
 578                 haltOnFailure = True
 579         ))
 580
 581         # factory.addStep(IfBuiltinShellCommand(
 582         factory.addStep(ShellCommand(
 583                 name = "pkginstall",
 584                 description = "Installing packages",
 585                 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "package/install", "V=s"],
 586                 env = MakeEnv(),
 587                 haltOnFailure = True
 588         ))
 589
 590         factory.addStep(ShellCommand(
 591                 name = "pkgindex",
 592                 description = "Indexing packages",
 593                 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "package/index", "V=s"],
 594                 env = MakeEnv(),
 595                 haltOnFailure = True
 596         ))
 597
 598         #factory.addStep(IfBuiltinShellCommand(
 599         factory.addStep(ShellCommand(
 600                 name = "images",
 601                 description = "Building images",
 602                 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "target/install", "V=s"],
 603                 env = MakeEnv(),
 604                 haltOnFailure = True
 605         ))
 606
 607         factory.addStep(ShellCommand(
 608                 name = "diffconfig",
 609                 description = "Generating config.seed",
 610                 command=["make", "-j1", "diffconfig", "V=s"],
 611                 env = MakeEnv(),
 612                 haltOnFailure = True
 613         ))
 614
 615         factory.addStep(ShellCommand(
 616                 name = "checksums",
 617                 description = "Calculating checksums",
 618                 command=["make", "-j1", "checksum", "V=s"],
 619                 env = MakeEnv(),
 620                 haltOnFailure = True
 621         ))
 622
 623         # sign
 624         if gpg_keyid is not None:
 625                 factory.addStep(MasterShellCommand(
 626                         name = "signprepare",
 627                         description = "Preparing temporary signing directory",
 628                         command = ["mkdir", "-p", "%s/signing" %(home_dir)],
 629                         haltOnFailure = True
 630                 ))
 631
 632                 factory.addStep(ShellCommand(
 633                         name = "signpack",
 634                         description = "Packing files to sign",
 635                         command = WithProperties("find bin/targets/%s/%s%%(libc)s/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz" %(ts[0], ts[1])),
 636                         haltOnFailure = True
 637                 ))
 638
 639                 factory.addStep(FileUpload(
 640                         slavesrc = "sign.tar.gz",
 641                         masterdest = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
 642                         haltOnFailure = True
 643                 ))
 644
 645                 factory.addStep(MasterShellCommand(
 646                         name = "signfiles",
 647                         description = "Signing files",
 648                         command = ["%s/signall.sh" %(home_dir), "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]), gpg_keyid, gpg_comment],
 649                         env = {'GNUPGHOME': gpg_home, 'PASSFILE': gpg_passfile},
 650                         haltOnFailure = True
 651                 ))
 652
 653                 factory.addStep(FileDownload(
 654                         mastersrc = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
 655                         slavedest = "sign.tar.gz",
 656                         haltOnFailure = True
 657                 ))
 658
 659                 factory.addStep(ShellCommand(
 660                         name = "signunpack",
 661                         description = "Unpacking signed files",
 662                         command = ["tar", "-xzf", "sign.tar.gz"],
 663                         haltOnFailure = True
 664                 ))
 665
 666         # upload
 667         factory.addStep(ShellCommand(
 668                 name = "dirprepare",
 669                 description = "Preparing upload directory structure",
 670                 command = ["mkdir", "-p", WithProperties("tmp/upload/%%(prefix)stargets/%s/%s" %(ts[0], ts[1]), prefix=GetVersionPrefix)],
 671                 haltOnFailure = True
 672         ))
 673
 674         factory.addStep(ShellCommand(
 675                 name = "linkprepare",
 676                 description = "Preparing repository symlink",
 677                 command = ["ln", "-s", "-f", WithProperties("../packages-%(basever)s", basever=GetBaseVersion), WithProperties("tmp/upload/%(prefix)spackages", prefix=GetVersionPrefix)],
 678                 doStepIf = IsNoMasterBuild,
 679                 haltOnFailure = True
 680         ))
 681
 682         factory.addStep(ShellCommand(
 683                 name = "dirupload",
 684                 description = "Uploading directory structure",
 685                 command = ["rsync", "-avz", "tmp/upload/", "%s/" %(rsync_bin_url)],
 686                 env={'RSYNC_PASSWORD': rsync_bin_key},
 687                 haltOnFailure = True,
 688                 logEnviron = False
 689         ))
 690
 691         factory.addStep(ShellCommand(
 692                 name = "targetupload",
 693                 description = "Uploading target files",
 694                 command=["rsync", "--progress", "--delete", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]),
 695                          "-avz", WithProperties("bin/targets/%s/%s%%(libc)s/" %(ts[0], ts[1])),
 696                          WithProperties("%s/%%(prefix)stargets/%s/%s/" %(rsync_bin_url, ts[0], ts[1]), prefix=GetVersionPrefix)],
 697                 env={'RSYNC_PASSWORD': rsync_bin_key},
 698                 haltOnFailure = True,
 699                 logEnviron = False
 700         ))
 701
 702         if rsync_src_url is not None:
 703                 factory.addStep(ShellCommand(
 704                         name = "sourceupload",
 705                         description = "Uploading source archives",
 706                         command=["rsync", "--progress", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "dl/", "%s/" %(rsync_src_url)],
 707                         env={'RSYNC_PASSWORD': rsync_src_key},
 708                         haltOnFailure = True,
 709                         logEnviron = False
 710                 ))
 711
 712         if False:
 713                 factory.addStep(ShellCommand(
 714                         name = "packageupload",
 715                         description = "Uploading package files",
 716                         command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "bin/packages/", "%s/packages/" %(rsync_bin_url)],
 717                         env={'RSYNC_PASSWORD': rsync_bin_key},
 718                         haltOnFailure = False,
 719                         logEnviron = False
 720                 ))
 721
 722         # logs
 723         if False:
 724                 factory.addStep(ShellCommand(
 725                         name = "upload",
 726                         description = "Uploading logs",
 727                         command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "logs/", "%s/logs/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
 728                         env={'RSYNC_PASSWORD': rsync_bin_key},
 729                         haltOnFailure = False,
 730                         alwaysRun = True,
 731                         logEnviron = False
 732                 ))
 733
 734         from buildbot.config import BuilderConfig
 735
 736         c['builders'].append(BuilderConfig(name=target, slavenames=slaveNames, factory=factory))
 737
 738
 739 ####### STATUS TARGETS
 740
 741 # 'status' is a list of Status Targets. The results of each build will be
 742 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
 743 # including web pages, email senders, and IRC bots.
 744
 745 c['status'] = []
 746
 747 from buildbot.status import html
 748 from buildbot.status.web import authz, auth
 749
 750 if ini.has_option("status", "bind"):
 751         if ini.has_option("status", "user") and ini.has_option("status", "password"):
 752                 authz_cfg=authz.Authz(
 753                         # change any of these to True to enable; see the manual for more
 754                         # options
 755                         auth=auth.BasicAuth([(ini.get("status", "user"), ini.get("status", "password"))]),
 756                         gracefulShutdown = 'auth',
 757                         forceBuild = 'auth', # use this to test your slave once it is set up
 758                         forceAllBuilds = 'auth',
 759                         pingBuilder = False,
 760                         stopBuild = 'auth',
 761                         stopAllBuilds = 'auth',
 762                         cancelPendingBuild = 'auth',
 763                 )
 764                 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind"), authz=authz_cfg))
 765         else:
 766                 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind")))
 767
 768
 769 from buildbot.status import words
 770
 771 if ini.has_option("irc", "host") and ini.has_option("irc", "nickname") and ini.has_option("irc", "channel"):
 772         irc_host = ini.get("irc", "host")
 773         irc_port = 6667
 774         irc_chan = ini.get("irc", "channel")
 775         irc_nick = ini.get("irc", "nickname")
 776         irc_pass = None
 777
 778         if ini.has_option("irc", "port"):
 779                 irc_port = ini.getint("irc", "port")
 780
 781         if ini.has_option("irc", "password"):
 782                 irc_pass = ini.get("irc", "password")
 783
 784         irc = words.IRC(irc_host, irc_nick, port = irc_port, password = irc_pass,
 785                         channels = [{ "channel": irc_chan }],
 786                         notify_events = {
 787                           'exception': 1,
 788                           'successToFailure': 1,
 789                           'failureToSuccess': 1
 790                         }
 791         )
 792
 793         c['status'].append(irc)
 794
 795
 796 ####### PROJECT IDENTITY
 797
 798 # the 'title' string will appear at the top of this buildbot
 799 # installation's html.WebStatus home page (linked to the
 800 # 'titleURL') and is embedded in the title of the waterfall HTML page.
 801
 802 c['title'] = ini.get("general", "title")
 803 c['titleURL'] = ini.get("general", "title_url")
 804
 805 # the 'buildbotURL' string should point to the location where the buildbot's
 806 # internal web server (usually the html.WebStatus page) is visible. This
 807 # typically uses the port number set in the Waterfall 'status' entry, but
 808 # with an externally-visible host name which the buildbot cannot figure out
 809 # without some help.
 810
 811 c['buildbotURL'] = ini.get("general", "buildbot_url")
 812
 813 ####### DB URL
 814
 815 c['db'] = {
 816         # This specifies what database buildbot uses to store its state.  You can leave
 817         # this at its default for all but the largest installations.
 818         'db_url' : "sqlite:///state.sqlite",
 819 }