phase1/master.cfg

   1 # -*- python -*-
   2 # ex: set syntax=python:
   3
   4 import os
   5 import re
   6 import subprocess
   7 import ConfigParser
   8
   9 from buildbot import locks
  10
  11 # This is a sample buildmaster config file. It must be installed as
  12 # 'master.cfg' in your buildmaster's base directory.
  13
  14 ini = ConfigParser.ConfigParser()
  15 ini.read("./config.ini")
  16
  17 # This is the dictionary that the buildmaster pays attention to. We also use
  18 # a shorter alias to save typing.
  19 c = BuildmasterConfig = {}
  20
  21 ####### BUILDSLAVES
  22
  23 # The 'slaves' list defines the set of recognized buildslaves. Each element is
  24 # a BuildSlave object, specifying a unique slave name and password.  The same
  25 # slave name and password must be configured on the slave.
  26 from buildbot.buildslave import BuildSlave
  27
  28 slave_port = 9989
  29
  30 if ini.has_option("general", "port"):
  31         slave_port = ini.getint("general", "port")
  32
  33 c['slaves'] = []
  34 max_builds = dict()
  35
  36 for section in ini.sections():
  37         if section.startswith("slave "):
  38                 if ini.has_option(section, "name") and ini.has_option(section, "password"):
  39                         name = ini.get(section, "name")
  40                         password = ini.get(section, "password")
  41                         max_builds[name] = 1
  42                         if ini.has_option(section, "builds"):
  43                                 max_builds[name] = ini.getint(section, "builds")
  44                         c['slaves'].append(BuildSlave(name, password, max_builds = max_builds[name]))
  45
  46 # 'slavePortnum' defines the TCP port to listen on for connections from slaves.
  47 # This must match the value configured into the buildslaves (with their
  48 # --master option)
  49 c['slavePortnum'] = slave_port
  50
  51 # coalesce builds
  52 c['mergeRequests'] = True
  53
  54 # Reduce amount of backlog data
  55 c['buildHorizon'] = 30
  56 c['logHorizon'] = 20
  57
  58 ####### CHANGESOURCES
  59
  60 home_dir = os.path.abspath(ini.get("general", "homedir"))
  61 tree_expire = 0
  62 other_builds = 0
  63 cc_version = None
  64
  65 cc_command = "gcc"
  66 cxx_command = "g++"
  67
  68 if ini.has_option("general", "expire"):
  69         tree_expire = ini.getint("general", "expire")
  70
  71 if ini.has_option("general", "other_builds"):
  72         other_builds = ini.getint("general", "other_builds")
  73
  74 if ini.has_option("general", "cc_version"):
  75         cc_version = ini.get("general", "cc_version").split()
  76         if len(cc_version) == 1:
  77                 cc_version = ["eq", cc_version[0]]
  78
  79 repo_url = ini.get("repo", "url")
  80 repo_branch = "master"
  81
  82 if ini.has_option("repo", "branch"):
  83         repo_branch = ini.get("repo", "branch")
  84
  85 rsync_bin_url = ini.get("rsync", "binary_url")
  86 rsync_bin_key = ini.get("rsync", "binary_password")
  87
  88 rsync_src_url = None
  89 rsync_src_key = None
  90
  91 if ini.has_option("rsync", "source_url"):
  92         rsync_src_url = ini.get("rsync", "source_url")
  93         rsync_src_key = ini.get("rsync", "source_password")
  94
  95 gpg_home = "~/.gnupg"
  96 gpg_keyid = None
  97 gpg_comment = "Unattended build signature"
  98 gpg_passfile = "/dev/null"
  99
 100 if ini.has_option("gpg", "home"):
 101         gpg_home = ini.get("gpg", "home")
 102
 103 if ini.has_option("gpg", "keyid"):
 104         gpg_keyid = ini.get("gpg", "keyid")
 105
 106 if ini.has_option("gpg", "comment"):
 107         gpg_comment = ini.get("gpg", "comment")
 108
 109 if ini.has_option("gpg", "passfile"):
 110         gpg_passfile = ini.get("gpg", "passfile")
 111
 112
 113 # find targets
 114 targets = [ ]
 115
 116 if not os.path.isdir(home_dir+'/source.git'):
 117         subprocess.call(["git", "clone", "--depth=1", "--branch="+repo_branch, repo_url, home_dir+'/source.git'])
 118 else:
 119         subprocess.call(["git", "pull"], cwd = home_dir+'/source.git')
 120
 121 findtargets = subprocess.Popen([home_dir+'/dumpinfo.pl', 'targets'],
 122         stdout = subprocess.PIPE, cwd = home_dir+'/source.git')
 123
 124 while True:
 125         line = findtargets.stdout.readline()
 126         if not line:
 127                 break
 128         ta = line.strip().split(' ')
 129         targets.append(ta[0])
 130
 131
 132 # the 'change_source' setting tells the buildmaster how it should find out
 133 # about source code changes.  Here we point to the buildbot clone of pyflakes.
 134
 135 from buildbot.changes.gitpoller import GitPoller
 136 c['change_source'] = []
 137 c['change_source'].append(GitPoller(
 138         repo_url,
 139         workdir=home_dir+'/work.git', branch=repo_branch,
 140         pollinterval=300))
 141
 142 ####### SCHEDULERS
 143
 144 # Configure the Schedulers, which decide how to react to incoming changes.  In this
 145 # case, just kick off a 'basebuild' build
 146
 147 from buildbot.schedulers.basic import SingleBranchScheduler
 148 from buildbot.schedulers.forcesched import ForceScheduler
 149 from buildbot.changes import filter
 150 c['schedulers'] = []
 151 c['schedulers'].append(SingleBranchScheduler(
 152         name="all",
 153         change_filter=filter.ChangeFilter(branch=repo_branch),
 154         treeStableTimer=60,
 155         builderNames=targets))
 156
 157 c['schedulers'].append(ForceScheduler(
 158         name="force",
 159         builderNames=targets))
 160
 161 ####### BUILDERS
 162
 163 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
 164 # what steps, and which slaves can execute them.  Note that any particular build will
 165 # only take place on one slave.
 166
 167 from buildbot.process.factory import BuildFactory
 168 from buildbot.steps.source.git import Git
 169 from buildbot.steps.shell import ShellCommand
 170 from buildbot.steps.shell import SetProperty
 171 from buildbot.steps.transfer import FileUpload
 172 from buildbot.steps.transfer import FileDownload
 173 from buildbot.steps.master import MasterShellCommand
 174 from buildbot.process.properties import WithProperties
 175
 176
 177 CleanTargetMap = [
 178         [ "tools",      "tools/clean"                   ],
 179         [ "chain",      "toolchain/clean"               ],
 180         [ "linux",      "target/linux/clean"    ],
 181         [ "dir",        "dirclean"                              ],
 182         [ "dist",       "distclean"                             ]
 183 ]
 184
 185 def IsCleanRequested(pattern):
 186         def CheckCleanProperty(step):
 187                 val = step.getProperty("clean")
 188                 if val and re.match(pattern, val):
 189                         return True
 190                 else:
 191                         return False
 192
 193         return CheckCleanProperty
 194
 195 def IsTaggingRequested(step):
 196         val = step.getProperty("tag")
 197         if val and re.match("^[0-9]+\.[0-9]+\.[0-9]+$", val):
 198                 return True
 199         else:
 200                 return False
 201
 202 def IsNoTaggingRequested(step):
 203         return not IsTaggingRequested(step)
 204
 205 def IsNoMasterBuild(step):
 206         return repo_branch != "master"
 207
 208 def GetBaseVersion(props):
 209         if re.match("^[^-]+-[0-9]+\.[0-9]+$", repo_branch):
 210                 return repo_branch.split('-')[1]
 211         else:
 212                 return "master"
 213
 214 def GetVersionPrefix(props):
 215         basever = GetBaseVersion(props)
 216         if props.hasProperty("tag") and re.match("^[0-9]+\.[0-9]+\.[0-9]+$", props["tag"]):
 217                 return "%s/" % props["tag"]
 218         elif basever != "master":
 219                 return "%s-SNAPSHOT/" % basever
 220         else:
 221                 return ""
 222
 223 def GetNumJobs(props):
 224         if props.hasProperty("slavename") and props.hasProperty("nproc"):
 225                 return ((int(props["nproc"]) / (max_builds[props["slavename"]] + other_builds)) + 1)
 226         else:
 227                 return 1
 228
 229 def GetCC(props):
 230         if props.hasProperty("cc_command"):
 231                 return props["cc_command"]
 232         else:
 233                 return "gcc"
 234
 235 def GetCXX(props):
 236         if props.hasProperty("cxx_command"):
 237                 return props["cxx_command"]
 238         else:
 239                 return "g++"
 240
 241 def MakeEnv(overrides=None):
 242         env = {
 243                 'CC': WithProperties("%(cc)s", cc=GetCC),
 244                 'CXX': WithProperties("%(cxx)s", cxx=GetCXX)
 245         }
 246         if overrides is not None:
 247                 env.update(overrides)
 248         return env
 249
 250
 251 c['builders'] = []
 252
 253 dlLock = locks.SlaveLock("slave_dl")
 254 tagLock = locks.MasterLock("make_tag")
 255
 256 checkBuiltin = re.sub('[\t\n ]+', ' ', """
 257         checkBuiltin() {
 258                 local symbol op path file;
 259                 for file in $CHANGED_FILES; do
 260                         case "$file" in
 261                                 package/*/*) : ;;
 262                                 *) return 0 ;;
 263                         esac;
 264                 done;
 265                 while read symbol op path; do
 266                         case "$symbol" in package-*)
 267                                 symbol="${symbol##*(}";
 268                                 symbol="${symbol%)}";
 269                                 for file in $CHANGED_FILES; do
 270                                         case "$file" in "package/$path/"*)
 271                                                 grep -qsx "$symbol=y" .config && return 0
 272                                         ;; esac;
 273                                 done;
 274                         esac;
 275                 done < tmp/.packagedeps;
 276                 return 1;
 277         }
 278 """).strip()
 279
 280
 281 class IfBuiltinShellCommand(ShellCommand):
 282         def _quote(self, str):
 283                 if re.search("[^a-zA-Z0-9/_.-]", str):
 284                         return "'%s'" %(re.sub("'", "'\"'\"'", str))
 285                 return str
 286
 287         def setCommand(self, command):
 288                 if not isinstance(command, (str, unicode)):
 289                         command = ' '.join(map(self._quote, command))
 290                 self.command = [
 291                         '/bin/sh', '-c',
 292                         '%s; if checkBuiltin; then %s; else exit 0; fi' %(checkBuiltin, command)
 293                 ]
 294
 295         def setupEnvironment(self, cmd):
 296                 slaveEnv = self.slaveEnvironment
 297                 if slaveEnv is None:
 298                         slaveEnv = { }
 299                 changedFiles = { }
 300                 for request in self.build.requests:
 301                         for source in request.sources:
 302                                 for change in source.changes:
 303                                         for file in change.files:
 304                                                 changedFiles[file] = True
 305                 fullSlaveEnv = slaveEnv.copy()
 306                 fullSlaveEnv['CHANGED_FILES'] = ' '.join(changedFiles.keys())
 307                 cmd.args['env'] = fullSlaveEnv
 308
 309 slaveNames = [ ]
 310
 311 for slave in c['slaves']:
 312         slaveNames.append(slave.slavename)
 313
 314 for target in targets:
 315         ts = target.split('/')
 316
 317         factory = BuildFactory()
 318
 319         # find number of cores
 320         factory.addStep(SetProperty(
 321                 name = "nproc",
 322                 property = "nproc",
 323                 description = "Finding number of CPUs",
 324                 command = ["nproc"]))
 325
 326         # find gcc and g++ compilers
 327         if cc_version is not None:
 328                 factory.addStep(FileDownload(
 329                         mastersrc = "findbin.pl",
 330                         slavedest = "../findbin.pl",
 331                         mode = 0755))
 332
 333                 factory.addStep(SetProperty(
 334                         name = "gcc",
 335                         property = "cc_command",
 336                         description = "Finding gcc command",
 337                         command = ["../findbin.pl", "gcc", cc_version[0], cc_version[1]],
 338                         haltOnFailure = True))
 339
 340                 factory.addStep(SetProperty(
 341                         name = "g++",
 342                         property = "cxx_command",
 343                         description = "Finding g++ command",
 344                         command = ["../findbin.pl", "g++", cc_version[0], cc_version[1]],
 345                         haltOnFailure = True))
 346
 347         # expire tree if needed
 348         if tree_expire > 0:
 349                 factory.addStep(FileDownload(
 350                         mastersrc = "expire.sh",
 351                         slavedest = "../expire.sh",
 352                         mode = 0755))
 353
 354                 factory.addStep(ShellCommand(
 355                         name = "expire",
 356                         description = "Checking for build tree expiry",
 357                         command = ["./expire.sh", str(tree_expire)],
 358                         workdir = ".",
 359                         haltOnFailure = True,
 360                         timeout = 2400))
 361
 362         # user-requested clean targets
 363         for tuple in CleanTargetMap:
 364                 factory.addStep(ShellCommand(
 365                         name = tuple[1],
 366                         description = 'User-requested "make %s"' % tuple[1],
 367                         command = ["make", tuple[1], "V=s"],
 368                         env = MakeEnv(),
 369                         doStepIf = IsCleanRequested(tuple[0])
 370                 ))
 371
 372         factory.addStep(MasterShellCommand(
 373                 name = "maketag",
 374                 description = "Tagging Git repository",
 375                 command = [home_dir+'/maketag.sh', '-i', '-k', str(gpg_keyid or ''),
 376                            '-p', str(gpg_passfile or ''), '-v', WithProperties("%(tag:-)s")],
 377                 path = home_dir+'/source.git',
 378                 env = {'GNUPGHOME': gpg_home},
 379                 haltOnFailure = True,
 380                 doStepIf = IsTaggingRequested,
 381                 locks = [tagLock.access('exclusive')]
 382         ))
 383
 384         # switch to branch
 385         factory.addStep(ShellCommand(
 386                 name = "switchbranch",
 387                 description = "Checking out Git branch",
 388                 command = "if [ -d .git ]; then git checkout '%s'; else exit 0; fi" % repo_branch,
 389                 haltOnFailure = True,
 390                 doStepIf = IsNoTaggingRequested
 391         ))
 392
 393         # check out the source
 394         factory.addStep(Git(
 395                 repourl = repo_url,
 396                 branch = repo_branch,
 397                 mode = 'incremental',
 398                 method = 'clean'))
 399
 400         # update remote refs
 401         factory.addStep(ShellCommand(
 402                 name = "fetchrefs",
 403                 description = "Fetching Git remote refs",
 404                 command = ["git", "fetch", "origin", "+refs/heads/%s:refs/remotes/origin/%s" %(repo_branch, repo_branch)],
 405                 haltOnFailure = True
 406         ))
 407
 408         # fetch tags
 409         factory.addStep(ShellCommand(
 410                 name = "fetchtag",
 411                 description = "Fetching Git tags",
 412                 command = ["git", "fetch", "--tags", "--", repo_url],
 413                 haltOnFailure = True,
 414                 doStepIf = IsTaggingRequested
 415         ))
 416
 417         # switch to tag
 418         factory.addStep(ShellCommand(
 419                 name = "switchtag",
 420                 description = "Checking out Git tag",
 421                 command = ["git", "checkout", WithProperties("tags/v%(tag:-)s")],
 422                 haltOnFailure = True,
 423                 doStepIf = IsTaggingRequested
 424         ))
 425
 426         factory.addStep(ShellCommand(
 427                 name = "rmtmp",
 428                 description = "Remove tmp folder",
 429                 command=["rm", "-rf", "tmp/"]))
 430
 431         # feed
 432 #       factory.addStep(ShellCommand(
 433 #               name = "feedsconf",
 434 #               description = "Copy the feeds.conf",
 435 #               command='''cp ~/feeds.conf ./feeds.conf''' ))
 436
 437         # feed
 438         factory.addStep(ShellCommand(
 439                 name = "rmfeedlinks",
 440                 description = "Remove feed symlinks",
 441                 command=["rm", "-rf", "package/feeds/"]))
 442
 443         # feed
 444         factory.addStep(ShellCommand(
 445                 name = "updatefeeds",
 446                 description = "Updating feeds",
 447                 command=["./scripts/feeds", "update"],
 448                 env = MakeEnv()))
 449
 450         # feed
 451         factory.addStep(ShellCommand(
 452                 name = "installfeeds",
 453                 description = "Installing feeds",
 454                 command=["./scripts/feeds", "install", "-a"],
 455                 env = MakeEnv()))
 456
 457         # ccache helper
 458         factory.addStep(FileDownload(
 459                 mastersrc = "ccache.sh",
 460                 slavedest = "ccache.sh",
 461                 mode = 0755
 462         ))
 463
 464         # ccache prepare
 465         factory.addStep(ShellCommand(
 466                 name = "prepccache",
 467                 description = "Preparing ccache",
 468                 command = ["./ccache.sh"]
 469         ))
 470
 471         # seed config
 472         factory.addStep(FileDownload(
 473                 mastersrc = "config.seed",
 474                 slavedest = ".config",
 475                 mode = 0644
 476         ))
 477
 478         # configure
 479         factory.addStep(ShellCommand(
 480                 name = "newconfig",
 481                 description = "Seeding .config",
 482                 command = "printf 'CONFIG_TARGET_%s=y\\nCONFIG_TARGET_%s_%s=y\\n' >> .config" %(ts[0], ts[0], ts[1])
 483         ))
 484
 485         factory.addStep(ShellCommand(
 486                 name = "delbin",
 487                 description = "Removing output directory",
 488                 command = ["rm", "-rf", "bin/"]
 489         ))
 490
 491         factory.addStep(ShellCommand(
 492                 name = "defconfig",
 493                 description = "Populating .config",
 494                 command = ["make", "defconfig"],
 495                 env = MakeEnv()
 496         ))
 497
 498         # check arch
 499         factory.addStep(ShellCommand(
 500                 name = "checkarch",
 501                 description = "Checking architecture",
 502                 command = ["grep", "-sq", "CONFIG_TARGET_%s=y" %(ts[0]), ".config"],
 503                 logEnviron = False,
 504                 want_stdout = False,
 505                 want_stderr = False,
 506                 haltOnFailure = True
 507         ))
 508
 509         # find libc suffix
 510         factory.addStep(SetProperty(
 511                 name = "libc",
 512                 property = "libc",
 513                 description = "Finding libc suffix",
 514                 command = ["sed", "-ne", '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }', ".config"]))
 515
 516         # install build key
 517         factory.addStep(FileDownload(mastersrc=home_dir+'/key-build', slavedest="key-build", mode=0600))
 518         factory.addStep(FileDownload(mastersrc=home_dir+'/key-build.pub', slavedest="key-build.pub", mode=0600))
 519
 520         # prepare dl
 521         factory.addStep(ShellCommand(
 522                 name = "dldir",
 523                 description = "Preparing dl/",
 524                 command = "mkdir -p $HOME/dl && rm -rf ./dl && ln -sf $HOME/dl ./dl",
 525                 logEnviron = False,
 526                 want_stdout = False
 527         ))
 528
 529         # prepare tar
 530         factory.addStep(ShellCommand(
 531                 name = "dltar",
 532                 description = "Building GNU tar",
 533                 command = ["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "tools/tar/install", "V=s"],
 534                 env = MakeEnv(),
 535                 haltOnFailure = True
 536         ))
 537
 538         # populate dl
 539         factory.addStep(ShellCommand(
 540                 name = "dlrun",
 541                 description = "Populating dl/",
 542                 command = ["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "download", "V=s"],
 543                 env = MakeEnv(),
 544                 logEnviron = False,
 545                 locks = [dlLock.access('exclusive')]
 546         ))
 547
 548         factory.addStep(ShellCommand(
 549                 name = "cleanbase",
 550                 description = "Cleaning base-files",
 551                 command=["make", "package/base-files/clean", "V=s"]
 552         ))
 553
 554         # build
 555         factory.addStep(ShellCommand(
 556                 name = "tools",
 557                 description = "Building tools",
 558                 command = ["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "tools/install", "V=s"],
 559                 env = MakeEnv(),
 560                 haltOnFailure = True
 561         ))
 562
 563         factory.addStep(ShellCommand(
 564                 name = "toolchain",
 565                 description = "Building toolchain",
 566                 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "toolchain/install", "V=s"],
 567                 env = MakeEnv(),
 568                 haltOnFailure = True
 569         ))
 570
 571         factory.addStep(ShellCommand(
 572                 name = "kmods",
 573                 description = "Building kmods",
 574                 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
 575                 env = MakeEnv(),
 576                 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
 577                 haltOnFailure = True
 578         ))
 579
 580         factory.addStep(ShellCommand(
 581                 name = "pkgclean",
 582                 description = "Cleaning up package build",
 583                 command=["make", "package/cleanup", "V=s"]
 584         ))
 585
 586         factory.addStep(ShellCommand(
 587                 name = "pkgbuild",
 588                 description = "Building packages",
 589                 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
 590                 env = MakeEnv(),
 591                 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
 592                 haltOnFailure = True
 593         ))
 594
 595         # factory.addStep(IfBuiltinShellCommand(
 596         factory.addStep(ShellCommand(
 597                 name = "pkginstall",
 598                 description = "Installing packages",
 599                 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "package/install", "V=s"],
 600                 env = MakeEnv(),
 601                 haltOnFailure = True
 602         ))
 603
 604         factory.addStep(ShellCommand(
 605                 name = "pkgindex",
 606                 description = "Indexing packages",
 607                 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "package/index", "V=s"],
 608                 env = MakeEnv(),
 609                 haltOnFailure = True
 610         ))
 611
 612         #factory.addStep(IfBuiltinShellCommand(
 613         factory.addStep(ShellCommand(
 614                 name = "images",
 615                 description = "Building images",
 616                 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "target/install", "V=s"],
 617                 env = MakeEnv(),
 618                 haltOnFailure = True
 619         ))
 620
 621         factory.addStep(ShellCommand(
 622                 name = "diffconfig",
 623                 description = "Generating config.seed",
 624                 command=["make", "-j1", "diffconfig", "V=s"],
 625                 env = MakeEnv(),
 626                 haltOnFailure = True
 627         ))
 628
 629         factory.addStep(ShellCommand(
 630                 name = "checksums",
 631                 description = "Calculating checksums",
 632                 command=["make", "-j1", "checksum", "V=s"],
 633                 env = MakeEnv(),
 634                 haltOnFailure = True
 635         ))
 636
 637         # sign
 638         if gpg_keyid is not None:
 639                 factory.addStep(MasterShellCommand(
 640                         name = "signprepare",
 641                         description = "Preparing temporary signing directory",
 642                         command = ["mkdir", "-p", "%s/signing" %(home_dir)],
 643                         haltOnFailure = True
 644                 ))
 645
 646                 factory.addStep(ShellCommand(
 647                         name = "signpack",
 648                         description = "Packing files to sign",
 649                         command = WithProperties("find bin/targets/%s/%s%%(libc)s/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz" %(ts[0], ts[1])),
 650                         haltOnFailure = True
 651                 ))
 652
 653                 factory.addStep(FileUpload(
 654                         slavesrc = "sign.tar.gz",
 655                         masterdest = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
 656                         haltOnFailure = True
 657                 ))
 658
 659                 factory.addStep(MasterShellCommand(
 660                         name = "signfiles",
 661                         description = "Signing files",
 662                         command = ["%s/signall.sh" %(home_dir), "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]), gpg_keyid, gpg_comment],
 663                         env = {'GNUPGHOME': gpg_home, 'PASSFILE': gpg_passfile},
 664                         haltOnFailure = True
 665                 ))
 666
 667                 factory.addStep(FileDownload(
 668                         mastersrc = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
 669                         slavedest = "sign.tar.gz",
 670                         haltOnFailure = True
 671                 ))
 672
 673                 factory.addStep(ShellCommand(
 674                         name = "signunpack",
 675                         description = "Unpacking signed files",
 676                         command = ["tar", "-xzf", "sign.tar.gz"],
 677                         haltOnFailure = True
 678                 ))
 679
 680         # upload
 681         factory.addStep(ShellCommand(
 682                 name = "dirprepare",
 683                 description = "Preparing upload directory structure",
 684                 command = ["mkdir", "-p", WithProperties("tmp/upload/%%(prefix)stargets/%s/%s" %(ts[0], ts[1]), prefix=GetVersionPrefix)],
 685                 haltOnFailure = True
 686         ))
 687
 688         factory.addStep(ShellCommand(
 689                 name = "linkprepare",
 690                 description = "Preparing repository symlink",
 691                 command = ["ln", "-s", "-f", WithProperties("../packages-%(basever)s", basever=GetBaseVersion), WithProperties("tmp/upload/%(prefix)spackages", prefix=GetVersionPrefix)],
 692                 doStepIf = IsNoMasterBuild,
 693                 haltOnFailure = True
 694         ))
 695
 696         factory.addStep(ShellCommand(
 697                 name = "dirupload",
 698                 description = "Uploading directory structure",
 699                 command = ["rsync", "-avz", "tmp/upload/", "%s/" %(rsync_bin_url)],
 700                 env={'RSYNC_PASSWORD': rsync_bin_key},
 701                 haltOnFailure = True,
 702                 logEnviron = False
 703         ))
 704
 705         factory.addStep(ShellCommand(
 706                 name = "targetupload",
 707                 description = "Uploading target files",
 708                 command=["rsync", "--progress", "--delete", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]),
 709                          "-avz", WithProperties("bin/targets/%s/%s%%(libc)s/" %(ts[0], ts[1])),
 710                          WithProperties("%s/%%(prefix)stargets/%s/%s/" %(rsync_bin_url, ts[0], ts[1]), prefix=GetVersionPrefix)],
 711                 env={'RSYNC_PASSWORD': rsync_bin_key},
 712                 haltOnFailure = True,
 713                 logEnviron = False
 714         ))
 715
 716         if rsync_src_url is not None:
 717                 factory.addStep(ShellCommand(
 718                         name = "sourceupload",
 719                         description = "Uploading source archives",
 720                         command=["rsync", "--progress", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "dl/", "%s/" %(rsync_src_url)],
 721                         env={'RSYNC_PASSWORD': rsync_src_key},
 722                         haltOnFailure = True,
 723                         logEnviron = False
 724                 ))
 725
 726         if False:
 727                 factory.addStep(ShellCommand(
 728                         name = "packageupload",
 729                         description = "Uploading package files",
 730                         command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "bin/packages/", "%s/packages/" %(rsync_bin_url)],
 731                         env={'RSYNC_PASSWORD': rsync_bin_key},
 732                         haltOnFailure = False,
 733                         logEnviron = False
 734                 ))
 735
 736         # logs
 737         if False:
 738                 factory.addStep(ShellCommand(
 739                         name = "upload",
 740                         description = "Uploading logs",
 741                         command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "logs/", "%s/logs/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
 742                         env={'RSYNC_PASSWORD': rsync_bin_key},
 743                         haltOnFailure = False,
 744                         alwaysRun = True,
 745                         logEnviron = False
 746                 ))
 747
 748         from buildbot.config import BuilderConfig
 749
 750         c['builders'].append(BuilderConfig(name=target, slavenames=slaveNames, factory=factory))
 751
 752
 753 ####### STATUS TARGETS
 754
 755 # 'status' is a list of Status Targets. The results of each build will be
 756 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
 757 # including web pages, email senders, and IRC bots.
 758
 759 c['status'] = []
 760
 761 from buildbot.status import html
 762 from buildbot.status.web import authz, auth
 763
 764 if ini.has_option("status", "bind"):
 765         if ini.has_option("status", "user") and ini.has_option("status", "password"):
 766                 authz_cfg=authz.Authz(
 767                         # change any of these to True to enable; see the manual for more
 768                         # options
 769                         auth=auth.BasicAuth([(ini.get("status", "user"), ini.get("status", "password"))]),
 770                         gracefulShutdown = 'auth',
 771                         forceBuild = 'auth', # use this to test your slave once it is set up
 772                         forceAllBuilds = 'auth',
 773                         pingBuilder = False,
 774                         stopBuild = 'auth',
 775                         stopAllBuilds = 'auth',
 776                         cancelPendingBuild = 'auth',
 777                 )
 778                 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind"), authz=authz_cfg))
 779         else:
 780                 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind")))
 781
 782
 783 from buildbot.status import words
 784
 785 if ini.has_option("irc", "host") and ini.has_option("irc", "nickname") and ini.has_option("irc", "channel"):
 786         irc_host = ini.get("irc", "host")
 787         irc_port = 6667
 788         irc_chan = ini.get("irc", "channel")
 789         irc_nick = ini.get("irc", "nickname")
 790         irc_pass = None
 791
 792         if ini.has_option("irc", "port"):
 793                 irc_port = ini.getint("irc", "port")
 794
 795         if ini.has_option("irc", "password"):
 796                 irc_pass = ini.get("irc", "password")
 797
 798         irc = words.IRC(irc_host, irc_nick, port = irc_port, password = irc_pass,
 799                         channels = [{ "channel": irc_chan }],
 800                         notify_events = {
 801                           'exception': 1,
 802                           'successToFailure': 1,
 803                           'failureToSuccess': 1
 804                         }
 805         )
 806
 807         c['status'].append(irc)
 808
 809
 810 ####### PROJECT IDENTITY
 811
 812 # the 'title' string will appear at the top of this buildbot
 813 # installation's html.WebStatus home page (linked to the
 814 # 'titleURL') and is embedded in the title of the waterfall HTML page.
 815
 816 c['title'] = ini.get("general", "title")
 817 c['titleURL'] = ini.get("general", "title_url")
 818
 819 # the 'buildbotURL' string should point to the location where the buildbot's
 820 # internal web server (usually the html.WebStatus page) is visible. This
 821 # typically uses the port number set in the Waterfall 'status' entry, but
 822 # with an externally-visible host name which the buildbot cannot figure out
 823 # without some help.
 824
 825 c['buildbotURL'] = ini.get("general", "buildbot_url")
 826
 827 ####### DB URL
 828
 829 c['db'] = {
 830         # This specifies what database buildbot uses to store its state.  You can leave
 831         # this at its default for all but the largest installations.
 832         'db_url' : "sqlite:///state.sqlite",
 833 }