phase1/master.cfg

   1 # -*- python -*-
   2 # ex: set syntax=python:
   3
   4 import os
   5 import re
   6 import subprocess
   7 import ConfigParser
   8
   9 from buildbot import locks
  10
  11 # This is a sample buildmaster config file. It must be installed as
  12 # 'master.cfg' in your buildmaster's base directory.
  13
  14 ini = ConfigParser.ConfigParser()
  15 ini.read("./config.ini")
  16
  17 # This is the dictionary that the buildmaster pays attention to. We also use
  18 # a shorter alias to save typing.
  19 c = BuildmasterConfig = {}
  20
  21 ####### BUILDSLAVES
  22
  23 # The 'slaves' list defines the set of recognized buildslaves. Each element is
  24 # a BuildSlave object, specifying a unique slave name and password.  The same
  25 # slave name and password must be configured on the slave.
  26 from buildbot.buildslave import BuildSlave
  27
  28 slave_port = 9989
  29
  30 if ini.has_option("general", "port"):
  31         slave_port = ini.getint("general", "port")
  32
  33 c['slaves'] = []
  34 max_builds = dict()
  35
  36 for section in ini.sections():
  37         if section.startswith("slave "):
  38                 if ini.has_option(section, "name") and ini.has_option(section, "password"):
  39                         name = ini.get(section, "name")
  40                         password = ini.get(section, "password")
  41                         max_builds[name] = 1
  42                         if ini.has_option(section, "builds"):
  43                                 max_builds[name] = ini.getint(section, "builds")
  44                         c['slaves'].append(BuildSlave(name, password, max_builds = max_builds[name]))
  45
  46 # 'slavePortnum' defines the TCP port to listen on for connections from slaves.
  47 # This must match the value configured into the buildslaves (with their
  48 # --master option)
  49 c['slavePortnum'] = slave_port
  50
  51 # coalesce builds
  52 c['mergeRequests'] = True
  53
  54 # Reduce amount of backlog data
  55 c['buildHorizon'] = 30
  56 c['logHorizon'] = 20
  57
  58 ####### CHANGESOURCES
  59
  60 home_dir = os.path.abspath(ini.get("general", "homedir"))
  61 tree_expire = 0
  62 other_builds = 0
  63 cc_version = None
  64
  65 cc_command = "gcc"
  66 cxx_command = "g++"
  67
  68 if ini.has_option("general", "expire"):
  69         tree_expire = ini.getint("general", "expire")
  70
  71 if ini.has_option("general", "other_builds"):
  72         other_builds = ini.getint("general", "other_builds")
  73
  74 if ini.has_option("general", "cc_version"):
  75         cc_version = ini.get("general", "cc_version").split()
  76         if len(cc_version) == 1:
  77                 cc_version = ["eq", cc_version[0]]
  78
  79 repo_url = ini.get("repo", "url")
  80 repo_branch = "master"
  81
  82 if ini.has_option("repo", "branch"):
  83         repo_branch = ini.get("repo", "branch")
  84
  85 rsync_bin_url = ini.get("rsync", "binary_url")
  86 rsync_bin_key = ini.get("rsync", "binary_password")
  87
  88 rsync_src_url = None
  89 rsync_src_key = None
  90
  91 if ini.has_option("rsync", "source_url"):
  92         rsync_src_url = ini.get("rsync", "source_url")
  93         rsync_src_key = ini.get("rsync", "source_password")
  94
  95 gpg_home = "~/.gnupg"
  96 gpg_keyid = None
  97 gpg_comment = "Unattended build signature"
  98 gpg_passfile = "/dev/null"
  99
 100 if ini.has_option("gpg", "home"):
 101         gpg_home = ini.get("gpg", "home")
 102
 103 if ini.has_option("gpg", "keyid"):
 104         gpg_keyid = ini.get("gpg", "keyid")
 105
 106 if ini.has_option("gpg", "comment"):
 107         gpg_comment = ini.get("gpg", "comment")
 108
 109 if ini.has_option("gpg", "passfile"):
 110         gpg_passfile = ini.get("gpg", "passfile")
 111
 112
 113 # find targets
 114 targets = [ ]
 115
 116 if not os.path.isdir(home_dir+'/source.git'):
 117         subprocess.call(["git", "clone", "--depth=1", "--branch="+repo_branch, repo_url, home_dir+'/source.git'])
 118 else:
 119         subprocess.call(["git", "pull"], cwd = home_dir+'/source.git')
 120
 121 findtargets = subprocess.Popen([home_dir+'/dumpinfo.pl', 'targets'],
 122         stdout = subprocess.PIPE, cwd = home_dir+'/source.git')
 123
 124 while True:
 125         line = findtargets.stdout.readline()
 126         if not line:
 127                 break
 128         ta = line.strip().split(' ')
 129         targets.append(ta[0])
 130
 131
 132 # the 'change_source' setting tells the buildmaster how it should find out
 133 # about source code changes.  Here we point to the buildbot clone of pyflakes.
 134
 135 from buildbot.changes.gitpoller import GitPoller
 136 c['change_source'] = []
 137 c['change_source'].append(GitPoller(
 138         repo_url,
 139         workdir=home_dir+'/work.git', branch=repo_branch,
 140         pollinterval=300))
 141
 142 ####### SCHEDULERS
 143
 144 # Configure the Schedulers, which decide how to react to incoming changes.  In this
 145 # case, just kick off a 'basebuild' build
 146
 147 from buildbot.schedulers.basic import SingleBranchScheduler
 148 from buildbot.schedulers.forcesched import ForceScheduler
 149 from buildbot.changes import filter
 150 c['schedulers'] = []
 151 c['schedulers'].append(SingleBranchScheduler(
 152         name="all",
 153         change_filter=filter.ChangeFilter(branch=repo_branch),
 154         treeStableTimer=60,
 155         builderNames=targets))
 156
 157 c['schedulers'].append(ForceScheduler(
 158         name="force",
 159         builderNames=targets))
 160
 161 ####### BUILDERS
 162
 163 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
 164 # what steps, and which slaves can execute them.  Note that any particular build will
 165 # only take place on one slave.
 166
 167 from buildbot.process.factory import BuildFactory
 168 from buildbot.steps.source.git import Git
 169 from buildbot.steps.shell import ShellCommand
 170 from buildbot.steps.shell import SetProperty
 171 from buildbot.steps.transfer import FileUpload
 172 from buildbot.steps.transfer import FileDownload
 173 from buildbot.steps.master import MasterShellCommand
 174 from buildbot.process.properties import WithProperties
 175
 176
 177 CleanTargetMap = [
 178         [ "tools",      "tools/clean"                   ],
 179         [ "chain",      "toolchain/clean"               ],
 180         [ "linux",      "target/linux/clean"    ],
 181         [ "dir",        "dirclean"                              ],
 182         [ "dist",       "distclean"                             ]
 183 ]
 184
 185 def IsCleanRequested(pattern):
 186         def CheckCleanProperty(step):
 187                 val = step.getProperty("clean")
 188                 if val and re.match(pattern, val):
 189                         return True
 190                 else:
 191                         return False
 192
 193         return CheckCleanProperty
 194
 195 def IsTaggingRequested(step):
 196         val = step.getProperty("tag")
 197         if val and re.match("^[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", val):
 198                 return True
 199         else:
 200                 return False
 201
 202 def IsNoTaggingRequested(step):
 203         return not IsTaggingRequested(step)
 204
 205 def IsNoMasterBuild(step):
 206         return repo_branch != "master"
 207
 208 def GetBaseVersion(props):
 209         if re.match("^[^-]+-[0-9]+\.[0-9]+$", repo_branch):
 210                 return repo_branch.split('-')[1]
 211         else:
 212                 return "master"
 213
 214 def GetVersionPrefix(props):
 215         basever = GetBaseVersion(props)
 216         if props.hasProperty("tag") and re.match("^[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", props["tag"]):
 217                 return "%s/" % props["tag"]
 218         elif basever != "master":
 219                 return "%s-SNAPSHOT/" % basever
 220         else:
 221                 return ""
 222
 223 def GetNumJobs(props):
 224         if props.hasProperty("slavename") and props.hasProperty("nproc"):
 225                 return ((int(props["nproc"]) / (max_builds[props["slavename"]] + other_builds)) + 1)
 226         else:
 227                 return 1
 228
 229 def GetCC(props):
 230         if props.hasProperty("cc_command"):
 231                 return props["cc_command"]
 232         else:
 233                 return "gcc"
 234
 235 def GetCXX(props):
 236         if props.hasProperty("cxx_command"):
 237                 return props["cxx_command"]
 238         else:
 239                 return "g++"
 240
 241 def GetCwd(props):
 242         if props.hasProperty("builddir"):
 243                 return props["builddir"]
 244         elif props.hasProperty("workdir"):
 245                 return props["workdir"]
 246         else:
 247                 return "/"
 248
 249 def GetNextBuild(builder, requests):
 250         for r in requests:
 251                 if r.properties and r.properties.hasProperty("tag"):
 252                         return r
 253         return requests[0]
 254
 255 def MakeEnv(overrides=None):
 256         env = {
 257                 'CC': WithProperties("%(cc)s", cc=GetCC),
 258                 'CXX': WithProperties("%(cxx)s", cxx=GetCXX),
 259                 'CCACHE_BASEDIR': WithProperties("%(cwd)s", cwd=GetCwd)
 260         }
 261         if overrides is not None:
 262                 env.update(overrides)
 263         return env
 264
 265
 266 c['builders'] = []
 267
 268 dlLock = locks.SlaveLock("slave_dl")
 269
 270 checkBuiltin = re.sub('[\t\n ]+', ' ', """
 271         checkBuiltin() {
 272                 local symbol op path file;
 273                 for file in $CHANGED_FILES; do
 274                         case "$file" in
 275                                 package/*/*) : ;;
 276                                 *) return 0 ;;
 277                         esac;
 278                 done;
 279                 while read symbol op path; do
 280                         case "$symbol" in package-*)
 281                                 symbol="${symbol##*(}";
 282                                 symbol="${symbol%)}";
 283                                 for file in $CHANGED_FILES; do
 284                                         case "$file" in "package/$path/"*)
 285                                                 grep -qsx "$symbol=y" .config && return 0
 286                                         ;; esac;
 287                                 done;
 288                         esac;
 289                 done < tmp/.packagedeps;
 290                 return 1;
 291         }
 292 """).strip()
 293
 294
 295 class IfBuiltinShellCommand(ShellCommand):
 296         def _quote(self, str):
 297                 if re.search("[^a-zA-Z0-9/_.-]", str):
 298                         return "'%s'" %(re.sub("'", "'\"'\"'", str))
 299                 return str
 300
 301         def setCommand(self, command):
 302                 if not isinstance(command, (str, unicode)):
 303                         command = ' '.join(map(self._quote, command))
 304                 self.command = [
 305                         '/bin/sh', '-c',
 306                         '%s; if checkBuiltin; then %s; else exit 0; fi' %(checkBuiltin, command)
 307                 ]
 308
 309         def setupEnvironment(self, cmd):
 310                 slaveEnv = self.slaveEnvironment
 311                 if slaveEnv is None:
 312                         slaveEnv = { }
 313                 changedFiles = { }
 314                 for request in self.build.requests:
 315                         for source in request.sources:
 316                                 for change in source.changes:
 317                                         for file in change.files:
 318                                                 changedFiles[file] = True
 319                 fullSlaveEnv = slaveEnv.copy()
 320                 fullSlaveEnv['CHANGED_FILES'] = ' '.join(changedFiles.keys())
 321                 cmd.args['env'] = fullSlaveEnv
 322
 323 slaveNames = [ ]
 324
 325 for slave in c['slaves']:
 326         slaveNames.append(slave.slavename)
 327
 328 for target in targets:
 329         ts = target.split('/')
 330
 331         factory = BuildFactory()
 332
 333         # find number of cores
 334         factory.addStep(SetProperty(
 335                 name = "nproc",
 336                 property = "nproc",
 337                 description = "Finding number of CPUs",
 338                 command = ["nproc"]))
 339
 340         # find gcc and g++ compilers
 341         if cc_version is not None:
 342                 factory.addStep(FileDownload(
 343                         mastersrc = "findbin.pl",
 344                         slavedest = "../findbin.pl",
 345                         mode = 0755))
 346
 347                 factory.addStep(SetProperty(
 348                         name = "gcc",
 349                         property = "cc_command",
 350                         description = "Finding gcc command",
 351                         command = ["../findbin.pl", "gcc", cc_version[0], cc_version[1]],
 352                         haltOnFailure = True))
 353
 354                 factory.addStep(SetProperty(
 355                         name = "g++",
 356                         property = "cxx_command",
 357                         description = "Finding g++ command",
 358                         command = ["../findbin.pl", "g++", cc_version[0], cc_version[1]],
 359                         haltOnFailure = True))
 360
 361         # expire tree if needed
 362         if tree_expire > 0:
 363                 factory.addStep(FileDownload(
 364                         mastersrc = "expire.sh",
 365                         slavedest = "../expire.sh",
 366                         mode = 0755))
 367
 368                 factory.addStep(ShellCommand(
 369                         name = "expire",
 370                         description = "Checking for build tree expiry",
 371                         command = ["./expire.sh", str(tree_expire)],
 372                         workdir = ".",
 373                         haltOnFailure = True,
 374                         timeout = 2400))
 375
 376         # user-requested clean targets
 377         for tuple in CleanTargetMap:
 378                 factory.addStep(ShellCommand(
 379                         name = tuple[1],
 380                         description = 'User-requested "make %s"' % tuple[1],
 381                         command = ["make", tuple[1], "V=s"],
 382                         env = MakeEnv(),
 383                         doStepIf = IsCleanRequested(tuple[0])
 384                 ))
 385
 386         # switch to branch
 387         factory.addStep(ShellCommand(
 388                 name = "switchbranch",
 389                 description = "Checking out Git branch",
 390                 command = "if [ -d .git ]; then git fetch && git checkout '%s'; else exit 0; fi" % repo_branch,
 391                 haltOnFailure = True,
 392                 doStepIf = IsNoTaggingRequested
 393         ))
 394
 395         # check out the source
 396         factory.addStep(Git(
 397                 repourl = repo_url,
 398                 branch = repo_branch,
 399                 mode = 'incremental',
 400                 method = 'clean'))
 401
 402         # update remote refs
 403         factory.addStep(ShellCommand(
 404                 name = "fetchrefs",
 405                 description = "Fetching Git remote refs",
 406                 command = ["git", "fetch", "origin", "+refs/heads/%s:refs/remotes/origin/%s" %(repo_branch, repo_branch)],
 407                 haltOnFailure = True
 408         ))
 409
 410         # fetch tags
 411         factory.addStep(ShellCommand(
 412                 name = "fetchtag",
 413                 description = "Fetching Git tags",
 414                 command = ["git", "fetch", "--tags", "--", repo_url],
 415                 haltOnFailure = True,
 416                 doStepIf = IsTaggingRequested
 417         ))
 418
 419         # switch to tag
 420         factory.addStep(ShellCommand(
 421                 name = "switchtag",
 422                 description = "Checking out Git tag",
 423                 command = ["git", "checkout", WithProperties("tags/v%(tag:-)s")],
 424                 haltOnFailure = True,
 425                 doStepIf = IsTaggingRequested
 426         ))
 427
 428         factory.addStep(ShellCommand(
 429                 name = "rmtmp",
 430                 description = "Remove tmp folder",
 431                 command=["rm", "-rf", "tmp/"]))
 432
 433         # feed
 434 #       factory.addStep(ShellCommand(
 435 #               name = "feedsconf",
 436 #               description = "Copy the feeds.conf",
 437 #               command='''cp ~/feeds.conf ./feeds.conf''' ))
 438
 439         # feed
 440         factory.addStep(ShellCommand(
 441                 name = "rmfeedlinks",
 442                 description = "Remove feed symlinks",
 443                 command=["rm", "-rf", "package/feeds/"]))
 444
 445         # feed
 446         factory.addStep(ShellCommand(
 447                 name = "updatefeeds",
 448                 description = "Updating feeds",
 449                 command=["./scripts/feeds", "update"],
 450                 env = MakeEnv()))
 451
 452         # feed
 453         factory.addStep(ShellCommand(
 454                 name = "installfeeds",
 455                 description = "Installing feeds",
 456                 command=["./scripts/feeds", "install", "-a"],
 457                 env = MakeEnv()))
 458
 459         # seed config
 460         factory.addStep(FileDownload(
 461                 mastersrc = "config.seed",
 462                 slavedest = ".config",
 463                 mode = 0644
 464         ))
 465
 466         # configure
 467         factory.addStep(ShellCommand(
 468                 name = "newconfig",
 469                 description = "Seeding .config",
 470                 command = "printf 'CONFIG_TARGET_%s=y\\nCONFIG_TARGET_%s_%s=y\\n' >> .config" %(ts[0], ts[0], ts[1])
 471         ))
 472
 473         factory.addStep(ShellCommand(
 474                 name = "delbin",
 475                 description = "Removing output directory",
 476                 command = ["rm", "-rf", "bin/"]
 477         ))
 478
 479         factory.addStep(ShellCommand(
 480                 name = "defconfig",
 481                 description = "Populating .config",
 482                 command = ["make", "defconfig"],
 483                 env = MakeEnv()
 484         ))
 485
 486         # check arch
 487         factory.addStep(ShellCommand(
 488                 name = "checkarch",
 489                 description = "Checking architecture",
 490                 command = ["grep", "-sq", "CONFIG_TARGET_%s=y" %(ts[0]), ".config"],
 491                 logEnviron = False,
 492                 want_stdout = False,
 493                 want_stderr = False,
 494                 haltOnFailure = True
 495         ))
 496
 497         # find libc suffix
 498         factory.addStep(SetProperty(
 499                 name = "libc",
 500                 property = "libc",
 501                 description = "Finding libc suffix",
 502                 command = ["sed", "-ne", '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }', ".config"]))
 503
 504         # ccache helper
 505         factory.addStep(FileDownload(
 506                 mastersrc = "ccache.sh",
 507                 slavedest = "ccache.sh",
 508                 mode = 0755
 509         ))
 510
 511         # ccache prepare
 512         factory.addStep(ShellCommand(
 513                 name = "prepccache",
 514                 description = "Preparing ccache",
 515                 command = ["./ccache.sh"]
 516         ))
 517
 518         # install build key
 519         factory.addStep(FileDownload(mastersrc=home_dir+'/key-build', slavedest="key-build", mode=0600))
 520         factory.addStep(FileDownload(mastersrc=home_dir+'/key-build.pub', slavedest="key-build.pub", mode=0600))
 521
 522         # prepare dl
 523         factory.addStep(ShellCommand(
 524                 name = "dldir",
 525                 description = "Preparing dl/",
 526                 command = "mkdir -p $HOME/dl && rm -rf ./dl && ln -sf $HOME/dl ./dl",
 527                 logEnviron = False,
 528                 want_stdout = False
 529         ))
 530
 531         # prepare tar
 532         factory.addStep(ShellCommand(
 533                 name = "dltar",
 534                 description = "Building GNU tar",
 535                 command = ["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "tools/tar/compile", "V=s"],
 536                 env = MakeEnv(),
 537                 haltOnFailure = True
 538         ))
 539
 540         # populate dl
 541         factory.addStep(ShellCommand(
 542                 name = "dlrun",
 543                 description = "Populating dl/",
 544                 command = ["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "download", "V=s"],
 545                 env = MakeEnv(),
 546                 logEnviron = False,
 547                 locks = [dlLock.access('exclusive')]
 548         ))
 549
 550         factory.addStep(ShellCommand(
 551                 name = "cleanbase",
 552                 description = "Cleaning base-files",
 553                 command=["make", "package/base-files/clean", "V=s"]
 554         ))
 555
 556         # build
 557         factory.addStep(ShellCommand(
 558                 name = "tools",
 559                 description = "Building tools",
 560                 command = ["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "tools/install", "V=s"],
 561                 env = MakeEnv(),
 562                 haltOnFailure = True
 563         ))
 564
 565         factory.addStep(ShellCommand(
 566                 name = "toolchain",
 567                 description = "Building toolchain",
 568                 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "toolchain/install", "V=s"],
 569                 env = MakeEnv(),
 570                 haltOnFailure = True
 571         ))
 572
 573         factory.addStep(ShellCommand(
 574                 name = "kmods",
 575                 description = "Building kmods",
 576                 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
 577                 env = MakeEnv(),
 578                 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
 579                 haltOnFailure = True
 580         ))
 581
 582         factory.addStep(ShellCommand(
 583                 name = "pkgclean",
 584                 description = "Cleaning up package build",
 585                 command=["make", "package/cleanup", "V=s"]
 586         ))
 587
 588         factory.addStep(ShellCommand(
 589                 name = "pkgbuild",
 590                 description = "Building packages",
 591                 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
 592                 env = MakeEnv(),
 593                 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
 594                 haltOnFailure = True
 595         ))
 596
 597         # factory.addStep(IfBuiltinShellCommand(
 598         factory.addStep(ShellCommand(
 599                 name = "pkginstall",
 600                 description = "Installing packages",
 601                 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "package/install", "V=s"],
 602                 env = MakeEnv(),
 603                 haltOnFailure = True
 604         ))
 605
 606         factory.addStep(ShellCommand(
 607                 name = "pkgindex",
 608                 description = "Indexing packages",
 609                 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "package/index", "V=s"],
 610                 env = MakeEnv(),
 611                 haltOnFailure = True
 612         ))
 613
 614         #factory.addStep(IfBuiltinShellCommand(
 615         factory.addStep(ShellCommand(
 616                 name = "images",
 617                 description = "Building images",
 618                 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "target/install", "V=s"],
 619                 env = MakeEnv(),
 620                 haltOnFailure = True
 621         ))
 622
 623         factory.addStep(ShellCommand(
 624                 name = "diffconfig",
 625                 description = "Generating config.seed",
 626                 command=["make", "-j1", "diffconfig", "V=s"],
 627                 env = MakeEnv(),
 628                 haltOnFailure = True
 629         ))
 630
 631         factory.addStep(ShellCommand(
 632                 name = "checksums",
 633                 description = "Calculating checksums",
 634                 command=["make", "-j1", "checksum", "V=s"],
 635                 env = MakeEnv(),
 636                 haltOnFailure = True
 637         ))
 638
 639         # sign
 640         if gpg_keyid is not None:
 641                 factory.addStep(MasterShellCommand(
 642                         name = "signprepare",
 643                         description = "Preparing temporary signing directory",
 644                         command = ["mkdir", "-p", "%s/signing" %(home_dir)],
 645                         haltOnFailure = True
 646                 ))
 647
 648                 factory.addStep(ShellCommand(
 649                         name = "signpack",
 650                         description = "Packing files to sign",
 651                         command = WithProperties("find bin/targets/%s/%s%%(libc)s/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz" %(ts[0], ts[1])),
 652                         haltOnFailure = True
 653                 ))
 654
 655                 factory.addStep(FileUpload(
 656                         slavesrc = "sign.tar.gz",
 657                         masterdest = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
 658                         haltOnFailure = True
 659                 ))
 660
 661                 factory.addStep(MasterShellCommand(
 662                         name = "signfiles",
 663                         description = "Signing files",
 664                         command = ["%s/signall.sh" %(home_dir), "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]), gpg_keyid, gpg_comment],
 665                         env = {'GNUPGHOME': gpg_home, 'PASSFILE': gpg_passfile},
 666                         haltOnFailure = True
 667                 ))
 668
 669                 factory.addStep(FileDownload(
 670                         mastersrc = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
 671                         slavedest = "sign.tar.gz",
 672                         haltOnFailure = True
 673                 ))
 674
 675                 factory.addStep(ShellCommand(
 676                         name = "signunpack",
 677                         description = "Unpacking signed files",
 678                         command = ["tar", "-xzf", "sign.tar.gz"],
 679                         haltOnFailure = True
 680                 ))
 681
 682         # upload
 683         factory.addStep(ShellCommand(
 684                 name = "dirprepare",
 685                 description = "Preparing upload directory structure",
 686                 command = ["mkdir", "-p", WithProperties("tmp/upload/%%(prefix)stargets/%s/%s" %(ts[0], ts[1]), prefix=GetVersionPrefix)],
 687                 haltOnFailure = True
 688         ))
 689
 690         factory.addStep(ShellCommand(
 691                 name = "linkprepare",
 692                 description = "Preparing repository symlink",
 693                 command = ["ln", "-s", "-f", WithProperties("../packages-%(basever)s", basever=GetBaseVersion), WithProperties("tmp/upload/%(prefix)spackages", prefix=GetVersionPrefix)],
 694                 doStepIf = IsNoMasterBuild,
 695                 haltOnFailure = True
 696         ))
 697
 698         factory.addStep(ShellCommand(
 699                 name = "dirupload",
 700                 description = "Uploading directory structure",
 701                 command = ["rsync", "-4", "-avz", "tmp/upload/", "%s/" %(rsync_bin_url)],
 702                 env={'RSYNC_PASSWORD': rsync_bin_key},
 703                 haltOnFailure = True,
 704                 logEnviron = False
 705         ))
 706
 707         factory.addStep(ShellCommand(
 708                 name = "targetupload",
 709                 description = "Uploading target files",
 710                 command=["rsync", "-4", "--progress", "--delete", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]),
 711                          "-avz", WithProperties("bin/targets/%s/%s%%(libc)s/" %(ts[0], ts[1])),
 712                          WithProperties("%s/%%(prefix)stargets/%s/%s/" %(rsync_bin_url, ts[0], ts[1]), prefix=GetVersionPrefix)],
 713                 env={'RSYNC_PASSWORD': rsync_bin_key},
 714                 haltOnFailure = True,
 715                 logEnviron = False
 716         ))
 717
 718         if rsync_src_url is not None:
 719                 factory.addStep(ShellCommand(
 720                         name = "sourceupload",
 721                         description = "Uploading source archives",
 722                         command=["rsync", "-4", "--progress", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "dl/", "%s/" %(rsync_src_url)],
 723                         env={'RSYNC_PASSWORD': rsync_src_key},
 724                         haltOnFailure = True,
 725                         logEnviron = False
 726                 ))
 727
 728         if False:
 729                 factory.addStep(ShellCommand(
 730                         name = "packageupload",
 731                         description = "Uploading package files",
 732                         command=["rsync", "-4", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "bin/packages/", "%s/packages/" %(rsync_bin_url)],
 733                         env={'RSYNC_PASSWORD': rsync_bin_key},
 734                         haltOnFailure = False,
 735                         logEnviron = False
 736                 ))
 737
 738         # logs
 739         if False:
 740                 factory.addStep(ShellCommand(
 741                         name = "upload",
 742                         description = "Uploading logs",
 743                         command=["rsync", "-4", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "logs/", "%s/logs/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
 744                         env={'RSYNC_PASSWORD': rsync_bin_key},
 745                         haltOnFailure = False,
 746                         alwaysRun = True,
 747                         logEnviron = False
 748                 ))
 749
 750         from buildbot.config import BuilderConfig
 751
 752         c['builders'].append(BuilderConfig(name=target, slavenames=slaveNames, factory=factory, nextBuild=GetNextBuild))
 753
 754
 755 ####### STATUS TARGETS
 756
 757 # 'status' is a list of Status Targets. The results of each build will be
 758 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
 759 # including web pages, email senders, and IRC bots.
 760
 761 c['status'] = []
 762
 763 from buildbot.status import html
 764 from buildbot.status.web import authz, auth
 765
 766 if ini.has_option("status", "bind"):
 767         if ini.has_option("status", "user") and ini.has_option("status", "password"):
 768                 authz_cfg=authz.Authz(
 769                         # change any of these to True to enable; see the manual for more
 770                         # options
 771                         auth=auth.BasicAuth([(ini.get("status", "user"), ini.get("status", "password"))]),
 772                         gracefulShutdown = 'auth',
 773                         forceBuild = 'auth', # use this to test your slave once it is set up
 774                         forceAllBuilds = 'auth',
 775                         pingBuilder = False,
 776                         stopBuild = 'auth',
 777                         stopAllBuilds = 'auth',
 778                         cancelPendingBuild = 'auth',
 779                 )
 780                 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind"), authz=authz_cfg))
 781         else:
 782                 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind")))
 783
 784
 785 from buildbot.status import words
 786
 787 if ini.has_option("irc", "host") and ini.has_option("irc", "nickname") and ini.has_option("irc", "channel"):
 788         irc_host = ini.get("irc", "host")
 789         irc_port = 6667
 790         irc_chan = ini.get("irc", "channel")
 791         irc_nick = ini.get("irc", "nickname")
 792         irc_pass = None
 793
 794         if ini.has_option("irc", "port"):
 795                 irc_port = ini.getint("irc", "port")
 796
 797         if ini.has_option("irc", "password"):
 798                 irc_pass = ini.get("irc", "password")
 799
 800         irc = words.IRC(irc_host, irc_nick, port = irc_port, password = irc_pass,
 801                         channels = [{ "channel": irc_chan }],
 802                         notify_events = {
 803                           'exception': 1,
 804                           'successToFailure': 1,
 805                           'failureToSuccess': 1
 806                         }
 807         )
 808
 809         c['status'].append(irc)
 810
 811
 812 ####### PROJECT IDENTITY
 813
 814 # the 'title' string will appear at the top of this buildbot
 815 # installation's html.WebStatus home page (linked to the
 816 # 'titleURL') and is embedded in the title of the waterfall HTML page.
 817
 818 c['title'] = ini.get("general", "title")
 819 c['titleURL'] = ini.get("general", "title_url")
 820
 821 # the 'buildbotURL' string should point to the location where the buildbot's
 822 # internal web server (usually the html.WebStatus page) is visible. This
 823 # typically uses the port number set in the Waterfall 'status' entry, but
 824 # with an externally-visible host name which the buildbot cannot figure out
 825 # without some help.
 826
 827 c['buildbotURL'] = ini.get("general", "buildbot_url")
 828
 829 ####### DB URL
 830
 831 c['db'] = {
 832         # This specifies what database buildbot uses to store its state.  You can leave
 833         # this at its default for all but the largest installations.
 834         'db_url' : "sqlite:///state.sqlite",
 835 }