phase1/master.cfg

   1 # -*- python -*-
   2 # ex: set syntax=python:
   3
   4 import os
   5 import re
   6 import subprocess
   7 import ConfigParser
   8
   9 from buildbot import locks
  10
  11 # This is a sample buildmaster config file. It must be installed as
  12 # 'master.cfg' in your buildmaster's base directory.
  13
  14 ini = ConfigParser.ConfigParser()
  15 ini.read("./config.ini")
  16
  17 # This is the dictionary that the buildmaster pays attention to. We also use
  18 # a shorter alias to save typing.
  19 c = BuildmasterConfig = {}
  20
  21 ####### BUILDSLAVES
  22
  23 # The 'slaves' list defines the set of recognized buildslaves. Each element is
  24 # a BuildSlave object, specifying a unique slave name and password.  The same
  25 # slave name and password must be configured on the slave.
  26 from buildbot.buildslave import BuildSlave
  27
  28 slave_port = 9989
  29
  30 if ini.has_option("general", "port"):
  31         slave_port = ini.getint("general", "port")
  32
  33 c['slaves'] = []
  34 max_builds = dict()
  35
  36 for section in ini.sections():
  37         if section.startswith("slave "):
  38                 if ini.has_option(section, "name") and ini.has_option(section, "password"):
  39                         name = ini.get(section, "name")
  40                         password = ini.get(section, "password")
  41                         max_builds[name] = 1
  42                         if ini.has_option(section, "builds"):
  43                                 max_builds[name] = ini.getint(section, "builds")
  44                         c['slaves'].append(BuildSlave(name, password, max_builds = max_builds[name]))
  45
  46 # 'slavePortnum' defines the TCP port to listen on for connections from slaves.
  47 # This must match the value configured into the buildslaves (with their
  48 # --master option)
  49 c['slavePortnum'] = slave_port
  50
  51 # coalesce builds
  52 c['mergeRequests'] = True
  53
  54 ####### CHANGESOURCES
  55
  56 home_dir = os.path.abspath(ini.get("general", "homedir"))
  57 tree_expire = 0
  58 other_builds = 0
  59 cc_version = None
  60
  61 cc_command = "gcc"
  62 cxx_command = "g++"
  63
  64 if ini.has_option("general", "expire"):
  65         tree_expire = ini.getint("general", "expire")
  66
  67 if ini.has_option("general", "other_builds"):
  68         other_builds = ini.getint("general", "other_builds")
  69
  70 if ini.has_option("general", "cc_version"):
  71         cc_version = ini.get("general", "cc_version").split()
  72         if len(cc_version) == 1:
  73                 cc_version = ["eq", cc_version[0]]
  74
  75 repo_url = ini.get("repo", "url")
  76 repo_branch = "master"
  77
  78 if ini.has_option("repo", "branch"):
  79         repo_branch = ini.get("repo", "branch")
  80
  81 rsync_bin_url = ini.get("rsync", "binary_url")
  82 rsync_bin_key = ini.get("rsync", "binary_password")
  83
  84 rsync_src_url = None
  85 rsync_src_key = None
  86
  87 if ini.has_option("rsync", "source_url"):
  88         rsync_src_url = ini.get("rsync", "source_url")
  89         rsync_src_key = ini.get("rsync", "source_password")
  90
  91 gpg_home = "~/.gnupg"
  92 gpg_keyid = None
  93 gpg_comment = "Unattended build signature"
  94 gpg_passfile = "/dev/null"
  95
  96 if ini.has_option("gpg", "home"):
  97         gpg_home = ini.get("gpg", "home")
  98
  99 if ini.has_option("gpg", "keyid"):
 100         gpg_keyid = ini.get("gpg", "keyid")
 101
 102 if ini.has_option("gpg", "comment"):
 103         gpg_comment = ini.get("gpg", "comment")
 104
 105 if ini.has_option("gpg", "passfile"):
 106         gpg_passfile = ini.get("gpg", "passfile")
 107
 108
 109 # find targets
 110 targets = [ ]
 111
 112 if not os.path.isdir(home_dir+'/source.git'):
 113         subprocess.call(["git", "clone", "--depth=1", "--branch="+repo_branch, repo_url, home_dir+'/source.git'])
 114 else:
 115         subprocess.call(["git", "pull"], cwd = home_dir+'/source.git')
 116
 117 findtargets = subprocess.Popen([home_dir+'/dumpinfo.pl', 'targets'],
 118         stdout = subprocess.PIPE, cwd = home_dir+'/source.git')
 119
 120 while True:
 121         line = findtargets.stdout.readline()
 122         if not line:
 123                 break
 124         ta = line.strip().split(' ')
 125         targets.append(ta[0])
 126
 127
 128 # the 'change_source' setting tells the buildmaster how it should find out
 129 # about source code changes.  Here we point to the buildbot clone of pyflakes.
 130
 131 from buildbot.changes.gitpoller import GitPoller
 132 c['change_source'] = []
 133 c['change_source'].append(GitPoller(
 134         repo_url,
 135         workdir=home_dir+'/work.git', branch=repo_branch,
 136         pollinterval=300))
 137
 138 ####### SCHEDULERS
 139
 140 # Configure the Schedulers, which decide how to react to incoming changes.  In this
 141 # case, just kick off a 'basebuild' build
 142
 143 from buildbot.schedulers.basic import SingleBranchScheduler
 144 from buildbot.schedulers.forcesched import ForceScheduler
 145 from buildbot.changes import filter
 146 c['schedulers'] = []
 147 c['schedulers'].append(SingleBranchScheduler(
 148         name="all",
 149         change_filter=filter.ChangeFilter(branch=repo_branch),
 150         treeStableTimer=60,
 151         builderNames=targets))
 152
 153 c['schedulers'].append(ForceScheduler(
 154         name="force",
 155         builderNames=targets))
 156
 157 ####### BUILDERS
 158
 159 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
 160 # what steps, and which slaves can execute them.  Note that any particular build will
 161 # only take place on one slave.
 162
 163 from buildbot.process.factory import BuildFactory
 164 from buildbot.steps.source.git import Git
 165 from buildbot.steps.shell import ShellCommand
 166 from buildbot.steps.shell import SetProperty
 167 from buildbot.steps.transfer import FileUpload
 168 from buildbot.steps.transfer import FileDownload
 169 from buildbot.steps.master import MasterShellCommand
 170 from buildbot.process.properties import WithProperties
 171
 172
 173 CleanTargetMap = [
 174         [ "tools",      "tools/clean"                   ],
 175         [ "chain",      "toolchain/clean"               ],
 176         [ "linux",      "target/linux/clean"    ],
 177         [ "dir",        "dirclean"                              ],
 178         [ "dist",       "distclean"                             ]
 179 ]
 180
 181 def IsCleanRequested(pattern):
 182         def CheckCleanProperty(step):
 183                 val = step.getProperty("clean")
 184                 if val and re.match(pattern, val):
 185                         return True
 186                 else:
 187                         return False
 188
 189         return CheckCleanProperty
 190
 191 def IsTaggingRequested(step):
 192         val = step.getProperty("tag")
 193         if val and re.match("^[0-9]+\.[0-9]+\.[0-9]+$", val):
 194                 return True
 195         else:
 196                 return False
 197
 198 def IsNoTaggingRequested(step):
 199         return not IsTaggingRequested(step)
 200
 201 def IsNoMasterBuild(step):
 202         return repo_branch != "master"
 203
 204 def GetBaseVersion(props):
 205         if re.match("^[^-]+-[0-9]+\.[0-9]+$", repo_branch):
 206                 return repo_branch.split('-')[1]
 207         else:
 208                 return "master"
 209
 210 def GetVersionPrefix(props):
 211         basever = GetBaseVersion(props)
 212         if props.hasProperty("tag") and re.match("^[0-9]+\.[0-9]+\.[0-9]+$", props["tag"]):
 213                 return "%s/" % props["tag"]
 214         elif basever != "master":
 215                 return "%s-SNAPSHOT/" % basever
 216         else:
 217                 return ""
 218
 219 def GetNumJobs(props):
 220         if props.hasProperty("slavename") and props.hasProperty("nproc"):
 221                 return ((int(props["nproc"]) / (max_builds[props["slavename"]] + other_builds)) + 1)
 222         else:
 223                 return 1
 224
 225 def GetCC(props):
 226         if props.hasProperty("cc_command"):
 227                 return props["cc_command"]
 228         else:
 229                 return "gcc"
 230
 231 def GetCXX(props):
 232         if props.hasProperty("cxx_command"):
 233                 return props["cxx_command"]
 234         else:
 235                 return "g++"
 236
 237 def MakeEnv(overrides=None):
 238         env = {
 239                 'CC': WithProperties("%(cc)s", cc=GetCC),
 240                 'CXX': WithProperties("%(cxx)s", cxx=GetCXX)
 241         }
 242         if overrides is not None:
 243                 env.update(overrides)
 244         return env
 245
 246
 247 c['builders'] = []
 248
 249 dlLock = locks.SlaveLock("slave_dl")
 250 tagLock = locks.MasterLock("make_tag")
 251
 252 checkBuiltin = re.sub('[\t\n ]+', ' ', """
 253         checkBuiltin() {
 254                 local symbol op path file;
 255                 for file in $CHANGED_FILES; do
 256                         case "$file" in
 257                                 package/*/*) : ;;
 258                                 *) return 0 ;;
 259                         esac;
 260                 done;
 261                 while read symbol op path; do
 262                         case "$symbol" in package-*)
 263                                 symbol="${symbol##*(}";
 264                                 symbol="${symbol%)}";
 265                                 for file in $CHANGED_FILES; do
 266                                         case "$file" in "package/$path/"*)
 267                                                 grep -qsx "$symbol=y" .config && return 0
 268                                         ;; esac;
 269                                 done;
 270                         esac;
 271                 done < tmp/.packagedeps;
 272                 return 1;
 273         }
 274 """).strip()
 275
 276
 277 class IfBuiltinShellCommand(ShellCommand):
 278         def _quote(self, str):
 279                 if re.search("[^a-zA-Z0-9/_.-]", str):
 280                         return "'%s'" %(re.sub("'", "'\"'\"'", str))
 281                 return str
 282
 283         def setCommand(self, command):
 284                 if not isinstance(command, (str, unicode)):
 285                         command = ' '.join(map(self._quote, command))
 286                 self.command = [
 287                         '/bin/sh', '-c',
 288                         '%s; if checkBuiltin; then %s; else exit 0; fi' %(checkBuiltin, command)
 289                 ]
 290
 291         def setupEnvironment(self, cmd):
 292                 slaveEnv = self.slaveEnvironment
 293                 if slaveEnv is None:
 294                         slaveEnv = { }
 295                 changedFiles = { }
 296                 for request in self.build.requests:
 297                         for source in request.sources:
 298                                 for change in source.changes:
 299                                         for file in change.files:
 300                                                 changedFiles[file] = True
 301                 fullSlaveEnv = slaveEnv.copy()
 302                 fullSlaveEnv['CHANGED_FILES'] = ' '.join(changedFiles.keys())
 303                 cmd.args['env'] = fullSlaveEnv
 304
 305 slaveNames = [ ]
 306
 307 for slave in c['slaves']:
 308         slaveNames.append(slave.slavename)
 309
 310 for target in targets:
 311         ts = target.split('/')
 312
 313         factory = BuildFactory()
 314
 315         # find number of cores
 316         factory.addStep(SetProperty(
 317                 name = "nproc",
 318                 property = "nproc",
 319                 description = "Finding number of CPUs",
 320                 command = ["nproc"]))
 321
 322         # find gcc and g++ compilers
 323         if cc_version is not None:
 324                 factory.addStep(FileDownload(
 325                         mastersrc = "findbin.pl",
 326                         slavedest = "../findbin.pl",
 327                         mode = 0755))
 328
 329                 factory.addStep(SetProperty(
 330                         name = "gcc",
 331                         property = "cc_command",
 332                         description = "Finding gcc command",
 333                         command = ["../findbin.pl", "gcc", cc_version[0], cc_version[1]],
 334                         haltOnFailure = True))
 335
 336                 factory.addStep(SetProperty(
 337                         name = "g++",
 338                         property = "cxx_command",
 339                         description = "Finding g++ command",
 340                         command = ["../findbin.pl", "g++", cc_version[0], cc_version[1]],
 341                         haltOnFailure = True))
 342
 343         # expire tree if needed
 344         if tree_expire > 0:
 345                 factory.addStep(FileDownload(
 346                         mastersrc = "expire.sh",
 347                         slavedest = "../expire.sh",
 348                         mode = 0755))
 349
 350                 factory.addStep(ShellCommand(
 351                         name = "expire",
 352                         description = "Checking for build tree expiry",
 353                         command = ["./expire.sh", str(tree_expire)],
 354                         workdir = ".",
 355                         haltOnFailure = True,
 356                         timeout = 2400))
 357
 358         # user-requested clean targets
 359         for tuple in CleanTargetMap:
 360                 factory.addStep(ShellCommand(
 361                         name = tuple[1],
 362                         description = 'User-requested "make %s"' % tuple[1],
 363                         command = ["make", tuple[1], "V=s"],
 364                         env = MakeEnv(),
 365                         doStepIf = IsCleanRequested(tuple[0])
 366                 ))
 367
 368         factory.addStep(MasterShellCommand(
 369                 name = "maketag",
 370                 description = "Tagging Git repository",
 371                 command = [home_dir+'/maketag.sh', '-i', '-k', str(gpg_keyid or ''),
 372                            '-p', str(gpg_passfile or ''), '-v', WithProperties("%(tag:-)s")],
 373                 path = home_dir+'/source.git',
 374                 env = {'GNUPGHOME': gpg_home},
 375                 haltOnFailure = True,
 376                 doStepIf = IsTaggingRequested,
 377                 locks = [tagLock.access('exclusive')]
 378         ))
 379
 380         # switch to branch
 381         factory.addStep(ShellCommand(
 382                 name = "switchbranch",
 383                 description = "Checking out Git branch",
 384                 command = "if [ -d .git ]; then git checkout '%s'; else exit 0; fi" % repo_branch,
 385                 haltOnFailure = True,
 386                 doStepIf = IsNoTaggingRequested
 387         ))
 388
 389         # check out the source
 390         factory.addStep(Git(
 391                 repourl = repo_url,
 392                 branch = repo_branch,
 393                 mode = 'incremental',
 394                 method = 'clean'))
 395
 396         # update remote refs
 397         factory.addStep(ShellCommand(
 398                 name = "fetchrefs",
 399                 description = "Fetching Git remote refs",
 400                 command = ["git", "fetch", "origin", "+refs/heads/%s:refs/remotes/origin/%s" %(repo_branch, repo_branch)],
 401                 haltOnFailure = True
 402         ))
 403
 404         # fetch tags
 405         factory.addStep(ShellCommand(
 406                 name = "fetchtag",
 407                 description = "Fetching Git tags",
 408                 command = ["git", "fetch", "--tags", "--", repo_url],
 409                 haltOnFailure = True,
 410                 doStepIf = IsTaggingRequested
 411         ))
 412
 413         # switch to tag
 414         factory.addStep(ShellCommand(
 415                 name = "switchtag",
 416                 description = "Checking out Git tag",
 417                 command = ["git", "checkout", WithProperties("tags/v%(tag:-)s")],
 418                 haltOnFailure = True,
 419                 doStepIf = IsTaggingRequested
 420         ))
 421
 422         factory.addStep(ShellCommand(
 423                 name = "rmtmp",
 424                 description = "Remove tmp folder",
 425                 command=["rm", "-rf", "tmp/"]))
 426
 427         # feed
 428 #       factory.addStep(ShellCommand(
 429 #               name = "feedsconf",
 430 #               description = "Copy the feeds.conf",
 431 #               command='''cp ~/feeds.conf ./feeds.conf''' ))
 432
 433         # feed
 434         factory.addStep(ShellCommand(
 435                 name = "rmfeedlinks",
 436                 description = "Remove feed symlinks",
 437                 command=["rm", "-rf", "package/feeds/"]))
 438
 439         # feed
 440         factory.addStep(ShellCommand(
 441                 name = "updatefeeds",
 442                 description = "Updating feeds",
 443                 command=["./scripts/feeds", "update"],
 444                 env = MakeEnv()))
 445
 446         # feed
 447         factory.addStep(ShellCommand(
 448                 name = "installfeeds",
 449                 description = "Installing feeds",
 450                 command=["./scripts/feeds", "install", "-a"],
 451                 env = MakeEnv()))
 452
 453         # seed config
 454         factory.addStep(FileDownload(
 455                 mastersrc = "config.seed",
 456                 slavedest = ".config",
 457                 mode = 0644
 458         ))
 459
 460         # configure
 461         factory.addStep(ShellCommand(
 462                 name = "newconfig",
 463                 description = "Seeding .config",
 464                 command = "printf 'CONFIG_TARGET_%s=y\\nCONFIG_TARGET_%s_%s=y\\n' >> .config" %(ts[0], ts[0], ts[1])
 465         ))
 466
 467         factory.addStep(ShellCommand(
 468                 name = "delbin",
 469                 description = "Removing output directory",
 470                 command = ["rm", "-rf", "bin/"]
 471         ))
 472
 473         factory.addStep(ShellCommand(
 474                 name = "defconfig",
 475                 description = "Populating .config",
 476                 command = ["make", "defconfig"],
 477                 env = MakeEnv()
 478         ))
 479
 480         # check arch
 481         factory.addStep(ShellCommand(
 482                 name = "checkarch",
 483                 description = "Checking architecture",
 484                 command = ["grep", "-sq", "CONFIG_TARGET_%s=y" %(ts[0]), ".config"],
 485                 logEnviron = False,
 486                 want_stdout = False,
 487                 want_stderr = False,
 488                 haltOnFailure = True
 489         ))
 490
 491         # find libc suffix
 492         factory.addStep(SetProperty(
 493                 name = "libc",
 494                 property = "libc",
 495                 description = "Finding libc suffix",
 496                 command = ["sed", "-ne", '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }', ".config"]))
 497
 498         # install build key
 499         factory.addStep(FileDownload(mastersrc=home_dir+'/key-build', slavedest="key-build", mode=0600))
 500         factory.addStep(FileDownload(mastersrc=home_dir+'/key-build.pub', slavedest="key-build.pub", mode=0600))
 501
 502         # prepare dl
 503         factory.addStep(ShellCommand(
 504                 name = "dldir",
 505                 description = "Preparing dl/",
 506                 command = "mkdir -p $HOME/dl && rm -rf ./dl && ln -sf $HOME/dl ./dl",
 507                 logEnviron = False,
 508                 want_stdout = False
 509         ))
 510
 511         # prepare tar
 512         factory.addStep(ShellCommand(
 513                 name = "dltar",
 514                 description = "Building GNU tar",
 515                 command = ["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "tools/tar/install", "V=s"],
 516                 env = MakeEnv(),
 517                 haltOnFailure = True
 518         ))
 519
 520         # populate dl
 521         factory.addStep(ShellCommand(
 522                 name = "dlrun",
 523                 description = "Populating dl/",
 524                 command = ["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "download", "V=s"],
 525                 env = MakeEnv(),
 526                 logEnviron = False,
 527                 locks = [dlLock.access('exclusive')]
 528         ))
 529
 530         factory.addStep(ShellCommand(
 531                 name = "cleanbase",
 532                 description = "Cleaning base-files",
 533                 command=["make", "package/base-files/clean", "V=s"]
 534         ))
 535
 536         # build
 537         factory.addStep(ShellCommand(
 538                 name = "tools",
 539                 description = "Building tools",
 540                 command = ["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "tools/install", "V=s"],
 541                 env = MakeEnv(),
 542                 haltOnFailure = True
 543         ))
 544
 545         factory.addStep(ShellCommand(
 546                 name = "toolchain",
 547                 description = "Building toolchain",
 548                 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "toolchain/install", "V=s"],
 549                 env = MakeEnv(),
 550                 haltOnFailure = True
 551         ))
 552
 553         factory.addStep(ShellCommand(
 554                 name = "kmods",
 555                 description = "Building kmods",
 556                 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
 557                 env = MakeEnv(),
 558                 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
 559                 haltOnFailure = True
 560         ))
 561
 562         factory.addStep(ShellCommand(
 563                 name = "pkgbuild",
 564                 description = "Building packages",
 565                 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
 566                 env = MakeEnv(),
 567                 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
 568                 haltOnFailure = True
 569         ))
 570
 571         # factory.addStep(IfBuiltinShellCommand(
 572         factory.addStep(ShellCommand(
 573                 name = "pkginstall",
 574                 description = "Installing packages",
 575                 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "package/install", "V=s"],
 576                 env = MakeEnv(),
 577                 haltOnFailure = True
 578         ))
 579
 580         factory.addStep(ShellCommand(
 581                 name = "pkgindex",
 582                 description = "Indexing packages",
 583                 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "package/index", "V=s"],
 584                 env = MakeEnv(),
 585                 haltOnFailure = True
 586         ))
 587
 588         #factory.addStep(IfBuiltinShellCommand(
 589         factory.addStep(ShellCommand(
 590                 name = "images",
 591                 description = "Building images",
 592                 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "target/install", "V=s"],
 593                 env = MakeEnv(),
 594                 haltOnFailure = True
 595         ))
 596
 597         factory.addStep(ShellCommand(
 598                 name = "diffconfig",
 599                 description = "Generating config.seed",
 600                 command=["make", "-j1", "diffconfig", "V=s"],
 601                 env = MakeEnv(),
 602                 haltOnFailure = True
 603         ))
 604
 605         factory.addStep(ShellCommand(
 606                 name = "checksums",
 607                 description = "Calculating checksums",
 608                 command=["make", "-j1", "checksum", "V=s"],
 609                 env = MakeEnv(),
 610                 haltOnFailure = True
 611         ))
 612
 613         # sign
 614         if gpg_keyid is not None:
 615                 factory.addStep(MasterShellCommand(
 616                         name = "signprepare",
 617                         description = "Preparing temporary signing directory",
 618                         command = ["mkdir", "-p", "%s/signing" %(home_dir)],
 619                         haltOnFailure = True
 620                 ))
 621
 622                 factory.addStep(ShellCommand(
 623                         name = "signpack",
 624                         description = "Packing files to sign",
 625                         command = WithProperties("find bin/targets/%s/%s%%(libc)s/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz" %(ts[0], ts[1])),
 626                         haltOnFailure = True
 627                 ))
 628
 629                 factory.addStep(FileUpload(
 630                         slavesrc = "sign.tar.gz",
 631                         masterdest = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
 632                         haltOnFailure = True
 633                 ))
 634
 635                 factory.addStep(MasterShellCommand(
 636                         name = "signfiles",
 637                         description = "Signing files",
 638                         command = ["%s/signall.sh" %(home_dir), "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]), gpg_keyid, gpg_comment],
 639                         env = {'GNUPGHOME': gpg_home, 'PASSFILE': gpg_passfile},
 640                         haltOnFailure = True
 641                 ))
 642
 643                 factory.addStep(FileDownload(
 644                         mastersrc = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
 645                         slavedest = "sign.tar.gz",
 646                         haltOnFailure = True
 647                 ))
 648
 649                 factory.addStep(ShellCommand(
 650                         name = "signunpack",
 651                         description = "Unpacking signed files",
 652                         command = ["tar", "-xzf", "sign.tar.gz"],
 653                         haltOnFailure = True
 654                 ))
 655
 656         # upload
 657         factory.addStep(ShellCommand(
 658                 name = "dirprepare",
 659                 description = "Preparing upload directory structure",
 660                 command = ["mkdir", "-p", WithProperties("tmp/upload/%%(prefix)stargets/%s/%s" %(ts[0], ts[1]), prefix=GetVersionPrefix)],
 661                 haltOnFailure = True
 662         ))
 663
 664         factory.addStep(ShellCommand(
 665                 name = "linkprepare",
 666                 description = "Preparing repository symlink",
 667                 command = ["ln", "-s", "-f", WithProperties("../packages-%(basever)s", basever=GetBaseVersion), WithProperties("tmp/upload/%(prefix)spackages", prefix=GetVersionPrefix)],
 668                 doStepIf = IsNoMasterBuild,
 669                 haltOnFailure = True
 670         ))
 671
 672         factory.addStep(ShellCommand(
 673                 name = "dirupload",
 674                 description = "Uploading directory structure",
 675                 command = ["rsync", "-avz", "tmp/upload/", "%s/" %(rsync_bin_url)],
 676                 env={'RSYNC_PASSWORD': rsync_bin_key},
 677                 haltOnFailure = True,
 678                 logEnviron = False
 679         ))
 680
 681         factory.addStep(ShellCommand(
 682                 name = "targetupload",
 683                 description = "Uploading target files",
 684                 command=["rsync", "--progress", "--delete", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]),
 685                          "-avz", WithProperties("bin/targets/%s/%s%%(libc)s/" %(ts[0], ts[1])),
 686                          WithProperties("%s/%%(prefix)stargets/%s/%s/" %(rsync_bin_url, ts[0], ts[1]), prefix=GetVersionPrefix)],
 687                 env={'RSYNC_PASSWORD': rsync_bin_key},
 688                 haltOnFailure = True,
 689                 logEnviron = False
 690         ))
 691
 692         if rsync_src_url is not None:
 693                 factory.addStep(ShellCommand(
 694                         name = "sourceupload",
 695                         description = "Uploading source archives",
 696                         command=["rsync", "--progress", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "dl/", "%s/" %(rsync_src_url)],
 697                         env={'RSYNC_PASSWORD': rsync_src_key},
 698                         haltOnFailure = True,
 699                         logEnviron = False
 700                 ))
 701
 702         if False:
 703                 factory.addStep(ShellCommand(
 704                         name = "packageupload",
 705                         description = "Uploading package files",
 706                         command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "bin/packages/", "%s/packages/" %(rsync_bin_url)],
 707                         env={'RSYNC_PASSWORD': rsync_bin_key},
 708                         haltOnFailure = False,
 709                         logEnviron = False
 710                 ))
 711
 712         # logs
 713         if False:
 714                 factory.addStep(ShellCommand(
 715                         name = "upload",
 716                         description = "Uploading logs",
 717                         command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "logs/", "%s/logs/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
 718                         env={'RSYNC_PASSWORD': rsync_bin_key},
 719                         haltOnFailure = False,
 720                         alwaysRun = True,
 721                         logEnviron = False
 722                 ))
 723
 724         from buildbot.config import BuilderConfig
 725
 726         c['builders'].append(BuilderConfig(name=target, slavenames=slaveNames, factory=factory))
 727
 728
 729 ####### STATUS TARGETS
 730
 731 # 'status' is a list of Status Targets. The results of each build will be
 732 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
 733 # including web pages, email senders, and IRC bots.
 734
 735 c['status'] = []
 736
 737 from buildbot.status import html
 738 from buildbot.status.web import authz, auth
 739
 740 if ini.has_option("status", "bind"):
 741         if ini.has_option("status", "user") and ini.has_option("status", "password"):
 742                 authz_cfg=authz.Authz(
 743                         # change any of these to True to enable; see the manual for more
 744                         # options
 745                         auth=auth.BasicAuth([(ini.get("status", "user"), ini.get("status", "password"))]),
 746                         gracefulShutdown = 'auth',
 747                         forceBuild = 'auth', # use this to test your slave once it is set up
 748                         forceAllBuilds = 'auth',
 749                         pingBuilder = False,
 750                         stopBuild = 'auth',
 751                         stopAllBuilds = 'auth',
 752                         cancelPendingBuild = 'auth',
 753                 )
 754                 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind"), authz=authz_cfg))
 755         else:
 756                 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind")))
 757
 758
 759 from buildbot.status import words
 760
 761 if ini.has_option("irc", "host") and ini.has_option("irc", "nickname") and ini.has_option("irc", "channel"):
 762         irc_host = ini.get("irc", "host")
 763         irc_port = 6667
 764         irc_chan = ini.get("irc", "channel")
 765         irc_nick = ini.get("irc", "nickname")
 766         irc_pass = None
 767
 768         if ini.has_option("irc", "port"):
 769                 irc_port = ini.getint("irc", "port")
 770
 771         if ini.has_option("irc", "password"):
 772                 irc_pass = ini.get("irc", "password")
 773
 774         irc = words.IRC(irc_host, irc_nick, port = irc_port, password = irc_pass,
 775                         channels = [{ "channel": irc_chan }],
 776                         notify_events = {
 777                           'exception': 1,
 778                           'successToFailure': 1,
 779                           'failureToSuccess': 1
 780                         }
 781         )
 782
 783         c['status'].append(irc)
 784
 785
 786 ####### PROJECT IDENTITY
 787
 788 # the 'title' string will appear at the top of this buildbot
 789 # installation's html.WebStatus home page (linked to the
 790 # 'titleURL') and is embedded in the title of the waterfall HTML page.
 791
 792 c['title'] = ini.get("general", "title")
 793 c['titleURL'] = ini.get("general", "title_url")
 794
 795 # the 'buildbotURL' string should point to the location where the buildbot's
 796 # internal web server (usually the html.WebStatus page) is visible. This
 797 # typically uses the port number set in the Waterfall 'status' entry, but
 798 # with an externally-visible host name which the buildbot cannot figure out
 799 # without some help.
 800
 801 c['buildbotURL'] = ini.get("general", "buildbot_url")
 802
 803 ####### DB URL
 804
 805 c['db'] = {
 806         # This specifies what database buildbot uses to store its state.  You can leave
 807         # this at its default for all but the largest installations.
 808         'db_url' : "sqlite:///state.sqlite",
 809 }