phase1/master.cfg

   1 # -*- python -*-
   2 # ex: set syntax=python:
   3
   4 import os
   5 import re
   6 import subprocess
   7 import ConfigParser
   8
   9 from buildbot import locks
  10
  11 # This is a sample buildmaster config file. It must be installed as
  12 # 'master.cfg' in your buildmaster's base directory.
  13
  14 ini = ConfigParser.ConfigParser()
  15 ini.read("./config.ini")
  16
  17 # This is the dictionary that the buildmaster pays attention to. We also use
  18 # a shorter alias to save typing.
  19 c = BuildmasterConfig = {}
  20
  21 ####### BUILDSLAVES
  22
  23 # The 'slaves' list defines the set of recognized buildslaves. Each element is
  24 # a BuildSlave object, specifying a unique slave name and password.  The same
  25 # slave name and password must be configured on the slave.
  26 from buildbot.buildslave import BuildSlave
  27
  28 slave_port = 9989
  29
  30 if ini.has_option("general", "port"):
  31         slave_port = ini.getint("general", "port")
  32
  33 c['slaves'] = []
  34 max_builds = dict()
  35
  36 for section in ini.sections():
  37         if section.startswith("slave "):
  38                 if ini.has_option(section, "name") and ini.has_option(section, "password"):
  39                         name = ini.get(section, "name")
  40                         password = ini.get(section, "password")
  41                         max_builds[name] = 1
  42                         if ini.has_option(section, "builds"):
  43                                 max_builds[name] = ini.getint(section, "builds")
  44                         c['slaves'].append(BuildSlave(name, password, max_builds = max_builds[name]))
  45
  46 # 'slavePortnum' defines the TCP port to listen on for connections from slaves.
  47 # This must match the value configured into the buildslaves (with their
  48 # --master option)
  49 c['slavePortnum'] = slave_port
  50
  51 # coalesce builds
  52 c['mergeRequests'] = True
  53
  54 # Reduce amount of backlog data
  55 c['buildHorizon'] = 30
  56 c['logHorizon'] = 20
  57
  58 ####### CHANGESOURCES
  59
  60 home_dir = os.path.abspath(ini.get("general", "homedir"))
  61 tree_expire = 0
  62 other_builds = 0
  63 cc_version = None
  64
  65 cc_command = "gcc"
  66 cxx_command = "g++"
  67
  68 if ini.has_option("general", "expire"):
  69         tree_expire = ini.getint("general", "expire")
  70
  71 if ini.has_option("general", "other_builds"):
  72         other_builds = ini.getint("general", "other_builds")
  73
  74 if ini.has_option("general", "cc_version"):
  75         cc_version = ini.get("general", "cc_version").split()
  76         if len(cc_version) == 1:
  77                 cc_version = ["eq", cc_version[0]]
  78
  79 repo_url = ini.get("repo", "url")
  80 repo_branch = "master"
  81
  82 if ini.has_option("repo", "branch"):
  83         repo_branch = ini.get("repo", "branch")
  84
  85 rsync_bin_url = ini.get("rsync", "binary_url")
  86 rsync_bin_key = ini.get("rsync", "binary_password")
  87
  88 rsync_src_url = None
  89 rsync_src_key = None
  90
  91 if ini.has_option("rsync", "source_url"):
  92         rsync_src_url = ini.get("rsync", "source_url")
  93         rsync_src_key = ini.get("rsync", "source_password")
  94
  95 gpg_home = "~/.gnupg"
  96 gpg_keyid = None
  97 gpg_comment = "Unattended build signature"
  98 gpg_passfile = "/dev/null"
  99
 100 if ini.has_option("gpg", "home"):
 101         gpg_home = ini.get("gpg", "home")
 102
 103 if ini.has_option("gpg", "keyid"):
 104         gpg_keyid = ini.get("gpg", "keyid")
 105
 106 if ini.has_option("gpg", "comment"):
 107         gpg_comment = ini.get("gpg", "comment")
 108
 109 if ini.has_option("gpg", "passfile"):
 110         gpg_passfile = ini.get("gpg", "passfile")
 111
 112
 113 # find targets
 114 targets = [ ]
 115
 116 if not os.path.isdir(home_dir+'/source.git'):
 117         subprocess.call(["git", "clone", "--depth=1", "--branch="+repo_branch, repo_url, home_dir+'/source.git'])
 118 else:
 119         subprocess.call(["git", "pull"], cwd = home_dir+'/source.git')
 120
 121 findtargets = subprocess.Popen([home_dir+'/dumpinfo.pl', 'targets'],
 122         stdout = subprocess.PIPE, cwd = home_dir+'/source.git')
 123
 124 while True:
 125         line = findtargets.stdout.readline()
 126         if not line:
 127                 break
 128         ta = line.strip().split(' ')
 129         targets.append(ta[0])
 130
 131
 132 # the 'change_source' setting tells the buildmaster how it should find out
 133 # about source code changes.  Here we point to the buildbot clone of pyflakes.
 134
 135 from buildbot.changes.gitpoller import GitPoller
 136 c['change_source'] = []
 137 c['change_source'].append(GitPoller(
 138         repo_url,
 139         workdir=home_dir+'/work.git', branch=repo_branch,
 140         pollinterval=300))
 141
 142 ####### SCHEDULERS
 143
 144 # Configure the Schedulers, which decide how to react to incoming changes.  In this
 145 # case, just kick off a 'basebuild' build
 146
 147 from buildbot.schedulers.basic import SingleBranchScheduler
 148 from buildbot.schedulers.forcesched import ForceScheduler
 149 from buildbot.changes import filter
 150 c['schedulers'] = []
 151 c['schedulers'].append(SingleBranchScheduler(
 152         name="all",
 153         change_filter=filter.ChangeFilter(branch=repo_branch),
 154         treeStableTimer=60,
 155         builderNames=targets))
 156
 157 c['schedulers'].append(ForceScheduler(
 158         name="force",
 159         builderNames=targets))
 160
 161 ####### BUILDERS
 162
 163 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
 164 # what steps, and which slaves can execute them.  Note that any particular build will
 165 # only take place on one slave.
 166
 167 from buildbot.process.factory import BuildFactory
 168 from buildbot.steps.source.git import Git
 169 from buildbot.steps.shell import ShellCommand
 170 from buildbot.steps.shell import SetProperty
 171 from buildbot.steps.transfer import FileUpload
 172 from buildbot.steps.transfer import FileDownload
 173 from buildbot.steps.master import MasterShellCommand
 174 from buildbot.process.properties import WithProperties
 175
 176
 177 CleanTargetMap = [
 178         [ "tools",      "tools/clean"                   ],
 179         [ "chain",      "toolchain/clean"               ],
 180         [ "linux",      "target/linux/clean"    ],
 181         [ "dir",        "dirclean"                              ],
 182         [ "dist",       "distclean"                             ]
 183 ]
 184
 185 def IsCleanRequested(pattern):
 186         def CheckCleanProperty(step):
 187                 val = step.getProperty("clean")
 188                 if val and re.match(pattern, val):
 189                         return True
 190                 else:
 191                         return False
 192
 193         return CheckCleanProperty
 194
 195 def IsTaggingRequested(step):
 196         val = step.getProperty("tag")
 197         if val and re.match("^[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", val):
 198                 return True
 199         else:
 200                 return False
 201
 202 def IsNoTaggingRequested(step):
 203         return not IsTaggingRequested(step)
 204
 205 def IsNoMasterBuild(step):
 206         return repo_branch != "master"
 207
 208 def GetBaseVersion(props):
 209         if re.match("^[^-]+-[0-9]+\.[0-9]+$", repo_branch):
 210                 return repo_branch.split('-')[1]
 211         else:
 212                 return "master"
 213
 214 def GetVersionPrefix(props):
 215         basever = GetBaseVersion(props)
 216         if props.hasProperty("tag") and re.match("^[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", props["tag"]):
 217                 return "%s/" % props["tag"]
 218         elif basever != "master":
 219                 return "%s-SNAPSHOT/" % basever
 220         else:
 221                 return ""
 222
 223 def GetNumJobs(props):
 224         if props.hasProperty("slavename") and props.hasProperty("nproc"):
 225                 return ((int(props["nproc"]) / (max_builds[props["slavename"]] + other_builds)) + 1)
 226         else:
 227                 return 1
 228
 229 def GetCC(props):
 230         if props.hasProperty("cc_command"):
 231                 return props["cc_command"]
 232         else:
 233                 return "gcc"
 234
 235 def GetCXX(props):
 236         if props.hasProperty("cxx_command"):
 237                 return props["cxx_command"]
 238         else:
 239                 return "g++"
 240
 241 def GetCwd(props):
 242         if props.hasProperty("builddir"):
 243                 return props["builddir"]
 244         elif props.hasProperty("workdir"):
 245                 return props["workdir"]
 246         else:
 247                 return "/"
 248
 249 def GetNextBuild(builder, requests):
 250         for r in requests:
 251                 if r.properties and r.properties.hasProperty("tag"):
 252                         return r
 253         return requests[0]
 254
 255 def MakeEnv(overrides=None):
 256         env = {
 257                 'CC': WithProperties("%(cc)s", cc=GetCC),
 258                 'CXX': WithProperties("%(cxx)s", cxx=GetCXX),
 259                 'CCACHE_BASEDIR': WithProperties("%(cwd)s", cwd=GetCwd)
 260         }
 261         if overrides is not None:
 262                 env.update(overrides)
 263         return env
 264
 265
 266 c['builders'] = []
 267
 268 dlLock = locks.SlaveLock("slave_dl")
 269 tagLock = locks.MasterLock("make_tag")
 270
 271 checkBuiltin = re.sub('[\t\n ]+', ' ', """
 272         checkBuiltin() {
 273                 local symbol op path file;
 274                 for file in $CHANGED_FILES; do
 275                         case "$file" in
 276                                 package/*/*) : ;;
 277                                 *) return 0 ;;
 278                         esac;
 279                 done;
 280                 while read symbol op path; do
 281                         case "$symbol" in package-*)
 282                                 symbol="${symbol##*(}";
 283                                 symbol="${symbol%)}";
 284                                 for file in $CHANGED_FILES; do
 285                                         case "$file" in "package/$path/"*)
 286                                                 grep -qsx "$symbol=y" .config && return 0
 287                                         ;; esac;
 288                                 done;
 289                         esac;
 290                 done < tmp/.packagedeps;
 291                 return 1;
 292         }
 293 """).strip()
 294
 295
 296 class IfBuiltinShellCommand(ShellCommand):
 297         def _quote(self, str):
 298                 if re.search("[^a-zA-Z0-9/_.-]", str):
 299                         return "'%s'" %(re.sub("'", "'\"'\"'", str))
 300                 return str
 301
 302         def setCommand(self, command):
 303                 if not isinstance(command, (str, unicode)):
 304                         command = ' '.join(map(self._quote, command))
 305                 self.command = [
 306                         '/bin/sh', '-c',
 307                         '%s; if checkBuiltin; then %s; else exit 0; fi' %(checkBuiltin, command)
 308                 ]
 309
 310         def setupEnvironment(self, cmd):
 311                 slaveEnv = self.slaveEnvironment
 312                 if slaveEnv is None:
 313                         slaveEnv = { }
 314                 changedFiles = { }
 315                 for request in self.build.requests:
 316                         for source in request.sources:
 317                                 for change in source.changes:
 318                                         for file in change.files:
 319                                                 changedFiles[file] = True
 320                 fullSlaveEnv = slaveEnv.copy()
 321                 fullSlaveEnv['CHANGED_FILES'] = ' '.join(changedFiles.keys())
 322                 cmd.args['env'] = fullSlaveEnv
 323
 324 slaveNames = [ ]
 325
 326 for slave in c['slaves']:
 327         slaveNames.append(slave.slavename)
 328
 329 for target in targets:
 330         ts = target.split('/')
 331
 332         factory = BuildFactory()
 333
 334         # find number of cores
 335         factory.addStep(SetProperty(
 336                 name = "nproc",
 337                 property = "nproc",
 338                 description = "Finding number of CPUs",
 339                 command = ["nproc"]))
 340
 341         # find gcc and g++ compilers
 342         if cc_version is not None:
 343                 factory.addStep(FileDownload(
 344                         mastersrc = "findbin.pl",
 345                         slavedest = "../findbin.pl",
 346                         mode = 0755))
 347
 348                 factory.addStep(SetProperty(
 349                         name = "gcc",
 350                         property = "cc_command",
 351                         description = "Finding gcc command",
 352                         command = ["../findbin.pl", "gcc", cc_version[0], cc_version[1]],
 353                         haltOnFailure = True))
 354
 355                 factory.addStep(SetProperty(
 356                         name = "g++",
 357                         property = "cxx_command",
 358                         description = "Finding g++ command",
 359                         command = ["../findbin.pl", "g++", cc_version[0], cc_version[1]],
 360                         haltOnFailure = True))
 361
 362         # expire tree if needed
 363         if tree_expire > 0:
 364                 factory.addStep(FileDownload(
 365                         mastersrc = "expire.sh",
 366                         slavedest = "../expire.sh",
 367                         mode = 0755))
 368
 369                 factory.addStep(ShellCommand(
 370                         name = "expire",
 371                         description = "Checking for build tree expiry",
 372                         command = ["./expire.sh", str(tree_expire)],
 373                         workdir = ".",
 374                         haltOnFailure = True,
 375                         timeout = 2400))
 376
 377         # user-requested clean targets
 378         for tuple in CleanTargetMap:
 379                 factory.addStep(ShellCommand(
 380                         name = tuple[1],
 381                         description = 'User-requested "make %s"' % tuple[1],
 382                         command = ["make", tuple[1], "V=s"],
 383                         env = MakeEnv(),
 384                         doStepIf = IsCleanRequested(tuple[0])
 385                 ))
 386
 387         factory.addStep(MasterShellCommand(
 388                 name = "maketag",
 389                 description = "Tagging Git repository",
 390                 command = [home_dir+'/maketag.sh', '-i', '-k', str(gpg_keyid or ''),
 391                            '-p', str(gpg_passfile or ''), '-v', WithProperties("%(tag:-)s")],
 392                 path = home_dir+'/source.git',
 393                 env = {'GNUPGHOME': gpg_home},
 394                 haltOnFailure = True,
 395                 doStepIf = IsTaggingRequested,
 396                 locks = [tagLock.access('exclusive')]
 397         ))
 398
 399         # switch to branch
 400         factory.addStep(ShellCommand(
 401                 name = "switchbranch",
 402                 description = "Checking out Git branch",
 403                 command = "if [ -d .git ]; then git fetch && git checkout '%s'; else exit 0; fi" % repo_branch,
 404                 haltOnFailure = True,
 405                 doStepIf = IsNoTaggingRequested
 406         ))
 407
 408         # check out the source
 409         factory.addStep(Git(
 410                 repourl = repo_url,
 411                 branch = repo_branch,
 412                 mode = 'incremental',
 413                 method = 'clean'))
 414
 415         # update remote refs
 416         factory.addStep(ShellCommand(
 417                 name = "fetchrefs",
 418                 description = "Fetching Git remote refs",
 419                 command = ["git", "fetch", "origin", "+refs/heads/%s:refs/remotes/origin/%s" %(repo_branch, repo_branch)],
 420                 haltOnFailure = True
 421         ))
 422
 423         # fetch tags
 424         factory.addStep(ShellCommand(
 425                 name = "fetchtag",
 426                 description = "Fetching Git tags",
 427                 command = ["git", "fetch", "--tags", "--", repo_url],
 428                 haltOnFailure = True,
 429                 doStepIf = IsTaggingRequested
 430         ))
 431
 432         # switch to tag
 433         factory.addStep(ShellCommand(
 434                 name = "switchtag",
 435                 description = "Checking out Git tag",
 436                 command = ["git", "checkout", WithProperties("tags/v%(tag:-)s")],
 437                 haltOnFailure = True,
 438                 doStepIf = IsTaggingRequested
 439         ))
 440
 441         factory.addStep(ShellCommand(
 442                 name = "rmtmp",
 443                 description = "Remove tmp folder",
 444                 command=["rm", "-rf", "tmp/"]))
 445
 446         # feed
 447 #       factory.addStep(ShellCommand(
 448 #               name = "feedsconf",
 449 #               description = "Copy the feeds.conf",
 450 #               command='''cp ~/feeds.conf ./feeds.conf''' ))
 451
 452         # feed
 453         factory.addStep(ShellCommand(
 454                 name = "rmfeedlinks",
 455                 description = "Remove feed symlinks",
 456                 command=["rm", "-rf", "package/feeds/"]))
 457
 458         # feed
 459         factory.addStep(ShellCommand(
 460                 name = "updatefeeds",
 461                 description = "Updating feeds",
 462                 command=["./scripts/feeds", "update"],
 463                 env = MakeEnv()))
 464
 465         # feed
 466         factory.addStep(ShellCommand(
 467                 name = "installfeeds",
 468                 description = "Installing feeds",
 469                 command=["./scripts/feeds", "install", "-a"],
 470                 env = MakeEnv()))
 471
 472         # seed config
 473         factory.addStep(FileDownload(
 474                 mastersrc = "config.seed",
 475                 slavedest = ".config",
 476                 mode = 0644
 477         ))
 478
 479         # configure
 480         factory.addStep(ShellCommand(
 481                 name = "newconfig",
 482                 description = "Seeding .config",
 483                 command = "printf 'CONFIG_TARGET_%s=y\\nCONFIG_TARGET_%s_%s=y\\n' >> .config" %(ts[0], ts[0], ts[1])
 484         ))
 485
 486         factory.addStep(ShellCommand(
 487                 name = "delbin",
 488                 description = "Removing output directory",
 489                 command = ["rm", "-rf", "bin/"]
 490         ))
 491
 492         factory.addStep(ShellCommand(
 493                 name = "defconfig",
 494                 description = "Populating .config",
 495                 command = ["make", "defconfig"],
 496                 env = MakeEnv()
 497         ))
 498
 499         # check arch
 500         factory.addStep(ShellCommand(
 501                 name = "checkarch",
 502                 description = "Checking architecture",
 503                 command = ["grep", "-sq", "CONFIG_TARGET_%s=y" %(ts[0]), ".config"],
 504                 logEnviron = False,
 505                 want_stdout = False,
 506                 want_stderr = False,
 507                 haltOnFailure = True
 508         ))
 509
 510         # find libc suffix
 511         factory.addStep(SetProperty(
 512                 name = "libc",
 513                 property = "libc",
 514                 description = "Finding libc suffix",
 515                 command = ["sed", "-ne", '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }', ".config"]))
 516
 517         # ccache helper
 518         factory.addStep(FileDownload(
 519                 mastersrc = "ccache.sh",
 520                 slavedest = "ccache.sh",
 521                 mode = 0755
 522         ))
 523
 524         # ccache prepare
 525         factory.addStep(ShellCommand(
 526                 name = "prepccache",
 527                 description = "Preparing ccache",
 528                 command = ["./ccache.sh"]
 529         ))
 530
 531         # install build key
 532         factory.addStep(FileDownload(mastersrc=home_dir+'/key-build', slavedest="key-build", mode=0600))
 533         factory.addStep(FileDownload(mastersrc=home_dir+'/key-build.pub', slavedest="key-build.pub", mode=0600))
 534
 535         # prepare dl
 536         factory.addStep(ShellCommand(
 537                 name = "dldir",
 538                 description = "Preparing dl/",
 539                 command = "mkdir -p $HOME/dl && rm -rf ./dl && ln -sf $HOME/dl ./dl",
 540                 logEnviron = False,
 541                 want_stdout = False
 542         ))
 543
 544         # prepare tar
 545         factory.addStep(ShellCommand(
 546                 name = "dltar",
 547                 description = "Building GNU tar",
 548                 command = ["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "tools/tar/compile", "V=s"],
 549                 env = MakeEnv(),
 550                 haltOnFailure = True
 551         ))
 552
 553         # populate dl
 554         factory.addStep(ShellCommand(
 555                 name = "dlrun",
 556                 description = "Populating dl/",
 557                 command = ["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "download", "V=s"],
 558                 env = MakeEnv(),
 559                 logEnviron = False,
 560                 locks = [dlLock.access('exclusive')]
 561         ))
 562
 563         factory.addStep(ShellCommand(
 564                 name = "cleanbase",
 565                 description = "Cleaning base-files",
 566                 command=["make", "package/base-files/clean", "V=s"]
 567         ))
 568
 569         # build
 570         factory.addStep(ShellCommand(
 571                 name = "tools",
 572                 description = "Building tools",
 573                 command = ["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "tools/install", "V=s"],
 574                 env = MakeEnv(),
 575                 haltOnFailure = True
 576         ))
 577
 578         factory.addStep(ShellCommand(
 579                 name = "toolchain",
 580                 description = "Building toolchain",
 581                 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "toolchain/install", "V=s"],
 582                 env = MakeEnv(),
 583                 haltOnFailure = True
 584         ))
 585
 586         factory.addStep(ShellCommand(
 587                 name = "kmods",
 588                 description = "Building kmods",
 589                 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
 590                 env = MakeEnv(),
 591                 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
 592                 haltOnFailure = True
 593         ))
 594
 595         factory.addStep(ShellCommand(
 596                 name = "pkgclean",
 597                 description = "Cleaning up package build",
 598                 command=["make", "package/cleanup", "V=s"]
 599         ))
 600
 601         factory.addStep(ShellCommand(
 602                 name = "pkgbuild",
 603                 description = "Building packages",
 604                 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
 605                 env = MakeEnv(),
 606                 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
 607                 haltOnFailure = True
 608         ))
 609
 610         # factory.addStep(IfBuiltinShellCommand(
 611         factory.addStep(ShellCommand(
 612                 name = "pkginstall",
 613                 description = "Installing packages",
 614                 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "package/install", "V=s"],
 615                 env = MakeEnv(),
 616                 haltOnFailure = True
 617         ))
 618
 619         factory.addStep(ShellCommand(
 620                 name = "pkgindex",
 621                 description = "Indexing packages",
 622                 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "package/index", "V=s"],
 623                 env = MakeEnv(),
 624                 haltOnFailure = True
 625         ))
 626
 627         #factory.addStep(IfBuiltinShellCommand(
 628         factory.addStep(ShellCommand(
 629                 name = "images",
 630                 description = "Building images",
 631                 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "target/install", "V=s"],
 632                 env = MakeEnv(),
 633                 haltOnFailure = True
 634         ))
 635
 636         factory.addStep(ShellCommand(
 637                 name = "diffconfig",
 638                 description = "Generating config.seed",
 639                 command=["make", "-j1", "diffconfig", "V=s"],
 640                 env = MakeEnv(),
 641                 haltOnFailure = True
 642         ))
 643
 644         factory.addStep(ShellCommand(
 645                 name = "checksums",
 646                 description = "Calculating checksums",
 647                 command=["make", "-j1", "checksum", "V=s"],
 648                 env = MakeEnv(),
 649                 haltOnFailure = True
 650         ))
 651
 652         # sign
 653         if gpg_keyid is not None:
 654                 factory.addStep(MasterShellCommand(
 655                         name = "signprepare",
 656                         description = "Preparing temporary signing directory",
 657                         command = ["mkdir", "-p", "%s/signing" %(home_dir)],
 658                         haltOnFailure = True
 659                 ))
 660
 661                 factory.addStep(ShellCommand(
 662                         name = "signpack",
 663                         description = "Packing files to sign",
 664                         command = WithProperties("find bin/targets/%s/%s%%(libc)s/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz" %(ts[0], ts[1])),
 665                         haltOnFailure = True
 666                 ))
 667
 668                 factory.addStep(FileUpload(
 669                         slavesrc = "sign.tar.gz",
 670                         masterdest = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
 671                         haltOnFailure = True
 672                 ))
 673
 674                 factory.addStep(MasterShellCommand(
 675                         name = "signfiles",
 676                         description = "Signing files",
 677                         command = ["%s/signall.sh" %(home_dir), "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]), gpg_keyid, gpg_comment],
 678                         env = {'GNUPGHOME': gpg_home, 'PASSFILE': gpg_passfile},
 679                         haltOnFailure = True
 680                 ))
 681
 682                 factory.addStep(FileDownload(
 683                         mastersrc = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
 684                         slavedest = "sign.tar.gz",
 685                         haltOnFailure = True
 686                 ))
 687
 688                 factory.addStep(ShellCommand(
 689                         name = "signunpack",
 690                         description = "Unpacking signed files",
 691                         command = ["tar", "-xzf", "sign.tar.gz"],
 692                         haltOnFailure = True
 693                 ))
 694
 695         # upload
 696         factory.addStep(ShellCommand(
 697                 name = "dirprepare",
 698                 description = "Preparing upload directory structure",
 699                 command = ["mkdir", "-p", WithProperties("tmp/upload/%%(prefix)stargets/%s/%s" %(ts[0], ts[1]), prefix=GetVersionPrefix)],
 700                 haltOnFailure = True
 701         ))
 702
 703         factory.addStep(ShellCommand(
 704                 name = "linkprepare",
 705                 description = "Preparing repository symlink",
 706                 command = ["ln", "-s", "-f", WithProperties("../packages-%(basever)s", basever=GetBaseVersion), WithProperties("tmp/upload/%(prefix)spackages", prefix=GetVersionPrefix)],
 707                 doStepIf = IsNoMasterBuild,
 708                 haltOnFailure = True
 709         ))
 710
 711         factory.addStep(ShellCommand(
 712                 name = "dirupload",
 713                 description = "Uploading directory structure",
 714                 command = ["rsync", "-4", "-avz", "tmp/upload/", "%s/" %(rsync_bin_url)],
 715                 env={'RSYNC_PASSWORD': rsync_bin_key},
 716                 haltOnFailure = True,
 717                 logEnviron = False
 718         ))
 719
 720         factory.addStep(ShellCommand(
 721                 name = "targetupload",
 722                 description = "Uploading target files",
 723                 command=["rsync", "-4", "--progress", "--delete", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]),
 724                          "-avz", WithProperties("bin/targets/%s/%s%%(libc)s/" %(ts[0], ts[1])),
 725                          WithProperties("%s/%%(prefix)stargets/%s/%s/" %(rsync_bin_url, ts[0], ts[1]), prefix=GetVersionPrefix)],
 726                 env={'RSYNC_PASSWORD': rsync_bin_key},
 727                 haltOnFailure = True,
 728                 logEnviron = False
 729         ))
 730
 731         if rsync_src_url is not None:
 732                 factory.addStep(ShellCommand(
 733                         name = "sourceupload",
 734                         description = "Uploading source archives",
 735                         command=["rsync", "-4", "--progress", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "dl/", "%s/" %(rsync_src_url)],
 736                         env={'RSYNC_PASSWORD': rsync_src_key},
 737                         haltOnFailure = True,
 738                         logEnviron = False
 739                 ))
 740
 741         if False:
 742                 factory.addStep(ShellCommand(
 743                         name = "packageupload",
 744                         description = "Uploading package files",
 745                         command=["rsync", "-4", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "bin/packages/", "%s/packages/" %(rsync_bin_url)],
 746                         env={'RSYNC_PASSWORD': rsync_bin_key},
 747                         haltOnFailure = False,
 748                         logEnviron = False
 749                 ))
 750
 751         # logs
 752         if False:
 753                 factory.addStep(ShellCommand(
 754                         name = "upload",
 755                         description = "Uploading logs",
 756                         command=["rsync", "-4", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "logs/", "%s/logs/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
 757                         env={'RSYNC_PASSWORD': rsync_bin_key},
 758                         haltOnFailure = False,
 759                         alwaysRun = True,
 760                         logEnviron = False
 761                 ))
 762
 763         from buildbot.config import BuilderConfig
 764
 765         c['builders'].append(BuilderConfig(name=target, slavenames=slaveNames, factory=factory, nextBuild=GetNextBuild))
 766
 767
 768 ####### STATUS TARGETS
 769
 770 # 'status' is a list of Status Targets. The results of each build will be
 771 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
 772 # including web pages, email senders, and IRC bots.
 773
 774 c['status'] = []
 775
 776 from buildbot.status import html
 777 from buildbot.status.web import authz, auth
 778
 779 if ini.has_option("status", "bind"):
 780         if ini.has_option("status", "user") and ini.has_option("status", "password"):
 781                 authz_cfg=authz.Authz(
 782                         # change any of these to True to enable; see the manual for more
 783                         # options
 784                         auth=auth.BasicAuth([(ini.get("status", "user"), ini.get("status", "password"))]),
 785                         gracefulShutdown = 'auth',
 786                         forceBuild = 'auth', # use this to test your slave once it is set up
 787                         forceAllBuilds = 'auth',
 788                         pingBuilder = False,
 789                         stopBuild = 'auth',
 790                         stopAllBuilds = 'auth',
 791                         cancelPendingBuild = 'auth',
 792                 )
 793                 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind"), authz=authz_cfg))
 794         else:
 795                 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind")))
 796
 797
 798 from buildbot.status import words
 799
 800 if ini.has_option("irc", "host") and ini.has_option("irc", "nickname") and ini.has_option("irc", "channel"):
 801         irc_host = ini.get("irc", "host")
 802         irc_port = 6667
 803         irc_chan = ini.get("irc", "channel")
 804         irc_nick = ini.get("irc", "nickname")
 805         irc_pass = None
 806
 807         if ini.has_option("irc", "port"):
 808                 irc_port = ini.getint("irc", "port")
 809
 810         if ini.has_option("irc", "password"):
 811                 irc_pass = ini.get("irc", "password")
 812
 813         irc = words.IRC(irc_host, irc_nick, port = irc_port, password = irc_pass,
 814                         channels = [{ "channel": irc_chan }],
 815                         notify_events = {
 816                           'exception': 1,
 817                           'successToFailure': 1,
 818                           'failureToSuccess': 1
 819                         }
 820         )
 821
 822         c['status'].append(irc)
 823
 824
 825 ####### PROJECT IDENTITY
 826
 827 # the 'title' string will appear at the top of this buildbot
 828 # installation's html.WebStatus home page (linked to the
 829 # 'titleURL') and is embedded in the title of the waterfall HTML page.
 830
 831 c['title'] = ini.get("general", "title")
 832 c['titleURL'] = ini.get("general", "title_url")
 833
 834 # the 'buildbotURL' string should point to the location where the buildbot's
 835 # internal web server (usually the html.WebStatus page) is visible. This
 836 # typically uses the port number set in the Waterfall 'status' entry, but
 837 # with an externally-visible host name which the buildbot cannot figure out
 838 # without some help.
 839
 840 c['buildbotURL'] = ini.get("general", "buildbot_url")
 841
 842 ####### DB URL
 843
 844 c['db'] = {
 845         # This specifies what database buildbot uses to store its state.  You can leave
 846         # this at its default for all but the largest installations.
 847         'db_url' : "sqlite:///state.sqlite",
 848 }