phase1/master.cfg

   1 # -*- python -*-
   2 # ex: set syntax=python:
   3
   4 import os
   5 import re
   6 import subprocess
   7 import ConfigParser
   8
   9 from buildbot import locks
  10
  11 # This is a sample buildmaster config file. It must be installed as
  12 # 'master.cfg' in your buildmaster's base directory.
  13
  14 ini = ConfigParser.ConfigParser()
  15 ini.read("./config.ini")
  16
  17 # This is the dictionary that the buildmaster pays attention to. We also use
  18 # a shorter alias to save typing.
  19 c = BuildmasterConfig = {}
  20
  21 ####### BUILDSLAVES
  22
  23 # The 'slaves' list defines the set of recognized buildslaves. Each element is
  24 # a BuildSlave object, specifying a unique slave name and password.  The same
  25 # slave name and password must be configured on the slave.
  26 from buildbot.buildslave import BuildSlave
  27
  28 slave_port = 9989
  29
  30 if ini.has_option("general", "port"):
  31         slave_port = ini.getint("general", "port")
  32
  33 c['slaves'] = []
  34 max_builds = dict()
  35
  36 for section in ini.sections():
  37         if section.startswith("slave "):
  38                 if ini.has_option(section, "name") and ini.has_option(section, "password"):
  39                         name = ini.get(section, "name")
  40                         password = ini.get(section, "password")
  41                         max_builds[name] = 1
  42                         if ini.has_option(section, "builds"):
  43                                 max_builds[name] = ini.getint(section, "builds")
  44                         c['slaves'].append(BuildSlave(name, password, max_builds = max_builds[name]))
  45
  46 # 'slavePortnum' defines the TCP port to listen on for connections from slaves.
  47 # This must match the value configured into the buildslaves (with their
  48 # --master option)
  49 c['slavePortnum'] = slave_port
  50
  51 # coalesce builds
  52 c['mergeRequests'] = True
  53
  54 # Reduce amount of backlog data
  55 c['buildHorizon'] = 30
  56 c['logHorizon'] = 20
  57
  58 ####### CHANGESOURCES
  59
  60 home_dir = os.path.abspath(ini.get("general", "homedir"))
  61 tree_expire = 0
  62 other_builds = 0
  63 cc_version = None
  64
  65 cc_command = "gcc"
  66 cxx_command = "g++"
  67
  68 if ini.has_option("general", "expire"):
  69         tree_expire = ini.getint("general", "expire")
  70
  71 if ini.has_option("general", "other_builds"):
  72         other_builds = ini.getint("general", "other_builds")
  73
  74 if ini.has_option("general", "cc_version"):
  75         cc_version = ini.get("general", "cc_version").split()
  76         if len(cc_version) == 1:
  77                 cc_version = ["eq", cc_version[0]]
  78
  79 repo_url = ini.get("repo", "url")
  80 repo_branch = "master"
  81
  82 if ini.has_option("repo", "branch"):
  83         repo_branch = ini.get("repo", "branch")
  84
  85 rsync_bin_url = ini.get("rsync", "binary_url")
  86 rsync_bin_key = ini.get("rsync", "binary_password")
  87
  88 rsync_src_url = None
  89 rsync_src_key = None
  90
  91 if ini.has_option("rsync", "source_url"):
  92         rsync_src_url = ini.get("rsync", "source_url")
  93         rsync_src_key = ini.get("rsync", "source_password")
  94
  95 gpg_home = "~/.gnupg"
  96 gpg_keyid = None
  97 gpg_comment = "Unattended build signature"
  98 gpg_passfile = "/dev/null"
  99
 100 if ini.has_option("gpg", "home"):
 101         gpg_home = ini.get("gpg", "home")
 102
 103 if ini.has_option("gpg", "keyid"):
 104         gpg_keyid = ini.get("gpg", "keyid")
 105
 106 if ini.has_option("gpg", "comment"):
 107         gpg_comment = ini.get("gpg", "comment")
 108
 109 if ini.has_option("gpg", "passfile"):
 110         gpg_passfile = ini.get("gpg", "passfile")
 111
 112
 113 # find targets
 114 targets = [ ]
 115
 116 if not os.path.isdir(home_dir+'/source.git'):
 117         subprocess.call(["git", "clone", "--depth=1", "--branch="+repo_branch, repo_url, home_dir+'/source.git'])
 118 else:
 119         subprocess.call(["git", "pull"], cwd = home_dir+'/source.git')
 120
 121 findtargets = subprocess.Popen([home_dir+'/dumpinfo.pl', 'targets'],
 122         stdout = subprocess.PIPE, cwd = home_dir+'/source.git')
 123
 124 while True:
 125         line = findtargets.stdout.readline()
 126         if not line:
 127                 break
 128         ta = line.strip().split(' ')
 129         targets.append(ta[0])
 130
 131
 132 # the 'change_source' setting tells the buildmaster how it should find out
 133 # about source code changes.  Here we point to the buildbot clone of pyflakes.
 134
 135 from buildbot.changes.gitpoller import GitPoller
 136 c['change_source'] = []
 137 c['change_source'].append(GitPoller(
 138         repo_url,
 139         workdir=home_dir+'/work.git', branch=repo_branch,
 140         pollinterval=300))
 141
 142 ####### SCHEDULERS
 143
 144 # Configure the Schedulers, which decide how to react to incoming changes.  In this
 145 # case, just kick off a 'basebuild' build
 146
 147 from buildbot.schedulers.basic import SingleBranchScheduler
 148 from buildbot.schedulers.forcesched import ForceScheduler
 149 from buildbot.changes import filter
 150 c['schedulers'] = []
 151 c['schedulers'].append(SingleBranchScheduler(
 152         name="all",
 153         change_filter=filter.ChangeFilter(branch=repo_branch),
 154         treeStableTimer=60,
 155         builderNames=targets))
 156
 157 c['schedulers'].append(ForceScheduler(
 158         name="force",
 159         builderNames=targets))
 160
 161 ####### BUILDERS
 162
 163 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
 164 # what steps, and which slaves can execute them.  Note that any particular build will
 165 # only take place on one slave.
 166
 167 from buildbot.process.factory import BuildFactory
 168 from buildbot.steps.source.git import Git
 169 from buildbot.steps.shell import ShellCommand
 170 from buildbot.steps.shell import SetProperty
 171 from buildbot.steps.transfer import FileUpload
 172 from buildbot.steps.transfer import FileDownload
 173 from buildbot.steps.master import MasterShellCommand
 174 from buildbot.process.properties import WithProperties
 175
 176
 177 CleanTargetMap = [
 178         [ "tools",      "tools/clean"                   ],
 179         [ "chain",      "toolchain/clean"               ],
 180         [ "linux",      "target/linux/clean"    ],
 181         [ "dir",        "dirclean"                              ],
 182         [ "dist",       "distclean"                             ]
 183 ]
 184
 185 def IsCleanRequested(pattern):
 186         def CheckCleanProperty(step):
 187                 val = step.getProperty("clean")
 188                 if val and re.match(pattern, val):
 189                         return True
 190                 else:
 191                         return False
 192
 193         return CheckCleanProperty
 194
 195 def IsTaggingRequested(step):
 196         val = step.getProperty("tag")
 197         if val and re.match("^[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", val):
 198                 return True
 199         else:
 200                 return False
 201
 202 def IsNoTaggingRequested(step):
 203         return not IsTaggingRequested(step)
 204
 205 def IsNoMasterBuild(step):
 206         return repo_branch != "master"
 207
 208 def GetBaseVersion(props):
 209         if re.match("^[^-]+-[0-9]+\.[0-9]+$", repo_branch):
 210                 return repo_branch.split('-')[1]
 211         else:
 212                 return "master"
 213
 214 def GetVersionPrefix(props):
 215         basever = GetBaseVersion(props)
 216         if props.hasProperty("tag") and re.match("^[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", props["tag"]):
 217                 return "%s/" % props["tag"]
 218         elif basever != "master":
 219                 return "%s-SNAPSHOT/" % basever
 220         else:
 221                 return ""
 222
 223 def GetNumJobs(props):
 224         if props.hasProperty("slavename") and props.hasProperty("nproc"):
 225                 return ((int(props["nproc"]) / (max_builds[props["slavename"]] + other_builds)) + 1)
 226         else:
 227                 return 1
 228
 229 def GetCC(props):
 230         if props.hasProperty("cc_command"):
 231                 return props["cc_command"]
 232         else:
 233                 return "gcc"
 234
 235 def GetCXX(props):
 236         if props.hasProperty("cxx_command"):
 237                 return props["cxx_command"]
 238         else:
 239                 return "g++"
 240
 241 def GetCwd(props):
 242         if props.hasProperty("builddir"):
 243                 return props["builddir"]
 244         elif props.hasProperty("workdir"):
 245                 return props["workdir"]
 246         else:
 247                 return "/"
 248
 249 def MakeEnv(overrides=None):
 250         env = {
 251                 'CC': WithProperties("%(cc)s", cc=GetCC),
 252                 'CXX': WithProperties("%(cxx)s", cxx=GetCXX),
 253                 'CCACHE_BASEDIR': WithProperties("%(cwd)s", cwd=GetCwd)
 254         }
 255         if overrides is not None:
 256                 env.update(overrides)
 257         return env
 258
 259
 260 c['builders'] = []
 261
 262 dlLock = locks.SlaveLock("slave_dl")
 263 tagLock = locks.MasterLock("make_tag")
 264
 265 checkBuiltin = re.sub('[\t\n ]+', ' ', """
 266         checkBuiltin() {
 267                 local symbol op path file;
 268                 for file in $CHANGED_FILES; do
 269                         case "$file" in
 270                                 package/*/*) : ;;
 271                                 *) return 0 ;;
 272                         esac;
 273                 done;
 274                 while read symbol op path; do
 275                         case "$symbol" in package-*)
 276                                 symbol="${symbol##*(}";
 277                                 symbol="${symbol%)}";
 278                                 for file in $CHANGED_FILES; do
 279                                         case "$file" in "package/$path/"*)
 280                                                 grep -qsx "$symbol=y" .config && return 0
 281                                         ;; esac;
 282                                 done;
 283                         esac;
 284                 done < tmp/.packagedeps;
 285                 return 1;
 286         }
 287 """).strip()
 288
 289
 290 class IfBuiltinShellCommand(ShellCommand):
 291         def _quote(self, str):
 292                 if re.search("[^a-zA-Z0-9/_.-]", str):
 293                         return "'%s'" %(re.sub("'", "'\"'\"'", str))
 294                 return str
 295
 296         def setCommand(self, command):
 297                 if not isinstance(command, (str, unicode)):
 298                         command = ' '.join(map(self._quote, command))
 299                 self.command = [
 300                         '/bin/sh', '-c',
 301                         '%s; if checkBuiltin; then %s; else exit 0; fi' %(checkBuiltin, command)
 302                 ]
 303
 304         def setupEnvironment(self, cmd):
 305                 slaveEnv = self.slaveEnvironment
 306                 if slaveEnv is None:
 307                         slaveEnv = { }
 308                 changedFiles = { }
 309                 for request in self.build.requests:
 310                         for source in request.sources:
 311                                 for change in source.changes:
 312                                         for file in change.files:
 313                                                 changedFiles[file] = True
 314                 fullSlaveEnv = slaveEnv.copy()
 315                 fullSlaveEnv['CHANGED_FILES'] = ' '.join(changedFiles.keys())
 316                 cmd.args['env'] = fullSlaveEnv
 317
 318 slaveNames = [ ]
 319
 320 for slave in c['slaves']:
 321         slaveNames.append(slave.slavename)
 322
 323 for target in targets:
 324         ts = target.split('/')
 325
 326         factory = BuildFactory()
 327
 328         # find number of cores
 329         factory.addStep(SetProperty(
 330                 name = "nproc",
 331                 property = "nproc",
 332                 description = "Finding number of CPUs",
 333                 command = ["nproc"]))
 334
 335         # find gcc and g++ compilers
 336         if cc_version is not None:
 337                 factory.addStep(FileDownload(
 338                         mastersrc = "findbin.pl",
 339                         slavedest = "../findbin.pl",
 340                         mode = 0755))
 341
 342                 factory.addStep(SetProperty(
 343                         name = "gcc",
 344                         property = "cc_command",
 345                         description = "Finding gcc command",
 346                         command = ["../findbin.pl", "gcc", cc_version[0], cc_version[1]],
 347                         haltOnFailure = True))
 348
 349                 factory.addStep(SetProperty(
 350                         name = "g++",
 351                         property = "cxx_command",
 352                         description = "Finding g++ command",
 353                         command = ["../findbin.pl", "g++", cc_version[0], cc_version[1]],
 354                         haltOnFailure = True))
 355
 356         # expire tree if needed
 357         if tree_expire > 0:
 358                 factory.addStep(FileDownload(
 359                         mastersrc = "expire.sh",
 360                         slavedest = "../expire.sh",
 361                         mode = 0755))
 362
 363                 factory.addStep(ShellCommand(
 364                         name = "expire",
 365                         description = "Checking for build tree expiry",
 366                         command = ["./expire.sh", str(tree_expire)],
 367                         workdir = ".",
 368                         haltOnFailure = True,
 369                         timeout = 2400))
 370
 371         # user-requested clean targets
 372         for tuple in CleanTargetMap:
 373                 factory.addStep(ShellCommand(
 374                         name = tuple[1],
 375                         description = 'User-requested "make %s"' % tuple[1],
 376                         command = ["make", tuple[1], "V=s"],
 377                         env = MakeEnv(),
 378                         doStepIf = IsCleanRequested(tuple[0])
 379                 ))
 380
 381         factory.addStep(MasterShellCommand(
 382                 name = "maketag",
 383                 description = "Tagging Git repository",
 384                 command = [home_dir+'/maketag.sh', '-i', '-k', str(gpg_keyid or ''),
 385                            '-p', str(gpg_passfile or ''), '-v', WithProperties("%(tag:-)s")],
 386                 workdir = home_dir+'/source.git',
 387                 env = {'GNUPGHOME': gpg_home},
 388                 haltOnFailure = True,
 389                 doStepIf = IsTaggingRequested,
 390                 locks = [tagLock.access('exclusive')]
 391         ))
 392
 393         # switch to branch
 394         factory.addStep(ShellCommand(
 395                 name = "switchbranch",
 396                 description = "Checking out Git branch",
 397                 command = "if [ -d .git ]; then git checkout '%s'; else exit 0; fi" % repo_branch,
 398                 haltOnFailure = True,
 399                 doStepIf = IsNoTaggingRequested
 400         ))
 401
 402         # check out the source
 403         factory.addStep(Git(
 404                 repourl = repo_url,
 405                 branch = repo_branch,
 406                 mode = 'incremental',
 407                 method = 'clean'))
 408
 409         # update remote refs
 410         factory.addStep(ShellCommand(
 411                 name = "fetchrefs",
 412                 description = "Fetching Git remote refs",
 413                 command = ["git", "fetch", "origin", "+refs/heads/%s:refs/remotes/origin/%s" %(repo_branch, repo_branch)],
 414                 haltOnFailure = True
 415         ))
 416
 417         # fetch tags
 418         factory.addStep(ShellCommand(
 419                 name = "fetchtag",
 420                 description = "Fetching Git tags",
 421                 command = ["git", "fetch", "--tags", "--", repo_url],
 422                 haltOnFailure = True,
 423                 doStepIf = IsTaggingRequested
 424         ))
 425
 426         # switch to tag
 427         factory.addStep(ShellCommand(
 428                 name = "switchtag",
 429                 description = "Checking out Git tag",
 430                 command = ["git", "checkout", WithProperties("tags/v%(tag:-)s")],
 431                 haltOnFailure = True,
 432                 doStepIf = IsTaggingRequested
 433         ))
 434
 435         factory.addStep(ShellCommand(
 436                 name = "rmtmp",
 437                 description = "Remove tmp folder",
 438                 command=["rm", "-rf", "tmp/"]))
 439
 440         # feed
 441 #       factory.addStep(ShellCommand(
 442 #               name = "feedsconf",
 443 #               description = "Copy the feeds.conf",
 444 #               command='''cp ~/feeds.conf ./feeds.conf''' ))
 445
 446         # feed
 447         factory.addStep(ShellCommand(
 448                 name = "rmfeedlinks",
 449                 description = "Remove feed symlinks",
 450                 command=["rm", "-rf", "package/feeds/"]))
 451
 452         # feed
 453         factory.addStep(ShellCommand(
 454                 name = "updatefeeds",
 455                 description = "Updating feeds",
 456                 command=["./scripts/feeds", "update"],
 457                 env = MakeEnv()))
 458
 459         # feed
 460         factory.addStep(ShellCommand(
 461                 name = "installfeeds",
 462                 description = "Installing feeds",
 463                 command=["./scripts/feeds", "install", "-a"],
 464                 env = MakeEnv()))
 465
 466         # seed config
 467         factory.addStep(FileDownload(
 468                 mastersrc = "config.seed",
 469                 slavedest = ".config",
 470                 mode = 0644
 471         ))
 472
 473         # configure
 474         factory.addStep(ShellCommand(
 475                 name = "newconfig",
 476                 description = "Seeding .config",
 477                 command = "printf 'CONFIG_TARGET_%s=y\\nCONFIG_TARGET_%s_%s=y\\n' >> .config" %(ts[0], ts[0], ts[1])
 478         ))
 479
 480         factory.addStep(ShellCommand(
 481                 name = "delbin",
 482                 description = "Removing output directory",
 483                 command = ["rm", "-rf", "bin/"]
 484         ))
 485
 486         factory.addStep(ShellCommand(
 487                 name = "defconfig",
 488                 description = "Populating .config",
 489                 command = ["make", "defconfig"],
 490                 env = MakeEnv()
 491         ))
 492
 493         # check arch
 494         factory.addStep(ShellCommand(
 495                 name = "checkarch",
 496                 description = "Checking architecture",
 497                 command = ["grep", "-sq", "CONFIG_TARGET_%s=y" %(ts[0]), ".config"],
 498                 logEnviron = False,
 499                 want_stdout = False,
 500                 want_stderr = False,
 501                 haltOnFailure = True
 502         ))
 503
 504         # find libc suffix
 505         factory.addStep(SetProperty(
 506                 name = "libc",
 507                 property = "libc",
 508                 description = "Finding libc suffix",
 509                 command = ["sed", "-ne", '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }', ".config"]))
 510
 511         # ccache helper
 512         factory.addStep(FileDownload(
 513                 mastersrc = "ccache.sh",
 514                 slavedest = "ccache.sh",
 515                 mode = 0755
 516         ))
 517
 518         # ccache prepare
 519         factory.addStep(ShellCommand(
 520                 name = "prepccache",
 521                 description = "Preparing ccache",
 522                 command = ["./ccache.sh"]
 523         ))
 524
 525         # install build key
 526         factory.addStep(FileDownload(mastersrc=home_dir+'/key-build', slavedest="key-build", mode=0600))
 527         factory.addStep(FileDownload(mastersrc=home_dir+'/key-build.pub', slavedest="key-build.pub", mode=0600))
 528
 529         # prepare dl
 530         factory.addStep(ShellCommand(
 531                 name = "dldir",
 532                 description = "Preparing dl/",
 533                 command = "mkdir -p $HOME/dl && rm -rf ./dl && ln -sf $HOME/dl ./dl",
 534                 logEnviron = False,
 535                 want_stdout = False
 536         ))
 537
 538         # prepare tar
 539         factory.addStep(ShellCommand(
 540                 name = "dltar",
 541                 description = "Building GNU tar",
 542                 command = ["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "tools/tar/compile", "V=s"],
 543                 env = MakeEnv(),
 544                 haltOnFailure = True
 545         ))
 546
 547         # populate dl
 548         factory.addStep(ShellCommand(
 549                 name = "dlrun",
 550                 description = "Populating dl/",
 551                 command = ["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "download", "V=s"],
 552                 env = MakeEnv(),
 553                 logEnviron = False,
 554                 locks = [dlLock.access('exclusive')]
 555         ))
 556
 557         factory.addStep(ShellCommand(
 558                 name = "cleanbase",
 559                 description = "Cleaning base-files",
 560                 command=["make", "package/base-files/clean", "V=s"]
 561         ))
 562
 563         # build
 564         factory.addStep(ShellCommand(
 565                 name = "tools",
 566                 description = "Building tools",
 567                 command = ["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "tools/install", "V=s"],
 568                 env = MakeEnv(),
 569                 haltOnFailure = True
 570         ))
 571
 572         factory.addStep(ShellCommand(
 573                 name = "toolchain",
 574                 description = "Building toolchain",
 575                 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "toolchain/install", "V=s"],
 576                 env = MakeEnv(),
 577                 haltOnFailure = True
 578         ))
 579
 580         factory.addStep(ShellCommand(
 581                 name = "kmods",
 582                 description = "Building kmods",
 583                 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
 584                 env = MakeEnv(),
 585                 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
 586                 haltOnFailure = True
 587         ))
 588
 589         factory.addStep(ShellCommand(
 590                 name = "pkgclean",
 591                 description = "Cleaning up package build",
 592                 command=["make", "package/cleanup", "V=s"]
 593         ))
 594
 595         factory.addStep(ShellCommand(
 596                 name = "pkgbuild",
 597                 description = "Building packages",
 598                 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
 599                 env = MakeEnv(),
 600                 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
 601                 haltOnFailure = True
 602         ))
 603
 604         # factory.addStep(IfBuiltinShellCommand(
 605         factory.addStep(ShellCommand(
 606                 name = "pkginstall",
 607                 description = "Installing packages",
 608                 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "package/install", "V=s"],
 609                 env = MakeEnv(),
 610                 haltOnFailure = True
 611         ))
 612
 613         factory.addStep(ShellCommand(
 614                 name = "pkgindex",
 615                 description = "Indexing packages",
 616                 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "package/index", "V=s"],
 617                 env = MakeEnv(),
 618                 haltOnFailure = True
 619         ))
 620
 621         #factory.addStep(IfBuiltinShellCommand(
 622         factory.addStep(ShellCommand(
 623                 name = "images",
 624                 description = "Building images",
 625                 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "target/install", "V=s"],
 626                 env = MakeEnv(),
 627                 haltOnFailure = True
 628         ))
 629
 630         factory.addStep(ShellCommand(
 631                 name = "diffconfig",
 632                 description = "Generating config.seed",
 633                 command=["make", "-j1", "diffconfig", "V=s"],
 634                 env = MakeEnv(),
 635                 haltOnFailure = True
 636         ))
 637
 638         factory.addStep(ShellCommand(
 639                 name = "checksums",
 640                 description = "Calculating checksums",
 641                 command=["make", "-j1", "checksum", "V=s"],
 642                 env = MakeEnv(),
 643                 haltOnFailure = True
 644         ))
 645
 646         # sign
 647         if gpg_keyid is not None:
 648                 factory.addStep(MasterShellCommand(
 649                         name = "signprepare",
 650                         description = "Preparing temporary signing directory",
 651                         command = ["mkdir", "-p", "%s/signing" %(home_dir)],
 652                         haltOnFailure = True
 653                 ))
 654
 655                 factory.addStep(ShellCommand(
 656                         name = "signpack",
 657                         description = "Packing files to sign",
 658                         command = WithProperties("find bin/targets/%s/%s%%(libc)s/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz" %(ts[0], ts[1])),
 659                         haltOnFailure = True
 660                 ))
 661
 662                 factory.addStep(FileUpload(
 663                         slavesrc = "sign.tar.gz",
 664                         masterdest = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
 665                         haltOnFailure = True
 666                 ))
 667
 668                 factory.addStep(MasterShellCommand(
 669                         name = "signfiles",
 670                         description = "Signing files",
 671                         command = ["%s/signall.sh" %(home_dir), "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]), gpg_keyid, gpg_comment],
 672                         env = {'GNUPGHOME': gpg_home, 'PASSFILE': gpg_passfile},
 673                         haltOnFailure = True
 674                 ))
 675
 676                 factory.addStep(FileDownload(
 677                         mastersrc = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
 678                         slavedest = "sign.tar.gz",
 679                         haltOnFailure = True
 680                 ))
 681
 682                 factory.addStep(ShellCommand(
 683                         name = "signunpack",
 684                         description = "Unpacking signed files",
 685                         command = ["tar", "-xzf", "sign.tar.gz"],
 686                         haltOnFailure = True
 687                 ))
 688
 689         # upload
 690         factory.addStep(ShellCommand(
 691                 name = "dirprepare",
 692                 description = "Preparing upload directory structure",
 693                 command = ["mkdir", "-p", WithProperties("tmp/upload/%%(prefix)stargets/%s/%s" %(ts[0], ts[1]), prefix=GetVersionPrefix)],
 694                 haltOnFailure = True
 695         ))
 696
 697         factory.addStep(ShellCommand(
 698                 name = "linkprepare",
 699                 description = "Preparing repository symlink",
 700                 command = ["ln", "-s", "-f", WithProperties("../packages-%(basever)s", basever=GetBaseVersion), WithProperties("tmp/upload/%(prefix)spackages", prefix=GetVersionPrefix)],
 701                 doStepIf = IsNoMasterBuild,
 702                 haltOnFailure = True
 703         ))
 704
 705         factory.addStep(ShellCommand(
 706                 name = "dirupload",
 707                 description = "Uploading directory structure",
 708                 command = ["rsync", "-avz", "tmp/upload/", "%s/" %(rsync_bin_url)],
 709                 env={'RSYNC_PASSWORD': rsync_bin_key},
 710                 haltOnFailure = True,
 711                 logEnviron = False
 712         ))
 713
 714         factory.addStep(ShellCommand(
 715                 name = "targetupload",
 716                 description = "Uploading target files",
 717                 command=["rsync", "--progress", "--delete", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]),
 718                          "-avz", WithProperties("bin/targets/%s/%s%%(libc)s/" %(ts[0], ts[1])),
 719                          WithProperties("%s/%%(prefix)stargets/%s/%s/" %(rsync_bin_url, ts[0], ts[1]), prefix=GetVersionPrefix)],
 720                 env={'RSYNC_PASSWORD': rsync_bin_key},
 721                 haltOnFailure = True,
 722                 logEnviron = False
 723         ))
 724
 725         if rsync_src_url is not None:
 726                 factory.addStep(ShellCommand(
 727                         name = "sourceupload",
 728                         description = "Uploading source archives",
 729                         command=["rsync", "--progress", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "dl/", "%s/" %(rsync_src_url)],
 730                         env={'RSYNC_PASSWORD': rsync_src_key},
 731                         haltOnFailure = True,
 732                         logEnviron = False
 733                 ))
 734
 735         if False:
 736                 factory.addStep(ShellCommand(
 737                         name = "packageupload",
 738                         description = "Uploading package files",
 739                         command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "bin/packages/", "%s/packages/" %(rsync_bin_url)],
 740                         env={'RSYNC_PASSWORD': rsync_bin_key},
 741                         haltOnFailure = False,
 742                         logEnviron = False
 743                 ))
 744
 745         # logs
 746         if False:
 747                 factory.addStep(ShellCommand(
 748                         name = "upload",
 749                         description = "Uploading logs",
 750                         command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "logs/", "%s/logs/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
 751                         env={'RSYNC_PASSWORD': rsync_bin_key},
 752                         haltOnFailure = False,
 753                         alwaysRun = True,
 754                         logEnviron = False
 755                 ))
 756
 757         from buildbot.config import BuilderConfig
 758
 759         c['builders'].append(BuilderConfig(name=target, slavenames=slaveNames, factory=factory))
 760
 761
 762 ####### STATUS TARGETS
 763
 764 # 'status' is a list of Status Targets. The results of each build will be
 765 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
 766 # including web pages, email senders, and IRC bots.
 767
 768 c['status'] = []
 769
 770 from buildbot.status import html
 771 from buildbot.status.web import authz, auth
 772
 773 if ini.has_option("status", "bind"):
 774         if ini.has_option("status", "user") and ini.has_option("status", "password"):
 775                 authz_cfg=authz.Authz(
 776                         # change any of these to True to enable; see the manual for more
 777                         # options
 778                         auth=auth.BasicAuth([(ini.get("status", "user"), ini.get("status", "password"))]),
 779                         gracefulShutdown = 'auth',
 780                         forceBuild = 'auth', # use this to test your slave once it is set up
 781                         forceAllBuilds = 'auth',
 782                         pingBuilder = False,
 783                         stopBuild = 'auth',
 784                         stopAllBuilds = 'auth',
 785                         cancelPendingBuild = 'auth',
 786                 )
 787                 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind"), authz=authz_cfg))
 788         else:
 789                 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind")))
 790
 791
 792 from buildbot.status import words
 793
 794 if ini.has_option("irc", "host") and ini.has_option("irc", "nickname") and ini.has_option("irc", "channel"):
 795         irc_host = ini.get("irc", "host")
 796         irc_port = 6667
 797         irc_chan = ini.get("irc", "channel")
 798         irc_nick = ini.get("irc", "nickname")
 799         irc_pass = None
 800
 801         if ini.has_option("irc", "port"):
 802                 irc_port = ini.getint("irc", "port")
 803
 804         if ini.has_option("irc", "password"):
 805                 irc_pass = ini.get("irc", "password")
 806
 807         irc = words.IRC(irc_host, irc_nick, port = irc_port, password = irc_pass,
 808                         channels = [{ "channel": irc_chan }],
 809                         notify_events = {
 810                           'exception': 1,
 811                           'successToFailure': 1,
 812                           'failureToSuccess': 1
 813                         }
 814         )
 815
 816         c['status'].append(irc)
 817
 818
 819 ####### PROJECT IDENTITY
 820
 821 # the 'title' string will appear at the top of this buildbot
 822 # installation's html.WebStatus home page (linked to the
 823 # 'titleURL') and is embedded in the title of the waterfall HTML page.
 824
 825 c['title'] = ini.get("general", "title")
 826 c['titleURL'] = ini.get("general", "title_url")
 827
 828 # the 'buildbotURL' string should point to the location where the buildbot's
 829 # internal web server (usually the html.WebStatus page) is visible. This
 830 # typically uses the port number set in the Waterfall 'status' entry, but
 831 # with an externally-visible host name which the buildbot cannot figure out
 832 # without some help.
 833
 834 c['buildbotURL'] = ini.get("general", "buildbot_url")
 835
 836 ####### DB URL
 837
 838 c['db'] = {
 839         # This specifies what database buildbot uses to store its state.  You can leave
 840         # this at its default for all but the largest installations.
 841         'db_url' : "sqlite:///state.sqlite",
 842 }