phase1/master.cfg

   1 # -*- python -*-
   2 # ex: set syntax=python:
   3
   4 import os
   5 import re
   6 import subprocess
   7 import ConfigParser
   8
   9 from buildbot import locks
  10
  11 # This is a sample buildmaster config file. It must be installed as
  12 # 'master.cfg' in your buildmaster's base directory.
  13
  14 ini = ConfigParser.ConfigParser()
  15 ini.read("./config.ini")
  16
  17 # This is the dictionary that the buildmaster pays attention to. We also use
  18 # a shorter alias to save typing.
  19 c = BuildmasterConfig = {}
  20
  21 ####### PROJECT IDENTITY
  22
  23 # the 'title' string will appear at the top of this buildbot
  24 # installation's html.WebStatus home page (linked to the
  25 # 'titleURL') and is embedded in the title of the waterfall HTML page.
  26
  27 c['title'] = ini.get("general", "title")
  28 c['titleURL'] = ini.get("general", "title_url")
  29
  30 # the 'buildbotURL' string should point to the location where the buildbot's
  31 # internal web server (usually the html.WebStatus page) is visible. This
  32 # typically uses the port number set in the Waterfall 'status' entry, but
  33 # with an externally-visible host name which the buildbot cannot figure out
  34 # without some help.
  35
  36 c['buildbotURL'] = ini.get("general", "buildbot_url")
  37
  38 ####### BUILDSLAVES
  39
  40 # The 'slaves' list defines the set of recognized buildslaves. Each element is
  41 # a BuildSlave object, specifying a unique slave name and password.  The same
  42 # slave name and password must be configured on the slave.
  43 from buildbot.buildslave import BuildSlave
  44
  45 slave_port = 9989
  46
  47 if ini.has_option("general", "port"):
  48         slave_port = ini.getint("general", "port")
  49
  50 c['slaves'] = []
  51 max_builds = dict()
  52 do_cleanup = dict()
  53 NetLocks = dict()
  54
  55 for section in ini.sections():
  56         if section.startswith("slave "):
  57                 if ini.has_option(section, "name") and ini.has_option(section, "password"):
  58                         sl_props = { 'dl_lock':None, 'ul_lock':None, }
  59                         name = ini.get(section, "name")
  60                         password = ini.get(section, "password")
  61                         max_builds[name] = 1
  62                         do_cleanup[name] = False
  63                         if ini.has_option(section, "builds"):
  64                                 max_builds[name] = ini.getint(section, "builds")
  65                         if ini.has_option(section, "cleanup"):
  66                                 do_cleanup[name] = ini.getboolean(section, "cleanup")
  67                         if ini.has_option(section, "dl_lock"):
  68                                 lockname = ini.get(section, "dl_lock")
  69                                 sl_props['dl_lock'] = lockname
  70                                 if lockname not in NetLocks:
  71                                         NetLocks[lockname] = locks.MasterLock(lockname)
  72                         if ini.has_option(section, "ul_lock"):
  73                                 lockname = ini.get(section, "dl_lock")
  74                                 sl_props['ul_lock'] = lockname
  75                                 if lockname not in NetLocks:
  76                                         NetLocks[lockname] = locks.MasterLock(lockname)
  77                         c['slaves'].append(BuildSlave(name, password, max_builds = max_builds[name], properties = sl_props))
  78
  79 # 'slavePortnum' defines the TCP port to listen on for connections from slaves.
  80 # This must match the value configured into the buildslaves (with their
  81 # --master option)
  82 c['slavePortnum'] = slave_port
  83
  84 # coalesce builds
  85 c['mergeRequests'] = True
  86
  87 # Reduce amount of backlog data
  88 c['buildHorizon'] = 30
  89 c['logHorizon'] = 20
  90
  91 ####### CHANGESOURCES
  92
  93 home_dir = os.path.abspath(ini.get("general", "homedir"))
  94 tree_expire = 0
  95 other_builds = 0
  96 cc_version = None
  97
  98 cc_command = "gcc"
  99 cxx_command = "g++"
 100
 101 if ini.has_option("general", "expire"):
 102         tree_expire = ini.getint("general", "expire")
 103
 104 if ini.has_option("general", "other_builds"):
 105         other_builds = ini.getint("general", "other_builds")
 106
 107 if ini.has_option("general", "cc_version"):
 108         cc_version = ini.get("general", "cc_version").split()
 109         if len(cc_version) == 1:
 110                 cc_version = ["eq", cc_version[0]]
 111
 112 repo_url = ini.get("repo", "url")
 113 repo_branch = "master"
 114
 115 if ini.has_option("repo", "branch"):
 116         repo_branch = ini.get("repo", "branch")
 117
 118 rsync_bin_url = ini.get("rsync", "binary_url")
 119 rsync_bin_key = ini.get("rsync", "binary_password")
 120
 121 rsync_src_url = None
 122 rsync_src_key = None
 123
 124 if ini.has_option("rsync", "source_url"):
 125         rsync_src_url = ini.get("rsync", "source_url")
 126         rsync_src_key = ini.get("rsync", "source_password")
 127
 128 gpg_home = "~/.gnupg"
 129 gpg_keyid = None
 130 gpg_comment = "Unattended build signature"
 131 gpg_passfile = "/dev/null"
 132
 133 if ini.has_option("gpg", "home"):
 134         gpg_home = ini.get("gpg", "home")
 135
 136 if ini.has_option("gpg", "keyid"):
 137         gpg_keyid = ini.get("gpg", "keyid")
 138
 139 if ini.has_option("gpg", "comment"):
 140         gpg_comment = ini.get("gpg", "comment")
 141
 142 if ini.has_option("gpg", "passfile"):
 143         gpg_passfile = ini.get("gpg", "passfile")
 144
 145 enable_kmod_archive = True
 146
 147
 148 # find targets
 149 targets = [ ]
 150
 151 if not os.path.isdir(home_dir+'/source.git'):
 152         subprocess.call(["git", "clone", "--depth=1", "--branch="+repo_branch, repo_url, home_dir+'/source.git'])
 153 else:
 154         subprocess.call(["git", "pull"], cwd = home_dir+'/source.git')
 155
 156 findtargets = subprocess.Popen([home_dir+'/dumpinfo.pl', 'targets'],
 157         stdout = subprocess.PIPE, cwd = home_dir+'/source.git')
 158
 159 while True:
 160         line = findtargets.stdout.readline()
 161         if not line:
 162                 break
 163         ta = line.strip().split(' ')
 164         targets.append(ta[0])
 165
 166
 167 # the 'change_source' setting tells the buildmaster how it should find out
 168 # about source code changes.  Here we point to the buildbot clone of pyflakes.
 169
 170 from buildbot.changes.gitpoller import GitPoller
 171 c['change_source'] = []
 172 c['change_source'].append(GitPoller(
 173         repo_url,
 174         workdir=home_dir+'/work.git', branch=repo_branch,
 175         pollinterval=300))
 176
 177 ####### SCHEDULERS
 178
 179 # Configure the Schedulers, which decide how to react to incoming changes.  In this
 180 # case, just kick off a 'basebuild' build
 181
 182 from buildbot.schedulers.basic import SingleBranchScheduler
 183 from buildbot.schedulers.forcesched import ForceScheduler
 184 from buildbot.changes import filter
 185 c['schedulers'] = []
 186 c['schedulers'].append(SingleBranchScheduler(
 187         name="all",
 188         change_filter=filter.ChangeFilter(branch=repo_branch),
 189         treeStableTimer=60,
 190         builderNames=targets))
 191
 192 c['schedulers'].append(ForceScheduler(
 193         name="force",
 194         builderNames=targets))
 195
 196 ####### BUILDERS
 197
 198 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
 199 # what steps, and which slaves can execute them.  Note that any particular build will
 200 # only take place on one slave.
 201
 202 from buildbot.process.factory import BuildFactory
 203 from buildbot.steps.source.git import Git
 204 from buildbot.steps.shell import ShellCommand
 205 from buildbot.steps.shell import SetProperty
 206 from buildbot.steps.transfer import FileUpload
 207 from buildbot.steps.transfer import FileDownload
 208 from buildbot.steps.master import MasterShellCommand
 209 from buildbot.process.properties import WithProperties
 210 from buildbot.process import properties
 211
 212
 213 CleanTargetMap = [
 214         [ "tools",      "tools/clean"                   ],
 215         [ "chain",      "toolchain/clean"               ],
 216         [ "linux",      "target/linux/clean"    ],
 217         [ "dir",        "dirclean"                              ],
 218         [ "dist",       "distclean"                             ]
 219 ]
 220
 221 def IsMakeCleanRequested(pattern):
 222         def CheckCleanProperty(step):
 223                 val = step.getProperty("clean")
 224                 if val and re.match(pattern, val):
 225                         return True
 226                 else:
 227                         return False
 228
 229         return CheckCleanProperty
 230
 231 def IsCleanupRequested(step):
 232         val = step.getProperty("slavename")
 233         if val and do_cleanup[val]:
 234                 return True
 235         else:
 236                 return False
 237
 238 def IsTaggingRequested(step):
 239         val = step.getProperty("tag")
 240         if val and re.match("^[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", val):
 241                 return True
 242         else:
 243                 return False
 244
 245 def IsNoTaggingRequested(step):
 246         return not IsTaggingRequested(step)
 247
 248 def IsNoMasterBuild(step):
 249         return repo_branch != "master"
 250
 251 def IsCleanupConfigured(step):
 252         slave = step.getProperty("slavename")
 253         if slave and slave in do_cleanup:
 254                 return do_cleanup[slave] > 0
 255         else:
 256                 return False
 257
 258 def GetBaseVersion(props):
 259         if re.match("^[^-]+-[0-9]+\.[0-9]+$", repo_branch):
 260                 return repo_branch.split('-')[1]
 261         else:
 262                 return "master"
 263
 264 def GetVersionPrefix(props):
 265         basever = GetBaseVersion(props)
 266         if props.hasProperty("tag") and re.match("^[0-9]+\.[0-9]+\.[0-9]+(?:-rc[0-9]+)?$", props["tag"]):
 267                 return "%s/" % props["tag"]
 268         elif basever != "master":
 269                 return "%s-SNAPSHOT/" % basever
 270         else:
 271                 return ""
 272
 273 def GetNumJobs(props):
 274         if props.hasProperty("slavename") and props.hasProperty("nproc"):
 275                 return ((int(props["nproc"]) / (max_builds[props["slavename"]] + other_builds)) + 1)
 276         else:
 277                 return 1
 278
 279 def GetCC(props):
 280         if props.hasProperty("cc_command"):
 281                 return props["cc_command"]
 282         else:
 283                 return "gcc"
 284
 285 def GetCXX(props):
 286         if props.hasProperty("cxx_command"):
 287                 return props["cxx_command"]
 288         else:
 289                 return "g++"
 290
 291 def GetCwd(props):
 292         if props.hasProperty("builddir"):
 293                 return props["builddir"]
 294         elif props.hasProperty("workdir"):
 295                 return props["workdir"]
 296         else:
 297                 return "/"
 298
 299 def GetNextBuild(builder, requests):
 300         for r in requests:
 301                 if r.properties and r.properties.hasProperty("tag"):
 302                         return r
 303         return requests[0]
 304
 305 def MakeEnv(overrides=None):
 306         env = {
 307                 'CC': WithProperties("%(cc)s", cc=GetCC),
 308                 'CXX': WithProperties("%(cxx)s", cxx=GetCXX),
 309                 'CCACHE_BASEDIR': WithProperties("%(cwd)s", cwd=GetCwd)
 310         }
 311         if overrides is not None:
 312                 env.update(overrides)
 313         return env
 314
 315 @properties.renderer
 316 def NetLockDl(props):
 317         lock = None
 318         if props.hasProperty("dl_lock"):
 319                 lock = NetLocks[props["dl_lock"]]
 320         if lock is not None:
 321                 return [lock.access('exclusive')]
 322         else:
 323                 return []
 324
 325 @properties.renderer
 326 def NetLockUl(props):
 327         lock = None
 328         if props.hasProperty("ul_lock"):
 329                 lock = NetLocks[props["ul_lock"]]
 330         if lock is not None:
 331                 return [lock.access('exclusive')]
 332         else:
 333                 return []
 334
 335 c['builders'] = []
 336
 337 dlLock = locks.SlaveLock("slave_dl")
 338
 339 checkBuiltin = re.sub('[\t\n ]+', ' ', """
 340         checkBuiltin() {
 341                 local symbol op path file;
 342                 for file in $CHANGED_FILES; do
 343                         case "$file" in
 344                                 package/*/*) : ;;
 345                                 *) return 0 ;;
 346                         esac;
 347                 done;
 348                 while read symbol op path; do
 349                         case "$symbol" in package-*)
 350                                 symbol="${symbol##*(}";
 351                                 symbol="${symbol%)}";
 352                                 for file in $CHANGED_FILES; do
 353                                         case "$file" in "package/$path/"*)
 354                                                 grep -qsx "$symbol=y" .config && return 0
 355                                         ;; esac;
 356                                 done;
 357                         esac;
 358                 done < tmp/.packagedeps;
 359                 return 1;
 360         }
 361 """).strip()
 362
 363
 364 class IfBuiltinShellCommand(ShellCommand):
 365         def _quote(self, str):
 366                 if re.search("[^a-zA-Z0-9/_.-]", str):
 367                         return "'%s'" %(re.sub("'", "'\"'\"'", str))
 368                 return str
 369
 370         def setCommand(self, command):
 371                 if not isinstance(command, (str, unicode)):
 372                         command = ' '.join(map(self._quote, command))
 373                 self.command = [
 374                         '/bin/sh', '-c',
 375                         '%s; if checkBuiltin; then %s; else exit 0; fi' %(checkBuiltin, command)
 376                 ]
 377
 378         def setupEnvironment(self, cmd):
 379                 slaveEnv = self.slaveEnvironment
 380                 if slaveEnv is None:
 381                         slaveEnv = { }
 382                 changedFiles = { }
 383                 for request in self.build.requests:
 384                         for source in request.sources:
 385                                 for change in source.changes:
 386                                         for file in change.files:
 387                                                 changedFiles[file] = True
 388                 fullSlaveEnv = slaveEnv.copy()
 389                 fullSlaveEnv['CHANGED_FILES'] = ' '.join(changedFiles.keys())
 390                 cmd.args['env'] = fullSlaveEnv
 391
 392 slaveNames = [ ]
 393
 394 for slave in c['slaves']:
 395         slaveNames.append(slave.slavename)
 396
 397 for target in targets:
 398         ts = target.split('/')
 399
 400         factory = BuildFactory()
 401
 402         # find number of cores
 403         factory.addStep(SetProperty(
 404                 name = "nproc",
 405                 property = "nproc",
 406                 description = "Finding number of CPUs",
 407                 command = ["nproc"]))
 408
 409         # find gcc and g++ compilers
 410         if cc_version is not None:
 411                 factory.addStep(FileDownload(
 412                         mastersrc = "findbin.pl",
 413                         slavedest = "../findbin.pl",
 414                         mode = 0755))
 415
 416                 factory.addStep(SetProperty(
 417                         name = "gcc",
 418                         property = "cc_command",
 419                         description = "Finding gcc command",
 420                         command = ["../findbin.pl", "gcc", cc_version[0], cc_version[1]],
 421                         haltOnFailure = True))
 422
 423                 factory.addStep(SetProperty(
 424                         name = "g++",
 425                         property = "cxx_command",
 426                         description = "Finding g++ command",
 427                         command = ["../findbin.pl", "g++", cc_version[0], cc_version[1]],
 428                         haltOnFailure = True))
 429
 430         # expire tree if needed
 431         if tree_expire > 0:
 432                 factory.addStep(FileDownload(
 433                         mastersrc = "expire.sh",
 434                         slavedest = "../expire.sh",
 435                         mode = 0755))
 436
 437                 factory.addStep(ShellCommand(
 438                         name = "expire",
 439                         description = "Checking for build tree expiry",
 440                         command = ["./expire.sh", str(tree_expire)],
 441                         workdir = ".",
 442                         haltOnFailure = True,
 443                         timeout = 2400))
 444
 445         # cleanup.sh if needed
 446         factory.addStep(FileDownload(
 447                 mastersrc = "cleanup.sh",
 448                 slavedest = "../cleanup.sh",
 449                 mode = 0755,
 450                 doStepIf = IsCleanupRequested))
 451
 452         factory.addStep(ShellCommand(
 453                 name = "cleanold",
 454                 description = "Cleaning previous builds",
 455                 command = ["./cleanup.sh", c['buildbotURL'], WithProperties("%(slavename)s"), WithProperties("%(buildername)s"), "full"],
 456                 workdir = ".",
 457                 haltOnFailure = True,
 458                 doStepIf = IsCleanupRequested,
 459                 timeout = 2400))
 460
 461         factory.addStep(ShellCommand(
 462                 name = "cleanup",
 463                 description = "Cleaning work area",
 464                 command = ["./cleanup.sh", c['buildbotURL'], WithProperties("%(slavename)s"), WithProperties("%(buildername)s"), "single"],
 465                 workdir = ".",
 466                 haltOnFailure = True,
 467                 doStepIf = IsCleanupRequested,
 468                 timeout = 2400))
 469
 470         # user-requested clean targets
 471         for tuple in CleanTargetMap:
 472                 factory.addStep(ShellCommand(
 473                         name = tuple[1],
 474                         description = 'User-requested "make %s"' % tuple[1],
 475                         command = ["make", tuple[1], "V=s"],
 476                         env = MakeEnv(),
 477                         doStepIf = IsMakeCleanRequested(tuple[0])
 478                 ))
 479
 480         # switch to branch
 481         factory.addStep(ShellCommand(
 482                 name = "switchbranch",
 483                 description = "Checking out Git branch",
 484                 command = "if [ -d .git ]; then git fetch && git checkout '%s'; else exit 0; fi" % repo_branch,
 485                 haltOnFailure = True,
 486                 doStepIf = IsNoTaggingRequested,
 487                 locks = NetLockDl,
 488         ))
 489
 490         # check out the source
 491         factory.addStep(Git(
 492                 repourl = repo_url,
 493                 branch = repo_branch,
 494                 mode = 'incremental',
 495                 method = 'clean',
 496                 locks = NetLockDl,
 497         ))
 498
 499         # update remote refs
 500         factory.addStep(ShellCommand(
 501                 name = "fetchrefs",
 502                 description = "Fetching Git remote refs",
 503                 command = ["git", "fetch", "origin", "+refs/heads/%s:refs/remotes/origin/%s" %(repo_branch, repo_branch)],
 504                 haltOnFailure = True
 505         ))
 506
 507         # fetch tags
 508         factory.addStep(ShellCommand(
 509                 name = "fetchtag",
 510                 description = "Fetching Git tags",
 511                 command = ["git", "fetch", "--tags", "--", repo_url],
 512                 haltOnFailure = True,
 513                 doStepIf = IsTaggingRequested,
 514                 locks = NetLockDl,
 515         ))
 516
 517         # switch to tag
 518         factory.addStep(ShellCommand(
 519                 name = "switchtag",
 520                 description = "Checking out Git tag",
 521                 command = ["git", "checkout", WithProperties("tags/v%(tag:-)s")],
 522                 haltOnFailure = True,
 523                 doStepIf = IsTaggingRequested
 524         ))
 525
 526         factory.addStep(ShellCommand(
 527                 name = "rmtmp",
 528                 description = "Remove tmp folder",
 529                 command=["rm", "-rf", "tmp/"]))
 530
 531         # feed
 532 #       factory.addStep(ShellCommand(
 533 #               name = "feedsconf",
 534 #               description = "Copy the feeds.conf",
 535 #               command='''cp ~/feeds.conf ./feeds.conf''' ))
 536
 537         # feed
 538         factory.addStep(ShellCommand(
 539                 name = "rmfeedlinks",
 540                 description = "Remove feed symlinks",
 541                 command=["rm", "-rf", "package/feeds/"]))
 542
 543         # feed
 544         factory.addStep(ShellCommand(
 545                 name = "updatefeeds",
 546                 description = "Updating feeds",
 547                 command=["./scripts/feeds", "update"],
 548                 env = MakeEnv(),
 549                 locks = NetLockDl,
 550         ))
 551
 552         # feed
 553         factory.addStep(ShellCommand(
 554                 name = "installfeeds",
 555                 description = "Installing feeds",
 556                 command=["./scripts/feeds", "install", "-a"],
 557                 env = MakeEnv()))
 558
 559         # seed config
 560         factory.addStep(FileDownload(
 561                 mastersrc = "config.seed",
 562                 slavedest = ".config",
 563                 mode = 0644
 564         ))
 565
 566         # configure
 567         factory.addStep(ShellCommand(
 568                 name = "newconfig",
 569                 description = "Seeding .config",
 570                 command = "printf 'CONFIG_TARGET_%s=y\\nCONFIG_TARGET_%s_%s=y\\n' >> .config" %(ts[0], ts[0], ts[1])
 571         ))
 572
 573         factory.addStep(ShellCommand(
 574                 name = "delbin",
 575                 description = "Removing output directory",
 576                 command = ["rm", "-rf", "bin/"]
 577         ))
 578
 579         factory.addStep(ShellCommand(
 580                 name = "defconfig",
 581                 description = "Populating .config",
 582                 command = ["make", "defconfig"],
 583                 env = MakeEnv()
 584         ))
 585
 586         # check arch
 587         factory.addStep(ShellCommand(
 588                 name = "checkarch",
 589                 description = "Checking architecture",
 590                 command = ["grep", "-sq", "CONFIG_TARGET_%s=y" %(ts[0]), ".config"],
 591                 logEnviron = False,
 592                 want_stdout = False,
 593                 want_stderr = False,
 594                 haltOnFailure = True
 595         ))
 596
 597         # find libc suffix
 598         factory.addStep(SetProperty(
 599                 name = "libc",
 600                 property = "libc",
 601                 description = "Finding libc suffix",
 602                 command = ["sed", "-ne", '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }', ".config"]))
 603
 604         # ccache helper
 605         factory.addStep(FileDownload(
 606                 mastersrc = "ccache.sh",
 607                 slavedest = "ccache.sh",
 608                 mode = 0755
 609         ))
 610
 611         # ccache prepare
 612         factory.addStep(ShellCommand(
 613                 name = "prepccache",
 614                 description = "Preparing ccache",
 615                 command = ["./ccache.sh"]
 616         ))
 617
 618         # install build key
 619         factory.addStep(FileDownload(mastersrc=home_dir+'/key-build', slavedest="key-build", mode=0600))
 620         factory.addStep(FileDownload(mastersrc=home_dir+'/key-build.pub', slavedest="key-build.pub", mode=0600))
 621
 622         # prepare dl
 623         factory.addStep(ShellCommand(
 624                 name = "dldir",
 625                 description = "Preparing dl/",
 626                 command = "mkdir -p $HOME/dl && rm -rf ./dl && ln -sf $HOME/dl ./dl",
 627                 logEnviron = False,
 628                 want_stdout = False
 629         ))
 630
 631         # prepare tar
 632         factory.addStep(ShellCommand(
 633                 name = "dltar",
 634                 description = "Building GNU tar",
 635                 command = ["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "tools/tar/compile", "V=s"],
 636                 env = MakeEnv(),
 637                 haltOnFailure = True
 638         ))
 639
 640         # populate dl
 641         factory.addStep(ShellCommand(
 642                 name = "dlrun",
 643                 description = "Populating dl/",
 644                 command = ["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "download", "V=s"],
 645                 env = MakeEnv(),
 646                 logEnviron = False,
 647                 locks = properties.FlattenList(NetLockDl, [dlLock.access('exclusive')]),
 648         ))
 649
 650         factory.addStep(ShellCommand(
 651                 name = "cleanbase",
 652                 description = "Cleaning base-files",
 653                 command=["make", "package/base-files/clean", "V=s"]
 654         ))
 655
 656         # build
 657         factory.addStep(ShellCommand(
 658                 name = "tools",
 659                 description = "Building tools",
 660                 command = ["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "tools/install", "V=s"],
 661                 env = MakeEnv(),
 662                 haltOnFailure = True
 663         ))
 664
 665         factory.addStep(ShellCommand(
 666                 name = "toolchain",
 667                 description = "Building toolchain",
 668                 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "toolchain/install", "V=s"],
 669                 env = MakeEnv(),
 670                 haltOnFailure = True
 671         ))
 672
 673         factory.addStep(ShellCommand(
 674                 name = "kmods",
 675                 description = "Building kmods",
 676                 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
 677                 env = MakeEnv(),
 678                 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
 679                 haltOnFailure = True
 680         ))
 681
 682         # find kernel version
 683         factory.addStep(SetProperty(
 684                 name = "kernelversion",
 685                 property = "kernelversion",
 686                 description = "Finding the effective Kernel version",
 687                 command = "make --no-print-directory -C target/linux/ val.LINUX_VERSION val.LINUX_RELEASE val.LINUX_VERMAGIC | xargs printf '%s-%s-%s'",
 688                 env = { 'TOPDIR': WithProperties("%(cwd)s/build", cwd=GetCwd) }
 689         ))
 690
 691         factory.addStep(ShellCommand(
 692                 name = "pkgclean",
 693                 description = "Cleaning up package build",
 694                 command=["make", "package/cleanup", "V=s"]
 695         ))
 696
 697         factory.addStep(ShellCommand(
 698                 name = "pkgbuild",
 699                 description = "Building packages",
 700                 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
 701                 env = MakeEnv(),
 702                 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
 703                 haltOnFailure = True
 704         ))
 705
 706         # factory.addStep(IfBuiltinShellCommand(
 707         factory.addStep(ShellCommand(
 708                 name = "pkginstall",
 709                 description = "Installing packages",
 710                 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "package/install", "V=s"],
 711                 env = MakeEnv(),
 712                 haltOnFailure = True
 713         ))
 714
 715         factory.addStep(ShellCommand(
 716                 name = "pkgindex",
 717                 description = "Indexing packages",
 718                 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "package/index", "V=s"],
 719                 env = MakeEnv(),
 720                 haltOnFailure = True
 721         ))
 722
 723         if enable_kmod_archive:
 724                 factory.addStep(ShellCommand(
 725                         name = "kmoddir",
 726                         description = "Creating kmod directory",
 727                         command=["mkdir", "-p", WithProperties("bin/targets/%s/%s%%(libc)s/kmods/%%(kernelversion)s" %(ts[0], ts[1]))],
 728                         haltOnFailure = True
 729                 ))
 730
 731                 factory.addStep(ShellCommand(
 732                         name = "kmodprepare",
 733                         description = "Preparing kmod archive",
 734                         command=["rsync", "--include=/kmod-*.ipk", "--exclude=*", "-va",
 735                                  WithProperties("bin/targets/%s/%s%%(libc)s/packages/" %(ts[0], ts[1])),
 736                                  WithProperties("bin/targets/%s/%s%%(libc)s/kmods/%%(kernelversion)s/" %(ts[0], ts[1]))],
 737                         haltOnFailure = True
 738                 ))
 739
 740                 factory.addStep(ShellCommand(
 741                         name = "kmodindex",
 742                         description = "Indexing kmod archive",
 743                         command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "package/index", "V=s",
 744                                  WithProperties("PACKAGE_SUBDIRS=bin/targets/%s/%s%%(libc)s/kmods/%%(kernelversion)s/" %(ts[0], ts[1]))],
 745                         env = MakeEnv(),
 746                         haltOnFailure = True
 747                 ))
 748
 749                 # find rootfs staging directory
 750                 factory.addStep(SetProperty(
 751                         name = "stageroot",
 752                         property = "stageroot",
 753                         description = "Finding the rootfs staging directory",
 754                         command=["make", "--no-print-directory", "val.STAGING_DIR_ROOT"],
 755                         env = { 'TOPDIR': WithProperties("%(cwd)s/build", cwd=GetCwd) }
 756                 ))
 757
 758                 factory.addStep(ShellCommand(
 759                         name = "filesdir",
 760                         description = "Creating file overlay directory",
 761                         command=["mkdir", "-p", "files/etc/opkg"],
 762                         haltOnFailure = True
 763                 ))
 764
 765                 factory.addStep(ShellCommand(
 766                         name = "kmodconfig",
 767                         description = "Embedding kmod repository configuration",
 768                         command=WithProperties("sed -e 's#^\\(src/gz .*\\)_core \\(.*\\)/packages$#&\\n\\1_kmods \\2/kmods/%(kernelversion)s#' " +
 769                                                "%(stageroot)s/etc/opkg/distfeeds.conf > files/etc/opkg/distfeeds.conf"),
 770                         haltOnFailure = True
 771                 ))
 772
 773         #factory.addStep(IfBuiltinShellCommand(
 774         factory.addStep(ShellCommand(
 775                 name = "images",
 776                 description = "Building images",
 777                 command=["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "target/install", "V=s"],
 778                 env = MakeEnv(),
 779                 haltOnFailure = True
 780         ))
 781
 782         factory.addStep(ShellCommand(
 783                 name = "diffconfig",
 784                 description = "Generating config.seed",
 785                 command=["make", "-j1", "diffconfig", "V=s"],
 786                 env = MakeEnv(),
 787                 haltOnFailure = True
 788         ))
 789
 790         factory.addStep(ShellCommand(
 791                 name = "checksums",
 792                 description = "Calculating checksums",
 793                 command=["make", "-j1", "checksum", "V=s"],
 794                 env = MakeEnv(),
 795                 haltOnFailure = True
 796         ))
 797
 798         # sign
 799         if gpg_keyid is not None:
 800                 factory.addStep(MasterShellCommand(
 801                         name = "signprepare",
 802                         description = "Preparing temporary signing directory",
 803                         command = ["mkdir", "-p", "%s/signing" %(home_dir)],
 804                         haltOnFailure = True
 805                 ))
 806
 807                 factory.addStep(ShellCommand(
 808                         name = "signpack",
 809                         description = "Packing files to sign",
 810                         command = WithProperties("find bin/targets/%s/%s%%(libc)s/ bin/targets/%s/%s%%(libc)s/kmods/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz" %(ts[0], ts[1], ts[0], ts[1])),
 811                         haltOnFailure = True
 812                 ))
 813
 814                 factory.addStep(FileUpload(
 815                         slavesrc = "sign.tar.gz",
 816                         masterdest = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
 817                         haltOnFailure = True
 818                 ))
 819
 820                 factory.addStep(MasterShellCommand(
 821                         name = "signfiles",
 822                         description = "Signing files",
 823                         command = ["%s/signall.sh" %(home_dir), "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]), gpg_keyid, gpg_comment],
 824                         env = {'GNUPGHOME': gpg_home, 'PASSFILE': gpg_passfile},
 825                         haltOnFailure = True
 826                 ))
 827
 828                 factory.addStep(FileDownload(
 829                         mastersrc = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
 830                         slavedest = "sign.tar.gz",
 831                         haltOnFailure = True
 832                 ))
 833
 834                 factory.addStep(ShellCommand(
 835                         name = "signunpack",
 836                         description = "Unpacking signed files",
 837                         command = ["tar", "-xzf", "sign.tar.gz"],
 838                         haltOnFailure = True
 839                 ))
 840
 841         # upload
 842         factory.addStep(ShellCommand(
 843                 name = "dirprepare",
 844                 description = "Preparing upload directory structure",
 845                 command = ["mkdir", "-p", WithProperties("tmp/upload/%%(prefix)stargets/%s/%s" %(ts[0], ts[1]), prefix=GetVersionPrefix)],
 846                 haltOnFailure = True
 847         ))
 848
 849         factory.addStep(ShellCommand(
 850                 name = "linkprepare",
 851                 description = "Preparing repository symlink",
 852                 command = ["ln", "-s", "-f", WithProperties("../packages-%(basever)s", basever=GetBaseVersion), WithProperties("tmp/upload/%(prefix)spackages", prefix=GetVersionPrefix)],
 853                 doStepIf = IsNoMasterBuild,
 854                 haltOnFailure = True
 855         ))
 856
 857         if enable_kmod_archive:
 858                 factory.addStep(ShellCommand(
 859                         name = "kmoddirprepare",
 860                         description = "Preparing kmod archive upload directory",
 861                         command = ["mkdir", "-p", WithProperties("tmp/upload/%%(prefix)stargets/%s/%s/kmods/%%(kernelversion)s" %(ts[0], ts[1]), prefix=GetVersionPrefix)],
 862                         haltOnFailure = True
 863                 ))
 864
 865         factory.addStep(ShellCommand(
 866                 name = "dirupload",
 867                 description = "Uploading directory structure",
 868                 command = ["rsync", "-4", "-avz", "tmp/upload/", "%s/" %(rsync_bin_url)],
 869                 env={'RSYNC_PASSWORD': rsync_bin_key},
 870                 haltOnFailure = True,
 871                 logEnviron = False,
 872                 locks = NetLockUl,
 873         ))
 874
 875         factory.addStep(ShellCommand(
 876                 name = "targetupload",
 877                 description = "Uploading target files",
 878                 command=["rsync", "-4", "--exclude=/kmods/", "--progress", "--delete", "--size-only", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]),
 879                          "-avz", WithProperties("bin/targets/%s/%s%%(libc)s/" %(ts[0], ts[1])),
 880                          WithProperties("%s/%%(prefix)stargets/%s/%s/" %(rsync_bin_url, ts[0], ts[1]), prefix=GetVersionPrefix)],
 881                 env={'RSYNC_PASSWORD': rsync_bin_key},
 882                 haltOnFailure = True,
 883                 logEnviron = False,
 884                 locks = NetLockUl,
 885         ))
 886
 887         if enable_kmod_archive:
 888                 factory.addStep(ShellCommand(
 889                         name = "kmodupload",
 890                         description = "Uploading kmod archive",
 891                         command=["rsync", "-4", "--progress", "--delete", "--size-only", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]),
 892                                  "-avz", WithProperties("bin/targets/%s/%s%%(libc)s/kmods/%%(kernelversion)s/" %(ts[0], ts[1])),
 893                                  WithProperties("%s/%%(prefix)stargets/%s/%s/kmods/%%(kernelversion)s/" %(rsync_bin_url, ts[0], ts[1]), prefix=GetVersionPrefix)],
 894                         env={'RSYNC_PASSWORD': rsync_bin_key},
 895                         haltOnFailure = True,
 896                         logEnviron = False,
 897                         locks = NetLockUl,
 898                 ))
 899
 900         if rsync_src_url is not None:
 901                 factory.addStep(ShellCommand(
 902                         name = "sourceupload",
 903                         description = "Uploading source archives",
 904                         command=["rsync", "-4", "--progress", "--size-only", "--delay-updates",
 905                                  WithProperties("--partial-dir=.~tmp~%s~%s~%%(slavename)s" %(ts[0], ts[1])), "-avz", "dl/", "%s/" %(rsync_src_url)],
 906                         env={'RSYNC_PASSWORD': rsync_src_key},
 907                         haltOnFailure = True,
 908                         logEnviron = False,
 909                         locks = NetLockUl,
 910                 ))
 911
 912         if False:
 913                 factory.addStep(ShellCommand(
 914                         name = "packageupload",
 915                         description = "Uploading package files",
 916                         command=["rsync", "-4", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "bin/packages/", "%s/packages/" %(rsync_bin_url)],
 917                         env={'RSYNC_PASSWORD': rsync_bin_key},
 918                         haltOnFailure = False,
 919                         logEnviron = False,
 920                         locks = NetLockUl,
 921                 ))
 922
 923         # logs
 924         if False:
 925                 factory.addStep(ShellCommand(
 926                         name = "upload",
 927                         description = "Uploading logs",
 928                         command=["rsync", "-4", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "logs/", "%s/logs/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
 929                         env={'RSYNC_PASSWORD': rsync_bin_key},
 930                         haltOnFailure = False,
 931                         alwaysRun = True,
 932                         logEnviron = False,
 933                         locks = NetLockUl,
 934                 ))
 935
 936         factory.addStep(ShellCommand(
 937                 name = "df",
 938                 description = "Reporting disk usage",
 939                 command=["df", "-h", "."],
 940                 env={'LC_ALL': 'C'},
 941                 haltOnFailure = False,
 942                 alwaysRun = True
 943         ))
 944
 945         from buildbot.config import BuilderConfig
 946
 947         c['builders'].append(BuilderConfig(name=target, slavenames=slaveNames, factory=factory, nextBuild=GetNextBuild))
 948
 949
 950 ####### STATUS TARGETS
 951
 952 # 'status' is a list of Status Targets. The results of each build will be
 953 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
 954 # including web pages, email senders, and IRC bots.
 955
 956 c['status'] = []
 957
 958 from buildbot.status import html
 959 from buildbot.status.web import authz, auth
 960
 961 if ini.has_option("status", "bind"):
 962         if ini.has_option("status", "user") and ini.has_option("status", "password"):
 963                 authz_cfg=authz.Authz(
 964                         # change any of these to True to enable; see the manual for more
 965                         # options
 966                         auth=auth.BasicAuth([(ini.get("status", "user"), ini.get("status", "password"))]),
 967                         gracefulShutdown = 'auth',
 968                         forceBuild = 'auth', # use this to test your slave once it is set up
 969                         forceAllBuilds = 'auth',
 970                         pingBuilder = False,
 971                         stopBuild = 'auth',
 972                         stopAllBuilds = 'auth',
 973                         cancelPendingBuild = 'auth',
 974                 )
 975                 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind"), authz=authz_cfg))
 976         else:
 977                 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind")))
 978
 979
 980 from buildbot.status import words
 981
 982 if ini.has_option("irc", "host") and ini.has_option("irc", "nickname") and ini.has_option("irc", "channel"):
 983         irc_host = ini.get("irc", "host")
 984         irc_port = 6667
 985         irc_chan = ini.get("irc", "channel")
 986         irc_nick = ini.get("irc", "nickname")
 987         irc_pass = None
 988
 989         if ini.has_option("irc", "port"):
 990                 irc_port = ini.getint("irc", "port")
 991
 992         if ini.has_option("irc", "password"):
 993                 irc_pass = ini.get("irc", "password")
 994
 995         irc = words.IRC(irc_host, irc_nick, port = irc_port, password = irc_pass,
 996                         channels = [{ "channel": irc_chan }],
 997                         notify_events = {
 998                           'exception': 1,
 999                           'successToFailure': 1,
1000                           'failureToSuccess': 1
1001                         }
1002         )
1003
1004         c['status'].append(irc)
1005
1006 ####### DB URL
1007
1008 c['db'] = {
1009         # This specifies what database buildbot uses to store its state.  You can leave
1010         # this at its default for all but the largest installations.
1011         'db_url' : "sqlite:///state.sqlite",
1012 }