phase1/master.cfg

   1 # -*- python -*-
   2 # ex: set syntax=python:
   3
   4 import os
   5 import re
   6 import subprocess
   7 import ConfigParser
   8
   9 from buildbot import locks
  10
  11 # This is a sample buildmaster config file. It must be installed as
  12 # 'master.cfg' in your buildmaster's base directory.
  13
  14 ini = ConfigParser.ConfigParser()
  15 ini.read("./config.ini")
  16
  17 # This is the dictionary that the buildmaster pays attention to. We also use
  18 # a shorter alias to save typing.
  19 c = BuildmasterConfig = {}
  20
  21 ####### BUILDSLAVES
  22
  23 # The 'slaves' list defines the set of recognized buildslaves. Each element is
  24 # a BuildSlave object, specifying a unique slave name and password.  The same
  25 # slave name and password must be configured on the slave.
  26 from buildbot.buildslave import BuildSlave
  27
  28 c['slaves'] = []
  29
  30 for section in ini.sections():
  31         if section.startswith("slave "):
  32                 if ini.has_option(section, "name") and ini.has_option(section, "password"):
  33                         name = ini.get(section, "name")
  34                         password = ini.get(section, "password")
  35                         max_builds = 1
  36                         if ini.has_option(section, "builds"):
  37                                 max_builds = ini.getint(section, "builds")
  38                         c['slaves'].append(BuildSlave(name, password, max_builds = max_builds))
  39
  40 # 'slavePortnum' defines the TCP port to listen on for connections from slaves.
  41 # This must match the value configured into the buildslaves (with their
  42 # --master option)
  43 c['slavePortnum'] = 9989
  44
  45 # coalesce builds
  46 c['mergeRequests'] = True
  47
  48 ####### CHANGESOURCES
  49
  50 home_dir = os.path.abspath(ini.get("general", "homedir"))
  51
  52 repo_url = ini.get("repo", "url")
  53
  54 rsync_url = ini.get("rsync", "url")
  55 rsync_key = ini.get("rsync", "password")
  56
  57 # find targets
  58 targets = [ ]
  59
  60 findtargets = subprocess.Popen([home_dir+'/dumpinfo.pl', 'targets'],
  61         stdout = subprocess.PIPE, cwd = home_dir+'/source.git')
  62
  63 while True:
  64         line = findtargets.stdout.readline()
  65         if not line:
  66                 break
  67         ta = line.strip().split(' ')
  68         targets.append(ta[0])
  69
  70
  71 # the 'change_source' setting tells the buildmaster how it should find out
  72 # about source code changes.  Here we point to the buildbot clone of pyflakes.
  73
  74 from buildbot.changes.gitpoller import GitPoller
  75 c['change_source'] = []
  76 c['change_source'].append(GitPoller(
  77         repo_url,
  78         workdir=home_dir+'/source.git', branch='master',
  79         pollinterval=300))
  80
  81 ####### SCHEDULERS
  82
  83 # Configure the Schedulers, which decide how to react to incoming changes.  In this
  84 # case, just kick off a 'basebuild' build
  85
  86 from buildbot.schedulers.basic import SingleBranchScheduler
  87 from buildbot.schedulers.forcesched import ForceScheduler
  88 from buildbot.changes import filter
  89 c['schedulers'] = []
  90 c['schedulers'].append(SingleBranchScheduler(
  91         name="all",
  92         change_filter=filter.ChangeFilter(branch='master'),
  93         treeStableTimer=60,
  94         builderNames=targets))
  95
  96 c['schedulers'].append(ForceScheduler(
  97         name="force",
  98         builderNames=targets))
  99
 100 ####### BUILDERS
 101
 102 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
 103 # what steps, and which slaves can execute them.  Note that any particular build will
 104 # only take place on one slave.
 105
 106 from buildbot.process.factory import BuildFactory
 107 from buildbot.steps.source import Git
 108 from buildbot.steps.shell import ShellCommand
 109 from buildbot.steps.transfer import FileDownload
 110
 111
 112 MakeTargetMap = {
 113         "^tools/":                              "tools/clean",
 114         "^toolchain/":                  "toolchain/clean",
 115         "^target/linux/":               "target/linux/clean",
 116         "^(config|include)/":   "dirclean"
 117 }
 118
 119 def IsAffected(pattern):
 120         def CheckAffected(change):
 121                 for request in change.build.requests:
 122                         for source in request.sources:
 123                                 for change in source.changes:
 124                                         for file in change.files:
 125                                                 if re.match(pattern, file):
 126                                                         return True
 127                 return False
 128         return CheckAffected
 129
 130 def isPathBuiltin(path):
 131         incl = {}
 132         pkgs = {}
 133         conf = open(".config", "r")
 134
 135         while True:
 136                 line = conf.readline()
 137                 if line == '':
 138                         break
 139                 m = re.match("^(CONFIG_PACKAGE_.+?)=y", line)
 140                 if m:
 141                         incl[m.group(1)] = True
 142
 143         conf.close()
 144
 145         deps = open("tmp/.packagedeps", "r")
 146
 147         while True:
 148                 line = deps.readline()
 149                 if line == '':
 150                         break
 151                 m = re.match("^package-\$\((CONFIG_PACKAGE_.+?)\) \+= (\S+)", line)
 152                 if m and incl.get(m.group(1)) == True:
 153                         pkgs["package/%s" % m.group(2)] = True
 154
 155         deps.close()
 156
 157         while path != '':
 158                 if pkgs.get(path) == True:
 159                         return True
 160                 path = os.path.dirname(path)
 161
 162         return False
 163
 164 def isChangeBuiltin(change):
 165         return True
 166 #       for request in change.build.requests:
 167 #               for source in request.sources:
 168 #                       for change in source.changes:
 169 #                               for file in change.files:
 170 #                                       if isPathBuiltin(file):
 171 #                                               return True
 172 #       return False
 173
 174
 175 c['builders'] = []
 176
 177 dlLock = locks.SlaveLock("slave_dl")
 178
 179 checkBuiltin = re.sub('[\t\n ]+', ' ', """
 180         checkBuiltin() {
 181                 local symbol op path file;
 182                 for file in $CHANGED_FILES; do
 183                         case "$file" in
 184                                 package/*/*) : ;;
 185                                 *) return 0 ;;
 186                         esac;
 187                 done;
 188                 while read symbol op path; do
 189                         case "$symbol" in package-*)
 190                                 symbol="${symbol##*(}";
 191                                 symbol="${symbol%)}";
 192                                 for file in $CHANGED_FILES; do
 193                                         case "$file" in "package/$path/"*)
 194                                                 grep -qsx "$symbol=y" .config && return 0
 195                                         ;; esac;
 196                                 done;
 197                         esac;
 198                 done < tmp/.packagedeps;
 199                 return 1;
 200         }
 201 """).strip()
 202
 203
 204 class IfBuiltinShellCommand(ShellCommand):
 205         def _quote(self, str):
 206                 if re.search("[^a-zA-Z0-9/_.-]", str):
 207                         return "'%s'" %(re.sub("'", "'\"'\"'", str))
 208                 return str
 209
 210         def setCommand(self, command):
 211                 if not isinstance(command, (str, unicode)):
 212                         command = ' '.join(map(self._quote, command))
 213                 self.command = [
 214                         '/bin/sh', '-c',
 215                         '%s; if checkBuiltin; then %s; else exit 0; fi' %(checkBuiltin, command)
 216                 ]
 217
 218         def setupEnvironment(self, cmd):
 219                 slaveEnv = self.slaveEnvironment
 220                 if slaveEnv is None:
 221                         slaveEnv = { }
 222                 changedFiles = { }
 223                 for request in self.build.requests:
 224                         for source in request.sources:
 225                                 for change in source.changes:
 226                                         for file in change.files:
 227                                                 changedFiles[file] = True
 228                 fullSlaveEnv = slaveEnv.copy()
 229                 fullSlaveEnv['CHANGED_FILES'] = ' '.join(changedFiles.keys())
 230                 cmd.args['env'] = fullSlaveEnv
 231
 232 slaveNames = [ ]
 233
 234 for slave in c['slaves']:
 235         slaveNames.append(slave.slavename)
 236
 237 for target in targets:
 238         ts = target.split('/')
 239
 240         factory = BuildFactory()
 241
 242         # check out the source
 243         factory.addStep(Git(repourl=repo_url, mode='update'))
 244
 245         factory.addStep(ShellCommand(
 246                 name = "rmtmp",
 247                 description = "Remove tmp folder",
 248                 command=["rm", "-rf", "tmp/"]))
 249
 250         # feed
 251 #       factory.addStep(ShellCommand(
 252 #               name = "feedsconf",
 253 #               description = "Copy the feeds.conf",
 254 #               command='''cp ~/feeds.conf ./feeds.conf''' ))
 255
 256         # feed
 257         factory.addStep(ShellCommand(
 258                 name = "updatefeeds",
 259                 description = "Updating feeds",
 260                 command=["./scripts/feeds", "update"]))
 261
 262         # feed
 263         factory.addStep(ShellCommand(
 264                 name = "installfeeds",
 265                 description = "Installing feeds",
 266                 command=["./scripts/feeds", "install", "-a"]))
 267
 268         # configure
 269         factory.addStep(ShellCommand(
 270                 name = "newconfig",
 271                 description = "Seeding .config",
 272                 command='''cat <<EOT > .config
 273 CONFIG_TARGET_%s=y
 274 CONFIG_TARGET_%s_%s=y
 275 CONFIG_ALL_NONSHARED=y
 276 CONFIG_SDK=y
 277 CONFIG_IB=y
 278 # CONFIG_IB_STANDALONE is not set
 279 CONFIG_DEVEL=y
 280 CONFIG_CCACHE=y
 281 CONFIG_SIGNED_PACKAGES=y
 282 # CONFIG_PER_FEED_REPO_ADD_COMMENTED is not set
 283 CONFIG_KERNEL_KALLSYMS=y
 284 CONFIG_COLLECT_KERNEL_DEBUG=y
 285 EOT''' %(ts[0], ts[0], ts[1]) ))
 286
 287         factory.addStep(ShellCommand(
 288                 name = "delbin",
 289                 description = "Removing output directory",
 290                 command = ["rm", "-rf", "bin/"]
 291         ))
 292
 293         factory.addStep(ShellCommand(
 294                 name = "defconfig",
 295                 description = "Populating .config",
 296                 command = ["make", "defconfig"]
 297         ))
 298
 299         # check arch
 300         factory.addStep(ShellCommand(
 301                 name = "checkarch",
 302                 description = "Checking architecture",
 303                 command = ["grep", "-sq", "CONFIG_TARGET_%s=y" %(ts[0]), ".config"],
 304                 logEnviron = False,
 305                 want_stdout = False,
 306                 want_stderr = False,
 307                 haltOnFailure = True
 308         ))
 309
 310         # install build key
 311         factory.addStep(FileDownload(mastersrc=home_dir+'/key-build', slavedest="key-build", mode=0400))
 312         factory.addStep(FileDownload(mastersrc=home_dir+'/key-build.pub', slavedest="key-build.pub", mode=0400))
 313
 314         # prepare dl
 315         factory.addStep(ShellCommand(
 316                 name = "dldir",
 317                 description = "Preparing dl/",
 318                 command = "mkdir -p $HOME/dl && ln -sf $HOME/dl ./dl",
 319                 logEnviron = False,
 320                 want_stdout = False
 321         ))
 322
 323         # populate dl
 324         factory.addStep(ShellCommand(
 325                 name = "dlrun",
 326                 description = "Populating dl/",
 327                 command = ["make", "-j4", "download", "V=s"],
 328                 logEnviron = False,
 329                 locks = [dlLock.access('exclusive')]
 330         ))
 331
 332         factory.addStep(ShellCommand(
 333                 name = "cleanbase",
 334                 description = "Cleaning base-files",
 335                 command=["make", "package/base-files/clean", "V=s"]
 336         ))
 337
 338         # optional clean steps
 339         for pattern, maketarget in MakeTargetMap.items():
 340                 factory.addStep(ShellCommand(
 341                         name = maketarget,
 342                         description = maketarget,
 343                         command=["make", maketarget, "V=s"], doStepIf=IsAffected(pattern)
 344                 ))
 345
 346         # build
 347         factory.addStep(ShellCommand(
 348                 name = "tools",
 349                 description = "Building tools",
 350                 command = ["make", "-j4", "tools/install", "V=s"],
 351                 haltOnFailure = True
 352         ))
 353
 354         factory.addStep(ShellCommand(
 355                 name = "toolchain",
 356                 description = "Building toolchain",
 357                 command=["make", "-j4", "toolchain/install", "V=s"],
 358                 haltOnFailure = True
 359         ))
 360
 361         factory.addStep(ShellCommand(
 362                 name = "kmods",
 363                 description = "Building kmods",
 364                 command=["make", "-j4", "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
 365                 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
 366                 haltOnFailure = True
 367         ))
 368
 369         factory.addStep(ShellCommand(
 370                 name = "pkgbuild",
 371                 description = "Building packages",
 372                 command=["make", "-j4", "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
 373                 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
 374                 haltOnFailure = True
 375         ))
 376
 377         # factory.addStep(IfBuiltinShellCommand(
 378         factory.addStep(ShellCommand(
 379                 name = "pkginstall",
 380                 description = "Installing packages",
 381                 command=["make", "-j4", "package/install", "V=s"],
 382                 doStepIf = isChangeBuiltin,
 383                 haltOnFailure = True
 384         ))
 385
 386         factory.addStep(ShellCommand(
 387                 name = "pkgindex",
 388                 description = "Indexing packages",
 389                 command=["make", "-j4", "package/index", "V=s"],
 390                 haltOnFailure = True
 391         ))
 392
 393         #factory.addStep(IfBuiltinShellCommand(
 394         factory.addStep(ShellCommand(
 395                 name = "images",
 396                 description = "Building images",
 397                 command=["make", "-j1", "target/install", "V=s"],
 398                 doStepIf = isChangeBuiltin,
 399                 haltOnFailure = True
 400         ))
 401
 402         # upload
 403         factory.addStep(ShellCommand(
 404                 name = "uploadprepare",
 405                 description = "Preparing target directory",
 406                 command=["rsync", "-av", "--include", "/%s/" %(ts[0]), "--include", "/%s/%s/" %(ts[0], ts[1]), "--exclude", "/*", "--exclude", "/*/*", "--exclude", "/%s/%s/*" %(ts[0], ts[1]), "bin/targets/", "%s/targets/" %(rsync_url)],
 407                 env={'RSYNC_PASSWORD': rsync_key},
 408                 haltOnFailure = True,
 409                 logEnviron = False
 410         ))
 411
 412         factory.addStep(ShellCommand(
 413                 name = "targetupload",
 414                 description = "Uploading target files",
 415                 command=["rsync", "--delete", "-avz", "bin/targets/%s/%s/" %(ts[0], ts[1]), "%s/targets/%s/%s/" %(rsync_url, ts[0], ts[1])],
 416                 env={'RSYNC_PASSWORD': rsync_key},
 417                 haltOnFailure = True,
 418                 logEnviron = False
 419         ))
 420
 421         if False:
 422                 factory.addStep(ShellCommand(
 423                         name = "packageupload",
 424                         description = "Uploading package files",
 425                         command=["rsync", "--delete", "-avz", "bin/packages/", "%s/packages/" %(rsync_url)],
 426                         env={'RSYNC_PASSWORD': rsync_key},
 427                         haltOnFailure = False,
 428                         logEnviron = False
 429                 ))
 430
 431         # logs
 432         if False:
 433                 factory.addStep(ShellCommand(
 434                         name = "upload",
 435                         description = "Uploading logs",
 436                         command=["rsync", "--delete", "-avz", "logs/", "%s/logs/%s/%s/" %(rsync_url, ts[0], ts[1])],
 437                         env={'RSYNC_PASSWORD': rsync_key},
 438                         haltOnFailure = False,
 439                         alwaysRun = True,
 440                         logEnviron = False
 441                 ))
 442
 443         from buildbot.config import BuilderConfig
 444
 445         c['builders'].append(BuilderConfig(name=target, slavenames=slaveNames, factory=factory))
 446
 447
 448 ####### STATUS TARGETS
 449
 450 # 'status' is a list of Status Targets. The results of each build will be
 451 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
 452 # including web pages, email senders, and IRC bots.
 453
 454 c['status'] = []
 455
 456 from buildbot.status import html
 457 from buildbot.status.web import authz, auth
 458
 459 if ini.has_option("status", "bind"):
 460         if ini.has_option("status", "user") and ini.has_option("status", "password"):
 461                 authz_cfg=authz.Authz(
 462                         # change any of these to True to enable; see the manual for more
 463                         # options
 464                         auth=auth.BasicAuth([(ini.get("status", "user"), ini.get("status", "password"))]),
 465                         gracefulShutdown = False,
 466                         forceBuild = 'auth', # use this to test your slave once it is set up
 467                         forceAllBuilds = 'auth',
 468                         pingBuilder = False,
 469                         stopBuild = 'auth',
 470                         stopAllBuilds = 'auth',
 471                         cancelPendingBuild = 'auth',
 472                 )
 473                 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind"), authz=authz_cfg))
 474         else:
 475                 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind")))
 476
 477 ####### PROJECT IDENTITY
 478
 479 # the 'title' string will appear at the top of this buildbot
 480 # installation's html.WebStatus home page (linked to the
 481 # 'titleURL') and is embedded in the title of the waterfall HTML page.
 482
 483 c['title'] = ini.get("general", "title")
 484 c['titleURL'] = ini.get("general", "title_url")
 485
 486 # the 'buildbotURL' string should point to the location where the buildbot's
 487 # internal web server (usually the html.WebStatus page) is visible. This
 488 # typically uses the port number set in the Waterfall 'status' entry, but
 489 # with an externally-visible host name which the buildbot cannot figure out
 490 # without some help.
 491
 492 c['buildbotURL'] = ini.get("general", "buildbot_url")
 493
 494 ####### DB URL
 495
 496 c['db'] = {
 497         # This specifies what database buildbot uses to store its state.  You can leave
 498         # this at its default for all but the largest installations.
 499         'db_url' : "sqlite:///state.sqlite",
 500 }
 501