phase1/master.cfg

   1 # -*- python -*-
   2 # ex: set syntax=python:
   3
   4 import os
   5 import re
   6 import subprocess
   7 import ConfigParser
   8
   9 from buildbot import locks
  10
  11 # This is a sample buildmaster config file. It must be installed as
  12 # 'master.cfg' in your buildmaster's base directory.
  13
  14 ini = ConfigParser.ConfigParser()
  15 ini.read("./config.ini")
  16
  17 # This is the dictionary that the buildmaster pays attention to. We also use
  18 # a shorter alias to save typing.
  19 c = BuildmasterConfig = {}
  20
  21 ####### BUILDSLAVES
  22
  23 # The 'slaves' list defines the set of recognized buildslaves. Each element is
  24 # a BuildSlave object, specifying a unique slave name and password.  The same
  25 # slave name and password must be configured on the slave.
  26 from buildbot.buildslave import BuildSlave
  27
  28 c['slaves'] = []
  29
  30 for section in ini.sections():
  31         if section.startswith("slave "):
  32                 if ini.has_option(section, "name") and ini.has_option(section, "password"):
  33                         name = ini.get(section, "name")
  34                         password = ini.get(section, "password")
  35                         max_builds = 1
  36                         if ini.has_option(section, "builds"):
  37                                 max_builds = ini.getint(section, "builds")
  38                         c['slaves'].append(BuildSlave(name, password, max_builds = max_builds))
  39
  40 # 'slavePortnum' defines the TCP port to listen on for connections from slaves.
  41 # This must match the value configured into the buildslaves (with their
  42 # --master option)
  43 c['slavePortnum'] = 9989
  44
  45 # coalesce builds
  46 c['mergeRequests'] = True
  47
  48 ####### CHANGESOURCES
  49
  50 home_dir = os.path.abspath(ini.get("general", "homedir"))
  51
  52 repo_url = ini.get("repo", "url")
  53
  54 rsync_url = ini.get("rsync", "url")
  55 rsync_key = ini.get("rsync", "password")
  56
  57 # find targets
  58 targets = [ ]
  59
  60 findtargets = subprocess.Popen([home_dir+'/dumpinfo.pl', 'targets'],
  61         stdout = subprocess.PIPE, cwd = home_dir+'/source.git')
  62
  63 while True:
  64         line = findtargets.stdout.readline()
  65         if not line:
  66                 break
  67         ta = line.strip().split(' ')
  68         targets.append(ta[0])
  69
  70
  71 # the 'change_source' setting tells the buildmaster how it should find out
  72 # about source code changes.  Here we point to the buildbot clone of pyflakes.
  73
  74 from buildbot.changes.gitpoller import GitPoller
  75 c['change_source'] = []
  76 c['change_source'].append(GitPoller(
  77         repo_url,
  78         workdir=home_dir+'/source.git', branch='master',
  79         pollinterval=300))
  80
  81 ####### SCHEDULERS
  82
  83 # Configure the Schedulers, which decide how to react to incoming changes.  In this
  84 # case, just kick off a 'basebuild' build
  85
  86 from buildbot.schedulers.basic import SingleBranchScheduler
  87 from buildbot.schedulers.forcesched import ForceScheduler
  88 from buildbot.changes import filter
  89 c['schedulers'] = []
  90 c['schedulers'].append(SingleBranchScheduler(
  91         name="all",
  92         change_filter=filter.ChangeFilter(branch='master'),
  93         treeStableTimer=60,
  94         builderNames=targets))
  95
  96 c['schedulers'].append(ForceScheduler(
  97         name="force",
  98         builderNames=targets))
  99
 100 ####### BUILDERS
 101
 102 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
 103 # what steps, and which slaves can execute them.  Note that any particular build will
 104 # only take place on one slave.
 105
 106 from buildbot.process.factory import BuildFactory
 107 from buildbot.steps.source import Git
 108 from buildbot.steps.shell import ShellCommand
 109 from buildbot.steps.shell import SetProperty
 110 from buildbot.steps.transfer import FileDownload
 111 from buildbot.process.properties import WithProperties
 112
 113
 114 MakeTargetMap = {
 115         "^tools/":                              "tools/clean",
 116         "^toolchain/":                  "toolchain/clean",
 117         "^target/linux/":               "target/linux/clean",
 118         "^(config|include)/":   "dirclean"
 119 }
 120
 121 def IsAffected(pattern):
 122         def CheckAffected(change):
 123                 for request in change.build.requests:
 124                         for source in request.sources:
 125                                 for change in source.changes:
 126                                         for file in change.files:
 127                                                 if re.match(pattern, file):
 128                                                         return True
 129                 return False
 130         return CheckAffected
 131
 132 def isPathBuiltin(path):
 133         incl = {}
 134         pkgs = {}
 135         conf = open(".config", "r")
 136
 137         while True:
 138                 line = conf.readline()
 139                 if line == '':
 140                         break
 141                 m = re.match("^(CONFIG_PACKAGE_.+?)=y", line)
 142                 if m:
 143                         incl[m.group(1)] = True
 144
 145         conf.close()
 146
 147         deps = open("tmp/.packagedeps", "r")
 148
 149         while True:
 150                 line = deps.readline()
 151                 if line == '':
 152                         break
 153                 m = re.match("^package-\$\((CONFIG_PACKAGE_.+?)\) \+= (\S+)", line)
 154                 if m and incl.get(m.group(1)) == True:
 155                         pkgs["package/%s" % m.group(2)] = True
 156
 157         deps.close()
 158
 159         while path != '':
 160                 if pkgs.get(path) == True:
 161                         return True
 162                 path = os.path.dirname(path)
 163
 164         return False
 165
 166 def isChangeBuiltin(change):
 167         return True
 168 #       for request in change.build.requests:
 169 #               for source in request.sources:
 170 #                       for change in source.changes:
 171 #                               for file in change.files:
 172 #                                       if isPathBuiltin(file):
 173 #                                               return True
 174 #       return False
 175
 176
 177 c['builders'] = []
 178
 179 dlLock = locks.SlaveLock("slave_dl")
 180
 181 checkBuiltin = re.sub('[\t\n ]+', ' ', """
 182         checkBuiltin() {
 183                 local symbol op path file;
 184                 for file in $CHANGED_FILES; do
 185                         case "$file" in
 186                                 package/*/*) : ;;
 187                                 *) return 0 ;;
 188                         esac;
 189                 done;
 190                 while read symbol op path; do
 191                         case "$symbol" in package-*)
 192                                 symbol="${symbol##*(}";
 193                                 symbol="${symbol%)}";
 194                                 for file in $CHANGED_FILES; do
 195                                         case "$file" in "package/$path/"*)
 196                                                 grep -qsx "$symbol=y" .config && return 0
 197                                         ;; esac;
 198                                 done;
 199                         esac;
 200                 done < tmp/.packagedeps;
 201                 return 1;
 202         }
 203 """).strip()
 204
 205
 206 class IfBuiltinShellCommand(ShellCommand):
 207         def _quote(self, str):
 208                 if re.search("[^a-zA-Z0-9/_.-]", str):
 209                         return "'%s'" %(re.sub("'", "'\"'\"'", str))
 210                 return str
 211
 212         def setCommand(self, command):
 213                 if not isinstance(command, (str, unicode)):
 214                         command = ' '.join(map(self._quote, command))
 215                 self.command = [
 216                         '/bin/sh', '-c',
 217                         '%s; if checkBuiltin; then %s; else exit 0; fi' %(checkBuiltin, command)
 218                 ]
 219
 220         def setupEnvironment(self, cmd):
 221                 slaveEnv = self.slaveEnvironment
 222                 if slaveEnv is None:
 223                         slaveEnv = { }
 224                 changedFiles = { }
 225                 for request in self.build.requests:
 226                         for source in request.sources:
 227                                 for change in source.changes:
 228                                         for file in change.files:
 229                                                 changedFiles[file] = True
 230                 fullSlaveEnv = slaveEnv.copy()
 231                 fullSlaveEnv['CHANGED_FILES'] = ' '.join(changedFiles.keys())
 232                 cmd.args['env'] = fullSlaveEnv
 233
 234 slaveNames = [ ]
 235
 236 for slave in c['slaves']:
 237         slaveNames.append(slave.slavename)
 238
 239 for target in targets:
 240         ts = target.split('/')
 241
 242         factory = BuildFactory()
 243
 244         # find number of cores
 245         factory.addStep(SetProperty(
 246                 name = "nproc",
 247                 description = "Finding number of CPUs",
 248                 command = ["nproc"]))
 249
 250         # check out the source
 251         factory.addStep(Git(repourl=repo_url, mode='update'))
 252
 253         factory.addStep(ShellCommand(
 254                 name = "rmtmp",
 255                 description = "Remove tmp folder",
 256                 command=["rm", "-rf", "tmp/"]))
 257
 258         # feed
 259 #       factory.addStep(ShellCommand(
 260 #               name = "feedsconf",
 261 #               description = "Copy the feeds.conf",
 262 #               command='''cp ~/feeds.conf ./feeds.conf''' ))
 263
 264         # feed
 265         factory.addStep(ShellCommand(
 266                 name = "updatefeeds",
 267                 description = "Updating feeds",
 268                 command=["./scripts/feeds", "update"]))
 269
 270         # feed
 271         factory.addStep(ShellCommand(
 272                 name = "installfeeds",
 273                 description = "Installing feeds",
 274                 command=["./scripts/feeds", "install", "-a"]))
 275
 276         # configure
 277         factory.addStep(ShellCommand(
 278                 name = "newconfig",
 279                 description = "Seeding .config",
 280                 command='''cat <<EOT > .config
 281 CONFIG_TARGET_%s=y
 282 CONFIG_TARGET_%s_%s=y
 283 CONFIG_ALL_NONSHARED=y
 284 CONFIG_SDK=y
 285 CONFIG_IB=y
 286 # CONFIG_IB_STANDALONE is not set
 287 CONFIG_DEVEL=y
 288 CONFIG_CCACHE=y
 289 CONFIG_SIGNED_PACKAGES=y
 290 # CONFIG_PER_FEED_REPO_ADD_COMMENTED is not set
 291 CONFIG_KERNEL_KALLSYMS=y
 292 CONFIG_COLLECT_KERNEL_DEBUG=y
 293 EOT''' %(ts[0], ts[0], ts[1]) ))
 294
 295         factory.addStep(ShellCommand(
 296                 name = "delbin",
 297                 description = "Removing output directory",
 298                 command = ["rm", "-rf", "bin/"]
 299         ))
 300
 301         factory.addStep(ShellCommand(
 302                 name = "defconfig",
 303                 description = "Populating .config",
 304                 command = ["make", "defconfig"]
 305         ))
 306
 307         # check arch / libc
 308         factory.addStep(ShellCommand(
 309                 name = "checkarch",
 310                 description = "Checking architecture",
 311                 command = ["grep", "-sq", "CONFIG_TARGET_%s=y" %(ts[0]), ".config"],
 312                 logEnviron = False,
 313                 want_stdout = False,
 314                 want_stderr = False,
 315                 haltOnFailure = True
 316         ))
 317
 318         factory.addStep(ShellCommand(
 319                 name = "checklibc",
 320                 description = "Checking libc flavor",
 321                 command = ["grep", "-sq", 'CONFIG_LIBC="musl"', ".config"],
 322                 logEnviron = False,
 323                 want_stdout = False,
 324                 want_stderr = False,
 325                 haltOnFailure = True
 326         ))
 327
 328         # install build key
 329         factory.addStep(FileDownload(mastersrc=home_dir+'/key-build', slavedest="key-build", mode=0600))
 330         factory.addStep(FileDownload(mastersrc=home_dir+'/key-build.pub', slavedest="key-build.pub", mode=0600))
 331
 332         # prepare dl
 333         factory.addStep(ShellCommand(
 334                 name = "dldir",
 335                 description = "Preparing dl/",
 336                 command = "mkdir -p $HOME/dl && ln -sf $HOME/dl ./dl",
 337                 logEnviron = False,
 338                 want_stdout = False
 339         ))
 340
 341         # populate dl
 342         factory.addStep(ShellCommand(
 343                 name = "dlrun",
 344                 description = "Populating dl/",
 345                 command = ["make", WithProperties("-j%(nproc:~4)s"), "download", "V=s"],
 346                 logEnviron = False,
 347                 locks = [dlLock.access('exclusive')]
 348         ))
 349
 350         factory.addStep(ShellCommand(
 351                 name = "cleanbase",
 352                 description = "Cleaning base-files",
 353                 command=["make", "package/base-files/clean", "V=s"]
 354         ))
 355
 356         # optional clean steps
 357         for pattern, maketarget in MakeTargetMap.items():
 358                 factory.addStep(ShellCommand(
 359                         name = maketarget,
 360                         description = maketarget,
 361                         command=["make", maketarget, "V=s"], doStepIf=IsAffected(pattern)
 362                 ))
 363
 364         # build
 365         factory.addStep(ShellCommand(
 366                 name = "tools",
 367                 description = "Building tools",
 368                 command = ["make", WithProperties("-j%(nproc:~4)s"), "tools/install", "V=s"],
 369                 haltOnFailure = True
 370         ))
 371
 372         factory.addStep(ShellCommand(
 373                 name = "toolchain",
 374                 description = "Building toolchain",
 375                 command=["make", WithProperties("-j%(nproc:~4)s"), "toolchain/install", "V=s"],
 376                 haltOnFailure = True
 377         ))
 378
 379         factory.addStep(ShellCommand(
 380                 name = "kmods",
 381                 description = "Building kmods",
 382                 command=["make", WithProperties("-j%(nproc:~4)s"), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
 383                 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
 384                 haltOnFailure = True
 385         ))
 386
 387         factory.addStep(ShellCommand(
 388                 name = "pkgbuild",
 389                 description = "Building packages",
 390                 command=["make", WithProperties("-j%(nproc:~4)s"), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
 391                 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
 392                 haltOnFailure = True
 393         ))
 394
 395         # factory.addStep(IfBuiltinShellCommand(
 396         factory.addStep(ShellCommand(
 397                 name = "pkginstall",
 398                 description = "Installing packages",
 399                 command=["make", WithProperties("-j%(nproc:~4)s"), "package/install", "V=s"],
 400                 doStepIf = isChangeBuiltin,
 401                 haltOnFailure = True
 402         ))
 403
 404         factory.addStep(ShellCommand(
 405                 name = "pkgindex",
 406                 description = "Indexing packages",
 407                 command=["make", WithProperties("-j%(nproc:~4)s"), "package/index", "V=s"],
 408                 haltOnFailure = True
 409         ))
 410
 411         #factory.addStep(IfBuiltinShellCommand(
 412         factory.addStep(ShellCommand(
 413                 name = "images",
 414                 description = "Building images",
 415                 command=["make", "-j1", "target/install", "V=s"],
 416                 doStepIf = isChangeBuiltin,
 417                 haltOnFailure = True
 418         ))
 419
 420         # upload
 421         factory.addStep(ShellCommand(
 422                 name = "uploadprepare",
 423                 description = "Preparing target directory",
 424                 command=["rsync", "-av", "--include", "/%s/" %(ts[0]), "--include", "/%s/%s/" %(ts[0], ts[1]), "--exclude", "/*", "--exclude", "/*/*", "--exclude", "/%s/%s/*" %(ts[0], ts[1]), "bin/targets/", "%s/targets/" %(rsync_url)],
 425                 env={'RSYNC_PASSWORD': rsync_key},
 426                 haltOnFailure = True,
 427                 logEnviron = False
 428         ))
 429
 430         factory.addStep(ShellCommand(
 431                 name = "targetupload",
 432                 description = "Uploading target files",
 433                 command=["rsync", "--delete", "--delay-updates", "-avz", "bin/targets/%s/%s/" %(ts[0], ts[1]), "%s/targets/%s/%s/" %(rsync_url, ts[0], ts[1])],
 434                 env={'RSYNC_PASSWORD': rsync_key},
 435                 haltOnFailure = True,
 436                 logEnviron = False
 437         ))
 438
 439         if False:
 440                 factory.addStep(ShellCommand(
 441                         name = "packageupload",
 442                         description = "Uploading package files",
 443                         command=["rsync", "--delete", "--delay-updates", "-avz", "bin/packages/", "%s/packages/" %(rsync_url)],
 444                         env={'RSYNC_PASSWORD': rsync_key},
 445                         haltOnFailure = False,
 446                         logEnviron = False
 447                 ))
 448
 449         # logs
 450         if False:
 451                 factory.addStep(ShellCommand(
 452                         name = "upload",
 453                         description = "Uploading logs",
 454                         command=["rsync", "--delete", "--delay-updates", "-avz", "logs/", "%s/logs/%s/%s/" %(rsync_url, ts[0], ts[1])],
 455                         env={'RSYNC_PASSWORD': rsync_key},
 456                         haltOnFailure = False,
 457                         alwaysRun = True,
 458                         logEnviron = False
 459                 ))
 460
 461         from buildbot.config import BuilderConfig
 462
 463         c['builders'].append(BuilderConfig(name=target, slavenames=slaveNames, factory=factory))
 464
 465
 466 ####### STATUS TARGETS
 467
 468 # 'status' is a list of Status Targets. The results of each build will be
 469 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
 470 # including web pages, email senders, and IRC bots.
 471
 472 c['status'] = []
 473
 474 from buildbot.status import html
 475 from buildbot.status.web import authz, auth
 476
 477 if ini.has_option("status", "bind"):
 478         if ini.has_option("status", "user") and ini.has_option("status", "password"):
 479                 authz_cfg=authz.Authz(
 480                         # change any of these to True to enable; see the manual for more
 481                         # options
 482                         auth=auth.BasicAuth([(ini.get("status", "user"), ini.get("status", "password"))]),
 483                         gracefulShutdown = False,
 484                         forceBuild = 'auth', # use this to test your slave once it is set up
 485                         forceAllBuilds = 'auth',
 486                         pingBuilder = False,
 487                         stopBuild = 'auth',
 488                         stopAllBuilds = 'auth',
 489                         cancelPendingBuild = 'auth',
 490                 )
 491                 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind"), authz=authz_cfg))
 492         else:
 493                 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind")))
 494
 495 ####### PROJECT IDENTITY
 496
 497 # the 'title' string will appear at the top of this buildbot
 498 # installation's html.WebStatus home page (linked to the
 499 # 'titleURL') and is embedded in the title of the waterfall HTML page.
 500
 501 c['title'] = ini.get("general", "title")
 502 c['titleURL'] = ini.get("general", "title_url")
 503
 504 # the 'buildbotURL' string should point to the location where the buildbot's
 505 # internal web server (usually the html.WebStatus page) is visible. This
 506 # typically uses the port number set in the Waterfall 'status' entry, but
 507 # with an externally-visible host name which the buildbot cannot figure out
 508 # without some help.
 509
 510 c['buildbotURL'] = ini.get("general", "buildbot_url")
 511
 512 ####### DB URL
 513
 514 c['db'] = {
 515         # This specifies what database buildbot uses to store its state.  You can leave
 516         # this at its default for all but the largest installations.
 517         'db_url' : "sqlite:///state.sqlite",
 518 }