phase1/master.cfg

   1 # -*- python -*-
   2 # ex: set syntax=python:
   3
   4 import os
   5 import re
   6 import subprocess
   7 import ConfigParser
   8
   9 from buildbot import locks
  10
  11 # This is a sample buildmaster config file. It must be installed as
  12 # 'master.cfg' in your buildmaster's base directory.
  13
  14 ini = ConfigParser.ConfigParser()
  15 ini.read("./config.ini")
  16
  17 # This is the dictionary that the buildmaster pays attention to. We also use
  18 # a shorter alias to save typing.
  19 c = BuildmasterConfig = {}
  20
  21 ####### BUILDSLAVES
  22
  23 # The 'slaves' list defines the set of recognized buildslaves. Each element is
  24 # a BuildSlave object, specifying a unique slave name and password.  The same
  25 # slave name and password must be configured on the slave.
  26 from buildbot.buildslave import BuildSlave
  27
  28 c['slaves'] = []
  29
  30 for section in ini.sections():
  31         if section.startswith("slave "):
  32                 if ini.has_option(section, "name") and ini.has_option(section, "password"):
  33                         name = ini.get(section, "name")
  34                         password = ini.get(section, "password")
  35                         max_builds = 1
  36                         if ini.has_option(section, "builds"):
  37                                 max_builds = ini.getint(section, "builds")
  38                         c['slaves'].append(BuildSlave(name, password, max_builds = max_builds))
  39
  40 # 'slavePortnum' defines the TCP port to listen on for connections from slaves.
  41 # This must match the value configured into the buildslaves (with their
  42 # --master option)
  43 c['slavePortnum'] = 9989
  44
  45 # coalesce builds
  46 c['mergeRequests'] = True
  47
  48 ####### CHANGESOURCES
  49
  50 home_dir = os.path.abspath(ini.get("general", "homedir"))
  51
  52 repo_url = ini.get("repo", "url")
  53
  54 rsync_url = ini.get("rsync", "url")
  55 rsync_key = ini.get("rsync", "password")
  56
  57 # find targets
  58 targets = [ ]
  59
  60 findtargets = subprocess.Popen([home_dir+'/dumpinfo.pl', 'targets'],
  61         stdout = subprocess.PIPE, cwd = home_dir+'/source.git')
  62
  63 while True:
  64         line = findtargets.stdout.readline()
  65         if not line:
  66                 break
  67         ta = line.strip().split(' ')
  68         targets.append(ta[0])
  69
  70
  71 # the 'change_source' setting tells the buildmaster how it should find out
  72 # about source code changes.  Here we point to the buildbot clone of pyflakes.
  73
  74 from buildbot.changes.gitpoller import GitPoller
  75 c['change_source'] = []
  76 c['change_source'].append(GitPoller(
  77         repo_url,
  78         workdir=home_dir+'/source.git', branch='master',
  79         pollinterval=300))
  80
  81 ####### SCHEDULERS
  82
  83 # Configure the Schedulers, which decide how to react to incoming changes.  In this
  84 # case, just kick off a 'basebuild' build
  85
  86 from buildbot.schedulers.basic import SingleBranchScheduler
  87 from buildbot.schedulers.forcesched import ForceScheduler
  88 from buildbot.changes import filter
  89 c['schedulers'] = []
  90 c['schedulers'].append(SingleBranchScheduler(
  91         name="all",
  92         change_filter=filter.ChangeFilter(branch='master'),
  93         treeStableTimer=60,
  94         builderNames=targets))
  95
  96 c['schedulers'].append(ForceScheduler(
  97         name="force",
  98         builderNames=targets))
  99
 100 ####### BUILDERS
 101
 102 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
 103 # what steps, and which slaves can execute them.  Note that any particular build will
 104 # only take place on one slave.
 105
 106 from buildbot.process.factory import BuildFactory
 107 from buildbot.steps.source import Git
 108 from buildbot.steps.shell import ShellCommand
 109 from buildbot.steps.shell import SetProperty
 110 from buildbot.steps.transfer import FileDownload
 111 from buildbot.process.properties import WithProperties
 112
 113
 114 MakeTargetMap = {
 115         "^tools/":                              "tools/clean",
 116         "^toolchain/":                  "toolchain/clean",
 117         "^target/linux/":               "target/linux/clean",
 118         "^(config|include)/":   "dirclean"
 119 }
 120
 121 def IsAffected(pattern):
 122         def CheckAffected(change):
 123                 for request in change.build.requests:
 124                         for source in request.sources:
 125                                 for change in source.changes:
 126                                         for file in change.files:
 127                                                 if re.match(pattern, file):
 128                                                         return True
 129                 return False
 130         return CheckAffected
 131
 132 def isPathBuiltin(path):
 133         incl = {}
 134         pkgs = {}
 135         conf = open(".config", "r")
 136
 137         while True:
 138                 line = conf.readline()
 139                 if line == '':
 140                         break
 141                 m = re.match("^(CONFIG_PACKAGE_.+?)=y", line)
 142                 if m:
 143                         incl[m.group(1)] = True
 144
 145         conf.close()
 146
 147         deps = open("tmp/.packagedeps", "r")
 148
 149         while True:
 150                 line = deps.readline()
 151                 if line == '':
 152                         break
 153                 m = re.match("^package-\$\((CONFIG_PACKAGE_.+?)\) \+= (\S+)", line)
 154                 if m and incl.get(m.group(1)) == True:
 155                         pkgs["package/%s" % m.group(2)] = True
 156
 157         deps.close()
 158
 159         while path != '':
 160                 if pkgs.get(path) == True:
 161                         return True
 162                 path = os.path.dirname(path)
 163
 164         return False
 165
 166 def isChangeBuiltin(change):
 167         return True
 168 #       for request in change.build.requests:
 169 #               for source in request.sources:
 170 #                       for change in source.changes:
 171 #                               for file in change.files:
 172 #                                       if isPathBuiltin(file):
 173 #                                               return True
 174 #       return False
 175
 176
 177 c['builders'] = []
 178
 179 dlLock = locks.SlaveLock("slave_dl")
 180
 181 checkBuiltin = re.sub('[\t\n ]+', ' ', """
 182         checkBuiltin() {
 183                 local symbol op path file;
 184                 for file in $CHANGED_FILES; do
 185                         case "$file" in
 186                                 package/*/*) : ;;
 187                                 *) return 0 ;;
 188                         esac;
 189                 done;
 190                 while read symbol op path; do
 191                         case "$symbol" in package-*)
 192                                 symbol="${symbol##*(}";
 193                                 symbol="${symbol%)}";
 194                                 for file in $CHANGED_FILES; do
 195                                         case "$file" in "package/$path/"*)
 196                                                 grep -qsx "$symbol=y" .config && return 0
 197                                         ;; esac;
 198                                 done;
 199                         esac;
 200                 done < tmp/.packagedeps;
 201                 return 1;
 202         }
 203 """).strip()
 204
 205
 206 class IfBuiltinShellCommand(ShellCommand):
 207         def _quote(self, str):
 208                 if re.search("[^a-zA-Z0-9/_.-]", str):
 209                         return "'%s'" %(re.sub("'", "'\"'\"'", str))
 210                 return str
 211
 212         def setCommand(self, command):
 213                 if not isinstance(command, (str, unicode)):
 214                         command = ' '.join(map(self._quote, command))
 215                 self.command = [
 216                         '/bin/sh', '-c',
 217                         '%s; if checkBuiltin; then %s; else exit 0; fi' %(checkBuiltin, command)
 218                 ]
 219
 220         def setupEnvironment(self, cmd):
 221                 slaveEnv = self.slaveEnvironment
 222                 if slaveEnv is None:
 223                         slaveEnv = { }
 224                 changedFiles = { }
 225                 for request in self.build.requests:
 226                         for source in request.sources:
 227                                 for change in source.changes:
 228                                         for file in change.files:
 229                                                 changedFiles[file] = True
 230                 fullSlaveEnv = slaveEnv.copy()
 231                 fullSlaveEnv['CHANGED_FILES'] = ' '.join(changedFiles.keys())
 232                 cmd.args['env'] = fullSlaveEnv
 233
 234 slaveNames = [ ]
 235
 236 for slave in c['slaves']:
 237         slaveNames.append(slave.slavename)
 238
 239 for target in targets:
 240         ts = target.split('/')
 241
 242         factory = BuildFactory()
 243
 244         # find number of cores
 245         factory.addStep(SetProperty(
 246                 name = "nproc",
 247                 property = "nproc",
 248                 description = "Finding number of CPUs",
 249                 command = ["nproc"]))
 250
 251         # check out the source
 252         factory.addStep(Git(repourl=repo_url, mode='update'))
 253
 254         factory.addStep(ShellCommand(
 255                 name = "rmtmp",
 256                 description = "Remove tmp folder",
 257                 command=["rm", "-rf", "tmp/"]))
 258
 259         # feed
 260 #       factory.addStep(ShellCommand(
 261 #               name = "feedsconf",
 262 #               description = "Copy the feeds.conf",
 263 #               command='''cp ~/feeds.conf ./feeds.conf''' ))
 264
 265         # feed
 266         factory.addStep(ShellCommand(
 267                 name = "updatefeeds",
 268                 description = "Updating feeds",
 269                 command=["./scripts/feeds", "update"]))
 270
 271         # feed
 272         factory.addStep(ShellCommand(
 273                 name = "installfeeds",
 274                 description = "Installing feeds",
 275                 command=["./scripts/feeds", "install", "-a"]))
 276
 277         # configure
 278         factory.addStep(ShellCommand(
 279                 name = "newconfig",
 280                 description = "Seeding .config",
 281                 command='''cat <<EOT > .config
 282 CONFIG_TARGET_%s=y
 283 CONFIG_TARGET_%s_%s=y
 284 CONFIG_ALL_NONSHARED=y
 285 CONFIG_SDK=y
 286 CONFIG_IB=y
 287 # CONFIG_IB_STANDALONE is not set
 288 CONFIG_DEVEL=y
 289 CONFIG_CCACHE=y
 290 CONFIG_SIGNED_PACKAGES=y
 291 # CONFIG_PER_FEED_REPO_ADD_COMMENTED is not set
 292 CONFIG_KERNEL_KALLSYMS=y
 293 CONFIG_COLLECT_KERNEL_DEBUG=y
 294 EOT''' %(ts[0], ts[0], ts[1]) ))
 295
 296         factory.addStep(ShellCommand(
 297                 name = "delbin",
 298                 description = "Removing output directory",
 299                 command = ["rm", "-rf", "bin/"]
 300         ))
 301
 302         factory.addStep(ShellCommand(
 303                 name = "defconfig",
 304                 description = "Populating .config",
 305                 command = ["make", "defconfig"]
 306         ))
 307
 308         # check arch / libc
 309         factory.addStep(ShellCommand(
 310                 name = "checkarch",
 311                 description = "Checking architecture",
 312                 command = ["grep", "-sq", "CONFIG_TARGET_%s=y" %(ts[0]), ".config"],
 313                 logEnviron = False,
 314                 want_stdout = False,
 315                 want_stderr = False,
 316                 haltOnFailure = True
 317         ))
 318
 319         factory.addStep(ShellCommand(
 320                 name = "checklibc",
 321                 description = "Checking libc flavor",
 322                 command = ["grep", "-sq", 'CONFIG_LIBC="musl"', ".config"],
 323                 logEnviron = False,
 324                 want_stdout = False,
 325                 want_stderr = False,
 326                 haltOnFailure = True
 327         ))
 328
 329         # install build key
 330         factory.addStep(FileDownload(mastersrc=home_dir+'/key-build', slavedest="key-build", mode=0600))
 331         factory.addStep(FileDownload(mastersrc=home_dir+'/key-build.pub', slavedest="key-build.pub", mode=0600))
 332
 333         # prepare dl
 334         factory.addStep(ShellCommand(
 335                 name = "dldir",
 336                 description = "Preparing dl/",
 337                 command = "mkdir -p $HOME/dl && ln -sf $HOME/dl ./dl",
 338                 logEnviron = False,
 339                 want_stdout = False
 340         ))
 341
 342         # populate dl
 343         factory.addStep(ShellCommand(
 344                 name = "dlrun",
 345                 description = "Populating dl/",
 346                 command = ["make", WithProperties("-j%(nproc:~4)s"), "download", "V=s"],
 347                 logEnviron = False,
 348                 locks = [dlLock.access('exclusive')]
 349         ))
 350
 351         factory.addStep(ShellCommand(
 352                 name = "cleanbase",
 353                 description = "Cleaning base-files",
 354                 command=["make", "package/base-files/clean", "V=s"]
 355         ))
 356
 357         # optional clean steps
 358         for pattern, maketarget in MakeTargetMap.items():
 359                 factory.addStep(ShellCommand(
 360                         name = maketarget,
 361                         description = maketarget,
 362                         command=["make", maketarget, "V=s"], doStepIf=IsAffected(pattern)
 363                 ))
 364
 365         # build
 366         factory.addStep(ShellCommand(
 367                 name = "tools",
 368                 description = "Building tools",
 369                 command = ["make", WithProperties("-j%(nproc:~4)s"), "tools/install", "V=s"],
 370                 haltOnFailure = True
 371         ))
 372
 373         factory.addStep(ShellCommand(
 374                 name = "toolchain",
 375                 description = "Building toolchain",
 376                 command=["make", WithProperties("-j%(nproc:~4)s"), "toolchain/install", "V=s"],
 377                 haltOnFailure = True
 378         ))
 379
 380         factory.addStep(ShellCommand(
 381                 name = "kmods",
 382                 description = "Building kmods",
 383                 command=["make", WithProperties("-j%(nproc:~4)s"), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
 384                 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
 385                 haltOnFailure = True
 386         ))
 387
 388         factory.addStep(ShellCommand(
 389                 name = "pkgbuild",
 390                 description = "Building packages",
 391                 command=["make", WithProperties("-j%(nproc:~4)s"), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
 392                 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
 393                 haltOnFailure = True
 394         ))
 395
 396         # factory.addStep(IfBuiltinShellCommand(
 397         factory.addStep(ShellCommand(
 398                 name = "pkginstall",
 399                 description = "Installing packages",
 400                 command=["make", WithProperties("-j%(nproc:~4)s"), "package/install", "V=s"],
 401                 doStepIf = isChangeBuiltin,
 402                 haltOnFailure = True
 403         ))
 404
 405         factory.addStep(ShellCommand(
 406                 name = "pkgindex",
 407                 description = "Indexing packages",
 408                 command=["make", WithProperties("-j%(nproc:~4)s"), "package/index", "V=s"],
 409                 haltOnFailure = True
 410         ))
 411
 412         #factory.addStep(IfBuiltinShellCommand(
 413         factory.addStep(ShellCommand(
 414                 name = "images",
 415                 description = "Building images",
 416                 command=["make", "-j1", "target/install", "V=s"],
 417                 doStepIf = isChangeBuiltin,
 418                 haltOnFailure = True
 419         ))
 420
 421         # upload
 422         factory.addStep(ShellCommand(
 423                 name = "uploadprepare",
 424                 description = "Preparing target directory",
 425                 command=["rsync", "-av", "--include", "/%s/" %(ts[0]), "--include", "/%s/%s/" %(ts[0], ts[1]), "--exclude", "/*", "--exclude", "/*/*", "--exclude", "/%s/%s/*" %(ts[0], ts[1]), "bin/targets/", "%s/targets/" %(rsync_url)],
 426                 env={'RSYNC_PASSWORD': rsync_key},
 427                 haltOnFailure = True,
 428                 logEnviron = False
 429         ))
 430
 431         factory.addStep(ShellCommand(
 432                 name = "targetupload",
 433                 description = "Uploading target files",
 434                 command=["rsync", "--delete", "--delay-updates", "-avz", "bin/targets/%s/%s/" %(ts[0], ts[1]), "%s/targets/%s/%s/" %(rsync_url, ts[0], ts[1])],
 435                 env={'RSYNC_PASSWORD': rsync_key},
 436                 haltOnFailure = True,
 437                 logEnviron = False
 438         ))
 439
 440         if False:
 441                 factory.addStep(ShellCommand(
 442                         name = "packageupload",
 443                         description = "Uploading package files",
 444                         command=["rsync", "--delete", "--delay-updates", "-avz", "bin/packages/", "%s/packages/" %(rsync_url)],
 445                         env={'RSYNC_PASSWORD': rsync_key},
 446                         haltOnFailure = False,
 447                         logEnviron = False
 448                 ))
 449
 450         # logs
 451         if False:
 452                 factory.addStep(ShellCommand(
 453                         name = "upload",
 454                         description = "Uploading logs",
 455                         command=["rsync", "--delete", "--delay-updates", "-avz", "logs/", "%s/logs/%s/%s/" %(rsync_url, ts[0], ts[1])],
 456                         env={'RSYNC_PASSWORD': rsync_key},
 457                         haltOnFailure = False,
 458                         alwaysRun = True,
 459                         logEnviron = False
 460                 ))
 461
 462         from buildbot.config import BuilderConfig
 463
 464         c['builders'].append(BuilderConfig(name=target, slavenames=slaveNames, factory=factory))
 465
 466
 467 ####### STATUS TARGETS
 468
 469 # 'status' is a list of Status Targets. The results of each build will be
 470 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
 471 # including web pages, email senders, and IRC bots.
 472
 473 c['status'] = []
 474
 475 from buildbot.status import html
 476 from buildbot.status.web import authz, auth
 477
 478 if ini.has_option("status", "bind"):
 479         if ini.has_option("status", "user") and ini.has_option("status", "password"):
 480                 authz_cfg=authz.Authz(
 481                         # change any of these to True to enable; see the manual for more
 482                         # options
 483                         auth=auth.BasicAuth([(ini.get("status", "user"), ini.get("status", "password"))]),
 484                         gracefulShutdown = False,
 485                         forceBuild = 'auth', # use this to test your slave once it is set up
 486                         forceAllBuilds = 'auth',
 487                         pingBuilder = False,
 488                         stopBuild = 'auth',
 489                         stopAllBuilds = 'auth',
 490                         cancelPendingBuild = 'auth',
 491                 )
 492                 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind"), authz=authz_cfg))
 493         else:
 494                 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind")))
 495
 496 ####### PROJECT IDENTITY
 497
 498 # the 'title' string will appear at the top of this buildbot
 499 # installation's html.WebStatus home page (linked to the
 500 # 'titleURL') and is embedded in the title of the waterfall HTML page.
 501
 502 c['title'] = ini.get("general", "title")
 503 c['titleURL'] = ini.get("general", "title_url")
 504
 505 # the 'buildbotURL' string should point to the location where the buildbot's
 506 # internal web server (usually the html.WebStatus page) is visible. This
 507 # typically uses the port number set in the Waterfall 'status' entry, but
 508 # with an externally-visible host name which the buildbot cannot figure out
 509 # without some help.
 510
 511 c['buildbotURL'] = ini.get("general", "buildbot_url")
 512
 513 ####### DB URL
 514
 515 c['db'] = {
 516         # This specifies what database buildbot uses to store its state.  You can leave
 517         # this at its default for all but the largest installations.
 518         'db_url' : "sqlite:///state.sqlite",
 519 }