phase1/master.cfg

   1 # -*- python -*-
   2 # ex: set syntax=python:
   3
   4 import os
   5 import re
   6 import subprocess
   7 import ConfigParser
   8
   9 from buildbot import locks
  10
  11 # This is a sample buildmaster config file. It must be installed as
  12 # 'master.cfg' in your buildmaster's base directory.
  13
  14 ini = ConfigParser.ConfigParser()
  15 ini.read("./config.ini")
  16
  17 # This is the dictionary that the buildmaster pays attention to. We also use
  18 # a shorter alias to save typing.
  19 c = BuildmasterConfig = {}
  20
  21 ####### BUILDSLAVES
  22
  23 # The 'slaves' list defines the set of recognized buildslaves. Each element is
  24 # a BuildSlave object, specifying a unique slave name and password.  The same
  25 # slave name and password must be configured on the slave.
  26 from buildbot.buildslave import BuildSlave
  27
  28 c['slaves'] = []
  29
  30 for section in ini.sections():
  31         if section.startswith("slave "):
  32                 if ini.has_option(section, "name") and ini.has_option(section, "password"):
  33                         name = ini.get(section, "name")
  34                         password = ini.get(section, "password")
  35                         max_builds = 1
  36                         if ini.has_option(section, "builds"):
  37                                 max_builds = ini.getint(section, "builds")
  38                         c['slaves'].append(BuildSlave(name, password, max_builds = max_builds))
  39
  40 # 'slavePortnum' defines the TCP port to listen on for connections from slaves.
  41 # This must match the value configured into the buildslaves (with their
  42 # --master option)
  43 c['slavePortnum'] = 9989
  44
  45 # coalesce builds
  46 c['mergeRequests'] = True
  47
  48 ####### CHANGESOURCES
  49
  50 home_dir = os.path.abspath(ini.get("general", "homedir"))
  51
  52 repo_url = ini.get("repo", "url")
  53
  54 rsync_bin_url = ini.get("rsync", "binary_url")
  55 rsync_bin_key = ini.get("rsync", "binary_password")
  56
  57 rsync_src_url = None
  58 rsync_src_key = None
  59
  60 if ini.has_option("rsync", "source_url"):
  61         rsync_src_url = ini.get("rsync", "source_url")
  62         rsync_src_key = ini.get("rsync", "source_password")
  63
  64 # find targets
  65 targets = [ ]
  66
  67 findtargets = subprocess.Popen([home_dir+'/dumpinfo.pl', 'targets'],
  68         stdout = subprocess.PIPE, cwd = home_dir+'/source.git')
  69
  70 while True:
  71         line = findtargets.stdout.readline()
  72         if not line:
  73                 break
  74         ta = line.strip().split(' ')
  75         targets.append(ta[0])
  76
  77
  78 # the 'change_source' setting tells the buildmaster how it should find out
  79 # about source code changes.  Here we point to the buildbot clone of pyflakes.
  80
  81 from buildbot.changes.gitpoller import GitPoller
  82 c['change_source'] = []
  83 c['change_source'].append(GitPoller(
  84         repo_url,
  85         workdir=home_dir+'/source.git', branch='master',
  86         pollinterval=300))
  87
  88 ####### SCHEDULERS
  89
  90 # Configure the Schedulers, which decide how to react to incoming changes.  In this
  91 # case, just kick off a 'basebuild' build
  92
  93 from buildbot.schedulers.basic import SingleBranchScheduler
  94 from buildbot.schedulers.forcesched import ForceScheduler
  95 from buildbot.changes import filter
  96 c['schedulers'] = []
  97 c['schedulers'].append(SingleBranchScheduler(
  98         name="all",
  99         change_filter=filter.ChangeFilter(branch='master'),
 100         treeStableTimer=60,
 101         builderNames=targets))
 102
 103 c['schedulers'].append(ForceScheduler(
 104         name="force",
 105         builderNames=targets))
 106
 107 ####### BUILDERS
 108
 109 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
 110 # what steps, and which slaves can execute them.  Note that any particular build will
 111 # only take place on one slave.
 112
 113 from buildbot.process.factory import BuildFactory
 114 from buildbot.steps.source import Git
 115 from buildbot.steps.shell import ShellCommand
 116 from buildbot.steps.shell import SetProperty
 117 from buildbot.steps.transfer import FileDownload
 118 from buildbot.process.properties import WithProperties
 119
 120
 121 MakeTargetMap = {
 122         "^tools/":                              "tools/clean",
 123         "^toolchain/":                  "toolchain/clean",
 124         "^target/linux/":               "target/linux/clean",
 125         "^(config|include)/":   "dirclean"
 126 }
 127
 128 def IsAffected(pattern):
 129         def CheckAffected(change):
 130                 for request in change.build.requests:
 131                         for source in request.sources:
 132                                 for change in source.changes:
 133                                         for file in change.files:
 134                                                 if re.match(pattern, file):
 135                                                         return True
 136                 return False
 137         return CheckAffected
 138
 139 def isPathBuiltin(path):
 140         incl = {}
 141         pkgs = {}
 142         conf = open(".config", "r")
 143
 144         while True:
 145                 line = conf.readline()
 146                 if line == '':
 147                         break
 148                 m = re.match("^(CONFIG_PACKAGE_.+?)=y", line)
 149                 if m:
 150                         incl[m.group(1)] = True
 151
 152         conf.close()
 153
 154         deps = open("tmp/.packagedeps", "r")
 155
 156         while True:
 157                 line = deps.readline()
 158                 if line == '':
 159                         break
 160                 m = re.match("^package-\$\((CONFIG_PACKAGE_.+?)\) \+= (\S+)", line)
 161                 if m and incl.get(m.group(1)) == True:
 162                         pkgs["package/%s" % m.group(2)] = True
 163
 164         deps.close()
 165
 166         while path != '':
 167                 if pkgs.get(path) == True:
 168                         return True
 169                 path = os.path.dirname(path)
 170
 171         return False
 172
 173 def isChangeBuiltin(change):
 174         return True
 175 #       for request in change.build.requests:
 176 #               for source in request.sources:
 177 #                       for change in source.changes:
 178 #                               for file in change.files:
 179 #                                       if isPathBuiltin(file):
 180 #                                               return True
 181 #       return False
 182
 183
 184 c['builders'] = []
 185
 186 dlLock = locks.SlaveLock("slave_dl")
 187
 188 checkBuiltin = re.sub('[\t\n ]+', ' ', """
 189         checkBuiltin() {
 190                 local symbol op path file;
 191                 for file in $CHANGED_FILES; do
 192                         case "$file" in
 193                                 package/*/*) : ;;
 194                                 *) return 0 ;;
 195                         esac;
 196                 done;
 197                 while read symbol op path; do
 198                         case "$symbol" in package-*)
 199                                 symbol="${symbol##*(}";
 200                                 symbol="${symbol%)}";
 201                                 for file in $CHANGED_FILES; do
 202                                         case "$file" in "package/$path/"*)
 203                                                 grep -qsx "$symbol=y" .config && return 0
 204                                         ;; esac;
 205                                 done;
 206                         esac;
 207                 done < tmp/.packagedeps;
 208                 return 1;
 209         }
 210 """).strip()
 211
 212
 213 class IfBuiltinShellCommand(ShellCommand):
 214         def _quote(self, str):
 215                 if re.search("[^a-zA-Z0-9/_.-]", str):
 216                         return "'%s'" %(re.sub("'", "'\"'\"'", str))
 217                 return str
 218
 219         def setCommand(self, command):
 220                 if not isinstance(command, (str, unicode)):
 221                         command = ' '.join(map(self._quote, command))
 222                 self.command = [
 223                         '/bin/sh', '-c',
 224                         '%s; if checkBuiltin; then %s; else exit 0; fi' %(checkBuiltin, command)
 225                 ]
 226
 227         def setupEnvironment(self, cmd):
 228                 slaveEnv = self.slaveEnvironment
 229                 if slaveEnv is None:
 230                         slaveEnv = { }
 231                 changedFiles = { }
 232                 for request in self.build.requests:
 233                         for source in request.sources:
 234                                 for change in source.changes:
 235                                         for file in change.files:
 236                                                 changedFiles[file] = True
 237                 fullSlaveEnv = slaveEnv.copy()
 238                 fullSlaveEnv['CHANGED_FILES'] = ' '.join(changedFiles.keys())
 239                 cmd.args['env'] = fullSlaveEnv
 240
 241 slaveNames = [ ]
 242
 243 for slave in c['slaves']:
 244         slaveNames.append(slave.slavename)
 245
 246 for target in targets:
 247         ts = target.split('/')
 248
 249         factory = BuildFactory()
 250
 251         # find number of cores
 252         factory.addStep(SetProperty(
 253                 name = "nproc",
 254                 property = "nproc",
 255                 description = "Finding number of CPUs",
 256                 command = ["nproc"]))
 257
 258         # check out the source
 259         factory.addStep(Git(repourl=repo_url, mode='update'))
 260
 261         factory.addStep(ShellCommand(
 262                 name = "rmtmp",
 263                 description = "Remove tmp folder",
 264                 command=["rm", "-rf", "tmp/"]))
 265
 266         # feed
 267 #       factory.addStep(ShellCommand(
 268 #               name = "feedsconf",
 269 #               description = "Copy the feeds.conf",
 270 #               command='''cp ~/feeds.conf ./feeds.conf''' ))
 271
 272         # feed
 273         factory.addStep(ShellCommand(
 274                 name = "updatefeeds",
 275                 description = "Updating feeds",
 276                 command=["./scripts/feeds", "update"]))
 277
 278         # feed
 279         factory.addStep(ShellCommand(
 280                 name = "installfeeds",
 281                 description = "Installing feeds",
 282                 command=["./scripts/feeds", "install", "-a"]))
 283
 284         # configure
 285         factory.addStep(ShellCommand(
 286                 name = "newconfig",
 287                 description = "Seeding .config",
 288                 command='''cat <<EOT > .config
 289 CONFIG_TARGET_%s=y
 290 CONFIG_TARGET_%s_%s=y
 291 CONFIG_ALL_NONSHARED=y
 292 CONFIG_SDK=y
 293 CONFIG_IB=y
 294 # CONFIG_IB_STANDALONE is not set
 295 CONFIG_DEVEL=y
 296 CONFIG_CCACHE=y
 297 CONFIG_SIGNED_PACKAGES=y
 298 # CONFIG_PER_FEED_REPO_ADD_COMMENTED is not set
 299 CONFIG_KERNEL_KALLSYMS=y
 300 CONFIG_COLLECT_KERNEL_DEBUG=y
 301 EOT''' %(ts[0], ts[0], ts[1]) ))
 302
 303         factory.addStep(ShellCommand(
 304                 name = "delbin",
 305                 description = "Removing output directory",
 306                 command = ["rm", "-rf", "bin/"]
 307         ))
 308
 309         factory.addStep(ShellCommand(
 310                 name = "defconfig",
 311                 description = "Populating .config",
 312                 command = ["make", "defconfig"]
 313         ))
 314
 315         # check arch
 316         factory.addStep(ShellCommand(
 317                 name = "checkarch",
 318                 description = "Checking architecture",
 319                 command = ["grep", "-sq", "CONFIG_TARGET_%s=y" %(ts[0]), ".config"],
 320                 logEnviron = False,
 321                 want_stdout = False,
 322                 want_stderr = False,
 323                 haltOnFailure = True
 324         ))
 325
 326         # find libc suffix
 327         factory.addStep(SetProperty(
 328                 name = "libc",
 329                 property = "libc",
 330                 description = "Finding libc suffix",
 331                 command = ["sed", "-ne", '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }', ".config"]))
 332
 333         # install build key
 334         factory.addStep(FileDownload(mastersrc=home_dir+'/key-build', slavedest="key-build", mode=0600))
 335         factory.addStep(FileDownload(mastersrc=home_dir+'/key-build.pub', slavedest="key-build.pub", mode=0600))
 336
 337         # prepare dl
 338         factory.addStep(ShellCommand(
 339                 name = "dldir",
 340                 description = "Preparing dl/",
 341                 command = "mkdir -p $HOME/dl && ln -sf $HOME/dl ./dl",
 342                 logEnviron = False,
 343                 want_stdout = False
 344         ))
 345
 346         # populate dl
 347         factory.addStep(ShellCommand(
 348                 name = "dlrun",
 349                 description = "Populating dl/",
 350                 command = ["make", WithProperties("-j%(nproc:~4)s"), "download", "V=s"],
 351                 logEnviron = False,
 352                 locks = [dlLock.access('exclusive')]
 353         ))
 354
 355         factory.addStep(ShellCommand(
 356                 name = "cleanbase",
 357                 description = "Cleaning base-files",
 358                 command=["make", "package/base-files/clean", "V=s"]
 359         ))
 360
 361         # optional clean steps
 362         for pattern, maketarget in MakeTargetMap.items():
 363                 factory.addStep(ShellCommand(
 364                         name = maketarget,
 365                         description = maketarget,
 366                         command=["make", maketarget, "V=s"], doStepIf=IsAffected(pattern)
 367                 ))
 368
 369         # build
 370         factory.addStep(ShellCommand(
 371                 name = "tools",
 372                 description = "Building tools",
 373                 command = ["make", WithProperties("-j%(nproc:~4)s"), "tools/install", "V=s"],
 374                 haltOnFailure = True
 375         ))
 376
 377         factory.addStep(ShellCommand(
 378                 name = "toolchain",
 379                 description = "Building toolchain",
 380                 command=["make", WithProperties("-j%(nproc:~4)s"), "toolchain/install", "V=s"],
 381                 haltOnFailure = True
 382         ))
 383
 384         factory.addStep(ShellCommand(
 385                 name = "kmods",
 386                 description = "Building kmods",
 387                 command=["make", WithProperties("-j%(nproc:~4)s"), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
 388                 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
 389                 haltOnFailure = True
 390         ))
 391
 392         factory.addStep(ShellCommand(
 393                 name = "pkgbuild",
 394                 description = "Building packages",
 395                 command=["make", WithProperties("-j%(nproc:~4)s"), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
 396                 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
 397                 haltOnFailure = True
 398         ))
 399
 400         # factory.addStep(IfBuiltinShellCommand(
 401         factory.addStep(ShellCommand(
 402                 name = "pkginstall",
 403                 description = "Installing packages",
 404                 command=["make", WithProperties("-j%(nproc:~4)s"), "package/install", "V=s"],
 405                 doStepIf = isChangeBuiltin,
 406                 haltOnFailure = True
 407         ))
 408
 409         factory.addStep(ShellCommand(
 410                 name = "pkgindex",
 411                 description = "Indexing packages",
 412                 command=["make", WithProperties("-j%(nproc:~4)s"), "package/index", "V=s"],
 413                 haltOnFailure = True
 414         ))
 415
 416         #factory.addStep(IfBuiltinShellCommand(
 417         factory.addStep(ShellCommand(
 418                 name = "images",
 419                 description = "Building images",
 420                 command=["make", "-j1", "target/install", "V=s"],
 421                 doStepIf = isChangeBuiltin,
 422                 haltOnFailure = True
 423         ))
 424
 425         # upload
 426         factory.addStep(ShellCommand(
 427                 name = "uploadprepare",
 428                 description = "Preparing target directory",
 429                 command=["rsync", "-av", "--include", "/%s/" %(ts[0]), "--include", "/%s/%s/" %(ts[0], ts[1]), "--exclude", "/*", "--exclude", "/*/*", "--exclude", "/%s/%s/*" %(ts[0], ts[1]), "bin/targets/", "%s/targets/" %(rsync_bin_url)],
 430                 env={'RSYNC_PASSWORD': rsync_bin_key},
 431                 haltOnFailure = True,
 432                 logEnviron = False
 433         ))
 434
 435         factory.addStep(ShellCommand(
 436                 name = "targetupload",
 437                 description = "Uploading target files",
 438                 command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "bin/targets/%s/%s%s/" %(ts[0], ts[1], WithProperties("%(libc)s")), "%s/targets/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
 439                 env={'RSYNC_PASSWORD': rsync_bin_key},
 440                 haltOnFailure = True,
 441                 logEnviron = False
 442         ))
 443
 444         if rsync_src_url is not None:
 445                 factory.addStep(ShellCommand(
 446                         name = "sourceupload",
 447                         description = "Uploading source archives",
 448                         command=["rsync", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "dl/", "%s/" %(rsync_src_url)],
 449                         env={'RSYNC_PASSWORD': rsync_src_key},
 450                         haltOnFailure = True,
 451                         logEnviron = False
 452                 ))
 453
 454         if False:
 455                 factory.addStep(ShellCommand(
 456                         name = "packageupload",
 457                         description = "Uploading package files",
 458                         command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "bin/packages/", "%s/packages/" %(rsync_bin_url)],
 459                         env={'RSYNC_PASSWORD': rsync_bin_key},
 460                         haltOnFailure = False,
 461                         logEnviron = False
 462                 ))
 463
 464         # logs
 465         if False:
 466                 factory.addStep(ShellCommand(
 467                         name = "upload",
 468                         description = "Uploading logs",
 469                         command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "logs/", "%s/logs/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
 470                         env={'RSYNC_PASSWORD': rsync_bin_key},
 471                         haltOnFailure = False,
 472                         alwaysRun = True,
 473                         logEnviron = False
 474                 ))
 475
 476         from buildbot.config import BuilderConfig
 477
 478         c['builders'].append(BuilderConfig(name=target, slavenames=slaveNames, factory=factory))
 479
 480
 481 ####### STATUS TARGETS
 482
 483 # 'status' is a list of Status Targets. The results of each build will be
 484 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
 485 # including web pages, email senders, and IRC bots.
 486
 487 c['status'] = []
 488
 489 from buildbot.status import html
 490 from buildbot.status.web import authz, auth
 491
 492 if ini.has_option("status", "bind"):
 493         if ini.has_option("status", "user") and ini.has_option("status", "password"):
 494                 authz_cfg=authz.Authz(
 495                         # change any of these to True to enable; see the manual for more
 496                         # options
 497                         auth=auth.BasicAuth([(ini.get("status", "user"), ini.get("status", "password"))]),
 498                         gracefulShutdown = 'auth',
 499                         forceBuild = 'auth', # use this to test your slave once it is set up
 500                         forceAllBuilds = 'auth',
 501                         pingBuilder = False,
 502                         stopBuild = 'auth',
 503                         stopAllBuilds = 'auth',
 504                         cancelPendingBuild = 'auth',
 505                 )
 506                 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind"), authz=authz_cfg))
 507         else:
 508                 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind")))
 509
 510
 511 from buildbot.status import words
 512
 513 if ini.has_option("irc", "host") and ini.has_option("irc", "nickname") and ini.has_option("irc", "channel"):
 514         irc_host = ini.get("irc", "host")
 515         irc_port = 6667
 516         irc_chan = ini.get("irc", "channel")
 517         irc_nick = ini.get("irc", "nickname")
 518         irc_pass = None
 519
 520         if ini.has_option("irc", "port"):
 521                 irc_port = ini.getint("irc", "port")
 522
 523         if ini.has_option("irc", "password"):
 524                 irc_pass = ini.get("irc", "password")
 525
 526         irc = words.IRC(irc_host, irc_nick, port = irc_port, password = irc_pass,
 527                         channels = [{ "channel": irc_chan }],
 528                         notify_events = {
 529                           'exception': 1,
 530                           'successToFailure': 1,
 531                           'failureToSuccess': 1
 532                         }
 533         )
 534
 535         c['status'].append(irc)
 536
 537
 538 ####### PROJECT IDENTITY
 539
 540 # the 'title' string will appear at the top of this buildbot
 541 # installation's html.WebStatus home page (linked to the
 542 # 'titleURL') and is embedded in the title of the waterfall HTML page.
 543
 544 c['title'] = ini.get("general", "title")
 545 c['titleURL'] = ini.get("general", "title_url")
 546
 547 # the 'buildbotURL' string should point to the location where the buildbot's
 548 # internal web server (usually the html.WebStatus page) is visible. This
 549 # typically uses the port number set in the Waterfall 'status' entry, but
 550 # with an externally-visible host name which the buildbot cannot figure out
 551 # without some help.
 552
 553 c['buildbotURL'] = ini.get("general", "buildbot_url")
 554
 555 ####### DB URL
 556
 557 c['db'] = {
 558         # This specifies what database buildbot uses to store its state.  You can leave
 559         # this at its default for all but the largest installations.
 560         'db_url' : "sqlite:///state.sqlite",
 561 }