phase1/master.cfg

   1 # -*- python -*-
   2 # ex: set syntax=python:
   3
   4 import os
   5 import re
   6 import subprocess
   7 import ConfigParser
   8
   9 from buildbot import locks
  10
  11 # This is a sample buildmaster config file. It must be installed as
  12 # 'master.cfg' in your buildmaster's base directory.
  13
  14 ini = ConfigParser.ConfigParser()
  15 ini.read("./config.ini")
  16
  17 # This is the dictionary that the buildmaster pays attention to. We also use
  18 # a shorter alias to save typing.
  19 c = BuildmasterConfig = {}
  20
  21 ####### BUILDSLAVES
  22
  23 # The 'slaves' list defines the set of recognized buildslaves. Each element is
  24 # a BuildSlave object, specifying a unique slave name and password.  The same
  25 # slave name and password must be configured on the slave.
  26 from buildbot.buildslave import BuildSlave
  27
  28 c['slaves'] = []
  29
  30 for section in ini.sections():
  31         if section.startswith("slave "):
  32                 if ini.has_option(section, "name") and ini.has_option(section, "password"):
  33                         name = ini.get(section, "name")
  34                         password = ini.get(section, "password")
  35                         max_builds = 1
  36                         if ini.has_option(section, "builds"):
  37                                 max_builds = ini.getint(section, "builds")
  38                         c['slaves'].append(BuildSlave(name, password, max_builds = max_builds))
  39
  40 # 'slavePortnum' defines the TCP port to listen on for connections from slaves.
  41 # This must match the value configured into the buildslaves (with their
  42 # --master option)
  43 c['slavePortnum'] = 9989
  44
  45 # coalesce builds
  46 c['mergeRequests'] = True
  47
  48 ####### CHANGESOURCES
  49
  50 home_dir = os.path.abspath(ini.get("general", "homedir"))
  51
  52 repo_url = ini.get("repo", "url")
  53
  54 rsync_bin_url = ini.get("rsync", "binary_url")
  55 rsync_bin_key = ini.get("rsync", "binary_password")
  56
  57 rsync_src_url = None
  58 rsync_src_key = None
  59
  60 if ini.has_option("rsync", "source_url"):
  61         rsync_src_url = ini.get("rsync", "source_url")
  62         rsync_src_key = ini.get("rsync", "source_password")
  63
  64 gpg_keyid = None
  65 gpg_comment = "Unattended build signature"
  66 gpg_passfile = "/dev/null"
  67
  68 if ini.has_option("gpg", "keyid"):
  69         gpg_keyid = ini.get("gpg", "keyid")
  70
  71 if ini.has_option("gpg", "comment"):
  72         gpg_comment = ini.get("gpg", "comment")
  73
  74 if ini.has_option("gpg", "passfile"):
  75         gpg_passfile = ini.get("gpg", "passfile")
  76
  77
  78 # find targets
  79 targets = [ ]
  80
  81 findtargets = subprocess.Popen([home_dir+'/dumpinfo.pl', 'targets'],
  82         stdout = subprocess.PIPE, cwd = home_dir+'/source.git')
  83
  84 while True:
  85         line = findtargets.stdout.readline()
  86         if not line:
  87                 break
  88         ta = line.strip().split(' ')
  89         targets.append(ta[0])
  90
  91
  92 # the 'change_source' setting tells the buildmaster how it should find out
  93 # about source code changes.  Here we point to the buildbot clone of pyflakes.
  94
  95 from buildbot.changes.gitpoller import GitPoller
  96 c['change_source'] = []
  97 c['change_source'].append(GitPoller(
  98         repo_url,
  99         workdir=home_dir+'/source.git', branch='master',
 100         pollinterval=300))
 101
 102 ####### SCHEDULERS
 103
 104 # Configure the Schedulers, which decide how to react to incoming changes.  In this
 105 # case, just kick off a 'basebuild' build
 106
 107 from buildbot.schedulers.basic import SingleBranchScheduler
 108 from buildbot.schedulers.forcesched import ForceScheduler
 109 from buildbot.changes import filter
 110 c['schedulers'] = []
 111 c['schedulers'].append(SingleBranchScheduler(
 112         name="all",
 113         change_filter=filter.ChangeFilter(branch='master'),
 114         treeStableTimer=60,
 115         builderNames=targets))
 116
 117 c['schedulers'].append(ForceScheduler(
 118         name="force",
 119         builderNames=targets))
 120
 121 ####### BUILDERS
 122
 123 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
 124 # what steps, and which slaves can execute them.  Note that any particular build will
 125 # only take place on one slave.
 126
 127 from buildbot.process.factory import BuildFactory
 128 from buildbot.steps.source import Git
 129 from buildbot.steps.shell import ShellCommand
 130 from buildbot.steps.shell import SetProperty
 131 from buildbot.steps.transfer import FileUpload
 132 from buildbot.steps.transfer import FileDownload
 133 from buildbot.steps.master import MasterShellCommand
 134 from buildbot.process.properties import WithProperties
 135
 136
 137 MakeTargetMap = {
 138         "^tools/":                              "tools/clean",
 139         "^toolchain/":                  "toolchain/clean",
 140         "^target/linux/":               "target/linux/clean",
 141         "^(config|include)/":   "dirclean"
 142 }
 143
 144 def IsAffected(pattern):
 145         def CheckAffected(change):
 146                 for request in change.build.requests:
 147                         for source in request.sources:
 148                                 for change in source.changes:
 149                                         for file in change.files:
 150                                                 if re.match(pattern, file):
 151                                                         return True
 152                 return False
 153         return CheckAffected
 154
 155 def isPathBuiltin(path):
 156         incl = {}
 157         pkgs = {}
 158         conf = open(".config", "r")
 159
 160         while True:
 161                 line = conf.readline()
 162                 if line == '':
 163                         break
 164                 m = re.match("^(CONFIG_PACKAGE_.+?)=y", line)
 165                 if m:
 166                         incl[m.group(1)] = True
 167
 168         conf.close()
 169
 170         deps = open("tmp/.packagedeps", "r")
 171
 172         while True:
 173                 line = deps.readline()
 174                 if line == '':
 175                         break
 176                 m = re.match("^package-\$\((CONFIG_PACKAGE_.+?)\) \+= (\S+)", line)
 177                 if m and incl.get(m.group(1)) == True:
 178                         pkgs["package/%s" % m.group(2)] = True
 179
 180         deps.close()
 181
 182         while path != '':
 183                 if pkgs.get(path) == True:
 184                         return True
 185                 path = os.path.dirname(path)
 186
 187         return False
 188
 189 def isChangeBuiltin(change):
 190         return True
 191 #       for request in change.build.requests:
 192 #               for source in request.sources:
 193 #                       for change in source.changes:
 194 #                               for file in change.files:
 195 #                                       if isPathBuiltin(file):
 196 #                                               return True
 197 #       return False
 198
 199
 200 c['builders'] = []
 201
 202 dlLock = locks.SlaveLock("slave_dl")
 203
 204 checkBuiltin = re.sub('[\t\n ]+', ' ', """
 205         checkBuiltin() {
 206                 local symbol op path file;
 207                 for file in $CHANGED_FILES; do
 208                         case "$file" in
 209                                 package/*/*) : ;;
 210                                 *) return 0 ;;
 211                         esac;
 212                 done;
 213                 while read symbol op path; do
 214                         case "$symbol" in package-*)
 215                                 symbol="${symbol##*(}";
 216                                 symbol="${symbol%)}";
 217                                 for file in $CHANGED_FILES; do
 218                                         case "$file" in "package/$path/"*)
 219                                                 grep -qsx "$symbol=y" .config && return 0
 220                                         ;; esac;
 221                                 done;
 222                         esac;
 223                 done < tmp/.packagedeps;
 224                 return 1;
 225         }
 226 """).strip()
 227
 228
 229 class IfBuiltinShellCommand(ShellCommand):
 230         def _quote(self, str):
 231                 if re.search("[^a-zA-Z0-9/_.-]", str):
 232                         return "'%s'" %(re.sub("'", "'\"'\"'", str))
 233                 return str
 234
 235         def setCommand(self, command):
 236                 if not isinstance(command, (str, unicode)):
 237                         command = ' '.join(map(self._quote, command))
 238                 self.command = [
 239                         '/bin/sh', '-c',
 240                         '%s; if checkBuiltin; then %s; else exit 0; fi' %(checkBuiltin, command)
 241                 ]
 242
 243         def setupEnvironment(self, cmd):
 244                 slaveEnv = self.slaveEnvironment
 245                 if slaveEnv is None:
 246                         slaveEnv = { }
 247                 changedFiles = { }
 248                 for request in self.build.requests:
 249                         for source in request.sources:
 250                                 for change in source.changes:
 251                                         for file in change.files:
 252                                                 changedFiles[file] = True
 253                 fullSlaveEnv = slaveEnv.copy()
 254                 fullSlaveEnv['CHANGED_FILES'] = ' '.join(changedFiles.keys())
 255                 cmd.args['env'] = fullSlaveEnv
 256
 257 slaveNames = [ ]
 258
 259 for slave in c['slaves']:
 260         slaveNames.append(slave.slavename)
 261
 262 for target in targets:
 263         ts = target.split('/')
 264
 265         factory = BuildFactory()
 266
 267         # find number of cores
 268         factory.addStep(SetProperty(
 269                 name = "nproc",
 270                 property = "nproc",
 271                 description = "Finding number of CPUs",
 272                 command = ["nproc"]))
 273
 274         # check out the source
 275         factory.addStep(Git(repourl=repo_url, mode='update'))
 276
 277         factory.addStep(ShellCommand(
 278                 name = "rmtmp",
 279                 description = "Remove tmp folder",
 280                 command=["rm", "-rf", "tmp/"]))
 281
 282         # feed
 283 #       factory.addStep(ShellCommand(
 284 #               name = "feedsconf",
 285 #               description = "Copy the feeds.conf",
 286 #               command='''cp ~/feeds.conf ./feeds.conf''' ))
 287
 288         # feed
 289         factory.addStep(ShellCommand(
 290                 name = "updatefeeds",
 291                 description = "Updating feeds",
 292                 command=["./scripts/feeds", "update"]))
 293
 294         # feed
 295         factory.addStep(ShellCommand(
 296                 name = "installfeeds",
 297                 description = "Installing feeds",
 298                 command=["./scripts/feeds", "install", "-a"]))
 299
 300         # configure
 301         factory.addStep(ShellCommand(
 302                 name = "newconfig",
 303                 description = "Seeding .config",
 304                 command='''cat <<EOT > .config
 305 CONFIG_TARGET_%s=y
 306 CONFIG_TARGET_%s_%s=y
 307 CONFIG_ALL_NONSHARED=y
 308 CONFIG_SDK=y
 309 CONFIG_IB=y
 310 # CONFIG_IB_STANDALONE is not set
 311 CONFIG_DEVEL=y
 312 CONFIG_CCACHE=y
 313 CONFIG_SIGNED_PACKAGES=y
 314 # CONFIG_PER_FEED_REPO_ADD_COMMENTED is not set
 315 CONFIG_KERNEL_KALLSYMS=y
 316 CONFIG_COLLECT_KERNEL_DEBUG=y
 317 CONFIG_TARGET_ALL_PROFILES=y
 318 CONFIG_TARGET_MULTI_PROFILE=y
 319 CONFIG_TARGET_PER_DEVICE_ROOTFS=y
 320 EOT''' %(ts[0], ts[0], ts[1]) ))
 321
 322         factory.addStep(ShellCommand(
 323                 name = "delbin",
 324                 description = "Removing output directory",
 325                 command = ["rm", "-rf", "bin/"]
 326         ))
 327
 328         factory.addStep(ShellCommand(
 329                 name = "defconfig",
 330                 description = "Populating .config",
 331                 command = ["make", "defconfig"]
 332         ))
 333
 334         # check arch
 335         factory.addStep(ShellCommand(
 336                 name = "checkarch",
 337                 description = "Checking architecture",
 338                 command = ["grep", "-sq", "CONFIG_TARGET_%s=y" %(ts[0]), ".config"],
 339                 logEnviron = False,
 340                 want_stdout = False,
 341                 want_stderr = False,
 342                 haltOnFailure = True
 343         ))
 344
 345         # find libc suffix
 346         factory.addStep(SetProperty(
 347                 name = "libc",
 348                 property = "libc",
 349                 description = "Finding libc suffix",
 350                 command = ["sed", "-ne", '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }', ".config"]))
 351
 352         # install build key
 353         factory.addStep(FileDownload(mastersrc=home_dir+'/key-build', slavedest="key-build", mode=0600))
 354         factory.addStep(FileDownload(mastersrc=home_dir+'/key-build.pub', slavedest="key-build.pub", mode=0600))
 355
 356         # prepare dl
 357         factory.addStep(ShellCommand(
 358                 name = "dldir",
 359                 description = "Preparing dl/",
 360                 command = "mkdir -p $HOME/dl && ln -sf $HOME/dl ./dl",
 361                 logEnviron = False,
 362                 want_stdout = False
 363         ))
 364
 365         # populate dl
 366         factory.addStep(ShellCommand(
 367                 name = "dlrun",
 368                 description = "Populating dl/",
 369                 command = ["make", WithProperties("-j%(nproc:~4)s"), "download", "V=s"],
 370                 logEnviron = False,
 371                 locks = [dlLock.access('exclusive')]
 372         ))
 373
 374         factory.addStep(ShellCommand(
 375                 name = "cleanbase",
 376                 description = "Cleaning base-files",
 377                 command=["make", "package/base-files/clean", "V=s"]
 378         ))
 379
 380         # optional clean steps
 381         for pattern, maketarget in MakeTargetMap.items():
 382                 factory.addStep(ShellCommand(
 383                         name = maketarget,
 384                         description = maketarget,
 385                         command=["make", maketarget, "V=s"], doStepIf=IsAffected(pattern)
 386                 ))
 387
 388         # build
 389         factory.addStep(ShellCommand(
 390                 name = "tools",
 391                 description = "Building tools",
 392                 command = ["make", WithProperties("-j%(nproc:~4)s"), "tools/install", "V=s"],
 393                 haltOnFailure = True
 394         ))
 395
 396         factory.addStep(ShellCommand(
 397                 name = "toolchain",
 398                 description = "Building toolchain",
 399                 command=["make", WithProperties("-j%(nproc:~4)s"), "toolchain/install", "V=s"],
 400                 haltOnFailure = True
 401         ))
 402
 403         factory.addStep(ShellCommand(
 404                 name = "kmods",
 405                 description = "Building kmods",
 406                 command=["make", WithProperties("-j%(nproc:~4)s"), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
 407                 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
 408                 haltOnFailure = True
 409         ))
 410
 411         factory.addStep(ShellCommand(
 412                 name = "pkgbuild",
 413                 description = "Building packages",
 414                 command=["make", WithProperties("-j%(nproc:~4)s"), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
 415                 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
 416                 haltOnFailure = True
 417         ))
 418
 419         # factory.addStep(IfBuiltinShellCommand(
 420         factory.addStep(ShellCommand(
 421                 name = "pkginstall",
 422                 description = "Installing packages",
 423                 command=["make", WithProperties("-j%(nproc:~4)s"), "package/install", "V=s"],
 424                 doStepIf = isChangeBuiltin,
 425                 haltOnFailure = True
 426         ))
 427
 428         factory.addStep(ShellCommand(
 429                 name = "pkgindex",
 430                 description = "Indexing packages",
 431                 command=["make", WithProperties("-j%(nproc:~4)s"), "package/index", "V=s"],
 432                 haltOnFailure = True
 433         ))
 434
 435         #factory.addStep(IfBuiltinShellCommand(
 436         factory.addStep(ShellCommand(
 437                 name = "images",
 438                 description = "Building images",
 439                 command=["make", "-j1", "target/install", "V=s"],
 440                 doStepIf = isChangeBuiltin,
 441                 haltOnFailure = True
 442         ))
 443
 444         # sign
 445         if gpg_keyid is not None:
 446                 factory.addStep(MasterShellCommand(
 447                         name = "signprepare",
 448                         description = "Preparing temporary signing directory",
 449                         command = ["mkdir", "-p", "%s/signing" %(home_dir)],
 450                         haltOnFailure = True
 451                 ))
 452
 453                 factory.addStep(ShellCommand(
 454                         name = "signpack",
 455                         description = "Packing files to sign",
 456                         command = WithProperties("find bin/targets/%s/%s%%(libc)s/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz" %(ts[0], ts[1])),
 457                         haltOnFailure = True
 458                 ))
 459
 460                 factory.addStep(FileUpload(
 461                         slavesrc = "sign.tar.gz",
 462                         masterdest = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
 463                         haltOnFailure = True
 464                 ))
 465
 466                 factory.addStep(MasterShellCommand(
 467                         name = "signfiles",
 468                         description = "Signing files",
 469                         command = ["%s/signall.sh" %(home_dir), "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]), gpg_keyid, gpg_passfile, gpg_comment],
 470                         haltOnFailure = True
 471                 ))
 472
 473                 factory.addStep(FileDownload(
 474                         mastersrc = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
 475                         slavedest = "sign.tar.gz",
 476                         haltOnFailure = True
 477                 ))
 478
 479                 factory.addStep(ShellCommand(
 480                         name = "signunpack",
 481                         description = "Unpacking signed files",
 482                         command = ["tar", "-xzf", "sign.tar.gz"],
 483                         haltOnFailure = True
 484                 ))
 485
 486         # upload
 487         factory.addStep(ShellCommand(
 488                 name = "uploadprepare",
 489                 description = "Preparing target directory",
 490                 command=["rsync", "-av", "--include", "/%s/" %(ts[0]), "--include", "/%s/%s/" %(ts[0], ts[1]), "--exclude", "/*", "--exclude", "/*/*", "--exclude", "/%s/%s/*" %(ts[0], ts[1]), "bin/targets/", "%s/targets/" %(rsync_bin_url)],
 491                 env={'RSYNC_PASSWORD': rsync_bin_key},
 492                 haltOnFailure = True,
 493                 logEnviron = False
 494         ))
 495
 496         factory.addStep(ShellCommand(
 497                 name = "targetupload",
 498                 description = "Uploading target files",
 499                 command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", WithProperties("bin/targets/%s/%s%%(libc)s/" %(ts[0], ts[1])), "%s/targets/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
 500                 env={'RSYNC_PASSWORD': rsync_bin_key},
 501                 haltOnFailure = True,
 502                 logEnviron = False
 503         ))
 504
 505         if rsync_src_url is not None:
 506                 factory.addStep(ShellCommand(
 507                         name = "sourceupload",
 508                         description = "Uploading source archives",
 509                         command=["rsync", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "dl/", "%s/" %(rsync_src_url)],
 510                         env={'RSYNC_PASSWORD': rsync_src_key},
 511                         haltOnFailure = True,
 512                         logEnviron = False
 513                 ))
 514
 515         if False:
 516                 factory.addStep(ShellCommand(
 517                         name = "packageupload",
 518                         description = "Uploading package files",
 519                         command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "bin/packages/", "%s/packages/" %(rsync_bin_url)],
 520                         env={'RSYNC_PASSWORD': rsync_bin_key},
 521                         haltOnFailure = False,
 522                         logEnviron = False
 523                 ))
 524
 525         # logs
 526         if False:
 527                 factory.addStep(ShellCommand(
 528                         name = "upload",
 529                         description = "Uploading logs",
 530                         command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "logs/", "%s/logs/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
 531                         env={'RSYNC_PASSWORD': rsync_bin_key},
 532                         haltOnFailure = False,
 533                         alwaysRun = True,
 534                         logEnviron = False
 535                 ))
 536
 537         from buildbot.config import BuilderConfig
 538
 539         c['builders'].append(BuilderConfig(name=target, slavenames=slaveNames, factory=factory))
 540
 541
 542 ####### STATUS TARGETS
 543
 544 # 'status' is a list of Status Targets. The results of each build will be
 545 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
 546 # including web pages, email senders, and IRC bots.
 547
 548 c['status'] = []
 549
 550 from buildbot.status import html
 551 from buildbot.status.web import authz, auth
 552
 553 if ini.has_option("status", "bind"):
 554         if ini.has_option("status", "user") and ini.has_option("status", "password"):
 555                 authz_cfg=authz.Authz(
 556                         # change any of these to True to enable; see the manual for more
 557                         # options
 558                         auth=auth.BasicAuth([(ini.get("status", "user"), ini.get("status", "password"))]),
 559                         gracefulShutdown = 'auth',
 560                         forceBuild = 'auth', # use this to test your slave once it is set up
 561                         forceAllBuilds = 'auth',
 562                         pingBuilder = False,
 563                         stopBuild = 'auth',
 564                         stopAllBuilds = 'auth',
 565                         cancelPendingBuild = 'auth',
 566                 )
 567                 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind"), authz=authz_cfg))
 568         else:
 569                 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind")))
 570
 571
 572 from buildbot.status import words
 573
 574 if ini.has_option("irc", "host") and ini.has_option("irc", "nickname") and ini.has_option("irc", "channel"):
 575         irc_host = ini.get("irc", "host")
 576         irc_port = 6667
 577         irc_chan = ini.get("irc", "channel")
 578         irc_nick = ini.get("irc", "nickname")
 579         irc_pass = None
 580
 581         if ini.has_option("irc", "port"):
 582                 irc_port = ini.getint("irc", "port")
 583
 584         if ini.has_option("irc", "password"):
 585                 irc_pass = ini.get("irc", "password")
 586
 587         irc = words.IRC(irc_host, irc_nick, port = irc_port, password = irc_pass,
 588                         channels = [{ "channel": irc_chan }],
 589                         notify_events = {
 590                           'exception': 1,
 591                           'successToFailure': 1,
 592                           'failureToSuccess': 1
 593                         }
 594         )
 595
 596         c['status'].append(irc)
 597
 598
 599 ####### PROJECT IDENTITY
 600
 601 # the 'title' string will appear at the top of this buildbot
 602 # installation's html.WebStatus home page (linked to the
 603 # 'titleURL') and is embedded in the title of the waterfall HTML page.
 604
 605 c['title'] = ini.get("general", "title")
 606 c['titleURL'] = ini.get("general", "title_url")
 607
 608 # the 'buildbotURL' string should point to the location where the buildbot's
 609 # internal web server (usually the html.WebStatus page) is visible. This
 610 # typically uses the port number set in the Waterfall 'status' entry, but
 611 # with an externally-visible host name which the buildbot cannot figure out
 612 # without some help.
 613
 614 c['buildbotURL'] = ini.get("general", "buildbot_url")
 615
 616 ####### DB URL
 617
 618 c['db'] = {
 619         # This specifies what database buildbot uses to store its state.  You can leave
 620         # this at its default for all but the largest installations.
 621         'db_url' : "sqlite:///state.sqlite",
 622 }