phase1/master.cfg

   1 # -*- python -*-
   2 # ex: set syntax=python:
   3
   4 import os
   5 import re
   6 import subprocess
   7 import ConfigParser
   8
   9 from buildbot import locks
  10
  11 # This is a sample buildmaster config file. It must be installed as
  12 # 'master.cfg' in your buildmaster's base directory.
  13
  14 ini = ConfigParser.ConfigParser()
  15 ini.read("./config.ini")
  16
  17 # This is the dictionary that the buildmaster pays attention to. We also use
  18 # a shorter alias to save typing.
  19 c = BuildmasterConfig = {}
  20
  21 ####### BUILDSLAVES
  22
  23 # The 'slaves' list defines the set of recognized buildslaves. Each element is
  24 # a BuildSlave object, specifying a unique slave name and password.  The same
  25 # slave name and password must be configured on the slave.
  26 from buildbot.buildslave import BuildSlave
  27
  28 c['slaves'] = []
  29
  30 for section in ini.sections():
  31         if section.startswith("slave "):
  32                 if ini.has_option(section, "name") and ini.has_option(section, "password"):
  33                         name = ini.get(section, "name")
  34                         password = ini.get(section, "password")
  35                         max_builds = 1
  36                         if ini.has_option(section, "builds"):
  37                                 max_builds = ini.getint(section, "builds")
  38                         c['slaves'].append(BuildSlave(name, password, max_builds = max_builds))
  39
  40 # 'slavePortnum' defines the TCP port to listen on for connections from slaves.
  41 # This must match the value configured into the buildslaves (with their
  42 # --master option)
  43 c['slavePortnum'] = 9989
  44
  45 # coalesce builds
  46 c['mergeRequests'] = True
  47
  48 ####### CHANGESOURCES
  49
  50 home_dir = os.path.abspath(ini.get("general", "homedir"))
  51 tree_expire = 0
  52
  53 if ini.has_option("general", "expire"):
  54         tree_expire = ini.getint("general", "expire")
  55
  56 repo_url = ini.get("repo", "url")
  57
  58 rsync_bin_url = ini.get("rsync", "binary_url")
  59 rsync_bin_key = ini.get("rsync", "binary_password")
  60
  61 rsync_src_url = None
  62 rsync_src_key = None
  63
  64 if ini.has_option("rsync", "source_url"):
  65         rsync_src_url = ini.get("rsync", "source_url")
  66         rsync_src_key = ini.get("rsync", "source_password")
  67
  68 gpg_keyid = None
  69 gpg_comment = "Unattended build signature"
  70 gpg_passfile = "/dev/null"
  71
  72 if ini.has_option("gpg", "keyid"):
  73         gpg_keyid = ini.get("gpg", "keyid")
  74
  75 if ini.has_option("gpg", "comment"):
  76         gpg_comment = ini.get("gpg", "comment")
  77
  78 if ini.has_option("gpg", "passfile"):
  79         gpg_passfile = ini.get("gpg", "passfile")
  80
  81
  82 # find targets
  83 targets = [ ]
  84
  85 if not os.path.isdir(home_dir+'/source.git'):
  86         subprocess.call(["git", "clone", "--depth=1", repo_url, home_dir+'/source.git'])
  87
  88 findtargets = subprocess.Popen([home_dir+'/dumpinfo.pl', 'targets'],
  89         stdout = subprocess.PIPE, cwd = home_dir+'/source.git')
  90
  91 while True:
  92         line = findtargets.stdout.readline()
  93         if not line:
  94                 break
  95         ta = line.strip().split(' ')
  96         targets.append(ta[0])
  97
  98
  99 # the 'change_source' setting tells the buildmaster how it should find out
 100 # about source code changes.  Here we point to the buildbot clone of pyflakes.
 101
 102 from buildbot.changes.gitpoller import GitPoller
 103 c['change_source'] = []
 104 c['change_source'].append(GitPoller(
 105         repo_url,
 106         workdir=home_dir+'/source.git', branch='master',
 107         pollinterval=300))
 108
 109 ####### SCHEDULERS
 110
 111 # Configure the Schedulers, which decide how to react to incoming changes.  In this
 112 # case, just kick off a 'basebuild' build
 113
 114 from buildbot.schedulers.basic import SingleBranchScheduler
 115 from buildbot.schedulers.forcesched import ForceScheduler
 116 from buildbot.changes import filter
 117 c['schedulers'] = []
 118 c['schedulers'].append(SingleBranchScheduler(
 119         name="all",
 120         change_filter=filter.ChangeFilter(branch='master'),
 121         treeStableTimer=60,
 122         builderNames=targets))
 123
 124 c['schedulers'].append(ForceScheduler(
 125         name="force",
 126         builderNames=targets))
 127
 128 ####### BUILDERS
 129
 130 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
 131 # what steps, and which slaves can execute them.  Note that any particular build will
 132 # only take place on one slave.
 133
 134 from buildbot.process.factory import BuildFactory
 135 from buildbot.steps.source import Git
 136 from buildbot.steps.shell import ShellCommand
 137 from buildbot.steps.shell import SetProperty
 138 from buildbot.steps.transfer import FileUpload
 139 from buildbot.steps.transfer import FileDownload
 140 from buildbot.steps.master import MasterShellCommand
 141 from buildbot.process.properties import WithProperties
 142
 143
 144 CleanTargetMap = [
 145         [ "tools",      "tools/clean"                   ],
 146         [ "chain",      "toolchain/clean"               ],
 147         [ "linux",      "target/linux/clean"    ],
 148         [ "dir",        "dirclean"                              ],
 149         [ "dist",       "distclean"                             ]
 150 ]
 151
 152 def IsCleanRequested(pattern):
 153         def CheckCleanProperty(step):
 154                 val = step.getProperty("clean")
 155                 if val and re.match(pattern, val):
 156                         return True
 157                 else:
 158                         return False
 159
 160         return CheckCleanProperty
 161
 162
 163 c['builders'] = []
 164
 165 dlLock = locks.SlaveLock("slave_dl")
 166
 167 checkBuiltin = re.sub('[\t\n ]+', ' ', """
 168         checkBuiltin() {
 169                 local symbol op path file;
 170                 for file in $CHANGED_FILES; do
 171                         case "$file" in
 172                                 package/*/*) : ;;
 173                                 *) return 0 ;;
 174                         esac;
 175                 done;
 176                 while read symbol op path; do
 177                         case "$symbol" in package-*)
 178                                 symbol="${symbol##*(}";
 179                                 symbol="${symbol%)}";
 180                                 for file in $CHANGED_FILES; do
 181                                         case "$file" in "package/$path/"*)
 182                                                 grep -qsx "$symbol=y" .config && return 0
 183                                         ;; esac;
 184                                 done;
 185                         esac;
 186                 done < tmp/.packagedeps;
 187                 return 1;
 188         }
 189 """).strip()
 190
 191
 192 class IfBuiltinShellCommand(ShellCommand):
 193         def _quote(self, str):
 194                 if re.search("[^a-zA-Z0-9/_.-]", str):
 195                         return "'%s'" %(re.sub("'", "'\"'\"'", str))
 196                 return str
 197
 198         def setCommand(self, command):
 199                 if not isinstance(command, (str, unicode)):
 200                         command = ' '.join(map(self._quote, command))
 201                 self.command = [
 202                         '/bin/sh', '-c',
 203                         '%s; if checkBuiltin; then %s; else exit 0; fi' %(checkBuiltin, command)
 204                 ]
 205
 206         def setupEnvironment(self, cmd):
 207                 slaveEnv = self.slaveEnvironment
 208                 if slaveEnv is None:
 209                         slaveEnv = { }
 210                 changedFiles = { }
 211                 for request in self.build.requests:
 212                         for source in request.sources:
 213                                 for change in source.changes:
 214                                         for file in change.files:
 215                                                 changedFiles[file] = True
 216                 fullSlaveEnv = slaveEnv.copy()
 217                 fullSlaveEnv['CHANGED_FILES'] = ' '.join(changedFiles.keys())
 218                 cmd.args['env'] = fullSlaveEnv
 219
 220 slaveNames = [ ]
 221
 222 for slave in c['slaves']:
 223         slaveNames.append(slave.slavename)
 224
 225 for target in targets:
 226         ts = target.split('/')
 227
 228         factory = BuildFactory()
 229
 230         # find number of cores
 231         factory.addStep(SetProperty(
 232                 name = "nproc",
 233                 property = "nproc",
 234                 description = "Finding number of CPUs",
 235                 command = ["nproc"]))
 236
 237         # expire tree if needed
 238         if tree_expire > 0:
 239                 factory.addStep(FileDownload(
 240                         mastersrc = "expire.sh",
 241                         slavedest = "../expire.sh",
 242                         mode = 0755))
 243
 244                 factory.addStep(ShellCommand(
 245                         name = "expire",
 246                         description = "Checking for build tree expiry",
 247                         command = ["./expire.sh", str(tree_expire)],
 248                         workdir = ".",
 249                         haltOnFailure = True,
 250                         timeout = 2400))
 251
 252         # user-requested clean targets
 253         for tuple in CleanTargetMap:
 254                 factory.addStep(ShellCommand(
 255                         name = tuple[1],
 256                         description = 'User-requested "make %s"' % tuple[1],
 257                         command = ["make", tuple[1], "V=s"],
 258                         doStepIf = IsCleanRequested(tuple[0])
 259                 ))
 260
 261         # check out the source
 262         factory.addStep(Git(repourl=repo_url, mode='update'))
 263
 264         factory.addStep(ShellCommand(
 265                 name = "rmtmp",
 266                 description = "Remove tmp folder",
 267                 command=["rm", "-rf", "tmp/"]))
 268
 269         # feed
 270 #       factory.addStep(ShellCommand(
 271 #               name = "feedsconf",
 272 #               description = "Copy the feeds.conf",
 273 #               command='''cp ~/feeds.conf ./feeds.conf''' ))
 274
 275         # feed
 276         factory.addStep(ShellCommand(
 277                 name = "rmfeedlinks",
 278                 description = "Remove feed symlinks",
 279                 command=["rm", "-rf", "package/feeds/"]))
 280
 281         # feed
 282         factory.addStep(ShellCommand(
 283                 name = "updatefeeds",
 284                 description = "Updating feeds",
 285                 command=["./scripts/feeds", "update"]))
 286
 287         # feed
 288         factory.addStep(ShellCommand(
 289                 name = "installfeeds",
 290                 description = "Installing feeds",
 291                 command=["./scripts/feeds", "install", "-a"]))
 292
 293         # configure
 294         factory.addStep(ShellCommand(
 295                 name = "newconfig",
 296                 description = "Seeding .config",
 297                 command='''cat <<EOT > .config
 298 CONFIG_TARGET_%s=y
 299 CONFIG_TARGET_%s_%s=y
 300 CONFIG_ALL_NONSHARED=y
 301 CONFIG_SDK=y
 302 CONFIG_IB=y
 303 # CONFIG_IB_STANDALONE is not set
 304 CONFIG_DEVEL=y
 305 CONFIG_CCACHE=y
 306 CONFIG_SIGNED_PACKAGES=y
 307 # CONFIG_PER_FEED_REPO_ADD_COMMENTED is not set
 308 CONFIG_KERNEL_KALLSYMS=y
 309 CONFIG_COLLECT_KERNEL_DEBUG=y
 310 CONFIG_TARGET_ALL_PROFILES=y
 311 CONFIG_TARGET_MULTI_PROFILE=y
 312 CONFIG_TARGET_PER_DEVICE_ROOTFS=y
 313 EOT''' %(ts[0], ts[0], ts[1]) ))
 314
 315         factory.addStep(ShellCommand(
 316                 name = "delbin",
 317                 description = "Removing output directory",
 318                 command = ["rm", "-rf", "bin/"]
 319         ))
 320
 321         factory.addStep(ShellCommand(
 322                 name = "defconfig",
 323                 description = "Populating .config",
 324                 command = ["make", "defconfig"]
 325         ))
 326
 327         # check arch
 328         factory.addStep(ShellCommand(
 329                 name = "checkarch",
 330                 description = "Checking architecture",
 331                 command = ["grep", "-sq", "CONFIG_TARGET_%s=y" %(ts[0]), ".config"],
 332                 logEnviron = False,
 333                 want_stdout = False,
 334                 want_stderr = False,
 335                 haltOnFailure = True
 336         ))
 337
 338         # find libc suffix
 339         factory.addStep(SetProperty(
 340                 name = "libc",
 341                 property = "libc",
 342                 description = "Finding libc suffix",
 343                 command = ["sed", "-ne", '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }', ".config"]))
 344
 345         # install build key
 346         factory.addStep(FileDownload(mastersrc=home_dir+'/key-build', slavedest="key-build", mode=0600))
 347         factory.addStep(FileDownload(mastersrc=home_dir+'/key-build.pub', slavedest="key-build.pub", mode=0600))
 348
 349         # prepare dl
 350         factory.addStep(ShellCommand(
 351                 name = "dldir",
 352                 description = "Preparing dl/",
 353                 command = "mkdir -p $HOME/dl && ln -sf $HOME/dl ./dl",
 354                 logEnviron = False,
 355                 want_stdout = False
 356         ))
 357
 358         # populate dl
 359         factory.addStep(ShellCommand(
 360                 name = "dlrun",
 361                 description = "Populating dl/",
 362                 command = ["make", WithProperties("-j%(nproc:~4)s"), "download", "V=s"],
 363                 logEnviron = False,
 364                 locks = [dlLock.access('exclusive')]
 365         ))
 366
 367         factory.addStep(ShellCommand(
 368                 name = "cleanbase",
 369                 description = "Cleaning base-files",
 370                 command=["make", "package/base-files/clean", "V=s"]
 371         ))
 372
 373         # build
 374         factory.addStep(ShellCommand(
 375                 name = "tools",
 376                 description = "Building tools",
 377                 command = ["make", WithProperties("-j%(nproc:~4)s"), "tools/install", "V=s"],
 378                 haltOnFailure = True
 379         ))
 380
 381         factory.addStep(ShellCommand(
 382                 name = "toolchain",
 383                 description = "Building toolchain",
 384                 command=["make", WithProperties("-j%(nproc:~4)s"), "toolchain/install", "V=s"],
 385                 haltOnFailure = True
 386         ))
 387
 388         factory.addStep(ShellCommand(
 389                 name = "kmods",
 390                 description = "Building kmods",
 391                 command=["make", WithProperties("-j%(nproc:~4)s"), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
 392                 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
 393                 haltOnFailure = True
 394         ))
 395
 396         factory.addStep(ShellCommand(
 397                 name = "pkgbuild",
 398                 description = "Building packages",
 399                 command=["make", WithProperties("-j%(nproc:~4)s"), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
 400                 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
 401                 haltOnFailure = True
 402         ))
 403
 404         # factory.addStep(IfBuiltinShellCommand(
 405         factory.addStep(ShellCommand(
 406                 name = "pkginstall",
 407                 description = "Installing packages",
 408                 command=["make", WithProperties("-j%(nproc:~4)s"), "package/install", "V=s"],
 409                 haltOnFailure = True
 410         ))
 411
 412         factory.addStep(ShellCommand(
 413                 name = "pkgindex",
 414                 description = "Indexing packages",
 415                 command=["make", WithProperties("-j%(nproc:~4)s"), "package/index", "V=s"],
 416                 haltOnFailure = True
 417         ))
 418
 419         #factory.addStep(IfBuiltinShellCommand(
 420         factory.addStep(ShellCommand(
 421                 name = "images",
 422                 description = "Building images",
 423                 command=["make", WithProperties("-j%(nproc:~4)s"), "target/install", "V=s"],
 424                 haltOnFailure = True
 425         ))
 426
 427         factory.addStep(ShellCommand(
 428                 name = "checksums",
 429                 description = "Calculating checksums",
 430                 command=["make", "-j1", "checksum", "V=s"],
 431                 haltOnFailure = True
 432         ))
 433
 434         # sign
 435         if gpg_keyid is not None:
 436                 factory.addStep(MasterShellCommand(
 437                         name = "signprepare",
 438                         description = "Preparing temporary signing directory",
 439                         command = ["mkdir", "-p", "%s/signing" %(home_dir)],
 440                         haltOnFailure = True
 441                 ))
 442
 443                 factory.addStep(ShellCommand(
 444                         name = "signpack",
 445                         description = "Packing files to sign",
 446                         command = WithProperties("find bin/targets/%s/%s%%(libc)s/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz" %(ts[0], ts[1])),
 447                         haltOnFailure = True
 448                 ))
 449
 450                 factory.addStep(FileUpload(
 451                         slavesrc = "sign.tar.gz",
 452                         masterdest = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
 453                         haltOnFailure = True
 454                 ))
 455
 456                 factory.addStep(MasterShellCommand(
 457                         name = "signfiles",
 458                         description = "Signing files",
 459                         command = ["%s/signall.sh" %(home_dir), "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]), gpg_keyid, gpg_passfile, gpg_comment],
 460                         haltOnFailure = True
 461                 ))
 462
 463                 factory.addStep(FileDownload(
 464                         mastersrc = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
 465                         slavedest = "sign.tar.gz",
 466                         haltOnFailure = True
 467                 ))
 468
 469                 factory.addStep(ShellCommand(
 470                         name = "signunpack",
 471                         description = "Unpacking signed files",
 472                         command = ["tar", "-xzf", "sign.tar.gz"],
 473                         haltOnFailure = True
 474                 ))
 475
 476         # upload
 477         factory.addStep(ShellCommand(
 478                 name = "uploadprepare",
 479                 description = "Preparing target directory",
 480                 command=["rsync", "-av", "--include", "/%s/" %(ts[0]), "--include", "/%s/%s/" %(ts[0], ts[1]), "--exclude", "/*", "--exclude", "/*/*", "--exclude", "/%s/%s/*" %(ts[0], ts[1]), "bin/targets/", "%s/targets/" %(rsync_bin_url)],
 481                 env={'RSYNC_PASSWORD': rsync_bin_key},
 482                 haltOnFailure = True,
 483                 logEnviron = False
 484         ))
 485
 486         factory.addStep(ShellCommand(
 487                 name = "targetupload",
 488                 description = "Uploading target files",
 489                 command=["rsync", "--delete", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", WithProperties("bin/targets/%s/%s%%(libc)s/" %(ts[0], ts[1])), "%s/targets/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
 490                 env={'RSYNC_PASSWORD': rsync_bin_key},
 491                 haltOnFailure = True,
 492                 logEnviron = False
 493         ))
 494
 495         if rsync_src_url is not None:
 496                 factory.addStep(ShellCommand(
 497                         name = "sourceupload",
 498                         description = "Uploading source archives",
 499                         command=["rsync", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "dl/", "%s/" %(rsync_src_url)],
 500                         env={'RSYNC_PASSWORD': rsync_src_key},
 501                         haltOnFailure = True,
 502                         logEnviron = False
 503                 ))
 504
 505         if False:
 506                 factory.addStep(ShellCommand(
 507                         name = "packageupload",
 508                         description = "Uploading package files",
 509                         command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "bin/packages/", "%s/packages/" %(rsync_bin_url)],
 510                         env={'RSYNC_PASSWORD': rsync_bin_key},
 511                         haltOnFailure = False,
 512                         logEnviron = False
 513                 ))
 514
 515         # logs
 516         if False:
 517                 factory.addStep(ShellCommand(
 518                         name = "upload",
 519                         description = "Uploading logs",
 520                         command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "logs/", "%s/logs/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
 521                         env={'RSYNC_PASSWORD': rsync_bin_key},
 522                         haltOnFailure = False,
 523                         alwaysRun = True,
 524                         logEnviron = False
 525                 ))
 526
 527         from buildbot.config import BuilderConfig
 528
 529         c['builders'].append(BuilderConfig(name=target, slavenames=slaveNames, factory=factory))
 530
 531
 532 ####### STATUS TARGETS
 533
 534 # 'status' is a list of Status Targets. The results of each build will be
 535 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
 536 # including web pages, email senders, and IRC bots.
 537
 538 c['status'] = []
 539
 540 from buildbot.status import html
 541 from buildbot.status.web import authz, auth
 542
 543 if ini.has_option("status", "bind"):
 544         if ini.has_option("status", "user") and ini.has_option("status", "password"):
 545                 authz_cfg=authz.Authz(
 546                         # change any of these to True to enable; see the manual for more
 547                         # options
 548                         auth=auth.BasicAuth([(ini.get("status", "user"), ini.get("status", "password"))]),
 549                         gracefulShutdown = 'auth',
 550                         forceBuild = 'auth', # use this to test your slave once it is set up
 551                         forceAllBuilds = 'auth',
 552                         pingBuilder = False,
 553                         stopBuild = 'auth',
 554                         stopAllBuilds = 'auth',
 555                         cancelPendingBuild = 'auth',
 556                 )
 557                 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind"), authz=authz_cfg))
 558         else:
 559                 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind")))
 560
 561
 562 from buildbot.status import words
 563
 564 if ini.has_option("irc", "host") and ini.has_option("irc", "nickname") and ini.has_option("irc", "channel"):
 565         irc_host = ini.get("irc", "host")
 566         irc_port = 6667
 567         irc_chan = ini.get("irc", "channel")
 568         irc_nick = ini.get("irc", "nickname")
 569         irc_pass = None
 570
 571         if ini.has_option("irc", "port"):
 572                 irc_port = ini.getint("irc", "port")
 573
 574         if ini.has_option("irc", "password"):
 575                 irc_pass = ini.get("irc", "password")
 576
 577         irc = words.IRC(irc_host, irc_nick, port = irc_port, password = irc_pass,
 578                         channels = [{ "channel": irc_chan }],
 579                         notify_events = {
 580                           'exception': 1,
 581                           'successToFailure': 1,
 582                           'failureToSuccess': 1
 583                         }
 584         )
 585
 586         c['status'].append(irc)
 587
 588
 589 ####### PROJECT IDENTITY
 590
 591 # the 'title' string will appear at the top of this buildbot
 592 # installation's html.WebStatus home page (linked to the
 593 # 'titleURL') and is embedded in the title of the waterfall HTML page.
 594
 595 c['title'] = ini.get("general", "title")
 596 c['titleURL'] = ini.get("general", "title_url")
 597
 598 # the 'buildbotURL' string should point to the location where the buildbot's
 599 # internal web server (usually the html.WebStatus page) is visible. This
 600 # typically uses the port number set in the Waterfall 'status' entry, but
 601 # with an externally-visible host name which the buildbot cannot figure out
 602 # without some help.
 603
 604 c['buildbotURL'] = ini.get("general", "buildbot_url")
 605
 606 ####### DB URL
 607
 608 c['db'] = {
 609         # This specifies what database buildbot uses to store its state.  You can leave
 610         # this at its default for all but the largest installations.
 611         'db_url' : "sqlite:///state.sqlite",
 612 }