phase1/master.cfg

   1 # -*- python -*-
   2 # ex: set syntax=python:
   3
   4 import os
   5 import re
   6 import subprocess
   7 import ConfigParser
   8
   9 from buildbot import locks
  10
  11 # This is a sample buildmaster config file. It must be installed as
  12 # 'master.cfg' in your buildmaster's base directory.
  13
  14 ini = ConfigParser.ConfigParser()
  15 ini.read("./config.ini")
  16
  17 # This is the dictionary that the buildmaster pays attention to. We also use
  18 # a shorter alias to save typing.
  19 c = BuildmasterConfig = {}
  20
  21 ####### BUILDSLAVES
  22
  23 # The 'slaves' list defines the set of recognized buildslaves. Each element is
  24 # a BuildSlave object, specifying a unique slave name and password.  The same
  25 # slave name and password must be configured on the slave.
  26 from buildbot.buildslave import BuildSlave
  27
  28 c['slaves'] = []
  29
  30 for section in ini.sections():
  31         if section.startswith("slave "):
  32                 if ini.has_option(section, "name") and ini.has_option(section, "password"):
  33                         name = ini.get(section, "name")
  34                         password = ini.get(section, "password")
  35                         max_builds = 1
  36                         if ini.has_option(section, "builds"):
  37                                 max_builds = ini.getint(section, "builds")
  38                         c['slaves'].append(BuildSlave(name, password, max_builds = max_builds))
  39
  40 # 'slavePortnum' defines the TCP port to listen on for connections from slaves.
  41 # This must match the value configured into the buildslaves (with their
  42 # --master option)
  43 c['slavePortnum'] = 9989
  44
  45 # coalesce builds
  46 c['mergeRequests'] = True
  47
  48 ####### CHANGESOURCES
  49
  50 home_dir = os.path.abspath(ini.get("general", "homedir"))
  51 tree_expire = 0
  52
  53 if ini.has_option("general", "expire"):
  54         tree_expire = ini.getint("general", "expire")
  55
  56 repo_url = ini.get("repo", "url")
  57
  58 rsync_bin_url = ini.get("rsync", "binary_url")
  59 rsync_bin_key = ini.get("rsync", "binary_password")
  60
  61 rsync_src_url = None
  62 rsync_src_key = None
  63
  64 if ini.has_option("rsync", "source_url"):
  65         rsync_src_url = ini.get("rsync", "source_url")
  66         rsync_src_key = ini.get("rsync", "source_password")
  67
  68 gpg_keyid = None
  69 gpg_comment = "Unattended build signature"
  70 gpg_passfile = "/dev/null"
  71
  72 if ini.has_option("gpg", "keyid"):
  73         gpg_keyid = ini.get("gpg", "keyid")
  74
  75 if ini.has_option("gpg", "comment"):
  76         gpg_comment = ini.get("gpg", "comment")
  77
  78 if ini.has_option("gpg", "passfile"):
  79         gpg_passfile = ini.get("gpg", "passfile")
  80
  81
  82 # find targets
  83 targets = [ ]
  84
  85 if not os.path.isdir(home_dir+'/source.git'):
  86         subprocess.call(["git", "clone", "--depth=1", repo_url, home_dir+'/source.git'])
  87
  88 findtargets = subprocess.Popen([home_dir+'/dumpinfo.pl', 'targets'],
  89         stdout = subprocess.PIPE, cwd = home_dir+'/source.git')
  90
  91 while True:
  92         line = findtargets.stdout.readline()
  93         if not line:
  94                 break
  95         ta = line.strip().split(' ')
  96         targets.append(ta[0])
  97
  98
  99 # the 'change_source' setting tells the buildmaster how it should find out
 100 # about source code changes.  Here we point to the buildbot clone of pyflakes.
 101
 102 from buildbot.changes.gitpoller import GitPoller
 103 c['change_source'] = []
 104 c['change_source'].append(GitPoller(
 105         repo_url,
 106         workdir=home_dir+'/source.git', branch='master',
 107         pollinterval=300))
 108
 109 ####### SCHEDULERS
 110
 111 # Configure the Schedulers, which decide how to react to incoming changes.  In this
 112 # case, just kick off a 'basebuild' build
 113
 114 from buildbot.schedulers.basic import SingleBranchScheduler
 115 from buildbot.schedulers.forcesched import ForceScheduler
 116 from buildbot.changes import filter
 117 c['schedulers'] = []
 118 c['schedulers'].append(SingleBranchScheduler(
 119         name="all",
 120         change_filter=filter.ChangeFilter(branch='master'),
 121         treeStableTimer=60,
 122         builderNames=targets))
 123
 124 c['schedulers'].append(ForceScheduler(
 125         name="force",
 126         builderNames=targets))
 127
 128 ####### BUILDERS
 129
 130 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
 131 # what steps, and which slaves can execute them.  Note that any particular build will
 132 # only take place on one slave.
 133
 134 from buildbot.process.factory import BuildFactory
 135 from buildbot.steps.source import Git
 136 from buildbot.steps.shell import ShellCommand
 137 from buildbot.steps.shell import SetProperty
 138 from buildbot.steps.transfer import FileUpload
 139 from buildbot.steps.transfer import FileDownload
 140 from buildbot.steps.master import MasterShellCommand
 141 from buildbot.process.properties import WithProperties
 142
 143
 144 CleanTargetMap = [
 145         [ "tools",      "tools/clean"                   ],
 146         [ "chain",      "toolchain/clean"               ],
 147         [ "linux",      "target/linux/clean"    ],
 148         [ "dir",        "dirclean"                              ],
 149         [ "dist",       "distclean"                             ]
 150 ]
 151
 152 def IsCleanRequested(pattern):
 153         def CheckCleanProperty(step):
 154                 val = step.getProperty("clean")
 155                 if val and re.match(pattern, val):
 156                         return True
 157                 else:
 158                         return False
 159
 160         return CheckCleanProperty
 161
 162
 163 c['builders'] = []
 164
 165 dlLock = locks.SlaveLock("slave_dl")
 166
 167 checkBuiltin = re.sub('[\t\n ]+', ' ', """
 168         checkBuiltin() {
 169                 local symbol op path file;
 170                 for file in $CHANGED_FILES; do
 171                         case "$file" in
 172                                 package/*/*) : ;;
 173                                 *) return 0 ;;
 174                         esac;
 175                 done;
 176                 while read symbol op path; do
 177                         case "$symbol" in package-*)
 178                                 symbol="${symbol##*(}";
 179                                 symbol="${symbol%)}";
 180                                 for file in $CHANGED_FILES; do
 181                                         case "$file" in "package/$path/"*)
 182                                                 grep -qsx "$symbol=y" .config && return 0
 183                                         ;; esac;
 184                                 done;
 185                         esac;
 186                 done < tmp/.packagedeps;
 187                 return 1;
 188         }
 189 """).strip()
 190
 191
 192 class IfBuiltinShellCommand(ShellCommand):
 193         def _quote(self, str):
 194                 if re.search("[^a-zA-Z0-9/_.-]", str):
 195                         return "'%s'" %(re.sub("'", "'\"'\"'", str))
 196                 return str
 197
 198         def setCommand(self, command):
 199                 if not isinstance(command, (str, unicode)):
 200                         command = ' '.join(map(self._quote, command))
 201                 self.command = [
 202                         '/bin/sh', '-c',
 203                         '%s; if checkBuiltin; then %s; else exit 0; fi' %(checkBuiltin, command)
 204                 ]
 205
 206         def setupEnvironment(self, cmd):
 207                 slaveEnv = self.slaveEnvironment
 208                 if slaveEnv is None:
 209                         slaveEnv = { }
 210                 changedFiles = { }
 211                 for request in self.build.requests:
 212                         for source in request.sources:
 213                                 for change in source.changes:
 214                                         for file in change.files:
 215                                                 changedFiles[file] = True
 216                 fullSlaveEnv = slaveEnv.copy()
 217                 fullSlaveEnv['CHANGED_FILES'] = ' '.join(changedFiles.keys())
 218                 cmd.args['env'] = fullSlaveEnv
 219
 220 slaveNames = [ ]
 221
 222 for slave in c['slaves']:
 223         slaveNames.append(slave.slavename)
 224
 225 for target in targets:
 226         ts = target.split('/')
 227
 228         factory = BuildFactory()
 229
 230         # find number of cores
 231         factory.addStep(SetProperty(
 232                 name = "nproc",
 233                 property = "nproc",
 234                 description = "Finding number of CPUs",
 235                 command = ["nproc"]))
 236
 237         # expire tree if needed
 238         if tree_expire > 0:
 239                 factory.addStep(FileDownload(
 240                         mastersrc = "expire.sh",
 241                         slavedest = "../expire.sh",
 242                         mode = 0755))
 243
 244                 factory.addStep(ShellCommand(
 245                         name = "expire",
 246                         description = "Checking for build tree expiry",
 247                         command = ["./expire.sh", str(tree_expire)],
 248                         workdir = ".",
 249                         haltOnFailure = True,
 250                         timeout = 2400))
 251
 252         # user-requested clean targets
 253         else:
 254                 for tuple in CleanTargetMap:
 255                         factory.addStep(ShellCommand(
 256                                 name = tuple[1],
 257                                 description = 'User-requested "make %s"' % tuple[1],
 258                                 command = ["make", tuple[1], "V=s"],
 259                                 doStepIf = IsCleanRequested(tuple[0])
 260                         ))
 261
 262         # check out the source
 263         factory.addStep(Git(repourl=repo_url, mode='update'))
 264
 265         factory.addStep(ShellCommand(
 266                 name = "rmtmp",
 267                 description = "Remove tmp folder",
 268                 command=["rm", "-rf", "tmp/"]))
 269
 270         # feed
 271 #       factory.addStep(ShellCommand(
 272 #               name = "feedsconf",
 273 #               description = "Copy the feeds.conf",
 274 #               command='''cp ~/feeds.conf ./feeds.conf''' ))
 275
 276         # feed
 277         factory.addStep(ShellCommand(
 278                 name = "rmfeedlinks",
 279                 description = "Remove feed symlinks",
 280                 command=["rm", "-rf", "package/feeds/"]))
 281
 282         # feed
 283         factory.addStep(ShellCommand(
 284                 name = "updatefeeds",
 285                 description = "Updating feeds",
 286                 command=["./scripts/feeds", "update"]))
 287
 288         # feed
 289         factory.addStep(ShellCommand(
 290                 name = "installfeeds",
 291                 description = "Installing feeds",
 292                 command=["./scripts/feeds", "install", "-a"]))
 293
 294         # configure
 295         factory.addStep(ShellCommand(
 296                 name = "newconfig",
 297                 description = "Seeding .config",
 298                 command='''cat <<EOT > .config
 299 CONFIG_TARGET_%s=y
 300 CONFIG_TARGET_%s_%s=y
 301 CONFIG_ALL_NONSHARED=y
 302 CONFIG_SDK=y
 303 CONFIG_IB=y
 304 # CONFIG_IB_STANDALONE is not set
 305 CONFIG_DEVEL=y
 306 CONFIG_CCACHE=y
 307 CONFIG_SIGNED_PACKAGES=y
 308 # CONFIG_PER_FEED_REPO_ADD_COMMENTED is not set
 309 CONFIG_KERNEL_KALLSYMS=y
 310 CONFIG_COLLECT_KERNEL_DEBUG=y
 311 CONFIG_TARGET_ALL_PROFILES=y
 312 CONFIG_TARGET_MULTI_PROFILE=y
 313 CONFIG_TARGET_PER_DEVICE_ROOTFS=y
 314 EOT''' %(ts[0], ts[0], ts[1]) ))
 315
 316         factory.addStep(ShellCommand(
 317                 name = "delbin",
 318                 description = "Removing output directory",
 319                 command = ["rm", "-rf", "bin/"]
 320         ))
 321
 322         factory.addStep(ShellCommand(
 323                 name = "defconfig",
 324                 description = "Populating .config",
 325                 command = ["make", "defconfig"]
 326         ))
 327
 328         # check arch
 329         factory.addStep(ShellCommand(
 330                 name = "checkarch",
 331                 description = "Checking architecture",
 332                 command = ["grep", "-sq", "CONFIG_TARGET_%s=y" %(ts[0]), ".config"],
 333                 logEnviron = False,
 334                 want_stdout = False,
 335                 want_stderr = False,
 336                 haltOnFailure = True
 337         ))
 338
 339         # find libc suffix
 340         factory.addStep(SetProperty(
 341                 name = "libc",
 342                 property = "libc",
 343                 description = "Finding libc suffix",
 344                 command = ["sed", "-ne", '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }', ".config"]))
 345
 346         # install build key
 347         factory.addStep(FileDownload(mastersrc=home_dir+'/key-build', slavedest="key-build", mode=0600))
 348         factory.addStep(FileDownload(mastersrc=home_dir+'/key-build.pub', slavedest="key-build.pub", mode=0600))
 349
 350         # prepare dl
 351         factory.addStep(ShellCommand(
 352                 name = "dldir",
 353                 description = "Preparing dl/",
 354                 command = "mkdir -p $HOME/dl && ln -sf $HOME/dl ./dl",
 355                 logEnviron = False,
 356                 want_stdout = False
 357         ))
 358
 359         # populate dl
 360         factory.addStep(ShellCommand(
 361                 name = "dlrun",
 362                 description = "Populating dl/",
 363                 command = ["make", WithProperties("-j%(nproc:~4)s"), "download", "V=s"],
 364                 logEnviron = False,
 365                 locks = [dlLock.access('exclusive')]
 366         ))
 367
 368         factory.addStep(ShellCommand(
 369                 name = "cleanbase",
 370                 description = "Cleaning base-files",
 371                 command=["make", "package/base-files/clean", "V=s"]
 372         ))
 373
 374         # build
 375         factory.addStep(ShellCommand(
 376                 name = "tools",
 377                 description = "Building tools",
 378                 command = ["make", WithProperties("-j%(nproc:~4)s"), "tools/install", "V=s"],
 379                 haltOnFailure = True
 380         ))
 381
 382         factory.addStep(ShellCommand(
 383                 name = "toolchain",
 384                 description = "Building toolchain",
 385                 command=["make", WithProperties("-j%(nproc:~4)s"), "toolchain/install", "V=s"],
 386                 haltOnFailure = True
 387         ))
 388
 389         factory.addStep(ShellCommand(
 390                 name = "kmods",
 391                 description = "Building kmods",
 392                 command=["make", WithProperties("-j%(nproc:~4)s"), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
 393                 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
 394                 haltOnFailure = True
 395         ))
 396
 397         factory.addStep(ShellCommand(
 398                 name = "pkgbuild",
 399                 description = "Building packages",
 400                 command=["make", WithProperties("-j%(nproc:~4)s"), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
 401                 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
 402                 haltOnFailure = True
 403         ))
 404
 405         # factory.addStep(IfBuiltinShellCommand(
 406         factory.addStep(ShellCommand(
 407                 name = "pkginstall",
 408                 description = "Installing packages",
 409                 command=["make", WithProperties("-j%(nproc:~4)s"), "package/install", "V=s"],
 410                 haltOnFailure = True
 411         ))
 412
 413         factory.addStep(ShellCommand(
 414                 name = "pkgindex",
 415                 description = "Indexing packages",
 416                 command=["make", WithProperties("-j%(nproc:~4)s"), "package/index", "V=s"],
 417                 haltOnFailure = True
 418         ))
 419
 420         #factory.addStep(IfBuiltinShellCommand(
 421         factory.addStep(ShellCommand(
 422                 name = "images",
 423                 description = "Building images",
 424                 command=["make", WithProperties("-j%(nproc:~4)s"), "target/install", "V=s"],
 425                 haltOnFailure = True
 426         ))
 427
 428         factory.addStep(ShellCommand(
 429                 name = "checksums",
 430                 description = "Calculating checksums",
 431                 command=["make", "-j1", "checksum", "V=s"],
 432                 haltOnFailure = True
 433         ))
 434
 435         # sign
 436         if gpg_keyid is not None:
 437                 factory.addStep(MasterShellCommand(
 438                         name = "signprepare",
 439                         description = "Preparing temporary signing directory",
 440                         command = ["mkdir", "-p", "%s/signing" %(home_dir)],
 441                         haltOnFailure = True
 442                 ))
 443
 444                 factory.addStep(ShellCommand(
 445                         name = "signpack",
 446                         description = "Packing files to sign",
 447                         command = WithProperties("find bin/targets/%s/%s%%(libc)s/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz" %(ts[0], ts[1])),
 448                         haltOnFailure = True
 449                 ))
 450
 451                 factory.addStep(FileUpload(
 452                         slavesrc = "sign.tar.gz",
 453                         masterdest = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
 454                         haltOnFailure = True
 455                 ))
 456
 457                 factory.addStep(MasterShellCommand(
 458                         name = "signfiles",
 459                         description = "Signing files",
 460                         command = ["%s/signall.sh" %(home_dir), "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]), gpg_keyid, gpg_passfile, gpg_comment],
 461                         haltOnFailure = True
 462                 ))
 463
 464                 factory.addStep(FileDownload(
 465                         mastersrc = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
 466                         slavedest = "sign.tar.gz",
 467                         haltOnFailure = True
 468                 ))
 469
 470                 factory.addStep(ShellCommand(
 471                         name = "signunpack",
 472                         description = "Unpacking signed files",
 473                         command = ["tar", "-xzf", "sign.tar.gz"],
 474                         haltOnFailure = True
 475                 ))
 476
 477         # upload
 478         factory.addStep(ShellCommand(
 479                 name = "uploadprepare",
 480                 description = "Preparing target directory",
 481                 command=["rsync", "-av", "--include", "/%s/" %(ts[0]), "--include", "/%s/%s/" %(ts[0], ts[1]), "--exclude", "/*", "--exclude", "/*/*", "--exclude", "/%s/%s/*" %(ts[0], ts[1]), "bin/targets/", "%s/targets/" %(rsync_bin_url)],
 482                 env={'RSYNC_PASSWORD': rsync_bin_key},
 483                 haltOnFailure = True,
 484                 logEnviron = False
 485         ))
 486
 487         factory.addStep(ShellCommand(
 488                 name = "targetupload",
 489                 description = "Uploading target files",
 490                 command=["rsync", "--delete", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", WithProperties("bin/targets/%s/%s%%(libc)s/" %(ts[0], ts[1])), "%s/targets/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
 491                 env={'RSYNC_PASSWORD': rsync_bin_key},
 492                 haltOnFailure = True,
 493                 logEnviron = False
 494         ))
 495
 496         if rsync_src_url is not None:
 497                 factory.addStep(ShellCommand(
 498                         name = "sourceupload",
 499                         description = "Uploading source archives",
 500                         command=["rsync", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "dl/", "%s/" %(rsync_src_url)],
 501                         env={'RSYNC_PASSWORD': rsync_src_key},
 502                         haltOnFailure = True,
 503                         logEnviron = False
 504                 ))
 505
 506         if False:
 507                 factory.addStep(ShellCommand(
 508                         name = "packageupload",
 509                         description = "Uploading package files",
 510                         command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "bin/packages/", "%s/packages/" %(rsync_bin_url)],
 511                         env={'RSYNC_PASSWORD': rsync_bin_key},
 512                         haltOnFailure = False,
 513                         logEnviron = False
 514                 ))
 515
 516         # logs
 517         if False:
 518                 factory.addStep(ShellCommand(
 519                         name = "upload",
 520                         description = "Uploading logs",
 521                         command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "logs/", "%s/logs/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
 522                         env={'RSYNC_PASSWORD': rsync_bin_key},
 523                         haltOnFailure = False,
 524                         alwaysRun = True,
 525                         logEnviron = False
 526                 ))
 527
 528         from buildbot.config import BuilderConfig
 529
 530         c['builders'].append(BuilderConfig(name=target, slavenames=slaveNames, factory=factory))
 531
 532
 533 ####### STATUS TARGETS
 534
 535 # 'status' is a list of Status Targets. The results of each build will be
 536 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
 537 # including web pages, email senders, and IRC bots.
 538
 539 c['status'] = []
 540
 541 from buildbot.status import html
 542 from buildbot.status.web import authz, auth
 543
 544 if ini.has_option("status", "bind"):
 545         if ini.has_option("status", "user") and ini.has_option("status", "password"):
 546                 authz_cfg=authz.Authz(
 547                         # change any of these to True to enable; see the manual for more
 548                         # options
 549                         auth=auth.BasicAuth([(ini.get("status", "user"), ini.get("status", "password"))]),
 550                         gracefulShutdown = 'auth',
 551                         forceBuild = 'auth', # use this to test your slave once it is set up
 552                         forceAllBuilds = 'auth',
 553                         pingBuilder = False,
 554                         stopBuild = 'auth',
 555                         stopAllBuilds = 'auth',
 556                         cancelPendingBuild = 'auth',
 557                 )
 558                 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind"), authz=authz_cfg))
 559         else:
 560                 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind")))
 561
 562
 563 from buildbot.status import words
 564
 565 if ini.has_option("irc", "host") and ini.has_option("irc", "nickname") and ini.has_option("irc", "channel"):
 566         irc_host = ini.get("irc", "host")
 567         irc_port = 6667
 568         irc_chan = ini.get("irc", "channel")
 569         irc_nick = ini.get("irc", "nickname")
 570         irc_pass = None
 571
 572         if ini.has_option("irc", "port"):
 573                 irc_port = ini.getint("irc", "port")
 574
 575         if ini.has_option("irc", "password"):
 576                 irc_pass = ini.get("irc", "password")
 577
 578         irc = words.IRC(irc_host, irc_nick, port = irc_port, password = irc_pass,
 579                         channels = [{ "channel": irc_chan }],
 580                         notify_events = {
 581                           'exception': 1,
 582                           'successToFailure': 1,
 583                           'failureToSuccess': 1
 584                         }
 585         )
 586
 587         c['status'].append(irc)
 588
 589
 590 ####### PROJECT IDENTITY
 591
 592 # the 'title' string will appear at the top of this buildbot
 593 # installation's html.WebStatus home page (linked to the
 594 # 'titleURL') and is embedded in the title of the waterfall HTML page.
 595
 596 c['title'] = ini.get("general", "title")
 597 c['titleURL'] = ini.get("general", "title_url")
 598
 599 # the 'buildbotURL' string should point to the location where the buildbot's
 600 # internal web server (usually the html.WebStatus page) is visible. This
 601 # typically uses the port number set in the Waterfall 'status' entry, but
 602 # with an externally-visible host name which the buildbot cannot figure out
 603 # without some help.
 604
 605 c['buildbotURL'] = ini.get("general", "buildbot_url")
 606
 607 ####### DB URL
 608
 609 c['db'] = {
 610         # This specifies what database buildbot uses to store its state.  You can leave
 611         # this at its default for all but the largest installations.
 612         'db_url' : "sqlite:///state.sqlite",
 613 }