phase1/master.cfg

   1 # -*- python -*-
   2 # ex: set syntax=python:
   3
   4 import os
   5 import re
   6 import subprocess
   7 import ConfigParser
   8
   9 from buildbot import locks
  10
  11 # This is a sample buildmaster config file. It must be installed as
  12 # 'master.cfg' in your buildmaster's base directory.
  13
  14 ini = ConfigParser.ConfigParser()
  15 ini.read("./config.ini")
  16
  17 # This is the dictionary that the buildmaster pays attention to. We also use
  18 # a shorter alias to save typing.
  19 c = BuildmasterConfig = {}
  20
  21 ####### BUILDSLAVES
  22
  23 # The 'slaves' list defines the set of recognized buildslaves. Each element is
  24 # a BuildSlave object, specifying a unique slave name and password.  The same
  25 # slave name and password must be configured on the slave.
  26 from buildbot.buildslave import BuildSlave
  27
  28 slave_port = 9989
  29
  30 if ini.has_option("general", "port"):
  31         slave_port = ini.getint("general", "port")
  32
  33 c['slaves'] = []
  34
  35 for section in ini.sections():
  36         if section.startswith("slave "):
  37                 if ini.has_option(section, "name") and ini.has_option(section, "password"):
  38                         name = ini.get(section, "name")
  39                         password = ini.get(section, "password")
  40                         max_builds = 1
  41                         if ini.has_option(section, "builds"):
  42                                 max_builds = ini.getint(section, "builds")
  43                         c['slaves'].append(BuildSlave(name, password, max_builds = max_builds))
  44
  45 # 'slavePortnum' defines the TCP port to listen on for connections from slaves.
  46 # This must match the value configured into the buildslaves (with their
  47 # --master option)
  48 c['slavePortnum'] = slave_port
  49
  50 # coalesce builds
  51 c['mergeRequests'] = True
  52
  53 ####### CHANGESOURCES
  54
  55 home_dir = os.path.abspath(ini.get("general", "homedir"))
  56 tree_expire = 0
  57
  58 if ini.has_option("general", "expire"):
  59         tree_expire = ini.getint("general", "expire")
  60
  61 repo_url = ini.get("repo", "url")
  62 repo_branch = "master"
  63
  64 if ini.has_option("repo", "branch"):
  65         repo_branch = ini.get("repo", "branch")
  66
  67 rsync_bin_url = ini.get("rsync", "binary_url")
  68 rsync_bin_key = ini.get("rsync", "binary_password")
  69
  70 rsync_src_url = None
  71 rsync_src_key = None
  72
  73 if ini.has_option("rsync", "source_url"):
  74         rsync_src_url = ini.get("rsync", "source_url")
  75         rsync_src_key = ini.get("rsync", "source_password")
  76
  77 gpg_keyid = None
  78 gpg_comment = "Unattended build signature"
  79 gpg_passfile = "/dev/null"
  80
  81 if ini.has_option("gpg", "keyid"):
  82         gpg_keyid = ini.get("gpg", "keyid")
  83
  84 if ini.has_option("gpg", "comment"):
  85         gpg_comment = ini.get("gpg", "comment")
  86
  87 if ini.has_option("gpg", "passfile"):
  88         gpg_passfile = ini.get("gpg", "passfile")
  89
  90
  91 # find targets
  92 targets = [ ]
  93
  94 if not os.path.isdir(home_dir+'/source.git'):
  95         subprocess.call(["git", "clone", "--depth=1", repo_url, home_dir+'/source.git'])
  96         subprocess.call(["git", "checkout", repo_branch], cwd = home_dir+'/source.git')
  97
  98 findtargets = subprocess.Popen([home_dir+'/dumpinfo.pl', 'targets'],
  99         stdout = subprocess.PIPE, cwd = home_dir+'/source.git')
 100
 101 while True:
 102         line = findtargets.stdout.readline()
 103         if not line:
 104                 break
 105         ta = line.strip().split(' ')
 106         targets.append(ta[0])
 107
 108
 109 # the 'change_source' setting tells the buildmaster how it should find out
 110 # about source code changes.  Here we point to the buildbot clone of pyflakes.
 111
 112 from buildbot.changes.gitpoller import GitPoller
 113 c['change_source'] = []
 114 c['change_source'].append(GitPoller(
 115         repo_url,
 116         workdir=home_dir+'/source.git', branch='master',
 117         pollinterval=300))
 118
 119 ####### SCHEDULERS
 120
 121 # Configure the Schedulers, which decide how to react to incoming changes.  In this
 122 # case, just kick off a 'basebuild' build
 123
 124 from buildbot.schedulers.basic import SingleBranchScheduler
 125 from buildbot.schedulers.forcesched import ForceScheduler
 126 from buildbot.changes import filter
 127 c['schedulers'] = []
 128 c['schedulers'].append(SingleBranchScheduler(
 129         name="all",
 130         change_filter=filter.ChangeFilter(branch='master'),
 131         treeStableTimer=60,
 132         builderNames=targets))
 133
 134 c['schedulers'].append(ForceScheduler(
 135         name="force",
 136         builderNames=targets))
 137
 138 ####### BUILDERS
 139
 140 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
 141 # what steps, and which slaves can execute them.  Note that any particular build will
 142 # only take place on one slave.
 143
 144 from buildbot.process.factory import BuildFactory
 145 from buildbot.steps.source import Git
 146 from buildbot.steps.shell import ShellCommand
 147 from buildbot.steps.shell import SetProperty
 148 from buildbot.steps.transfer import FileUpload
 149 from buildbot.steps.transfer import FileDownload
 150 from buildbot.steps.master import MasterShellCommand
 151 from buildbot.process.properties import WithProperties
 152
 153
 154 CleanTargetMap = [
 155         [ "tools",      "tools/clean"                   ],
 156         [ "chain",      "toolchain/clean"               ],
 157         [ "linux",      "target/linux/clean"    ],
 158         [ "dir",        "dirclean"                              ],
 159         [ "dist",       "distclean"                             ]
 160 ]
 161
 162 def IsCleanRequested(pattern):
 163         def CheckCleanProperty(step):
 164                 val = step.getProperty("clean")
 165                 if val and re.match(pattern, val):
 166                         return True
 167                 else:
 168                         return False
 169
 170         return CheckCleanProperty
 171
 172
 173 c['builders'] = []
 174
 175 dlLock = locks.SlaveLock("slave_dl")
 176
 177 checkBuiltin = re.sub('[\t\n ]+', ' ', """
 178         checkBuiltin() {
 179                 local symbol op path file;
 180                 for file in $CHANGED_FILES; do
 181                         case "$file" in
 182                                 package/*/*) : ;;
 183                                 *) return 0 ;;
 184                         esac;
 185                 done;
 186                 while read symbol op path; do
 187                         case "$symbol" in package-*)
 188                                 symbol="${symbol##*(}";
 189                                 symbol="${symbol%)}";
 190                                 for file in $CHANGED_FILES; do
 191                                         case "$file" in "package/$path/"*)
 192                                                 grep -qsx "$symbol=y" .config && return 0
 193                                         ;; esac;
 194                                 done;
 195                         esac;
 196                 done < tmp/.packagedeps;
 197                 return 1;
 198         }
 199 """).strip()
 200
 201
 202 class IfBuiltinShellCommand(ShellCommand):
 203         def _quote(self, str):
 204                 if re.search("[^a-zA-Z0-9/_.-]", str):
 205                         return "'%s'" %(re.sub("'", "'\"'\"'", str))
 206                 return str
 207
 208         def setCommand(self, command):
 209                 if not isinstance(command, (str, unicode)):
 210                         command = ' '.join(map(self._quote, command))
 211                 self.command = [
 212                         '/bin/sh', '-c',
 213                         '%s; if checkBuiltin; then %s; else exit 0; fi' %(checkBuiltin, command)
 214                 ]
 215
 216         def setupEnvironment(self, cmd):
 217                 slaveEnv = self.slaveEnvironment
 218                 if slaveEnv is None:
 219                         slaveEnv = { }
 220                 changedFiles = { }
 221                 for request in self.build.requests:
 222                         for source in request.sources:
 223                                 for change in source.changes:
 224                                         for file in change.files:
 225                                                 changedFiles[file] = True
 226                 fullSlaveEnv = slaveEnv.copy()
 227                 fullSlaveEnv['CHANGED_FILES'] = ' '.join(changedFiles.keys())
 228                 cmd.args['env'] = fullSlaveEnv
 229
 230 slaveNames = [ ]
 231
 232 for slave in c['slaves']:
 233         slaveNames.append(slave.slavename)
 234
 235 for target in targets:
 236         ts = target.split('/')
 237
 238         factory = BuildFactory()
 239
 240         # find number of cores
 241         factory.addStep(SetProperty(
 242                 name = "nproc",
 243                 property = "nproc",
 244                 description = "Finding number of CPUs",
 245                 command = ["nproc"]))
 246
 247         # expire tree if needed
 248         if tree_expire > 0:
 249                 factory.addStep(FileDownload(
 250                         mastersrc = "expire.sh",
 251                         slavedest = "../expire.sh",
 252                         mode = 0755))
 253
 254                 factory.addStep(ShellCommand(
 255                         name = "expire",
 256                         description = "Checking for build tree expiry",
 257                         command = ["./expire.sh", str(tree_expire)],
 258                         workdir = ".",
 259                         haltOnFailure = True,
 260                         timeout = 2400))
 261
 262         # user-requested clean targets
 263         for tuple in CleanTargetMap:
 264                 factory.addStep(ShellCommand(
 265                         name = tuple[1],
 266                         description = 'User-requested "make %s"' % tuple[1],
 267                         command = ["make", tuple[1], "V=s"],
 268                         doStepIf = IsCleanRequested(tuple[0])
 269                 ))
 270
 271         # check out the source
 272         factory.addStep(Git(repourl=repo_url, branch=repo_branch, mode='update'))
 273
 274         factory.addStep(ShellCommand(
 275                 name = "rmtmp",
 276                 description = "Remove tmp folder",
 277                 command=["rm", "-rf", "tmp/"]))
 278
 279         # feed
 280 #       factory.addStep(ShellCommand(
 281 #               name = "feedsconf",
 282 #               description = "Copy the feeds.conf",
 283 #               command='''cp ~/feeds.conf ./feeds.conf''' ))
 284
 285         # feed
 286         factory.addStep(ShellCommand(
 287                 name = "rmfeedlinks",
 288                 description = "Remove feed symlinks",
 289                 command=["rm", "-rf", "package/feeds/"]))
 290
 291         # feed
 292         factory.addStep(ShellCommand(
 293                 name = "updatefeeds",
 294                 description = "Updating feeds",
 295                 command=["./scripts/feeds", "update"]))
 296
 297         # feed
 298         factory.addStep(ShellCommand(
 299                 name = "installfeeds",
 300                 description = "Installing feeds",
 301                 command=["./scripts/feeds", "install", "-a"]))
 302
 303         # seed config
 304         factory.addStep(FileDownload(
 305                 mastersrc = "config.seed",
 306                 slavedest = ".config",
 307                 mode = 0644
 308         ))
 309
 310         # configure
 311         factory.addStep(ShellCommand(
 312                 name = "newconfig",
 313                 description = "Seeding .config",
 314                 command = "printf 'CONFIG_TARGET_%s=y\\nCONFIG_TARGET_%s_%s=y\\n' >> .config" %(ts[0], ts[0], ts[1])
 315         ))
 316
 317         factory.addStep(ShellCommand(
 318                 name = "delbin",
 319                 description = "Removing output directory",
 320                 command = ["rm", "-rf", "bin/"]
 321         ))
 322
 323         factory.addStep(ShellCommand(
 324                 name = "defconfig",
 325                 description = "Populating .config",
 326                 command = ["make", "defconfig"]
 327         ))
 328
 329         # check arch
 330         factory.addStep(ShellCommand(
 331                 name = "checkarch",
 332                 description = "Checking architecture",
 333                 command = ["grep", "-sq", "CONFIG_TARGET_%s=y" %(ts[0]), ".config"],
 334                 logEnviron = False,
 335                 want_stdout = False,
 336                 want_stderr = False,
 337                 haltOnFailure = True
 338         ))
 339
 340         # find libc suffix
 341         factory.addStep(SetProperty(
 342                 name = "libc",
 343                 property = "libc",
 344                 description = "Finding libc suffix",
 345                 command = ["sed", "-ne", '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }', ".config"]))
 346
 347         # install build key
 348         factory.addStep(FileDownload(mastersrc=home_dir+'/key-build', slavedest="key-build", mode=0600))
 349         factory.addStep(FileDownload(mastersrc=home_dir+'/key-build.pub', slavedest="key-build.pub", mode=0600))
 350
 351         # prepare dl
 352         factory.addStep(ShellCommand(
 353                 name = "dldir",
 354                 description = "Preparing dl/",
 355                 command = "mkdir -p $HOME/dl && rm -rf ./dl && ln -sf $HOME/dl ./dl",
 356                 logEnviron = False,
 357                 want_stdout = False
 358         ))
 359
 360         # prepare tar
 361         factory.addStep(ShellCommand(
 362                 name = "dltar",
 363                 description = "Building GNU tar",
 364                 command = ["make", WithProperties("-j%(nproc:~4)s"), "tools/tar/install", "V=s"],
 365                 haltOnFailure = True
 366         ))
 367
 368         # populate dl
 369         factory.addStep(ShellCommand(
 370                 name = "dlrun",
 371                 description = "Populating dl/",
 372                 command = ["make", WithProperties("-j%(nproc:~4)s"), "download", "V=s"],
 373                 logEnviron = False,
 374                 locks = [dlLock.access('exclusive')]
 375         ))
 376
 377         factory.addStep(ShellCommand(
 378                 name = "cleanbase",
 379                 description = "Cleaning base-files",
 380                 command=["make", "package/base-files/clean", "V=s"]
 381         ))
 382
 383         # build
 384         factory.addStep(ShellCommand(
 385                 name = "tools",
 386                 description = "Building tools",
 387                 command = ["make", WithProperties("-j%(nproc:~4)s"), "tools/install", "V=s"],
 388                 haltOnFailure = True
 389         ))
 390
 391         factory.addStep(ShellCommand(
 392                 name = "toolchain",
 393                 description = "Building toolchain",
 394                 command=["make", WithProperties("-j%(nproc:~4)s"), "toolchain/install", "V=s"],
 395                 haltOnFailure = True
 396         ))
 397
 398         factory.addStep(ShellCommand(
 399                 name = "kmods",
 400                 description = "Building kmods",
 401                 command=["make", WithProperties("-j%(nproc:~4)s"), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
 402                 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
 403                 haltOnFailure = True
 404         ))
 405
 406         factory.addStep(ShellCommand(
 407                 name = "pkgbuild",
 408                 description = "Building packages",
 409                 command=["make", WithProperties("-j%(nproc:~4)s"), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
 410                 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
 411                 haltOnFailure = True
 412         ))
 413
 414         # factory.addStep(IfBuiltinShellCommand(
 415         factory.addStep(ShellCommand(
 416                 name = "pkginstall",
 417                 description = "Installing packages",
 418                 command=["make", WithProperties("-j%(nproc:~4)s"), "package/install", "V=s"],
 419                 haltOnFailure = True
 420         ))
 421
 422         factory.addStep(ShellCommand(
 423                 name = "pkgindex",
 424                 description = "Indexing packages",
 425                 command=["make", WithProperties("-j%(nproc:~4)s"), "package/index", "V=s"],
 426                 haltOnFailure = True
 427         ))
 428
 429         #factory.addStep(IfBuiltinShellCommand(
 430         factory.addStep(ShellCommand(
 431                 name = "images",
 432                 description = "Building images",
 433                 command=["make", WithProperties("-j%(nproc:~4)s"), "target/install", "V=s"],
 434                 haltOnFailure = True
 435         ))
 436
 437         factory.addStep(ShellCommand(
 438                 name = "checksums",
 439                 description = "Calculating checksums",
 440                 command=["make", "-j1", "checksum", "V=s"],
 441                 haltOnFailure = True
 442         ))
 443
 444         # sign
 445         if gpg_keyid is not None:
 446                 factory.addStep(MasterShellCommand(
 447                         name = "signprepare",
 448                         description = "Preparing temporary signing directory",
 449                         command = ["mkdir", "-p", "%s/signing" %(home_dir)],
 450                         haltOnFailure = True
 451                 ))
 452
 453                 factory.addStep(ShellCommand(
 454                         name = "signpack",
 455                         description = "Packing files to sign",
 456                         command = WithProperties("find bin/targets/%s/%s%%(libc)s/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz" %(ts[0], ts[1])),
 457                         haltOnFailure = True
 458                 ))
 459
 460                 factory.addStep(FileUpload(
 461                         slavesrc = "sign.tar.gz",
 462                         masterdest = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
 463                         haltOnFailure = True
 464                 ))
 465
 466                 factory.addStep(MasterShellCommand(
 467                         name = "signfiles",
 468                         description = "Signing files",
 469                         command = ["%s/signall.sh" %(home_dir), "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]), gpg_keyid, gpg_passfile, gpg_comment],
 470                         haltOnFailure = True
 471                 ))
 472
 473                 factory.addStep(FileDownload(
 474                         mastersrc = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
 475                         slavedest = "sign.tar.gz",
 476                         haltOnFailure = True
 477                 ))
 478
 479                 factory.addStep(ShellCommand(
 480                         name = "signunpack",
 481                         description = "Unpacking signed files",
 482                         command = ["tar", "-xzf", "sign.tar.gz"],
 483                         haltOnFailure = True
 484                 ))
 485
 486         # upload
 487         factory.addStep(ShellCommand(
 488                 name = "uploadprepare",
 489                 description = "Preparing target directory",
 490                 command=["rsync", "-av", "--include", "/%s/" %(ts[0]), "--include", "/%s/%s/" %(ts[0], ts[1]), "--exclude", "/*", "--exclude", "/*/*", "--exclude", "/%s/%s/*" %(ts[0], ts[1]), "bin/targets/", "%s/targets/" %(rsync_bin_url)],
 491                 env={'RSYNC_PASSWORD': rsync_bin_key},
 492                 haltOnFailure = True,
 493                 logEnviron = False
 494         ))
 495
 496         factory.addStep(ShellCommand(
 497                 name = "targetupload",
 498                 description = "Uploading target files",
 499                 command=["rsync", "--delete", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", WithProperties("bin/targets/%s/%s%%(libc)s/" %(ts[0], ts[1])), "%s/targets/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
 500                 env={'RSYNC_PASSWORD': rsync_bin_key},
 501                 haltOnFailure = True,
 502                 logEnviron = False
 503         ))
 504
 505         if rsync_src_url is not None:
 506                 factory.addStep(ShellCommand(
 507                         name = "sourceupload",
 508                         description = "Uploading source archives",
 509                         command=["rsync", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "dl/", "%s/" %(rsync_src_url)],
 510                         env={'RSYNC_PASSWORD': rsync_src_key},
 511                         haltOnFailure = True,
 512                         logEnviron = False
 513                 ))
 514
 515         if False:
 516                 factory.addStep(ShellCommand(
 517                         name = "packageupload",
 518                         description = "Uploading package files",
 519                         command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "bin/packages/", "%s/packages/" %(rsync_bin_url)],
 520                         env={'RSYNC_PASSWORD': rsync_bin_key},
 521                         haltOnFailure = False,
 522                         logEnviron = False
 523                 ))
 524
 525         # logs
 526         if False:
 527                 factory.addStep(ShellCommand(
 528                         name = "upload",
 529                         description = "Uploading logs",
 530                         command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "logs/", "%s/logs/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
 531                         env={'RSYNC_PASSWORD': rsync_bin_key},
 532                         haltOnFailure = False,
 533                         alwaysRun = True,
 534                         logEnviron = False
 535                 ))
 536
 537         from buildbot.config import BuilderConfig
 538
 539         c['builders'].append(BuilderConfig(name=target, slavenames=slaveNames, factory=factory))
 540
 541
 542 ####### STATUS TARGETS
 543
 544 # 'status' is a list of Status Targets. The results of each build will be
 545 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
 546 # including web pages, email senders, and IRC bots.
 547
 548 c['status'] = []
 549
 550 from buildbot.status import html
 551 from buildbot.status.web import authz, auth
 552
 553 if ini.has_option("status", "bind"):
 554         if ini.has_option("status", "user") and ini.has_option("status", "password"):
 555                 authz_cfg=authz.Authz(
 556                         # change any of these to True to enable; see the manual for more
 557                         # options
 558                         auth=auth.BasicAuth([(ini.get("status", "user"), ini.get("status", "password"))]),
 559                         gracefulShutdown = 'auth',
 560                         forceBuild = 'auth', # use this to test your slave once it is set up
 561                         forceAllBuilds = 'auth',
 562                         pingBuilder = False,
 563                         stopBuild = 'auth',
 564                         stopAllBuilds = 'auth',
 565                         cancelPendingBuild = 'auth',
 566                 )
 567                 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind"), authz=authz_cfg))
 568         else:
 569                 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind")))
 570
 571
 572 from buildbot.status import words
 573
 574 if ini.has_option("irc", "host") and ini.has_option("irc", "nickname") and ini.has_option("irc", "channel"):
 575         irc_host = ini.get("irc", "host")
 576         irc_port = 6667
 577         irc_chan = ini.get("irc", "channel")
 578         irc_nick = ini.get("irc", "nickname")
 579         irc_pass = None
 580
 581         if ini.has_option("irc", "port"):
 582                 irc_port = ini.getint("irc", "port")
 583
 584         if ini.has_option("irc", "password"):
 585                 irc_pass = ini.get("irc", "password")
 586
 587         irc = words.IRC(irc_host, irc_nick, port = irc_port, password = irc_pass,
 588                         channels = [{ "channel": irc_chan }],
 589                         notify_events = {
 590                           'exception': 1,
 591                           'successToFailure': 1,
 592                           'failureToSuccess': 1
 593                         }
 594         )
 595
 596         c['status'].append(irc)
 597
 598
 599 ####### PROJECT IDENTITY
 600
 601 # the 'title' string will appear at the top of this buildbot
 602 # installation's html.WebStatus home page (linked to the
 603 # 'titleURL') and is embedded in the title of the waterfall HTML page.
 604
 605 c['title'] = ini.get("general", "title")
 606 c['titleURL'] = ini.get("general", "title_url")
 607
 608 # the 'buildbotURL' string should point to the location where the buildbot's
 609 # internal web server (usually the html.WebStatus page) is visible. This
 610 # typically uses the port number set in the Waterfall 'status' entry, but
 611 # with an externally-visible host name which the buildbot cannot figure out
 612 # without some help.
 613
 614 c['buildbotURL'] = ini.get("general", "buildbot_url")
 615
 616 ####### DB URL
 617
 618 c['db'] = {
 619         # This specifies what database buildbot uses to store its state.  You can leave
 620         # this at its default for all but the largest installations.
 621         'db_url' : "sqlite:///state.sqlite",
 622 }