phase1/master.cfg

   1 # -*- python -*-
   2 # ex: set syntax=python:
   3
   4 import os
   5 import re
   6 import subprocess
   7 import ConfigParser
   8
   9 from buildbot import locks
  10
  11 # This is a sample buildmaster config file. It must be installed as
  12 # 'master.cfg' in your buildmaster's base directory.
  13
  14 ini = ConfigParser.ConfigParser()
  15 ini.read("./config.ini")
  16
  17 # This is the dictionary that the buildmaster pays attention to. We also use
  18 # a shorter alias to save typing.
  19 c = BuildmasterConfig = {}
  20
  21 ####### BUILDSLAVES
  22
  23 # The 'slaves' list defines the set of recognized buildslaves. Each element is
  24 # a BuildSlave object, specifying a unique slave name and password.  The same
  25 # slave name and password must be configured on the slave.
  26 from buildbot.buildslave import BuildSlave
  27
  28 c['slaves'] = []
  29
  30 for section in ini.sections():
  31         if section.startswith("slave "):
  32                 if ini.has_option(section, "name") and ini.has_option(section, "password"):
  33                         name = ini.get(section, "name")
  34                         password = ini.get(section, "password")
  35                         max_builds = 1
  36                         if ini.has_option(section, "builds"):
  37                                 max_builds = ini.getint(section, "builds")
  38                         c['slaves'].append(BuildSlave(name, password, max_builds = max_builds))
  39
  40 # 'slavePortnum' defines the TCP port to listen on for connections from slaves.
  41 # This must match the value configured into the buildslaves (with their
  42 # --master option)
  43 c['slavePortnum'] = 9989
  44
  45 # coalesce builds
  46 c['mergeRequests'] = True
  47
  48 ####### CHANGESOURCES
  49
  50 home_dir = os.path.abspath(ini.get("general", "homedir"))
  51 tree_expire = 0
  52
  53 if ini.has_option("general", "expire"):
  54         tree_expire = ini.getint("general", "expire")
  55
  56 repo_url = ini.get("repo", "url")
  57
  58 rsync_bin_url = ini.get("rsync", "binary_url")
  59 rsync_bin_key = ini.get("rsync", "binary_password")
  60
  61 rsync_src_url = None
  62 rsync_src_key = None
  63
  64 if ini.has_option("rsync", "source_url"):
  65         rsync_src_url = ini.get("rsync", "source_url")
  66         rsync_src_key = ini.get("rsync", "source_password")
  67
  68 gpg_keyid = None
  69 gpg_comment = "Unattended build signature"
  70 gpg_passfile = "/dev/null"
  71
  72 if ini.has_option("gpg", "keyid"):
  73         gpg_keyid = ini.get("gpg", "keyid")
  74
  75 if ini.has_option("gpg", "comment"):
  76         gpg_comment = ini.get("gpg", "comment")
  77
  78 if ini.has_option("gpg", "passfile"):
  79         gpg_passfile = ini.get("gpg", "passfile")
  80
  81
  82 # find targets
  83 targets = [ ]
  84
  85 if not os.path.isdir(home_dir+'/source.git'):
  86         subprocess.call(["git", "clone", "--depth=1", repo_url, home_dir+'/source.git'])
  87
  88 findtargets = subprocess.Popen([home_dir+'/dumpinfo.pl', 'targets'],
  89         stdout = subprocess.PIPE, cwd = home_dir+'/source.git')
  90
  91 while True:
  92         line = findtargets.stdout.readline()
  93         if not line:
  94                 break
  95         ta = line.strip().split(' ')
  96         targets.append(ta[0])
  97
  98
  99 # the 'change_source' setting tells the buildmaster how it should find out
 100 # about source code changes.  Here we point to the buildbot clone of pyflakes.
 101
 102 from buildbot.changes.gitpoller import GitPoller
 103 c['change_source'] = []
 104 c['change_source'].append(GitPoller(
 105         repo_url,
 106         workdir=home_dir+'/source.git', branch='master',
 107         pollinterval=300))
 108
 109 ####### SCHEDULERS
 110
 111 # Configure the Schedulers, which decide how to react to incoming changes.  In this
 112 # case, just kick off a 'basebuild' build
 113
 114 from buildbot.schedulers.basic import SingleBranchScheduler
 115 from buildbot.schedulers.forcesched import ForceScheduler
 116 from buildbot.changes import filter
 117 c['schedulers'] = []
 118 c['schedulers'].append(SingleBranchScheduler(
 119         name="all",
 120         change_filter=filter.ChangeFilter(branch='master'),
 121         treeStableTimer=60,
 122         builderNames=targets))
 123
 124 c['schedulers'].append(ForceScheduler(
 125         name="force",
 126         builderNames=targets))
 127
 128 ####### BUILDERS
 129
 130 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
 131 # what steps, and which slaves can execute them.  Note that any particular build will
 132 # only take place on one slave.
 133
 134 from buildbot.process.factory import BuildFactory
 135 from buildbot.steps.source import Git
 136 from buildbot.steps.shell import ShellCommand
 137 from buildbot.steps.shell import SetProperty
 138 from buildbot.steps.transfer import FileUpload
 139 from buildbot.steps.transfer import FileDownload
 140 from buildbot.steps.master import MasterShellCommand
 141 from buildbot.process.properties import WithProperties
 142
 143
 144 CleanTargetMap = [
 145         [ "tools",      "tools/clean"                   ],
 146         [ "chain",      "toolchain/clean"               ],
 147         [ "linux",      "target/linux/clean"    ],
 148         [ "dir",        "dirclean"                              ],
 149         [ "dist",       "distclean"                             ]
 150 ]
 151
 152 def IsCleanRequested(pattern):
 153         def CheckCleanProperty(step):
 154                 val = step.getProperty("clean")
 155                 if val and re.match(pattern, val):
 156                         return True
 157                 else:
 158                         return False
 159
 160         return CheckCleanProperty
 161
 162
 163 c['builders'] = []
 164
 165 dlLock = locks.SlaveLock("slave_dl")
 166
 167 checkBuiltin = re.sub('[\t\n ]+', ' ', """
 168         checkBuiltin() {
 169                 local symbol op path file;
 170                 for file in $CHANGED_FILES; do
 171                         case "$file" in
 172                                 package/*/*) : ;;
 173                                 *) return 0 ;;
 174                         esac;
 175                 done;
 176                 while read symbol op path; do
 177                         case "$symbol" in package-*)
 178                                 symbol="${symbol##*(}";
 179                                 symbol="${symbol%)}";
 180                                 for file in $CHANGED_FILES; do
 181                                         case "$file" in "package/$path/"*)
 182                                                 grep -qsx "$symbol=y" .config && return 0
 183                                         ;; esac;
 184                                 done;
 185                         esac;
 186                 done < tmp/.packagedeps;
 187                 return 1;
 188         }
 189 """).strip()
 190
 191
 192 class IfBuiltinShellCommand(ShellCommand):
 193         def _quote(self, str):
 194                 if re.search("[^a-zA-Z0-9/_.-]", str):
 195                         return "'%s'" %(re.sub("'", "'\"'\"'", str))
 196                 return str
 197
 198         def setCommand(self, command):
 199                 if not isinstance(command, (str, unicode)):
 200                         command = ' '.join(map(self._quote, command))
 201                 self.command = [
 202                         '/bin/sh', '-c',
 203                         '%s; if checkBuiltin; then %s; else exit 0; fi' %(checkBuiltin, command)
 204                 ]
 205
 206         def setupEnvironment(self, cmd):
 207                 slaveEnv = self.slaveEnvironment
 208                 if slaveEnv is None:
 209                         slaveEnv = { }
 210                 changedFiles = { }
 211                 for request in self.build.requests:
 212                         for source in request.sources:
 213                                 for change in source.changes:
 214                                         for file in change.files:
 215                                                 changedFiles[file] = True
 216                 fullSlaveEnv = slaveEnv.copy()
 217                 fullSlaveEnv['CHANGED_FILES'] = ' '.join(changedFiles.keys())
 218                 cmd.args['env'] = fullSlaveEnv
 219
 220 slaveNames = [ ]
 221
 222 for slave in c['slaves']:
 223         slaveNames.append(slave.slavename)
 224
 225 for target in targets:
 226         ts = target.split('/')
 227
 228         factory = BuildFactory()
 229
 230         # find number of cores
 231         factory.addStep(SetProperty(
 232                 name = "nproc",
 233                 property = "nproc",
 234                 description = "Finding number of CPUs",
 235                 command = ["nproc"]))
 236
 237         # expire tree if needed
 238         if tree_expire > 0:
 239                 factory.addStep(FileDownload(
 240                         mastersrc = "expire.sh",
 241                         slavedest = "../expire.sh",
 242                         mode = 0755))
 243
 244                 factory.addStep(ShellCommand(
 245                         name = "expire",
 246                         description = "Checking for build tree expiry",
 247                         command = ["./expire.sh", str(tree_expire)],
 248                         workdir = ".",
 249                         haltOnFailure = True,
 250                         timeout = 2400))
 251
 252         # user-requested clean targets
 253         for tuple in CleanTargetMap:
 254                 factory.addStep(ShellCommand(
 255                         name = tuple[1],
 256                         description = 'User-requested "make %s"' % tuple[1],
 257                         command = ["make", tuple[1], "V=s"],
 258                         doStepIf = IsCleanRequested(tuple[0])
 259                 ))
 260
 261         # check out the source
 262         factory.addStep(Git(repourl=repo_url, mode='update'))
 263
 264         factory.addStep(ShellCommand(
 265                 name = "rmtmp",
 266                 description = "Remove tmp folder",
 267                 command=["rm", "-rf", "tmp/"]))
 268
 269         # feed
 270 #       factory.addStep(ShellCommand(
 271 #               name = "feedsconf",
 272 #               description = "Copy the feeds.conf",
 273 #               command='''cp ~/feeds.conf ./feeds.conf''' ))
 274
 275         # feed
 276         factory.addStep(ShellCommand(
 277                 name = "rmfeedlinks",
 278                 description = "Remove feed symlinks",
 279                 command=["rm", "-rf", "package/feeds/"]))
 280
 281         # feed
 282         factory.addStep(ShellCommand(
 283                 name = "updatefeeds",
 284                 description = "Updating feeds",
 285                 command=["./scripts/feeds", "update"]))
 286
 287         # feed
 288         factory.addStep(ShellCommand(
 289                 name = "installfeeds",
 290                 description = "Installing feeds",
 291                 command=["./scripts/feeds", "install", "-a"]))
 292
 293         # configure
 294         factory.addStep(ShellCommand(
 295                 name = "newconfig",
 296                 description = "Seeding .config",
 297                 command='''cat <<EOT > .config
 298 CONFIG_TARGET_%s=y
 299 CONFIG_TARGET_%s_%s=y
 300 CONFIG_ALL_NONSHARED=y
 301 CONFIG_SDK=y
 302 CONFIG_IB=y
 303 # CONFIG_IB_STANDALONE is not set
 304 CONFIG_DEVEL=y
 305 CONFIG_CCACHE=y
 306 CONFIG_SIGNED_PACKAGES=y
 307 # CONFIG_PER_FEED_REPO_ADD_COMMENTED is not set
 308 CONFIG_KERNEL_KALLSYMS=y
 309 CONFIG_COLLECT_KERNEL_DEBUG=y
 310 CONFIG_TARGET_ALL_PROFILES=y
 311 CONFIG_TARGET_MULTI_PROFILE=y
 312 CONFIG_TARGET_PER_DEVICE_ROOTFS=y
 313 EOT''' %(ts[0], ts[0], ts[1]) ))
 314
 315         factory.addStep(ShellCommand(
 316                 name = "delbin",
 317                 description = "Removing output directory",
 318                 command = ["rm", "-rf", "bin/"]
 319         ))
 320
 321         factory.addStep(ShellCommand(
 322                 name = "defconfig",
 323                 description = "Populating .config",
 324                 command = ["make", "defconfig"]
 325         ))
 326
 327         # check arch
 328         factory.addStep(ShellCommand(
 329                 name = "checkarch",
 330                 description = "Checking architecture",
 331                 command = ["grep", "-sq", "CONFIG_TARGET_%s=y" %(ts[0]), ".config"],
 332                 logEnviron = False,
 333                 want_stdout = False,
 334                 want_stderr = False,
 335                 haltOnFailure = True
 336         ))
 337
 338         # find libc suffix
 339         factory.addStep(SetProperty(
 340                 name = "libc",
 341                 property = "libc",
 342                 description = "Finding libc suffix",
 343                 command = ["sed", "-ne", '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }', ".config"]))
 344
 345         # install build key
 346         factory.addStep(FileDownload(mastersrc=home_dir+'/key-build', slavedest="key-build", mode=0600))
 347         factory.addStep(FileDownload(mastersrc=home_dir+'/key-build.pub', slavedest="key-build.pub", mode=0600))
 348
 349         # prepare dl
 350         factory.addStep(ShellCommand(
 351                 name = "dldir",
 352                 description = "Preparing dl/",
 353                 command = "mkdir -p $HOME/dl && ln -sf $HOME/dl ./dl",
 354                 logEnviron = False,
 355                 want_stdout = False
 356         ))
 357
 358         # prepare tar
 359         factory.addStep(ShellCommand(
 360                 name = "dltar",
 361                 description = "Building GNU tar",
 362                 command = ["make", WithProperties("-j%(nproc:~4)s"), "tools/tar/install", "V=s"],
 363                 haltOnFailure = True
 364         ))
 365
 366         # populate dl
 367         factory.addStep(ShellCommand(
 368                 name = "dlrun",
 369                 description = "Populating dl/",
 370                 command = ["make", WithProperties("-j%(nproc:~4)s"), "download", "V=s"],
 371                 logEnviron = False,
 372                 locks = [dlLock.access('exclusive')]
 373         ))
 374
 375         factory.addStep(ShellCommand(
 376                 name = "cleanbase",
 377                 description = "Cleaning base-files",
 378                 command=["make", "package/base-files/clean", "V=s"]
 379         ))
 380
 381         # build
 382         factory.addStep(ShellCommand(
 383                 name = "tools",
 384                 description = "Building tools",
 385                 command = ["make", WithProperties("-j%(nproc:~4)s"), "tools/install", "V=s"],
 386                 haltOnFailure = True
 387         ))
 388
 389         factory.addStep(ShellCommand(
 390                 name = "toolchain",
 391                 description = "Building toolchain",
 392                 command=["make", WithProperties("-j%(nproc:~4)s"), "toolchain/install", "V=s"],
 393                 haltOnFailure = True
 394         ))
 395
 396         factory.addStep(ShellCommand(
 397                 name = "kmods",
 398                 description = "Building kmods",
 399                 command=["make", WithProperties("-j%(nproc:~4)s"), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
 400                 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
 401                 haltOnFailure = True
 402         ))
 403
 404         factory.addStep(ShellCommand(
 405                 name = "pkgbuild",
 406                 description = "Building packages",
 407                 command=["make", WithProperties("-j%(nproc:~4)s"), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
 408                 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
 409                 haltOnFailure = True
 410         ))
 411
 412         # factory.addStep(IfBuiltinShellCommand(
 413         factory.addStep(ShellCommand(
 414                 name = "pkginstall",
 415                 description = "Installing packages",
 416                 command=["make", WithProperties("-j%(nproc:~4)s"), "package/install", "V=s"],
 417                 haltOnFailure = True
 418         ))
 419
 420         factory.addStep(ShellCommand(
 421                 name = "pkgindex",
 422                 description = "Indexing packages",
 423                 command=["make", WithProperties("-j%(nproc:~4)s"), "package/index", "V=s"],
 424                 haltOnFailure = True
 425         ))
 426
 427         #factory.addStep(IfBuiltinShellCommand(
 428         factory.addStep(ShellCommand(
 429                 name = "images",
 430                 description = "Building images",
 431                 command=["make", WithProperties("-j%(nproc:~4)s"), "target/install", "V=s"],
 432                 haltOnFailure = True
 433         ))
 434
 435         factory.addStep(ShellCommand(
 436                 name = "checksums",
 437                 description = "Calculating checksums",
 438                 command=["make", "-j1", "checksum", "V=s"],
 439                 haltOnFailure = True
 440         ))
 441
 442         # sign
 443         if gpg_keyid is not None:
 444                 factory.addStep(MasterShellCommand(
 445                         name = "signprepare",
 446                         description = "Preparing temporary signing directory",
 447                         command = ["mkdir", "-p", "%s/signing" %(home_dir)],
 448                         haltOnFailure = True
 449                 ))
 450
 451                 factory.addStep(ShellCommand(
 452                         name = "signpack",
 453                         description = "Packing files to sign",
 454                         command = WithProperties("find bin/targets/%s/%s%%(libc)s/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz" %(ts[0], ts[1])),
 455                         haltOnFailure = True
 456                 ))
 457
 458                 factory.addStep(FileUpload(
 459                         slavesrc = "sign.tar.gz",
 460                         masterdest = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
 461                         haltOnFailure = True
 462                 ))
 463
 464                 factory.addStep(MasterShellCommand(
 465                         name = "signfiles",
 466                         description = "Signing files",
 467                         command = ["%s/signall.sh" %(home_dir), "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]), gpg_keyid, gpg_passfile, gpg_comment],
 468                         haltOnFailure = True
 469                 ))
 470
 471                 factory.addStep(FileDownload(
 472                         mastersrc = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
 473                         slavedest = "sign.tar.gz",
 474                         haltOnFailure = True
 475                 ))
 476
 477                 factory.addStep(ShellCommand(
 478                         name = "signunpack",
 479                         description = "Unpacking signed files",
 480                         command = ["tar", "-xzf", "sign.tar.gz"],
 481                         haltOnFailure = True
 482                 ))
 483
 484         # upload
 485         factory.addStep(ShellCommand(
 486                 name = "uploadprepare",
 487                 description = "Preparing target directory",
 488                 command=["rsync", "-av", "--include", "/%s/" %(ts[0]), "--include", "/%s/%s/" %(ts[0], ts[1]), "--exclude", "/*", "--exclude", "/*/*", "--exclude", "/%s/%s/*" %(ts[0], ts[1]), "bin/targets/", "%s/targets/" %(rsync_bin_url)],
 489                 env={'RSYNC_PASSWORD': rsync_bin_key},
 490                 haltOnFailure = True,
 491                 logEnviron = False
 492         ))
 493
 494         factory.addStep(ShellCommand(
 495                 name = "targetupload",
 496                 description = "Uploading target files",
 497                 command=["rsync", "--delete", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", WithProperties("bin/targets/%s/%s%%(libc)s/" %(ts[0], ts[1])), "%s/targets/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
 498                 env={'RSYNC_PASSWORD': rsync_bin_key},
 499                 haltOnFailure = True,
 500                 logEnviron = False
 501         ))
 502
 503         if rsync_src_url is not None:
 504                 factory.addStep(ShellCommand(
 505                         name = "sourceupload",
 506                         description = "Uploading source archives",
 507                         command=["rsync", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "dl/", "%s/" %(rsync_src_url)],
 508                         env={'RSYNC_PASSWORD': rsync_src_key},
 509                         haltOnFailure = True,
 510                         logEnviron = False
 511                 ))
 512
 513         if False:
 514                 factory.addStep(ShellCommand(
 515                         name = "packageupload",
 516                         description = "Uploading package files",
 517                         command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "bin/packages/", "%s/packages/" %(rsync_bin_url)],
 518                         env={'RSYNC_PASSWORD': rsync_bin_key},
 519                         haltOnFailure = False,
 520                         logEnviron = False
 521                 ))
 522
 523         # logs
 524         if False:
 525                 factory.addStep(ShellCommand(
 526                         name = "upload",
 527                         description = "Uploading logs",
 528                         command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "logs/", "%s/logs/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
 529                         env={'RSYNC_PASSWORD': rsync_bin_key},
 530                         haltOnFailure = False,
 531                         alwaysRun = True,
 532                         logEnviron = False
 533                 ))
 534
 535         from buildbot.config import BuilderConfig
 536
 537         c['builders'].append(BuilderConfig(name=target, slavenames=slaveNames, factory=factory))
 538
 539
 540 ####### STATUS TARGETS
 541
 542 # 'status' is a list of Status Targets. The results of each build will be
 543 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
 544 # including web pages, email senders, and IRC bots.
 545
 546 c['status'] = []
 547
 548 from buildbot.status import html
 549 from buildbot.status.web import authz, auth
 550
 551 if ini.has_option("status", "bind"):
 552         if ini.has_option("status", "user") and ini.has_option("status", "password"):
 553                 authz_cfg=authz.Authz(
 554                         # change any of these to True to enable; see the manual for more
 555                         # options
 556                         auth=auth.BasicAuth([(ini.get("status", "user"), ini.get("status", "password"))]),
 557                         gracefulShutdown = 'auth',
 558                         forceBuild = 'auth', # use this to test your slave once it is set up
 559                         forceAllBuilds = 'auth',
 560                         pingBuilder = False,
 561                         stopBuild = 'auth',
 562                         stopAllBuilds = 'auth',
 563                         cancelPendingBuild = 'auth',
 564                 )
 565                 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind"), authz=authz_cfg))
 566         else:
 567                 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind")))
 568
 569
 570 from buildbot.status import words
 571
 572 if ini.has_option("irc", "host") and ini.has_option("irc", "nickname") and ini.has_option("irc", "channel"):
 573         irc_host = ini.get("irc", "host")
 574         irc_port = 6667
 575         irc_chan = ini.get("irc", "channel")
 576         irc_nick = ini.get("irc", "nickname")
 577         irc_pass = None
 578
 579         if ini.has_option("irc", "port"):
 580                 irc_port = ini.getint("irc", "port")
 581
 582         if ini.has_option("irc", "password"):
 583                 irc_pass = ini.get("irc", "password")
 584
 585         irc = words.IRC(irc_host, irc_nick, port = irc_port, password = irc_pass,
 586                         channels = [{ "channel": irc_chan }],
 587                         notify_events = {
 588                           'exception': 1,
 589                           'successToFailure': 1,
 590                           'failureToSuccess': 1
 591                         }
 592         )
 593
 594         c['status'].append(irc)
 595
 596
 597 ####### PROJECT IDENTITY
 598
 599 # the 'title' string will appear at the top of this buildbot
 600 # installation's html.WebStatus home page (linked to the
 601 # 'titleURL') and is embedded in the title of the waterfall HTML page.
 602
 603 c['title'] = ini.get("general", "title")
 604 c['titleURL'] = ini.get("general", "title_url")
 605
 606 # the 'buildbotURL' string should point to the location where the buildbot's
 607 # internal web server (usually the html.WebStatus page) is visible. This
 608 # typically uses the port number set in the Waterfall 'status' entry, but
 609 # with an externally-visible host name which the buildbot cannot figure out
 610 # without some help.
 611
 612 c['buildbotURL'] = ini.get("general", "buildbot_url")
 613
 614 ####### DB URL
 615
 616 c['db'] = {
 617         # This specifies what database buildbot uses to store its state.  You can leave
 618         # this at its default for all but the largest installations.
 619         'db_url' : "sqlite:///state.sqlite",
 620 }