phase1/master.cfg

   1 # -*- python -*-
   2 # ex: set syntax=python:
   3
   4 import os
   5 import re
   6 import subprocess
   7 import ConfigParser
   8
   9 from buildbot import locks
  10
  11 # This is a sample buildmaster config file. It must be installed as
  12 # 'master.cfg' in your buildmaster's base directory.
  13
  14 ini = ConfigParser.ConfigParser()
  15 ini.read("./config.ini")
  16
  17 # This is the dictionary that the buildmaster pays attention to. We also use
  18 # a shorter alias to save typing.
  19 c = BuildmasterConfig = {}
  20
  21 ####### BUILDSLAVES
  22
  23 # The 'slaves' list defines the set of recognized buildslaves. Each element is
  24 # a BuildSlave object, specifying a unique slave name and password.  The same
  25 # slave name and password must be configured on the slave.
  26 from buildbot.buildslave import BuildSlave
  27
  28 c['slaves'] = []
  29
  30 for section in ini.sections():
  31         if section.startswith("slave "):
  32                 if ini.has_option(section, "name") and ini.has_option(section, "password"):
  33                         name = ini.get(section, "name")
  34                         password = ini.get(section, "password")
  35                         max_builds = 1
  36                         if ini.has_option(section, "builds"):
  37                                 max_builds = ini.getint(section, "builds")
  38                         c['slaves'].append(BuildSlave(name, password, max_builds = max_builds))
  39
  40 # 'slavePortnum' defines the TCP port to listen on for connections from slaves.
  41 # This must match the value configured into the buildslaves (with their
  42 # --master option)
  43 c['slavePortnum'] = 9989
  44
  45 # coalesce builds
  46 c['mergeRequests'] = True
  47
  48 ####### CHANGESOURCES
  49
  50 home_dir = os.path.abspath(ini.get("general", "homedir"))
  51 tree_expire = 0
  52
  53 if ini.has_option("general", "expire"):
  54         tree_expire = ini.getint("general", "expire")
  55
  56 repo_url = ini.get("repo", "url")
  57
  58 rsync_bin_url = ini.get("rsync", "binary_url")
  59 rsync_bin_key = ini.get("rsync", "binary_password")
  60
  61 rsync_src_url = None
  62 rsync_src_key = None
  63
  64 if ini.has_option("rsync", "source_url"):
  65         rsync_src_url = ini.get("rsync", "source_url")
  66         rsync_src_key = ini.get("rsync", "source_password")
  67
  68 gpg_keyid = None
  69 gpg_comment = "Unattended build signature"
  70 gpg_passfile = "/dev/null"
  71
  72 if ini.has_option("gpg", "keyid"):
  73         gpg_keyid = ini.get("gpg", "keyid")
  74
  75 if ini.has_option("gpg", "comment"):
  76         gpg_comment = ini.get("gpg", "comment")
  77
  78 if ini.has_option("gpg", "passfile"):
  79         gpg_passfile = ini.get("gpg", "passfile")
  80
  81
  82 # find targets
  83 targets = [ ]
  84
  85 if not os.path.isdir(home_dir+'/source.git'):
  86         subprocess.call(["git", "clone", "--depth=1", repo_url, home_dir+'/source.git'])
  87
  88 findtargets = subprocess.Popen([home_dir+'/dumpinfo.pl', 'targets'],
  89         stdout = subprocess.PIPE, cwd = home_dir+'/source.git')
  90
  91 while True:
  92         line = findtargets.stdout.readline()
  93         if not line:
  94                 break
  95         ta = line.strip().split(' ')
  96         targets.append(ta[0])
  97
  98
  99 # the 'change_source' setting tells the buildmaster how it should find out
 100 # about source code changes.  Here we point to the buildbot clone of pyflakes.
 101
 102 from buildbot.changes.gitpoller import GitPoller
 103 c['change_source'] = []
 104 c['change_source'].append(GitPoller(
 105         repo_url,
 106         workdir=home_dir+'/source.git', branch='master',
 107         pollinterval=300))
 108
 109 ####### SCHEDULERS
 110
 111 # Configure the Schedulers, which decide how to react to incoming changes.  In this
 112 # case, just kick off a 'basebuild' build
 113
 114 from buildbot.schedulers.basic import SingleBranchScheduler
 115 from buildbot.schedulers.forcesched import ForceScheduler
 116 from buildbot.changes import filter
 117 c['schedulers'] = []
 118 c['schedulers'].append(SingleBranchScheduler(
 119         name="all",
 120         change_filter=filter.ChangeFilter(branch='master'),
 121         treeStableTimer=60,
 122         builderNames=targets))
 123
 124 c['schedulers'].append(ForceScheduler(
 125         name="force",
 126         builderNames=targets))
 127
 128 ####### BUILDERS
 129
 130 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
 131 # what steps, and which slaves can execute them.  Note that any particular build will
 132 # only take place on one slave.
 133
 134 from buildbot.process.factory import BuildFactory
 135 from buildbot.steps.source import Git
 136 from buildbot.steps.shell import ShellCommand
 137 from buildbot.steps.shell import SetProperty
 138 from buildbot.steps.transfer import FileUpload
 139 from buildbot.steps.transfer import FileDownload
 140 from buildbot.steps.master import MasterShellCommand
 141 from buildbot.process.properties import WithProperties
 142
 143
 144 MakeTargetMap = {
 145         "^tools/":                              "tools/clean",
 146         "^toolchain/":                  "toolchain/clean",
 147         "^target/linux/":               "target/linux/clean",
 148         "^(config|include)/":   "dirclean"
 149 }
 150
 151 def IsAffected(pattern):
 152         def CheckAffected(change):
 153                 for request in change.build.requests:
 154                         for source in request.sources:
 155                                 for change in source.changes:
 156                                         for file in change.files:
 157                                                 if re.match(pattern, file):
 158                                                         return True
 159                 return False
 160         return CheckAffected
 161
 162
 163 c['builders'] = []
 164
 165 dlLock = locks.SlaveLock("slave_dl")
 166
 167 checkBuiltin = re.sub('[\t\n ]+', ' ', """
 168         checkBuiltin() {
 169                 local symbol op path file;
 170                 for file in $CHANGED_FILES; do
 171                         case "$file" in
 172                                 package/*/*) : ;;
 173                                 *) return 0 ;;
 174                         esac;
 175                 done;
 176                 while read symbol op path; do
 177                         case "$symbol" in package-*)
 178                                 symbol="${symbol##*(}";
 179                                 symbol="${symbol%)}";
 180                                 for file in $CHANGED_FILES; do
 181                                         case "$file" in "package/$path/"*)
 182                                                 grep -qsx "$symbol=y" .config && return 0
 183                                         ;; esac;
 184                                 done;
 185                         esac;
 186                 done < tmp/.packagedeps;
 187                 return 1;
 188         }
 189 """).strip()
 190
 191
 192 class IfBuiltinShellCommand(ShellCommand):
 193         def _quote(self, str):
 194                 if re.search("[^a-zA-Z0-9/_.-]", str):
 195                         return "'%s'" %(re.sub("'", "'\"'\"'", str))
 196                 return str
 197
 198         def setCommand(self, command):
 199                 if not isinstance(command, (str, unicode)):
 200                         command = ' '.join(map(self._quote, command))
 201                 self.command = [
 202                         '/bin/sh', '-c',
 203                         '%s; if checkBuiltin; then %s; else exit 0; fi' %(checkBuiltin, command)
 204                 ]
 205
 206         def setupEnvironment(self, cmd):
 207                 slaveEnv = self.slaveEnvironment
 208                 if slaveEnv is None:
 209                         slaveEnv = { }
 210                 changedFiles = { }
 211                 for request in self.build.requests:
 212                         for source in request.sources:
 213                                 for change in source.changes:
 214                                         for file in change.files:
 215                                                 changedFiles[file] = True
 216                 fullSlaveEnv = slaveEnv.copy()
 217                 fullSlaveEnv['CHANGED_FILES'] = ' '.join(changedFiles.keys())
 218                 cmd.args['env'] = fullSlaveEnv
 219
 220 slaveNames = [ ]
 221
 222 for slave in c['slaves']:
 223         slaveNames.append(slave.slavename)
 224
 225 for target in targets:
 226         ts = target.split('/')
 227
 228         factory = BuildFactory()
 229
 230         # find number of cores
 231         factory.addStep(SetProperty(
 232                 name = "nproc",
 233                 property = "nproc",
 234                 description = "Finding number of CPUs",
 235                 command = ["nproc"]))
 236
 237         # expire tree if needed
 238         if tree_expire > 0:
 239                 factory.addStep(FileDownload(
 240                         mastersrc = "expire.sh",
 241                         slavedest = "../expire.sh",
 242                         mode = 0755))
 243
 244                 factory.addStep(ShellCommand(
 245                         name = "expire",
 246                         description = "Checking for build tree expiry",
 247                         command = ["./expire.sh", str(tree_expire)],
 248                         workdir = ".",
 249                         haltOnFailure = True,
 250                         timeout = 2400))
 251
 252         # check out the source
 253         factory.addStep(Git(repourl=repo_url, mode='update'))
 254
 255         factory.addStep(ShellCommand(
 256                 name = "rmtmp",
 257                 description = "Remove tmp folder",
 258                 command=["rm", "-rf", "tmp/"]))
 259
 260         # feed
 261 #       factory.addStep(ShellCommand(
 262 #               name = "feedsconf",
 263 #               description = "Copy the feeds.conf",
 264 #               command='''cp ~/feeds.conf ./feeds.conf''' ))
 265
 266         # feed
 267         factory.addStep(ShellCommand(
 268                 name = "rmfeedlinks",
 269                 description = "Remove feed symlinks",
 270                 command=["rm", "-rf", "package/feeds/"]))
 271
 272         # feed
 273         factory.addStep(ShellCommand(
 274                 name = "updatefeeds",
 275                 description = "Updating feeds",
 276                 command=["./scripts/feeds", "update"]))
 277
 278         # feed
 279         factory.addStep(ShellCommand(
 280                 name = "installfeeds",
 281                 description = "Installing feeds",
 282                 command=["./scripts/feeds", "install", "-a"]))
 283
 284         # configure
 285         factory.addStep(ShellCommand(
 286                 name = "newconfig",
 287                 description = "Seeding .config",
 288                 command='''cat <<EOT > .config
 289 CONFIG_TARGET_%s=y
 290 CONFIG_TARGET_%s_%s=y
 291 CONFIG_ALL_NONSHARED=y
 292 CONFIG_SDK=y
 293 CONFIG_IB=y
 294 # CONFIG_IB_STANDALONE is not set
 295 CONFIG_DEVEL=y
 296 CONFIG_CCACHE=y
 297 CONFIG_SIGNED_PACKAGES=y
 298 # CONFIG_PER_FEED_REPO_ADD_COMMENTED is not set
 299 CONFIG_KERNEL_KALLSYMS=y
 300 CONFIG_COLLECT_KERNEL_DEBUG=y
 301 CONFIG_TARGET_ALL_PROFILES=y
 302 CONFIG_TARGET_MULTI_PROFILE=y
 303 CONFIG_TARGET_PER_DEVICE_ROOTFS=y
 304 EOT''' %(ts[0], ts[0], ts[1]) ))
 305
 306         factory.addStep(ShellCommand(
 307                 name = "delbin",
 308                 description = "Removing output directory",
 309                 command = ["rm", "-rf", "bin/"]
 310         ))
 311
 312         factory.addStep(ShellCommand(
 313                 name = "defconfig",
 314                 description = "Populating .config",
 315                 command = ["make", "defconfig"]
 316         ))
 317
 318         # check arch
 319         factory.addStep(ShellCommand(
 320                 name = "checkarch",
 321                 description = "Checking architecture",
 322                 command = ["grep", "-sq", "CONFIG_TARGET_%s=y" %(ts[0]), ".config"],
 323                 logEnviron = False,
 324                 want_stdout = False,
 325                 want_stderr = False,
 326                 haltOnFailure = True
 327         ))
 328
 329         # find libc suffix
 330         factory.addStep(SetProperty(
 331                 name = "libc",
 332                 property = "libc",
 333                 description = "Finding libc suffix",
 334                 command = ["sed", "-ne", '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }', ".config"]))
 335
 336         # install build key
 337         factory.addStep(FileDownload(mastersrc=home_dir+'/key-build', slavedest="key-build", mode=0600))
 338         factory.addStep(FileDownload(mastersrc=home_dir+'/key-build.pub', slavedest="key-build.pub", mode=0600))
 339
 340         # prepare dl
 341         factory.addStep(ShellCommand(
 342                 name = "dldir",
 343                 description = "Preparing dl/",
 344                 command = "mkdir -p $HOME/dl && ln -sf $HOME/dl ./dl",
 345                 logEnviron = False,
 346                 want_stdout = False
 347         ))
 348
 349         # populate dl
 350         factory.addStep(ShellCommand(
 351                 name = "dlrun",
 352                 description = "Populating dl/",
 353                 command = ["make", WithProperties("-j%(nproc:~4)s"), "download", "V=s"],
 354                 logEnviron = False,
 355                 locks = [dlLock.access('exclusive')]
 356         ))
 357
 358         factory.addStep(ShellCommand(
 359                 name = "cleanbase",
 360                 description = "Cleaning base-files",
 361                 command=["make", "package/base-files/clean", "V=s"]
 362         ))
 363
 364         # optional clean steps
 365         for pattern, maketarget in MakeTargetMap.items():
 366                 factory.addStep(ShellCommand(
 367                         name = maketarget,
 368                         description = maketarget,
 369                         command=["make", maketarget, "V=s"], doStepIf=IsAffected(pattern)
 370                 ))
 371
 372         # build
 373         factory.addStep(ShellCommand(
 374                 name = "tools",
 375                 description = "Building tools",
 376                 command = ["make", WithProperties("-j%(nproc:~4)s"), "tools/install", "V=s"],
 377                 haltOnFailure = True
 378         ))
 379
 380         factory.addStep(ShellCommand(
 381                 name = "toolchain",
 382                 description = "Building toolchain",
 383                 command=["make", WithProperties("-j%(nproc:~4)s"), "toolchain/install", "V=s"],
 384                 haltOnFailure = True
 385         ))
 386
 387         factory.addStep(ShellCommand(
 388                 name = "kmods",
 389                 description = "Building kmods",
 390                 command=["make", WithProperties("-j%(nproc:~4)s"), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
 391                 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
 392                 haltOnFailure = True
 393         ))
 394
 395         factory.addStep(ShellCommand(
 396                 name = "pkgbuild",
 397                 description = "Building packages",
 398                 command=["make", WithProperties("-j%(nproc:~4)s"), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
 399                 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
 400                 haltOnFailure = True
 401         ))
 402
 403         # factory.addStep(IfBuiltinShellCommand(
 404         factory.addStep(ShellCommand(
 405                 name = "pkginstall",
 406                 description = "Installing packages",
 407                 command=["make", WithProperties("-j%(nproc:~4)s"), "package/install", "V=s"],
 408                 haltOnFailure = True
 409         ))
 410
 411         factory.addStep(ShellCommand(
 412                 name = "pkgindex",
 413                 description = "Indexing packages",
 414                 command=["make", WithProperties("-j%(nproc:~4)s"), "package/index", "V=s"],
 415                 haltOnFailure = True
 416         ))
 417
 418         #factory.addStep(IfBuiltinShellCommand(
 419         factory.addStep(ShellCommand(
 420                 name = "images",
 421                 description = "Building images",
 422                 command=["make", WithProperties("-j%(nproc:~4)s"), "target/install", "V=s"],
 423                 haltOnFailure = True
 424         ))
 425
 426         factory.addStep(ShellCommand(
 427                 name = "checksums",
 428                 description = "Calculating checksums",
 429                 command=["make", "-j1", "checksum", "V=s"],
 430                 haltOnFailure = True
 431         ))
 432
 433         # sign
 434         if gpg_keyid is not None:
 435                 factory.addStep(MasterShellCommand(
 436                         name = "signprepare",
 437                         description = "Preparing temporary signing directory",
 438                         command = ["mkdir", "-p", "%s/signing" %(home_dir)],
 439                         haltOnFailure = True
 440                 ))
 441
 442                 factory.addStep(ShellCommand(
 443                         name = "signpack",
 444                         description = "Packing files to sign",
 445                         command = WithProperties("find bin/targets/%s/%s%%(libc)s/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz" %(ts[0], ts[1])),
 446                         haltOnFailure = True
 447                 ))
 448
 449                 factory.addStep(FileUpload(
 450                         slavesrc = "sign.tar.gz",
 451                         masterdest = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
 452                         haltOnFailure = True
 453                 ))
 454
 455                 factory.addStep(MasterShellCommand(
 456                         name = "signfiles",
 457                         description = "Signing files",
 458                         command = ["%s/signall.sh" %(home_dir), "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]), gpg_keyid, gpg_passfile, gpg_comment],
 459                         haltOnFailure = True
 460                 ))
 461
 462                 factory.addStep(FileDownload(
 463                         mastersrc = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
 464                         slavedest = "sign.tar.gz",
 465                         haltOnFailure = True
 466                 ))
 467
 468                 factory.addStep(ShellCommand(
 469                         name = "signunpack",
 470                         description = "Unpacking signed files",
 471                         command = ["tar", "-xzf", "sign.tar.gz"],
 472                         haltOnFailure = True
 473                 ))
 474
 475         # upload
 476         factory.addStep(ShellCommand(
 477                 name = "uploadprepare",
 478                 description = "Preparing target directory",
 479                 command=["rsync", "-av", "--include", "/%s/" %(ts[0]), "--include", "/%s/%s/" %(ts[0], ts[1]), "--exclude", "/*", "--exclude", "/*/*", "--exclude", "/%s/%s/*" %(ts[0], ts[1]), "bin/targets/", "%s/targets/" %(rsync_bin_url)],
 480                 env={'RSYNC_PASSWORD': rsync_bin_key},
 481                 haltOnFailure = True,
 482                 logEnviron = False
 483         ))
 484
 485         factory.addStep(ShellCommand(
 486                 name = "targetupload",
 487                 description = "Uploading target files",
 488                 command=["rsync", "--delete", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", WithProperties("bin/targets/%s/%s%%(libc)s/" %(ts[0], ts[1])), "%s/targets/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
 489                 env={'RSYNC_PASSWORD': rsync_bin_key},
 490                 haltOnFailure = True,
 491                 logEnviron = False
 492         ))
 493
 494         if rsync_src_url is not None:
 495                 factory.addStep(ShellCommand(
 496                         name = "sourceupload",
 497                         description = "Uploading source archives",
 498                         command=["rsync", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "dl/", "%s/" %(rsync_src_url)],
 499                         env={'RSYNC_PASSWORD': rsync_src_key},
 500                         haltOnFailure = True,
 501                         logEnviron = False
 502                 ))
 503
 504         if False:
 505                 factory.addStep(ShellCommand(
 506                         name = "packageupload",
 507                         description = "Uploading package files",
 508                         command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "bin/packages/", "%s/packages/" %(rsync_bin_url)],
 509                         env={'RSYNC_PASSWORD': rsync_bin_key},
 510                         haltOnFailure = False,
 511                         logEnviron = False
 512                 ))
 513
 514         # logs
 515         if False:
 516                 factory.addStep(ShellCommand(
 517                         name = "upload",
 518                         description = "Uploading logs",
 519                         command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "logs/", "%s/logs/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
 520                         env={'RSYNC_PASSWORD': rsync_bin_key},
 521                         haltOnFailure = False,
 522                         alwaysRun = True,
 523                         logEnviron = False
 524                 ))
 525
 526         from buildbot.config import BuilderConfig
 527
 528         c['builders'].append(BuilderConfig(name=target, slavenames=slaveNames, factory=factory))
 529
 530
 531 ####### STATUS TARGETS
 532
 533 # 'status' is a list of Status Targets. The results of each build will be
 534 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
 535 # including web pages, email senders, and IRC bots.
 536
 537 c['status'] = []
 538
 539 from buildbot.status import html
 540 from buildbot.status.web import authz, auth
 541
 542 if ini.has_option("status", "bind"):
 543         if ini.has_option("status", "user") and ini.has_option("status", "password"):
 544                 authz_cfg=authz.Authz(
 545                         # change any of these to True to enable; see the manual for more
 546                         # options
 547                         auth=auth.BasicAuth([(ini.get("status", "user"), ini.get("status", "password"))]),
 548                         gracefulShutdown = 'auth',
 549                         forceBuild = 'auth', # use this to test your slave once it is set up
 550                         forceAllBuilds = 'auth',
 551                         pingBuilder = False,
 552                         stopBuild = 'auth',
 553                         stopAllBuilds = 'auth',
 554                         cancelPendingBuild = 'auth',
 555                 )
 556                 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind"), authz=authz_cfg))
 557         else:
 558                 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind")))
 559
 560
 561 from buildbot.status import words
 562
 563 if ini.has_option("irc", "host") and ini.has_option("irc", "nickname") and ini.has_option("irc", "channel"):
 564         irc_host = ini.get("irc", "host")
 565         irc_port = 6667
 566         irc_chan = ini.get("irc", "channel")
 567         irc_nick = ini.get("irc", "nickname")
 568         irc_pass = None
 569
 570         if ini.has_option("irc", "port"):
 571                 irc_port = ini.getint("irc", "port")
 572
 573         if ini.has_option("irc", "password"):
 574                 irc_pass = ini.get("irc", "password")
 575
 576         irc = words.IRC(irc_host, irc_nick, port = irc_port, password = irc_pass,
 577                         channels = [{ "channel": irc_chan }],
 578                         notify_events = {
 579                           'exception': 1,
 580                           'successToFailure': 1,
 581                           'failureToSuccess': 1
 582                         }
 583         )
 584
 585         c['status'].append(irc)
 586
 587
 588 ####### PROJECT IDENTITY
 589
 590 # the 'title' string will appear at the top of this buildbot
 591 # installation's html.WebStatus home page (linked to the
 592 # 'titleURL') and is embedded in the title of the waterfall HTML page.
 593
 594 c['title'] = ini.get("general", "title")
 595 c['titleURL'] = ini.get("general", "title_url")
 596
 597 # the 'buildbotURL' string should point to the location where the buildbot's
 598 # internal web server (usually the html.WebStatus page) is visible. This
 599 # typically uses the port number set in the Waterfall 'status' entry, but
 600 # with an externally-visible host name which the buildbot cannot figure out
 601 # without some help.
 602
 603 c['buildbotURL'] = ini.get("general", "buildbot_url")
 604
 605 ####### DB URL
 606
 607 c['db'] = {
 608         # This specifies what database buildbot uses to store its state.  You can leave
 609         # this at its default for all but the largest installations.
 610         'db_url' : "sqlite:///state.sqlite",
 611 }