phase1/master.cfg

   1 # -*- python -*-
   2 # ex: set syntax=python:
   3
   4 import os
   5 import re
   6 import subprocess
   7 import ConfigParser
   8
   9 from buildbot import locks
  10
  11 # This is a sample buildmaster config file. It must be installed as
  12 # 'master.cfg' in your buildmaster's base directory.
  13
  14 ini = ConfigParser.ConfigParser()
  15 ini.read("./config.ini")
  16
  17 # This is the dictionary that the buildmaster pays attention to. We also use
  18 # a shorter alias to save typing.
  19 c = BuildmasterConfig = {}
  20
  21 ####### BUILDSLAVES
  22
  23 # The 'slaves' list defines the set of recognized buildslaves. Each element is
  24 # a BuildSlave object, specifying a unique slave name and password.  The same
  25 # slave name and password must be configured on the slave.
  26 from buildbot.buildslave import BuildSlave
  27
  28 c['slaves'] = []
  29
  30 for section in ini.sections():
  31         if section.startswith("slave "):
  32                 if ini.has_option(section, "name") and ini.has_option(section, "password"):
  33                         name = ini.get(section, "name")
  34                         password = ini.get(section, "password")
  35                         max_builds = 1
  36                         if ini.has_option(section, "builds"):
  37                                 max_builds = ini.getint(section, "builds")
  38                         c['slaves'].append(BuildSlave(name, password, max_builds = max_builds))
  39
  40 # 'slavePortnum' defines the TCP port to listen on for connections from slaves.
  41 # This must match the value configured into the buildslaves (with their
  42 # --master option)
  43 c['slavePortnum'] = 9989
  44
  45 # coalesce builds
  46 c['mergeRequests'] = True
  47
  48 ####### CHANGESOURCES
  49
  50 home_dir = os.path.abspath(ini.get("general", "homedir"))
  51
  52 repo_url = ini.get("repo", "url")
  53
  54 rsync_bin_url = ini.get("rsync", "binary_url")
  55 rsync_bin_key = ini.get("rsync", "binary_password")
  56
  57 rsync_src_url = None
  58 rsync_src_key = None
  59
  60 if ini.has_option("rsync", "source_url"):
  61         rsync_src_url = ini.get("rsync", "source_url")
  62         rsync_src_key = ini.get("rsync", "source_password")
  63
  64 gpg_keyid = None
  65 gpg_comment = "Unattended build signature"
  66 gpg_passfile = "/dev/null"
  67
  68 if ini.has_option("gpg", "keyid"):
  69         gpg_keyid = ini.get("gpg", "keyid")
  70
  71 if ini.has_option("gpg", "comment"):
  72         gpg_comment = ini.get("gpg", "comment")
  73
  74 if ini.has_option("gpg", "passfile"):
  75         gpg_passfile = ini.get("gpg", "passfile")
  76
  77
  78 # find targets
  79 targets = [ ]
  80
  81 findtargets = subprocess.Popen([home_dir+'/dumpinfo.pl', 'targets'],
  82         stdout = subprocess.PIPE, cwd = home_dir+'/source.git')
  83
  84 while True:
  85         line = findtargets.stdout.readline()
  86         if not line:
  87                 break
  88         ta = line.strip().split(' ')
  89         targets.append(ta[0])
  90
  91
  92 # the 'change_source' setting tells the buildmaster how it should find out
  93 # about source code changes.  Here we point to the buildbot clone of pyflakes.
  94
  95 from buildbot.changes.gitpoller import GitPoller
  96 c['change_source'] = []
  97 c['change_source'].append(GitPoller(
  98         repo_url,
  99         workdir=home_dir+'/source.git', branch='master',
 100         pollinterval=300))
 101
 102 ####### SCHEDULERS
 103
 104 # Configure the Schedulers, which decide how to react to incoming changes.  In this
 105 # case, just kick off a 'basebuild' build
 106
 107 from buildbot.schedulers.basic import SingleBranchScheduler
 108 from buildbot.schedulers.forcesched import ForceScheduler
 109 from buildbot.changes import filter
 110 c['schedulers'] = []
 111 c['schedulers'].append(SingleBranchScheduler(
 112         name="all",
 113         change_filter=filter.ChangeFilter(branch='master'),
 114         treeStableTimer=60,
 115         builderNames=targets))
 116
 117 c['schedulers'].append(ForceScheduler(
 118         name="force",
 119         builderNames=targets))
 120
 121 ####### BUILDERS
 122
 123 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
 124 # what steps, and which slaves can execute them.  Note that any particular build will
 125 # only take place on one slave.
 126
 127 from buildbot.process.factory import BuildFactory
 128 from buildbot.steps.source import Git
 129 from buildbot.steps.shell import ShellCommand
 130 from buildbot.steps.shell import SetProperty
 131 from buildbot.steps.transfer import FileUpload
 132 from buildbot.steps.transfer import FileDownload
 133 from buildbot.steps.master import MasterShellCommand
 134 from buildbot.process.properties import WithProperties
 135
 136
 137 MakeTargetMap = {
 138         "^tools/":                              "tools/clean",
 139         "^toolchain/":                  "toolchain/clean",
 140         "^target/linux/":               "target/linux/clean",
 141         "^(config|include)/":   "dirclean"
 142 }
 143
 144 def IsAffected(pattern):
 145         def CheckAffected(change):
 146                 for request in change.build.requests:
 147                         for source in request.sources:
 148                                 for change in source.changes:
 149                                         for file in change.files:
 150                                                 if re.match(pattern, file):
 151                                                         return True
 152                 return False
 153         return CheckAffected
 154
 155 def isPathBuiltin(path):
 156         incl = {}
 157         pkgs = {}
 158         conf = open(".config", "r")
 159
 160         while True:
 161                 line = conf.readline()
 162                 if line == '':
 163                         break
 164                 m = re.match("^(CONFIG_PACKAGE_.+?)=y", line)
 165                 if m:
 166                         incl[m.group(1)] = True
 167
 168         conf.close()
 169
 170         deps = open("tmp/.packagedeps", "r")
 171
 172         while True:
 173                 line = deps.readline()
 174                 if line == '':
 175                         break
 176                 m = re.match("^package-\$\((CONFIG_PACKAGE_.+?)\) \+= (\S+)", line)
 177                 if m and incl.get(m.group(1)) == True:
 178                         pkgs["package/%s" % m.group(2)] = True
 179
 180         deps.close()
 181
 182         while path != '':
 183                 if pkgs.get(path) == True:
 184                         return True
 185                 path = os.path.dirname(path)
 186
 187         return False
 188
 189 def isChangeBuiltin(change):
 190         return True
 191 #       for request in change.build.requests:
 192 #               for source in request.sources:
 193 #                       for change in source.changes:
 194 #                               for file in change.files:
 195 #                                       if isPathBuiltin(file):
 196 #                                               return True
 197 #       return False
 198
 199
 200 c['builders'] = []
 201
 202 dlLock = locks.SlaveLock("slave_dl")
 203
 204 checkBuiltin = re.sub('[\t\n ]+', ' ', """
 205         checkBuiltin() {
 206                 local symbol op path file;
 207                 for file in $CHANGED_FILES; do
 208                         case "$file" in
 209                                 package/*/*) : ;;
 210                                 *) return 0 ;;
 211                         esac;
 212                 done;
 213                 while read symbol op path; do
 214                         case "$symbol" in package-*)
 215                                 symbol="${symbol##*(}";
 216                                 symbol="${symbol%)}";
 217                                 for file in $CHANGED_FILES; do
 218                                         case "$file" in "package/$path/"*)
 219                                                 grep -qsx "$symbol=y" .config && return 0
 220                                         ;; esac;
 221                                 done;
 222                         esac;
 223                 done < tmp/.packagedeps;
 224                 return 1;
 225         }
 226 """).strip()
 227
 228
 229 class IfBuiltinShellCommand(ShellCommand):
 230         def _quote(self, str):
 231                 if re.search("[^a-zA-Z0-9/_.-]", str):
 232                         return "'%s'" %(re.sub("'", "'\"'\"'", str))
 233                 return str
 234
 235         def setCommand(self, command):
 236                 if not isinstance(command, (str, unicode)):
 237                         command = ' '.join(map(self._quote, command))
 238                 self.command = [
 239                         '/bin/sh', '-c',
 240                         '%s; if checkBuiltin; then %s; else exit 0; fi' %(checkBuiltin, command)
 241                 ]
 242
 243         def setupEnvironment(self, cmd):
 244                 slaveEnv = self.slaveEnvironment
 245                 if slaveEnv is None:
 246                         slaveEnv = { }
 247                 changedFiles = { }
 248                 for request in self.build.requests:
 249                         for source in request.sources:
 250                                 for change in source.changes:
 251                                         for file in change.files:
 252                                                 changedFiles[file] = True
 253                 fullSlaveEnv = slaveEnv.copy()
 254                 fullSlaveEnv['CHANGED_FILES'] = ' '.join(changedFiles.keys())
 255                 cmd.args['env'] = fullSlaveEnv
 256
 257 slaveNames = [ ]
 258
 259 for slave in c['slaves']:
 260         slaveNames.append(slave.slavename)
 261
 262 for target in targets:
 263         ts = target.split('/')
 264
 265         factory = BuildFactory()
 266
 267         # find number of cores
 268         factory.addStep(SetProperty(
 269                 name = "nproc",
 270                 property = "nproc",
 271                 description = "Finding number of CPUs",
 272                 command = ["nproc"]))
 273
 274         # check out the source
 275         factory.addStep(Git(repourl=repo_url, mode='update'))
 276
 277         factory.addStep(ShellCommand(
 278                 name = "rmtmp",
 279                 description = "Remove tmp folder",
 280                 command=["rm", "-rf", "tmp/"]))
 281
 282         # feed
 283 #       factory.addStep(ShellCommand(
 284 #               name = "feedsconf",
 285 #               description = "Copy the feeds.conf",
 286 #               command='''cp ~/feeds.conf ./feeds.conf''' ))
 287
 288         # feed
 289         factory.addStep(ShellCommand(
 290                 name = "updatefeeds",
 291                 description = "Updating feeds",
 292                 command=["./scripts/feeds", "update"]))
 293
 294         # feed
 295         factory.addStep(ShellCommand(
 296                 name = "installfeeds",
 297                 description = "Installing feeds",
 298                 command=["./scripts/feeds", "install", "-a"]))
 299
 300         # configure
 301         factory.addStep(ShellCommand(
 302                 name = "newconfig",
 303                 description = "Seeding .config",
 304                 command='''cat <<EOT > .config
 305 CONFIG_TARGET_%s=y
 306 CONFIG_TARGET_%s_%s=y
 307 CONFIG_ALL_NONSHARED=y
 308 CONFIG_SDK=y
 309 CONFIG_IB=y
 310 # CONFIG_IB_STANDALONE is not set
 311 CONFIG_DEVEL=y
 312 CONFIG_CCACHE=y
 313 CONFIG_SIGNED_PACKAGES=y
 314 # CONFIG_PER_FEED_REPO_ADD_COMMENTED is not set
 315 CONFIG_KERNEL_KALLSYMS=y
 316 CONFIG_COLLECT_KERNEL_DEBUG=y
 317 EOT''' %(ts[0], ts[0], ts[1]) ))
 318
 319         factory.addStep(ShellCommand(
 320                 name = "delbin",
 321                 description = "Removing output directory",
 322                 command = ["rm", "-rf", "bin/"]
 323         ))
 324
 325         factory.addStep(ShellCommand(
 326                 name = "defconfig",
 327                 description = "Populating .config",
 328                 command = ["make", "defconfig"]
 329         ))
 330
 331         # check arch
 332         factory.addStep(ShellCommand(
 333                 name = "checkarch",
 334                 description = "Checking architecture",
 335                 command = ["grep", "-sq", "CONFIG_TARGET_%s=y" %(ts[0]), ".config"],
 336                 logEnviron = False,
 337                 want_stdout = False,
 338                 want_stderr = False,
 339                 haltOnFailure = True
 340         ))
 341
 342         # find libc suffix
 343         factory.addStep(SetProperty(
 344                 name = "libc",
 345                 property = "libc",
 346                 description = "Finding libc suffix",
 347                 command = ["sed", "-ne", '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }', ".config"]))
 348
 349         # install build key
 350         factory.addStep(FileDownload(mastersrc=home_dir+'/key-build', slavedest="key-build", mode=0600))
 351         factory.addStep(FileDownload(mastersrc=home_dir+'/key-build.pub', slavedest="key-build.pub", mode=0600))
 352
 353         # prepare dl
 354         factory.addStep(ShellCommand(
 355                 name = "dldir",
 356                 description = "Preparing dl/",
 357                 command = "mkdir -p $HOME/dl && ln -sf $HOME/dl ./dl",
 358                 logEnviron = False,
 359                 want_stdout = False
 360         ))
 361
 362         # populate dl
 363         factory.addStep(ShellCommand(
 364                 name = "dlrun",
 365                 description = "Populating dl/",
 366                 command = ["make", WithProperties("-j%(nproc:~4)s"), "download", "V=s"],
 367                 logEnviron = False,
 368                 locks = [dlLock.access('exclusive')]
 369         ))
 370
 371         factory.addStep(ShellCommand(
 372                 name = "cleanbase",
 373                 description = "Cleaning base-files",
 374                 command=["make", "package/base-files/clean", "V=s"]
 375         ))
 376
 377         # optional clean steps
 378         for pattern, maketarget in MakeTargetMap.items():
 379                 factory.addStep(ShellCommand(
 380                         name = maketarget,
 381                         description = maketarget,
 382                         command=["make", maketarget, "V=s"], doStepIf=IsAffected(pattern)
 383                 ))
 384
 385         # build
 386         factory.addStep(ShellCommand(
 387                 name = "tools",
 388                 description = "Building tools",
 389                 command = ["make", WithProperties("-j%(nproc:~4)s"), "tools/install", "V=s"],
 390                 haltOnFailure = True
 391         ))
 392
 393         factory.addStep(ShellCommand(
 394                 name = "toolchain",
 395                 description = "Building toolchain",
 396                 command=["make", WithProperties("-j%(nproc:~4)s"), "toolchain/install", "V=s"],
 397                 haltOnFailure = True
 398         ))
 399
 400         factory.addStep(ShellCommand(
 401                 name = "kmods",
 402                 description = "Building kmods",
 403                 command=["make", WithProperties("-j%(nproc:~4)s"), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
 404                 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
 405                 haltOnFailure = True
 406         ))
 407
 408         factory.addStep(ShellCommand(
 409                 name = "pkgbuild",
 410                 description = "Building packages",
 411                 command=["make", WithProperties("-j%(nproc:~4)s"), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
 412                 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
 413                 haltOnFailure = True
 414         ))
 415
 416         # factory.addStep(IfBuiltinShellCommand(
 417         factory.addStep(ShellCommand(
 418                 name = "pkginstall",
 419                 description = "Installing packages",
 420                 command=["make", WithProperties("-j%(nproc:~4)s"), "package/install", "V=s"],
 421                 doStepIf = isChangeBuiltin,
 422                 haltOnFailure = True
 423         ))
 424
 425         factory.addStep(ShellCommand(
 426                 name = "pkgindex",
 427                 description = "Indexing packages",
 428                 command=["make", WithProperties("-j%(nproc:~4)s"), "package/index", "V=s"],
 429                 haltOnFailure = True
 430         ))
 431
 432         #factory.addStep(IfBuiltinShellCommand(
 433         factory.addStep(ShellCommand(
 434                 name = "images",
 435                 description = "Building images",
 436                 command=["make", "-j1", "target/install", "V=s"],
 437                 doStepIf = isChangeBuiltin,
 438                 haltOnFailure = True
 439         ))
 440
 441         # sign
 442         if gpg_keyid is not None:
 443                 factory.addStep(MasterShellCommand(
 444                         name = "signprepare",
 445                         description = "Preparing temporary signing directory",
 446                         command = ["mkdir", "-p", "%s/signing" %(home_dir)],
 447                         haltOnFailure = True
 448                 ))
 449
 450                 factory.addStep(ShellCommand(
 451                         name = "signpack",
 452                         description = "Packing files to sign",
 453                         command = ["sh", "-c", WithProperties("find bin/targets/%s/%s%%(libc)s/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz" %(ts[0], ts[1]))],
 454                         haltOnFailure = True
 455                 ))
 456
 457                 factory.addStep(FileUpload(
 458                         slavesrc = "sign.tar.gz",
 459                         masterdest = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
 460                         haltOnFailure = True
 461                 ))
 462
 463                 factory.addStep(MasterShellCommand(
 464                         name = "signfiles",
 465                         description = "Signing files",
 466                         command = ["%s/signall.sh" %(home_dir), "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]), gpg_keyid, gpg_passfile, gpg_comment],
 467                         haltOnFailure = True
 468                 ))
 469
 470                 factory.addStep(FileDownload(
 471                         mastersrc = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]),
 472                         slavedest = "sign.tar.gz",
 473                         haltOnFailure = True
 474                 ))
 475
 476                 factory.addStep(ShellCommand(
 477                         name = "signunpack",
 478                         description = "Unpacking signed files",
 479                         command = ["tar", "-xzf", "sign.tar.gz"],
 480                         haltOnFailure = True
 481                 ))
 482
 483         # upload
 484         factory.addStep(ShellCommand(
 485                 name = "uploadprepare",
 486                 description = "Preparing target directory",
 487                 command=["rsync", "-av", "--include", "/%s/" %(ts[0]), "--include", "/%s/%s/" %(ts[0], ts[1]), "--exclude", "/*", "--exclude", "/*/*", "--exclude", "/%s/%s/*" %(ts[0], ts[1]), "bin/targets/", "%s/targets/" %(rsync_bin_url)],
 488                 env={'RSYNC_PASSWORD': rsync_bin_key},
 489                 haltOnFailure = True,
 490                 logEnviron = False
 491         ))
 492
 493         factory.addStep(ShellCommand(
 494                 name = "targetupload",
 495                 description = "Uploading target files",
 496                 command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", WithProperties("bin/targets/%s/%s%%(libc)s/" %(ts[0], ts[1])), "%s/targets/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
 497                 env={'RSYNC_PASSWORD': rsync_bin_key},
 498                 haltOnFailure = True,
 499                 logEnviron = False
 500         ))
 501
 502         if rsync_src_url is not None:
 503                 factory.addStep(ShellCommand(
 504                         name = "sourceupload",
 505                         description = "Uploading source archives",
 506                         command=["rsync", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "dl/", "%s/" %(rsync_src_url)],
 507                         env={'RSYNC_PASSWORD': rsync_src_key},
 508                         haltOnFailure = True,
 509                         logEnviron = False
 510                 ))
 511
 512         if False:
 513                 factory.addStep(ShellCommand(
 514                         name = "packageupload",
 515                         description = "Uploading package files",
 516                         command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "bin/packages/", "%s/packages/" %(rsync_bin_url)],
 517                         env={'RSYNC_PASSWORD': rsync_bin_key},
 518                         haltOnFailure = False,
 519                         logEnviron = False
 520                 ))
 521
 522         # logs
 523         if False:
 524                 factory.addStep(ShellCommand(
 525                         name = "upload",
 526                         description = "Uploading logs",
 527                         command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "logs/", "%s/logs/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
 528                         env={'RSYNC_PASSWORD': rsync_bin_key},
 529                         haltOnFailure = False,
 530                         alwaysRun = True,
 531                         logEnviron = False
 532                 ))
 533
 534         from buildbot.config import BuilderConfig
 535
 536         c['builders'].append(BuilderConfig(name=target, slavenames=slaveNames, factory=factory))
 537
 538
 539 ####### STATUS TARGETS
 540
 541 # 'status' is a list of Status Targets. The results of each build will be
 542 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
 543 # including web pages, email senders, and IRC bots.
 544
 545 c['status'] = []
 546
 547 from buildbot.status import html
 548 from buildbot.status.web import authz, auth
 549
 550 if ini.has_option("status", "bind"):
 551         if ini.has_option("status", "user") and ini.has_option("status", "password"):
 552                 authz_cfg=authz.Authz(
 553                         # change any of these to True to enable; see the manual for more
 554                         # options
 555                         auth=auth.BasicAuth([(ini.get("status", "user"), ini.get("status", "password"))]),
 556                         gracefulShutdown = 'auth',
 557                         forceBuild = 'auth', # use this to test your slave once it is set up
 558                         forceAllBuilds = 'auth',
 559                         pingBuilder = False,
 560                         stopBuild = 'auth',
 561                         stopAllBuilds = 'auth',
 562                         cancelPendingBuild = 'auth',
 563                 )
 564                 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind"), authz=authz_cfg))
 565         else:
 566                 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind")))
 567
 568
 569 from buildbot.status import words
 570
 571 if ini.has_option("irc", "host") and ini.has_option("irc", "nickname") and ini.has_option("irc", "channel"):
 572         irc_host = ini.get("irc", "host")
 573         irc_port = 6667
 574         irc_chan = ini.get("irc", "channel")
 575         irc_nick = ini.get("irc", "nickname")
 576         irc_pass = None
 577
 578         if ini.has_option("irc", "port"):
 579                 irc_port = ini.getint("irc", "port")
 580
 581         if ini.has_option("irc", "password"):
 582                 irc_pass = ini.get("irc", "password")
 583
 584         irc = words.IRC(irc_host, irc_nick, port = irc_port, password = irc_pass,
 585                         channels = [{ "channel": irc_chan }],
 586                         notify_events = {
 587                           'exception': 1,
 588                           'successToFailure': 1,
 589                           'failureToSuccess': 1
 590                         }
 591         )
 592
 593         c['status'].append(irc)
 594
 595
 596 ####### PROJECT IDENTITY
 597
 598 # the 'title' string will appear at the top of this buildbot
 599 # installation's html.WebStatus home page (linked to the
 600 # 'titleURL') and is embedded in the title of the waterfall HTML page.
 601
 602 c['title'] = ini.get("general", "title")
 603 c['titleURL'] = ini.get("general", "title_url")
 604
 605 # the 'buildbotURL' string should point to the location where the buildbot's
 606 # internal web server (usually the html.WebStatus page) is visible. This
 607 # typically uses the port number set in the Waterfall 'status' entry, but
 608 # with an externally-visible host name which the buildbot cannot figure out
 609 # without some help.
 610
 611 c['buildbotURL'] = ini.get("general", "buildbot_url")
 612
 613 ####### DB URL
 614
 615 c['db'] = {
 616         # This specifies what database buildbot uses to store its state.  You can leave
 617         # this at its default for all but the largest installations.
 618         'db_url' : "sqlite:///state.sqlite",
 619 }