phase1/master.cfg

   1 # -*- python -*-
   2 # ex: set syntax=python:
   3
   4 import os
   5 import re
   6 import subprocess
   7 import ConfigParser
   8
   9 from buildbot import locks
  10
  11 # This is a sample buildmaster config file. It must be installed as
  12 # 'master.cfg' in your buildmaster's base directory.
  13
  14 ini = ConfigParser.ConfigParser()
  15 ini.read("./config.ini")
  16
  17 # This is the dictionary that the buildmaster pays attention to. We also use
  18 # a shorter alias to save typing.
  19 c = BuildmasterConfig = {}
  20
  21 ####### BUILDSLAVES
  22
  23 # The 'slaves' list defines the set of recognized buildslaves. Each element is
  24 # a BuildSlave object, specifying a unique slave name and password.  The same
  25 # slave name and password must be configured on the slave.
  26 from buildbot.buildslave import BuildSlave
  27
  28 c['slaves'] = []
  29
  30 for section in ini.sections():
  31         if section.startswith("slave "):
  32                 if ini.has_option(section, "name") and ini.has_option(section, "password"):
  33                         name = ini.get(section, "name")
  34                         password = ini.get(section, "password")
  35                         max_builds = 1
  36                         if ini.has_option(section, "builds"):
  37                                 max_builds = ini.getint(section, "builds")
  38                         c['slaves'].append(BuildSlave(name, password, max_builds = max_builds))
  39
  40 # 'slavePortnum' defines the TCP port to listen on for connections from slaves.
  41 # This must match the value configured into the buildslaves (with their
  42 # --master option)
  43 c['slavePortnum'] = 9989
  44
  45 # coalesce builds
  46 c['mergeRequests'] = True
  47
  48 ####### CHANGESOURCES
  49
  50 home_dir = os.path.abspath(ini.get("general", "homedir"))
  51
  52 repo_url = ini.get("repo", "url")
  53
  54 rsync_bin_url = ini.get("rsync", "binary_url")
  55 rsync_bin_key = ini.get("rsync", "binary_password")
  56
  57 rsync_src_url = None
  58 rsync_src_key = None
  59
  60 if ini.has_option("rsync", "source_url"):
  61         rsync_src_url = ini.get("rsync", "source_url")
  62         rsync_src_key = ini.get("rsync", "source_password")
  63
  64 # find targets
  65 targets = [ ]
  66
  67 findtargets = subprocess.Popen([home_dir+'/dumpinfo.pl', 'targets'],
  68         stdout = subprocess.PIPE, cwd = home_dir+'/source.git')
  69
  70 while True:
  71         line = findtargets.stdout.readline()
  72         if not line:
  73                 break
  74         ta = line.strip().split(' ')
  75         targets.append(ta[0])
  76
  77
  78 # the 'change_source' setting tells the buildmaster how it should find out
  79 # about source code changes.  Here we point to the buildbot clone of pyflakes.
  80
  81 from buildbot.changes.gitpoller import GitPoller
  82 c['change_source'] = []
  83 c['change_source'].append(GitPoller(
  84         repo_url,
  85         workdir=home_dir+'/source.git', branch='master',
  86         pollinterval=300))
  87
  88 ####### SCHEDULERS
  89
  90 # Configure the Schedulers, which decide how to react to incoming changes.  In this
  91 # case, just kick off a 'basebuild' build
  92
  93 from buildbot.schedulers.basic import SingleBranchScheduler
  94 from buildbot.schedulers.forcesched import ForceScheduler
  95 from buildbot.changes import filter
  96 c['schedulers'] = []
  97 c['schedulers'].append(SingleBranchScheduler(
  98         name="all",
  99         change_filter=filter.ChangeFilter(branch='master'),
 100         treeStableTimer=60,
 101         builderNames=targets))
 102
 103 c['schedulers'].append(ForceScheduler(
 104         name="force",
 105         builderNames=targets))
 106
 107 ####### BUILDERS
 108
 109 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
 110 # what steps, and which slaves can execute them.  Note that any particular build will
 111 # only take place on one slave.
 112
 113 from buildbot.process.factory import BuildFactory
 114 from buildbot.steps.source import Git
 115 from buildbot.steps.shell import ShellCommand
 116 from buildbot.steps.shell import SetProperty
 117 from buildbot.steps.transfer import FileDownload
 118 from buildbot.process.properties import WithProperties
 119
 120
 121 MakeTargetMap = {
 122         "^tools/":                              "tools/clean",
 123         "^toolchain/":                  "toolchain/clean",
 124         "^target/linux/":               "target/linux/clean",
 125         "^(config|include)/":   "dirclean"
 126 }
 127
 128 def IsAffected(pattern):
 129         def CheckAffected(change):
 130                 for request in change.build.requests:
 131                         for source in request.sources:
 132                                 for change in source.changes:
 133                                         for file in change.files:
 134                                                 if re.match(pattern, file):
 135                                                         return True
 136                 return False
 137         return CheckAffected
 138
 139 def isPathBuiltin(path):
 140         incl = {}
 141         pkgs = {}
 142         conf = open(".config", "r")
 143
 144         while True:
 145                 line = conf.readline()
 146                 if line == '':
 147                         break
 148                 m = re.match("^(CONFIG_PACKAGE_.+?)=y", line)
 149                 if m:
 150                         incl[m.group(1)] = True
 151
 152         conf.close()
 153
 154         deps = open("tmp/.packagedeps", "r")
 155
 156         while True:
 157                 line = deps.readline()
 158                 if line == '':
 159                         break
 160                 m = re.match("^package-\$\((CONFIG_PACKAGE_.+?)\) \+= (\S+)", line)
 161                 if m and incl.get(m.group(1)) == True:
 162                         pkgs["package/%s" % m.group(2)] = True
 163
 164         deps.close()
 165
 166         while path != '':
 167                 if pkgs.get(path) == True:
 168                         return True
 169                 path = os.path.dirname(path)
 170
 171         return False
 172
 173 def isChangeBuiltin(change):
 174         return True
 175 #       for request in change.build.requests:
 176 #               for source in request.sources:
 177 #                       for change in source.changes:
 178 #                               for file in change.files:
 179 #                                       if isPathBuiltin(file):
 180 #                                               return True
 181 #       return False
 182
 183
 184 c['builders'] = []
 185
 186 dlLock = locks.SlaveLock("slave_dl")
 187
 188 checkBuiltin = re.sub('[\t\n ]+', ' ', """
 189         checkBuiltin() {
 190                 local symbol op path file;
 191                 for file in $CHANGED_FILES; do
 192                         case "$file" in
 193                                 package/*/*) : ;;
 194                                 *) return 0 ;;
 195                         esac;
 196                 done;
 197                 while read symbol op path; do
 198                         case "$symbol" in package-*)
 199                                 symbol="${symbol##*(}";
 200                                 symbol="${symbol%)}";
 201                                 for file in $CHANGED_FILES; do
 202                                         case "$file" in "package/$path/"*)
 203                                                 grep -qsx "$symbol=y" .config && return 0
 204                                         ;; esac;
 205                                 done;
 206                         esac;
 207                 done < tmp/.packagedeps;
 208                 return 1;
 209         }
 210 """).strip()
 211
 212
 213 class IfBuiltinShellCommand(ShellCommand):
 214         def _quote(self, str):
 215                 if re.search("[^a-zA-Z0-9/_.-]", str):
 216                         return "'%s'" %(re.sub("'", "'\"'\"'", str))
 217                 return str
 218
 219         def setCommand(self, command):
 220                 if not isinstance(command, (str, unicode)):
 221                         command = ' '.join(map(self._quote, command))
 222                 self.command = [
 223                         '/bin/sh', '-c',
 224                         '%s; if checkBuiltin; then %s; else exit 0; fi' %(checkBuiltin, command)
 225                 ]
 226
 227         def setupEnvironment(self, cmd):
 228                 slaveEnv = self.slaveEnvironment
 229                 if slaveEnv is None:
 230                         slaveEnv = { }
 231                 changedFiles = { }
 232                 for request in self.build.requests:
 233                         for source in request.sources:
 234                                 for change in source.changes:
 235                                         for file in change.files:
 236                                                 changedFiles[file] = True
 237                 fullSlaveEnv = slaveEnv.copy()
 238                 fullSlaveEnv['CHANGED_FILES'] = ' '.join(changedFiles.keys())
 239                 cmd.args['env'] = fullSlaveEnv
 240
 241 slaveNames = [ ]
 242
 243 for slave in c['slaves']:
 244         slaveNames.append(slave.slavename)
 245
 246 for target in targets:
 247         ts = target.split('/')
 248
 249         factory = BuildFactory()
 250
 251         # find number of cores
 252         factory.addStep(SetProperty(
 253                 name = "nproc",
 254                 property = "nproc",
 255                 description = "Finding number of CPUs",
 256                 command = ["nproc"]))
 257
 258         # check out the source
 259         factory.addStep(Git(repourl=repo_url, mode='update'))
 260
 261         factory.addStep(ShellCommand(
 262                 name = "rmtmp",
 263                 description = "Remove tmp folder",
 264                 command=["rm", "-rf", "tmp/"]))
 265
 266         # feed
 267 #       factory.addStep(ShellCommand(
 268 #               name = "feedsconf",
 269 #               description = "Copy the feeds.conf",
 270 #               command='''cp ~/feeds.conf ./feeds.conf''' ))
 271
 272         # feed
 273         factory.addStep(ShellCommand(
 274                 name = "updatefeeds",
 275                 description = "Updating feeds",
 276                 command=["./scripts/feeds", "update"]))
 277
 278         # feed
 279         factory.addStep(ShellCommand(
 280                 name = "installfeeds",
 281                 description = "Installing feeds",
 282                 command=["./scripts/feeds", "install", "-a"]))
 283
 284         # configure
 285         factory.addStep(ShellCommand(
 286                 name = "newconfig",
 287                 description = "Seeding .config",
 288                 command='''cat <<EOT > .config
 289 CONFIG_TARGET_%s=y
 290 CONFIG_TARGET_%s_%s=y
 291 CONFIG_ALL_NONSHARED=y
 292 CONFIG_SDK=y
 293 CONFIG_IB=y
 294 # CONFIG_IB_STANDALONE is not set
 295 CONFIG_DEVEL=y
 296 CONFIG_CCACHE=y
 297 CONFIG_SIGNED_PACKAGES=y
 298 # CONFIG_PER_FEED_REPO_ADD_COMMENTED is not set
 299 CONFIG_KERNEL_KALLSYMS=y
 300 CONFIG_COLLECT_KERNEL_DEBUG=y
 301 EOT''' %(ts[0], ts[0], ts[1]) ))
 302
 303         factory.addStep(ShellCommand(
 304                 name = "delbin",
 305                 description = "Removing output directory",
 306                 command = ["rm", "-rf", "bin/"]
 307         ))
 308
 309         factory.addStep(ShellCommand(
 310                 name = "defconfig",
 311                 description = "Populating .config",
 312                 command = ["make", "defconfig"]
 313         ))
 314
 315         # check arch / libc
 316         factory.addStep(ShellCommand(
 317                 name = "checkarch",
 318                 description = "Checking architecture",
 319                 command = ["grep", "-sq", "CONFIG_TARGET_%s=y" %(ts[0]), ".config"],
 320                 logEnviron = False,
 321                 want_stdout = False,
 322                 want_stderr = False,
 323                 haltOnFailure = True
 324         ))
 325
 326         factory.addStep(ShellCommand(
 327                 name = "checklibc",
 328                 description = "Checking libc flavor",
 329                 command = ["grep", "-sq", 'CONFIG_LIBC="musl"', ".config"],
 330                 logEnviron = False,
 331                 want_stdout = False,
 332                 want_stderr = False,
 333                 haltOnFailure = True
 334         ))
 335
 336         # install build key
 337         factory.addStep(FileDownload(mastersrc=home_dir+'/key-build', slavedest="key-build", mode=0600))
 338         factory.addStep(FileDownload(mastersrc=home_dir+'/key-build.pub', slavedest="key-build.pub", mode=0600))
 339
 340         # prepare dl
 341         factory.addStep(ShellCommand(
 342                 name = "dldir",
 343                 description = "Preparing dl/",
 344                 command = "mkdir -p $HOME/dl && ln -sf $HOME/dl ./dl",
 345                 logEnviron = False,
 346                 want_stdout = False
 347         ))
 348
 349         # populate dl
 350         factory.addStep(ShellCommand(
 351                 name = "dlrun",
 352                 description = "Populating dl/",
 353                 command = ["make", WithProperties("-j%(nproc:~4)s"), "download", "V=s"],
 354                 logEnviron = False,
 355                 locks = [dlLock.access('exclusive')]
 356         ))
 357
 358         factory.addStep(ShellCommand(
 359                 name = "cleanbase",
 360                 description = "Cleaning base-files",
 361                 command=["make", "package/base-files/clean", "V=s"]
 362         ))
 363
 364         # optional clean steps
 365         for pattern, maketarget in MakeTargetMap.items():
 366                 factory.addStep(ShellCommand(
 367                         name = maketarget,
 368                         description = maketarget,
 369                         command=["make", maketarget, "V=s"], doStepIf=IsAffected(pattern)
 370                 ))
 371
 372         # build
 373         factory.addStep(ShellCommand(
 374                 name = "tools",
 375                 description = "Building tools",
 376                 command = ["make", WithProperties("-j%(nproc:~4)s"), "tools/install", "V=s"],
 377                 haltOnFailure = True
 378         ))
 379
 380         factory.addStep(ShellCommand(
 381                 name = "toolchain",
 382                 description = "Building toolchain",
 383                 command=["make", WithProperties("-j%(nproc:~4)s"), "toolchain/install", "V=s"],
 384                 haltOnFailure = True
 385         ))
 386
 387         factory.addStep(ShellCommand(
 388                 name = "kmods",
 389                 description = "Building kmods",
 390                 command=["make", WithProperties("-j%(nproc:~4)s"), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
 391                 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
 392                 haltOnFailure = True
 393         ))
 394
 395         factory.addStep(ShellCommand(
 396                 name = "pkgbuild",
 397                 description = "Building packages",
 398                 command=["make", WithProperties("-j%(nproc:~4)s"), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"],
 399                 #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])},
 400                 haltOnFailure = True
 401         ))
 402
 403         # factory.addStep(IfBuiltinShellCommand(
 404         factory.addStep(ShellCommand(
 405                 name = "pkginstall",
 406                 description = "Installing packages",
 407                 command=["make", WithProperties("-j%(nproc:~4)s"), "package/install", "V=s"],
 408                 doStepIf = isChangeBuiltin,
 409                 haltOnFailure = True
 410         ))
 411
 412         factory.addStep(ShellCommand(
 413                 name = "pkgindex",
 414                 description = "Indexing packages",
 415                 command=["make", WithProperties("-j%(nproc:~4)s"), "package/index", "V=s"],
 416                 haltOnFailure = True
 417         ))
 418
 419         #factory.addStep(IfBuiltinShellCommand(
 420         factory.addStep(ShellCommand(
 421                 name = "images",
 422                 description = "Building images",
 423                 command=["make", "-j1", "target/install", "V=s"],
 424                 doStepIf = isChangeBuiltin,
 425                 haltOnFailure = True
 426         ))
 427
 428         # upload
 429         factory.addStep(ShellCommand(
 430                 name = "uploadprepare",
 431                 description = "Preparing target directory",
 432                 command=["rsync", "-av", "--include", "/%s/" %(ts[0]), "--include", "/%s/%s/" %(ts[0], ts[1]), "--exclude", "/*", "--exclude", "/*/*", "--exclude", "/%s/%s/*" %(ts[0], ts[1]), "bin/targets/", "%s/targets/" %(rsync_bin_url)],
 433                 env={'RSYNC_PASSWORD': rsync_bin_key},
 434                 haltOnFailure = True,
 435                 logEnviron = False
 436         ))
 437
 438         factory.addStep(ShellCommand(
 439                 name = "targetupload",
 440                 description = "Uploading target files",
 441                 command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "bin/targets/%s/%s/" %(ts[0], ts[1]), "%s/targets/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
 442                 env={'RSYNC_PASSWORD': rsync_bin_key},
 443                 haltOnFailure = True,
 444                 logEnviron = False
 445         ))
 446
 447         if rsync_src_url is not None:
 448                 factory.addStep(ShellCommand(
 449                         name = "sourceupload",
 450                         description = "Uploading source archives",
 451                         command=["rsync", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "dl/", "%s/" %(rsync_src_url)],
 452                         env={'RSYNC_PASSWORD': rsync_src_key},
 453                         haltOnFailure = True,
 454                         logEnviron = False
 455                 ))
 456
 457         if False:
 458                 factory.addStep(ShellCommand(
 459                         name = "packageupload",
 460                         description = "Uploading package files",
 461                         command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "bin/packages/", "%s/packages/" %(rsync_bin_url)],
 462                         env={'RSYNC_PASSWORD': rsync_bin_key},
 463                         haltOnFailure = False,
 464                         logEnviron = False
 465                 ))
 466
 467         # logs
 468         if False:
 469                 factory.addStep(ShellCommand(
 470                         name = "upload",
 471                         description = "Uploading logs",
 472                         command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "logs/", "%s/logs/%s/%s/" %(rsync_bin_url, ts[0], ts[1])],
 473                         env={'RSYNC_PASSWORD': rsync_bin_key},
 474                         haltOnFailure = False,
 475                         alwaysRun = True,
 476                         logEnviron = False
 477                 ))
 478
 479         from buildbot.config import BuilderConfig
 480
 481         c['builders'].append(BuilderConfig(name=target, slavenames=slaveNames, factory=factory))
 482
 483
 484 ####### STATUS TARGETS
 485
 486 # 'status' is a list of Status Targets. The results of each build will be
 487 # pushed to these targets. buildbot/status/*.py has a variety to choose from,
 488 # including web pages, email senders, and IRC bots.
 489
 490 c['status'] = []
 491
 492 from buildbot.status import html
 493 from buildbot.status.web import authz, auth
 494
 495 if ini.has_option("status", "bind"):
 496         if ini.has_option("status", "user") and ini.has_option("status", "password"):
 497                 authz_cfg=authz.Authz(
 498                         # change any of these to True to enable; see the manual for more
 499                         # options
 500                         auth=auth.BasicAuth([(ini.get("status", "user"), ini.get("status", "password"))]),
 501                         gracefulShutdown = 'auth',
 502                         forceBuild = 'auth', # use this to test your slave once it is set up
 503                         forceAllBuilds = 'auth',
 504                         pingBuilder = False,
 505                         stopBuild = 'auth',
 506                         stopAllBuilds = 'auth',
 507                         cancelPendingBuild = 'auth',
 508                 )
 509                 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind"), authz=authz_cfg))
 510         else:
 511                 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind")))
 512
 513
 514 from buildbot.status import words
 515
 516 if ini.has_option("irc", "host") and ini.has_option("irc", "nickname") and ini.has_option("irc", "channel"):
 517         irc_host = ini.get("irc", "host")
 518         irc_port = 6667
 519         irc_chan = ini.get("irc", "channel")
 520         irc_nick = ini.get("irc", "nickname")
 521         irc_pass = None
 522
 523         if ini.has_option("irc", "port"):
 524                 irc_port = ini.getint("irc", "port")
 525
 526         if ini.has_option("irc", "password"):
 527                 irc_pass = ini.get("irc", "password")
 528
 529         irc = words.IRC(irc_host, irc_nick, port = irc_port, password = irc_pass,
 530                         channels = [{ "channel": irc_chan }],
 531                         notify_events = {
 532                           'exception': 1,
 533                           'successToFailure': 1,
 534                           'failureToSuccess': 1
 535                         }
 536         )
 537
 538         c['status'].append(irc)
 539
 540
 541 ####### PROJECT IDENTITY
 542
 543 # the 'title' string will appear at the top of this buildbot
 544 # installation's html.WebStatus home page (linked to the
 545 # 'titleURL') and is embedded in the title of the waterfall HTML page.
 546
 547 c['title'] = ini.get("general", "title")
 548 c['titleURL'] = ini.get("general", "title_url")
 549
 550 # the 'buildbotURL' string should point to the location where the buildbot's
 551 # internal web server (usually the html.WebStatus page) is visible. This
 552 # typically uses the port number set in the Waterfall 'status' entry, but
 553 # with an externally-visible host name which the buildbot cannot figure out
 554 # without some help.
 555
 556 c['buildbotURL'] = ini.get("general", "buildbot_url")
 557
 558 ####### DB URL
 559
 560 c['db'] = {
 561         # This specifies what database buildbot uses to store its state.  You can leave
 562         # this at its default for all but the largest installations.
 563         'db_url' : "sqlite:///state.sqlite",
 564 }