phase2/master.cfg

   1 # -*- python -*-
   2 # ex: set syntax=python:
   3
   4 import os
   5 import re
   6 import subprocess
   7 import ConfigParser
   8
   9 from buildbot import locks
  10
  11 ini = ConfigParser.ConfigParser()
  12 ini.read("./config.ini")
  13
  14 # This is a sample buildmaster config file. It must be installed as
  15 # 'master.cfg' in your buildmaster's base directory.
  16
  17 # This is the dictionary that the buildmaster pays attention to. We also use
  18 # a shorter alias to save typing.
  19 c = BuildmasterConfig = {}
  20
  21 ####### BUILDSLAVES
  22
  23 # The 'slaves' list defines the set of recognized buildslaves. Each element is
  24 # a BuildSlave object, specifying a unique slave name and password.  The same
  25 # slave name and password must be configured on the slave.
  26 from buildbot.buildslave import BuildSlave
  27
  28 slave_port = 9990
  29
  30 if ini.has_option("general", "port"):
  31         slave_port = ini.getint("general", "port")
  32
  33 c['slaves'] = []
  34
  35 for section in ini.sections():
  36         if section.startswith("slave "):
  37                 if ini.has_option(section, "name") and ini.has_option(section, "password"):
  38                         name = ini.get(section, "name")
  39                         password = ini.get(section, "password")
  40                         max_builds = 1
  41                         if ini.has_option(section, "builds"):
  42                                 max_builds = ini.getint(section, "builds")
  43                         c['slaves'].append(BuildSlave(name, password, max_builds = max_builds))
  44
  45 # 'slavePortnum' defines the TCP port to listen on for connections from slaves.
  46 # This must match the value configured into the buildslaves (with their
  47 # --master option)
  48 c['slavePortnum'] = slave_port
  49
  50 # coalesce builds
  51 c['mergeRequests'] = True
  52
  53 ####### CHANGESOURCES
  54
  55 home_dir = os.path.abspath(ini.get("general", "homedir"))
  56
  57 rsync_bin_url = ini.get("rsync", "binary_url")
  58 rsync_bin_key = ini.get("rsync", "binary_password")
  59
  60 rsync_src_url = None
  61 rsync_src_key = None
  62
  63 if ini.has_option("rsync", "source_url"):
  64         rsync_src_url = ini.get("rsync", "source_url")
  65         rsync_src_key = ini.get("rsync", "source_password")
  66
  67 gpg_keyid = None
  68 gpg_comment = "Unattended build signature"
  69 gpg_passfile = "/dev/null"
  70
  71 if ini.has_option("gpg", "keyid"):
  72         gpg_keyid = ini.get("gpg", "keyid")
  73
  74 if ini.has_option("gpg", "comment"):
  75         gpg_comment = ini.get("gpg", "comment")
  76
  77 if ini.has_option("gpg", "passfile"):
  78         gpg_passfile = ini.get("gpg", "passfile")
  79
  80
  81 # find arches
  82 arches = [ ]
  83 archnames = [ ]
  84
  85 findarches = subprocess.Popen([home_dir+'/dumpinfo.pl', 'architectures'],
  86         stdout = subprocess.PIPE, cwd = home_dir+'/source.git')
  87
  88 while True:
  89         line = findarches.stdout.readline()
  90         if not line:
  91                 break
  92         at = line.strip().split()
  93         arches.append(at)
  94         archnames.append(at[0])
  95
  96
  97 # find feeds
  98 feeds = []
  99
 100 from buildbot.changes.gitpoller import GitPoller
 101 c['change_source'] = []
 102
 103 with open(home_dir+'/source.git/feeds.conf.default', 'r') as f:
 104         for line in f:
 105                 parts = line.strip().split()
 106                 if parts[0] == "src-git":
 107                         feeds.append(parts)
 108                         c['change_source'].append(GitPoller(parts[2], workdir='%s/%s.git' %(os.getcwd(), parts[1]), branch='master', pollinterval=300))
 109
 110
 111 ####### SCHEDULERS
 112
 113 # Configure the Schedulers, which decide how to react to incoming changes.  In this
 114 # case, just kick off a 'basebuild' build
 115
 116 from buildbot.schedulers.basic import SingleBranchScheduler
 117 from buildbot.schedulers.forcesched import ForceScheduler
 118 from buildbot.changes import filter
 119 c['schedulers'] = []
 120 c['schedulers'].append(SingleBranchScheduler(
 121         name="all",
 122         change_filter=filter.ChangeFilter(branch='master'),
 123         treeStableTimer=60,
 124         builderNames=archnames))
 125
 126 c['schedulers'].append(ForceScheduler(
 127         name="force",
 128         builderNames=archnames))
 129
 130 ####### BUILDERS
 131
 132 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
 133 # what steps, and which slaves can execute them.  Note that any particular build will
 134 # only take place on one slave.
 135
 136 from buildbot.process.factory import BuildFactory
 137 from buildbot.steps.source import Git
 138 from buildbot.steps.shell import ShellCommand
 139 from buildbot.steps.shell import SetProperty
 140 from buildbot.steps.transfer import FileUpload
 141 from buildbot.steps.transfer import FileDownload
 142 from buildbot.steps.master import MasterShellCommand
 143 from buildbot.process.properties import WithProperties
 144
 145 c['builders'] = []
 146
 147 dlLock = locks.SlaveLock("slave_dl")
 148
 149 slaveNames = [ ]
 150
 151 for slave in c['slaves']:
 152         slaveNames.append(slave.slavename)
 153
 154 for arch in arches:
 155         ts = arch[1].split('/')
 156
 157         factory = BuildFactory()
 158
 159         # find number of cores
 160         factory.addStep(SetProperty(
 161                 name = "nproc",
 162                 property = "nproc",
 163                 description = "Finding number of CPUs",
 164                 command = ["nproc"]))
 165
 166         # prepare workspace
 167         factory.addStep(FileDownload(mastersrc="cleanup.sh", slavedest="cleanup.sh", mode=0755))
 168
 169         factory.addStep(ShellCommand(
 170                 name = "cleanold",
 171                 description = "Cleaning previous builds",
 172                 command = ["./cleanup.sh", WithProperties("%(slavename)s"), WithProperties("%(buildername)s"), "full"],
 173                 haltOnFailure = True,
 174                 timeout = 2400))
 175
 176         factory.addStep(ShellCommand(
 177                 name = "cleanup",
 178                 description = "Cleaning work area",
 179                 command = ["./cleanup.sh", WithProperties("%(slavename)s"), WithProperties("%(buildername)s"), "single"],
 180                 haltOnFailure = True,
 181                 timeout = 2400))
 182
 183         factory.addStep(ShellCommand(
 184                 name = "mksdkdir",
 185                 description = "Preparing SDK directory",
 186                 command = ["mkdir", "sdk"],
 187                 haltOnFailure = True))
 188
 189         factory.addStep(ShellCommand(
 190                 name = "downloadsdk",
 191                 description = "Downloading SDK archive",
 192                 command = ["rsync", "-va", "downloads.lede-project.org::downloads/snapshots/targets/%s/%s/lede-sdk-*.tar.xz" %(ts[0], ts[1]), "sdk.tar.xz"],
 193                 haltOnFailure = True))
 194
 195         factory.addStep(ShellCommand(
 196                 name = "unpacksdk",
 197                 description = "Unpacking SDK archive",
 198                 command = ["tar", "--strip-components=1", "-C", "sdk/", "-vxJf", "sdk.tar.xz"],
 199                 haltOnFailure = True))
 200
 201         factory.addStep(FileDownload(mastersrc=home_dir+'/key-build', slavedest="sdk/key-build", mode=0600))
 202         factory.addStep(FileDownload(mastersrc=home_dir+'/key-build.pub', slavedest="sdk/key-build.pub", mode=0600))
 203
 204         factory.addStep(ShellCommand(
 205                 name = "mkdldir",
 206                 description = "Preparing download directory",
 207                 command = ["sh", "-c", "mkdir -p $HOME/dl && rmdir ./sdk/dl && ln -sf $HOME/dl ./sdk/dl"]))
 208
 209         factory.addStep(ShellCommand(
 210                 name = "mkconf",
 211                 description = "Preparing SDK configuration",
 212                 workdir = "build/sdk",
 213                 command = ["sh", "-c", "rm -f .config && make defconfig"]))
 214
 215         factory.addStep(ShellCommand(
 216                 name = "updatefeeds",
 217                 description = "Updating feeds",
 218                 workdir = "build/sdk",
 219                 command = ["./scripts/feeds", "update"]))
 220
 221         factory.addStep(ShellCommand(
 222                 name = "installfeeds",
 223                 description = "Installing feeds",
 224                 workdir = "build/sdk",
 225                 command = ["./scripts/feeds", "install", "-a"]))
 226
 227         factory.addStep(ShellCommand(
 228                 name = "compile",
 229                 description = "Building packages",
 230                 workdir = "build/sdk",
 231                 command = ["make", WithProperties("-j%(nproc:~4)s"), "V=s", "IGNORE_ERRORS=n m y", "BUILD_LOG=1", "CONFIG_SIGNED_PACKAGES=y"]))
 232
 233         if gpg_keyid is not None:
 234                 factory.addStep(MasterShellCommand(
 235                         name = "signprepare",
 236                         description = "Preparing temporary signing directory",
 237                         command = ["mkdir", "-p", "%s/signing" %(home_dir)],
 238                         haltOnFailure = True
 239                 ))
 240
 241                 factory.addStep(ShellCommand(
 242                         name = "signpack",
 243                         description = "Packing files to sign",
 244                         workdir = "build/sdk",
 245                         command = "find bin/packages/%s/ -mindepth 2 -maxdepth 2 -type f -name Packages -print0 | xargs -0 tar -czf sign.tar.gz" %(arch[0]),
 246                         haltOnFailure = True
 247                 ))
 248
 249                 factory.addStep(FileUpload(
 250                         slavesrc = "sdk/sign.tar.gz",
 251                         masterdest = "%s/signing/%s.tar.gz" %(home_dir, arch[0]),
 252                         haltOnFailure = True
 253                 ))
 254
 255                 factory.addStep(MasterShellCommand(
 256                         name = "signfiles",
 257                         description = "Signing files",
 258                         command = ["%s/signall.sh" %(home_dir), "%s/signing/%s.tar.gz" %(home_dir, arch[0]), gpg_keyid, gpg_passfile, gpg_comment],
 259                         haltOnFailure = True
 260                 ))
 261
 262                 factory.addStep(FileDownload(
 263                         mastersrc = "%s/signing/%s.tar.gz" %(home_dir, arch[0]),
 264                         slavedest = "sdk/sign.tar.gz",
 265                         haltOnFailure = True
 266                 ))
 267
 268                 factory.addStep(ShellCommand(
 269                         name = "signunpack",
 270                         description = "Unpacking signed files",
 271                         workdir = "build/sdk",
 272                         command = ["tar", "-xzf", "sign.tar.gz"],
 273                         haltOnFailure = True
 274                 ))
 275
 276         factory.addStep(ShellCommand(
 277                 name = "uploadprepare",
 278                 description = "Preparing package directory",
 279                 workdir = "build/sdk",
 280                 command = ["rsync", "-av", "--include", "/%s/" %(arch[0]), "--exclude", "/*", "--exclude", "/%s/*" %(arch[0]), "bin/packages/", "%s/packages/" %(rsync_bin_url)],
 281                 env={'RSYNC_PASSWORD': rsync_bin_key},
 282                 haltOnFailure = True,
 283                 logEnviron = False
 284         ))
 285
 286         factory.addStep(ShellCommand(
 287                 name = "packageupload",
 288                 description = "Uploading package files",
 289                 workdir = "build/sdk",
 290                 command = ["rsync", "--delete", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-avz", "bin/packages/%s/" %(arch[0]), "%s/packages/%s/" %(rsync_bin_url, arch[0])],
 291                 env={'RSYNC_PASSWORD': rsync_bin_key},
 292                 haltOnFailure = True,
 293                 logEnviron = False
 294         ))
 295
 296         factory.addStep(ShellCommand(
 297                 name = "logprepare",
 298                 description = "Preparing log directory",
 299                 workdir = "build/sdk",
 300                 command = ["rsync", "-av", "--include", "/%s/" %(arch[0]), "--exclude", "/*", "--exclude", "/%s/*" %(arch[0]), "bin/packages/", "%s/faillogs/" %(rsync_bin_url)],
 301                 env={'RSYNC_PASSWORD': rsync_bin_key},
 302                 haltOnFailure = True,
 303                 logEnviron = False
 304         ))
 305
 306         factory.addStep(ShellCommand(
 307                 name = "logfind",
 308                 description = "Finding failure logs",
 309                 workdir = "build/sdk/logs/package/feeds",
 310                 command = ["sh", "-c", "sed -ne 's!^ *ERROR: package/feeds/\\([^ ]*\\) .*$!\\1!p' ../error.txt | sort -u | xargs -r find > ../../../logs.txt"],
 311                 haltOnFailure = False
 312         ))
 313
 314         factory.addStep(ShellCommand(
 315                 name = "logcollect",
 316                 description = "Collecting failure logs",
 317                 workdir = "build/sdk",
 318                 command = ["rsync", "-av", "--files-from=logs.txt", "logs/package/feeds/", "faillogs/"],
 319                 haltOnFailure = False
 320         ))
 321
 322         factory.addStep(ShellCommand(
 323                 name = "logupload",
 324                 description = "Uploading failure logs",
 325                 workdir = "build/sdk",
 326                 command = ["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-avz", "faillogs/", "%s/faillogs/%s/" %(rsync_bin_url, arch[0])],
 327                 env={'RSYNC_PASSWORD': rsync_bin_key},
 328                 haltOnFailure = False,
 329                 logEnviron = False
 330         ))
 331
 332         if rsync_src_url is not None:
 333                 factory.addStep(ShellCommand(
 334                         name = "sourceupload",
 335                         description = "Uploading source archives",
 336                         workdir = "build/sdk",
 337                         command = ["rsync", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-avz", "dl/", "%s/" %(rsync_src_url)],
 338                         env={'RSYNC_PASSWORD': rsync_src_key},
 339                         haltOnFailure = False,
 340                         logEnviron = False
 341                 ))
 342
 343         from buildbot.config import BuilderConfig
 344
 345         c['builders'].append(BuilderConfig(name=arch[0], slavenames=slaveNames, factory=factory))
 346
 347
 348 ####### STATUS arches
 349
 350 # 'status' is a list of Status arches. The results of each build will be
 351 # pushed to these arches. buildbot/status/*.py has a variety to choose from,
 352 # including web pages, email senders, and IRC bots.
 353
 354 c['status'] = []
 355
 356 from buildbot.status import html
 357 from buildbot.status.web import authz, auth
 358
 359 if ini.has_option("status", "bind"):
 360         if ini.has_option("status", "user") and ini.has_option("status", "password"):
 361                 authz_cfg=authz.Authz(
 362                         # change any of these to True to enable; see the manual for more
 363                         # options
 364                         auth=auth.BasicAuth([(ini.get("status", "user"), ini.get("status", "password"))]),
 365                         gracefulShutdown = 'auth',
 366                         forceBuild = 'auth', # use this to test your slave once it is set up
 367                         forceAllBuilds = 'auth',
 368                         pingBuilder = False,
 369                         stopBuild = 'auth',
 370                         stopAllBuilds = 'auth',
 371                         cancelPendingBuild = 'auth',
 372                 )
 373                 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind"), authz=authz_cfg))
 374         else:
 375                 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind")))
 376
 377 ####### PROJECT IDENTITY
 378
 379 # the 'title' string will appear at the top of this buildbot
 380 # installation's html.WebStatus home page (linked to the
 381 # 'titleURL') and is embedded in the title of the waterfall HTML page.
 382
 383 c['title'] = ini.get("general", "title")
 384 c['titleURL'] = ini.get("general", "title_url")
 385
 386 # the 'buildbotURL' string should point to the location where the buildbot's
 387 # internal web server (usually the html.WebStatus page) is visible. This
 388 # typically uses the port number set in the Waterfall 'status' entry, but
 389 # with an externally-visible host name which the buildbot cannot figure out
 390 # without some help.
 391
 392 c['buildbotURL'] = ini.get("general", "buildbot_url")
 393
 394 ####### DB URL
 395
 396 c['db'] = {
 397         # This specifies what database buildbot uses to store its state.  You can leave
 398         # this at its default for all but the largest installations.
 399         'db_url' : "sqlite:///state.sqlite",
 400 }