phase2/master.cfg

   1 # -*- python -*-
   2 # ex: set syntax=python:
   3
   4 import os
   5 import re
   6 import subprocess
   7 import ConfigParser
   8
   9 from buildbot import locks
  10
  11 ini = ConfigParser.ConfigParser()
  12 ini.read("./config.ini")
  13
  14 # This is a sample buildmaster config file. It must be installed as
  15 # 'master.cfg' in your buildmaster's base directory.
  16
  17 # This is the dictionary that the buildmaster pays attention to. We also use
  18 # a shorter alias to save typing.
  19 c = BuildmasterConfig = {}
  20
  21 ####### BUILDSLAVES
  22
  23 # The 'slaves' list defines the set of recognized buildslaves. Each element is
  24 # a BuildSlave object, specifying a unique slave name and password.  The same
  25 # slave name and password must be configured on the slave.
  26 from buildbot.buildslave import BuildSlave
  27
  28 c['slaves'] = []
  29
  30 for section in ini.sections():
  31         if section.startswith("slave "):
  32                 if ini.has_option(section, "name") and ini.has_option(section, "password"):
  33                         name = ini.get(section, "name")
  34                         password = ini.get(section, "password")
  35                         max_builds = 1
  36                         if ini.has_option(section, "builds"):
  37                                 max_builds = ini.getint(section, "builds")
  38                         c['slaves'].append(BuildSlave(name, password, max_builds = max_builds))
  39
  40 # 'slavePortnum' defines the TCP port to listen on for connections from slaves.
  41 # This must match the value configured into the buildslaves (with their
  42 # --master option)
  43 c['slavePortnum'] = 9990
  44
  45 # coalesce builds
  46 c['mergeRequests'] = True
  47
  48 ####### CHANGESOURCES
  49
  50 home_dir = os.path.abspath(ini.get("general", "homedir"))
  51
  52 repo_url = ini.get("repo", "url")
  53
  54 rsync_url = ini.get("rsync", "url")
  55 rsync_key = ini.get("rsync", "password")
  56
  57 # find arches
  58 arches = [ ]
  59 archnames = [ ]
  60
  61 findarches = subprocess.Popen([home_dir+'/dumpinfo.pl', 'architectures'],
  62         stdout = subprocess.PIPE, cwd = home_dir+'/source.git')
  63
  64 while True:
  65         line = findarches.stdout.readline()
  66         if not line:
  67                 break
  68         at = line.strip().split()
  69         arches.append(at)
  70         archnames.append(at[0])
  71
  72
  73 # find feeds
  74 feeds = []
  75
  76 from buildbot.changes.gitpoller import GitPoller
  77 c['change_source'] = []
  78
  79 with open(home_dir+'/source.git/feeds.conf.default', 'r') as f:
  80         for line in f:
  81                 parts = line.strip().split()
  82                 if parts[0] == "src-git":
  83                         feeds.append(parts)
  84                         c['change_source'].append(GitPoller(parts[2], workdir='%s/%s.git' %(os.getcwd(), parts[1]), branch='master', pollinterval=300))
  85
  86
  87 ####### SCHEDULERS
  88
  89 # Configure the Schedulers, which decide how to react to incoming changes.  In this
  90 # case, just kick off a 'basebuild' build
  91
  92 from buildbot.schedulers.basic import SingleBranchScheduler
  93 from buildbot.schedulers.forcesched import ForceScheduler
  94 from buildbot.changes import filter
  95 c['schedulers'] = []
  96 c['schedulers'].append(SingleBranchScheduler(
  97         name="all",
  98         change_filter=filter.ChangeFilter(branch='master'),
  99         treeStableTimer=60,
 100         builderNames=archnames))
 101
 102 c['schedulers'].append(ForceScheduler(
 103         name="force",
 104         builderNames=archnames))
 105
 106 ####### BUILDERS
 107
 108 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
 109 # what steps, and which slaves can execute them.  Note that any particular build will
 110 # only take place on one slave.
 111
 112 from buildbot.process.factory import BuildFactory
 113 from buildbot.steps.source import Git
 114 from buildbot.steps.shell import ShellCommand
 115 from buildbot.steps.transfer import FileDownload
 116 from buildbot.process.properties import WithProperties
 117
 118 c['builders'] = []
 119
 120 dlLock = locks.SlaveLock("slave_dl")
 121
 122 slaveNames = [ ]
 123
 124 for slave in c['slaves']:
 125         slaveNames.append(slave.slavename)
 126
 127 for arch in arches:
 128         ts = arch[1].split('/')
 129
 130         factory = BuildFactory()
 131
 132         # prepare workspace
 133         factory.addStep(FileDownload(mastersrc="cleanup.sh", slavedest="cleanup.sh", mode=0755))
 134
 135         factory.addStep(ShellCommand(
 136                 name = "cleanold",
 137                 description = "Cleaning previous builds",
 138                 command = ["./cleanup.sh", WithProperties("%(slavename)s"), WithProperties("%(buildername)s"), "full"],
 139                 haltOnFailure = True,
 140                 timeout = 2400))
 141
 142         factory.addStep(ShellCommand(
 143                 name = "cleanup",
 144                 description = "Cleaning work area",
 145                 command = ["./cleanup.sh", WithProperties("%(slavename)s"), WithProperties("%(buildername)s"), "single"],
 146                 haltOnFailure = True,
 147                 timeout = 2400))
 148
 149         factory.addStep(ShellCommand(
 150                 name = "mksdkdir",
 151                 description = "Preparing SDK directory",
 152                 command = ["mkdir", "sdk"],
 153                 haltOnFailure = True))
 154
 155         factory.addStep(ShellCommand(
 156                 name = "downloadsdk",
 157                 description = "Downloading SDK archive",
 158                 command = ["rsync", "-va", "downloads.lede-project.org::downloads/snapshots/targets/%s/%s/LEDE-SDK-*.tar.bz2" %(ts[0], ts[1]), "sdk.tar.bz2"],
 159                 haltOnFailure = True))
 160
 161         factory.addStep(ShellCommand(
 162                 name = "unpacksdk",
 163                 description = "Unpacking SDK archive",
 164                 command = ["tar", "--strip-components=1", "-C", "sdk/", "-vxjf", "sdk.tar.bz2"],
 165                 haltOnFailure = True))
 166
 167         factory.addStep(FileDownload(mastersrc=home_dir+'/key-build', slavedest="sdk/key-build", mode=0400))
 168         factory.addStep(FileDownload(mastersrc=home_dir+'/key-build.pub', slavedest="sdk/key-build.pub", mode=0400))
 169
 170         factory.addStep(ShellCommand(
 171                 name = "mkdldir",
 172                 description = "Preparing download directory",
 173                 command = ["sh", "-c", "mkdir -p $HOME/dl && rmdir ./sdk/dl && ln -sf $HOME/dl ./sdk/dl"]))
 174
 175         factory.addStep(ShellCommand(
 176                 name = "updatefeeds",
 177                 description = "Updating feeds",
 178                 workdir = "build/sdk",
 179                 command = ["./scripts/feeds", "update"]))
 180
 181         factory.addStep(ShellCommand(
 182                 name = "installfeeds",
 183                 description = "Installing feeds",
 184                 workdir = "build/sdk",
 185                 command = ["./scripts/feeds", "install", "-a"]))
 186
 187         factory.addStep(ShellCommand(
 188                 name = "compile",
 189                 description = "Building packages",
 190                 workdir = "build/sdk",
 191                 command = ["make", "-j4", "V=s", "IGNORE_ERRORS=n m y", "BUILD_LOG=1", "CONFIG_SIGNED_PACKAGES=y"]))
 192
 193         factory.addStep(ShellCommand(
 194                 name = "uploadprepare",
 195                 description = "Preparing package directory",
 196                 workdir = "build/sdk",
 197                 command = ["rsync", "-av", "--include", "/%s/" %(arch[0]), "--exclude", "/*", "--exclude", "/%s/*" %(arch[0]), "bin/packages/", "%s/packages/" %(rsync_url)],
 198                 env={'RSYNC_PASSWORD': rsync_key},
 199                 haltOnFailure = True,
 200                 logEnviron = False
 201         ))
 202
 203         factory.addStep(ShellCommand(
 204                 name = "packageupload",
 205                 description = "Uploading package files",
 206                 workdir = "build/sdk",
 207                 command = ["rsync", "--delete", "--delay-updates", "-avz", "bin/packages/%s/" %(arch[0]), "%s/packages/%s/" %(rsync_url, arch[0])],
 208                 env={'RSYNC_PASSWORD': rsync_key},
 209                 haltOnFailure = True,
 210                 logEnviron = False
 211         ))
 212
 213         factory.addStep(ShellCommand(
 214                 name = "logprepare",
 215                 description = "Preparing log directory",
 216                 workdir = "build/sdk",
 217                 command = ["rsync", "-av", "--include", "/%s/" %(arch[0]), "--exclude", "/*", "--exclude", "/%s/*" %(arch[0]), "bin/packages/", "%s/faillogs/" %(rsync_url)],
 218                 env={'RSYNC_PASSWORD': rsync_key},
 219                 haltOnFailure = True,
 220                 logEnviron = False
 221         ))
 222
 223         factory.addStep(ShellCommand(
 224                 name = "logfind",
 225                 description = "Finding failure logs",
 226                 workdir = "build/sdk/logs/package/feeds",
 227                 command = ["sh", "-c", "sed -ne 's!^ *ERROR: package/feeds/\\([^ ]*\\) .*$!\\1!p' ../error.txt | sort -u | xargs -r find > ../../../logs.txt"],
 228                 haltOnFailure = False
 229         ))
 230
 231         factory.addStep(ShellCommand(
 232                 name = "logcollect",
 233                 description = "Collecting failure logs",
 234                 workdir = "build/sdk",
 235                 command = ["rsync", "-av", "--files-from=logs.txt", "logs/package/feeds/", "faillogs/"],
 236                 haltOnFailure = False
 237         ))
 238
 239         factory.addStep(ShellCommand(
 240                 name = "logupload",
 241                 description = "Uploading failure logs",
 242                 workdir = "build/sdk",
 243                 command = ["rsync", "--delete", "--delay-updates", "-avz", "faillogs/", "%s/faillogs/%s/" %(rsync_url, arch[0])],
 244                 env={'RSYNC_PASSWORD': rsync_key},
 245                 haltOnFailure = False,
 246                 logEnviron = False
 247         ))
 248
 249         from buildbot.config import BuilderConfig
 250
 251         c['builders'].append(BuilderConfig(name=arch[0], slavenames=slaveNames, factory=factory))
 252
 253
 254 ####### STATUS arches
 255
 256 # 'status' is a list of Status arches. The results of each build will be
 257 # pushed to these arches. buildbot/status/*.py has a variety to choose from,
 258 # including web pages, email senders, and IRC bots.
 259
 260 c['status'] = []
 261
 262 from buildbot.status import html
 263 from buildbot.status.web import authz, auth
 264
 265 if ini.has_option("status", "bind"):
 266         if ini.has_option("status", "user") and ini.has_option("status", "password"):
 267                 authz_cfg=authz.Authz(
 268                         # change any of these to True to enable; see the manual for more
 269                         # options
 270                         auth=auth.BasicAuth([(ini.get("status", "user"), ini.get("status", "password"))]),
 271                         gracefulShutdown = False,
 272                         forceBuild = 'auth', # use this to test your slave once it is set up
 273                         forceAllBuilds = 'auth',
 274                         pingBuilder = False,
 275                         stopBuild = 'auth',
 276                         stopAllBuilds = 'auth',
 277                         cancelPendingBuild = 'auth',
 278                 )
 279                 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind"), authz=authz_cfg))
 280         else:
 281                 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind")))
 282
 283 ####### PROJECT IDENTITY
 284
 285 # the 'title' string will appear at the top of this buildbot
 286 # installation's html.WebStatus home page (linked to the
 287 # 'titleURL') and is embedded in the title of the waterfall HTML page.
 288
 289 c['title'] = ini.get("general", "title")
 290 c['titleURL'] = ini.get("general", "title_url")
 291
 292 # the 'buildbotURL' string should point to the location where the buildbot's
 293 # internal web server (usually the html.WebStatus page) is visible. This
 294 # typically uses the port number set in the Waterfall 'status' entry, but
 295 # with an externally-visible host name which the buildbot cannot figure out
 296 # without some help.
 297
 298 c['buildbotURL'] = ini.get("general", "buildbot_url")
 299
 300 ####### DB URL
 301
 302 c['db'] = {
 303         # This specifies what database buildbot uses to store its state.  You can leave
 304         # this at its default for all but the largest installations.
 305         'db_url' : "sqlite:///state.sqlite",
 306 }
 307