phase2/master.cfg

   1 # -*- python -*-
   2 # ex: set syntax=python:
   3
   4 import os
   5 import re
   6 import subprocess
   7 import ConfigParser
   8
   9 from buildbot import locks
  10
  11 ini = ConfigParser.ConfigParser()
  12 ini.read("./config.ini")
  13
  14 # This is a sample buildmaster config file. It must be installed as
  15 # 'master.cfg' in your buildmaster's base directory.
  16
  17 # This is the dictionary that the buildmaster pays attention to. We also use
  18 # a shorter alias to save typing.
  19 c = BuildmasterConfig = {}
  20
  21 ####### BUILDSLAVES
  22
  23 # The 'slaves' list defines the set of recognized buildslaves. Each element is
  24 # a BuildSlave object, specifying a unique slave name and password.  The same
  25 # slave name and password must be configured on the slave.
  26 from buildbot.buildslave import BuildSlave
  27
  28 c['slaves'] = []
  29
  30 for section in ini.sections():
  31         if section.startswith("slave "):
  32                 if ini.has_option(section, "name") and ini.has_option(section, "password"):
  33                         name = ini.get(section, "name")
  34                         password = ini.get(section, "password")
  35                         max_builds = 1
  36                         if ini.has_option(section, "builds"):
  37                                 max_builds = ini.getint(section, "builds")
  38                         c['slaves'].append(BuildSlave(name, password, max_builds = max_builds))
  39
  40 # 'slavePortnum' defines the TCP port to listen on for connections from slaves.
  41 # This must match the value configured into the buildslaves (with their
  42 # --master option)
  43 c['slavePortnum'] = 9990
  44
  45 # coalesce builds
  46 c['mergeRequests'] = True
  47
  48 ####### CHANGESOURCES
  49
  50 home_dir = os.path.abspath(ini.get("general", "homedir"))
  51
  52 repo_url = ini.get("repo", "url")
  53
  54 rsync_url = ini.get("rsync", "url")
  55 rsync_key = ini.get("rsync", "password")
  56
  57 # find arches
  58 arches = [ ]
  59 archnames = [ ]
  60
  61 findarches = subprocess.Popen([home_dir+'/dumpinfo.pl', 'architectures'],
  62         stdout = subprocess.PIPE, cwd = home_dir+'/source.git')
  63
  64 while True:
  65         line = findarches.stdout.readline()
  66         if not line:
  67                 break
  68         at = line.strip().split()
  69         arches.append(at)
  70         archnames.append(at[0])
  71
  72
  73 # find feeds
  74 feeds = []
  75
  76 from buildbot.changes.gitpoller import GitPoller
  77 c['change_source'] = []
  78
  79 with open(home_dir+'/source.git/feeds.conf.default', 'r') as f:
  80         for line in f:
  81                 parts = line.strip().split()
  82                 if parts[0] == "src-git":
  83                         feeds.append(parts)
  84                         c['change_source'].append(GitPoller(parts[2], workdir='%s/%s.git' %(os.getcwd(), parts[1]), branch='master', pollinterval=300))
  85
  86
  87 ####### SCHEDULERS
  88
  89 # Configure the Schedulers, which decide how to react to incoming changes.  In this
  90 # case, just kick off a 'basebuild' build
  91
  92 from buildbot.schedulers.basic import SingleBranchScheduler
  93 from buildbot.schedulers.forcesched import ForceScheduler
  94 from buildbot.changes import filter
  95 c['schedulers'] = []
  96 c['schedulers'].append(SingleBranchScheduler(
  97         name="all",
  98         change_filter=filter.ChangeFilter(branch='master'),
  99         treeStableTimer=60,
 100         builderNames=archnames))
 101
 102 c['schedulers'].append(ForceScheduler(
 103         name="force",
 104         builderNames=archnames))
 105
 106 ####### BUILDERS
 107
 108 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
 109 # what steps, and which slaves can execute them.  Note that any particular build will
 110 # only take place on one slave.
 111
 112 from buildbot.process.factory import BuildFactory
 113 from buildbot.steps.source import Git
 114 from buildbot.steps.shell import ShellCommand
 115 from buildbot.steps.shell import SetProperty
 116 from buildbot.steps.transfer import FileDownload
 117 from buildbot.process.properties import WithProperties
 118
 119 c['builders'] = []
 120
 121 dlLock = locks.SlaveLock("slave_dl")
 122
 123 slaveNames = [ ]
 124
 125 for slave in c['slaves']:
 126         slaveNames.append(slave.slavename)
 127
 128 for arch in arches:
 129         ts = arch[1].split('/')
 130
 131         factory = BuildFactory()
 132
 133         # find number of cores
 134         factory.addStep(SetProperty(
 135                 name = "nproc",
 136                 property = "nproc",
 137                 description = "Finding number of CPUs",
 138                 command = ["nproc"]))
 139
 140         # prepare workspace
 141         factory.addStep(FileDownload(mastersrc="cleanup.sh", slavedest="cleanup.sh", mode=0755))
 142
 143         factory.addStep(ShellCommand(
 144                 name = "cleanold",
 145                 description = "Cleaning previous builds",
 146                 command = ["./cleanup.sh", WithProperties("%(slavename)s"), WithProperties("%(buildername)s"), "full"],
 147                 haltOnFailure = True,
 148                 timeout = 2400))
 149
 150         factory.addStep(ShellCommand(
 151                 name = "cleanup",
 152                 description = "Cleaning work area",
 153                 command = ["./cleanup.sh", WithProperties("%(slavename)s"), WithProperties("%(buildername)s"), "single"],
 154                 haltOnFailure = True,
 155                 timeout = 2400))
 156
 157         factory.addStep(ShellCommand(
 158                 name = "mksdkdir",
 159                 description = "Preparing SDK directory",
 160                 command = ["mkdir", "sdk"],
 161                 haltOnFailure = True))
 162
 163         factory.addStep(ShellCommand(
 164                 name = "downloadsdk",
 165                 description = "Downloading SDK archive",
 166                 command = ["rsync", "-va", "downloads.lede-project.org::downloads/snapshots/targets/%s/%s/[Ll][Ee][Dd][Ee]-[Ss][Dd][Kk]-*.tar.bz2" %(ts[0], ts[1]), "sdk.tar.bz2"],
 167                 haltOnFailure = True))
 168
 169         factory.addStep(ShellCommand(
 170                 name = "unpacksdk",
 171                 description = "Unpacking SDK archive",
 172                 command = ["tar", "--strip-components=1", "-C", "sdk/", "-vxjf", "sdk.tar.bz2"],
 173                 haltOnFailure = True))
 174
 175         factory.addStep(FileDownload(mastersrc=home_dir+'/key-build', slavedest="sdk/key-build", mode=0600))
 176         factory.addStep(FileDownload(mastersrc=home_dir+'/key-build.pub', slavedest="sdk/key-build.pub", mode=0600))
 177
 178         factory.addStep(ShellCommand(
 179                 name = "mkdldir",
 180                 description = "Preparing download directory",
 181                 command = ["sh", "-c", "mkdir -p $HOME/dl && rmdir ./sdk/dl && ln -sf $HOME/dl ./sdk/dl"]))
 182
 183         factory.addStep(ShellCommand(
 184                 name = "mkconf",
 185                 description = "Preparing SDK configuration",
 186                 workdir = "build/sdk",
 187                 command = ["sh", "-c", "rm -f .config && make defconfig"]))
 188
 189         factory.addStep(ShellCommand(
 190                 name = "updatefeeds",
 191                 description = "Updating feeds",
 192                 workdir = "build/sdk",
 193                 command = ["./scripts/feeds", "update"]))
 194
 195         factory.addStep(ShellCommand(
 196                 name = "installfeeds",
 197                 description = "Installing feeds",
 198                 workdir = "build/sdk",
 199                 command = ["./scripts/feeds", "install", "-a"]))
 200
 201         factory.addStep(ShellCommand(
 202                 name = "compile",
 203                 description = "Building packages",
 204                 workdir = "build/sdk",
 205                 command = ["make", WithProperties("-j%(nproc:~4)s"), "V=s", "IGNORE_ERRORS=n m y", "BUILD_LOG=1", "CONFIG_SIGNED_PACKAGES=y"]))
 206
 207         factory.addStep(ShellCommand(
 208                 name = "uploadprepare",
 209                 description = "Preparing package directory",
 210                 workdir = "build/sdk",
 211                 command = ["rsync", "-av", "--include", "/%s/" %(arch[0]), "--exclude", "/*", "--exclude", "/%s/*" %(arch[0]), "bin/packages/", "%s/packages/" %(rsync_url)],
 212                 env={'RSYNC_PASSWORD': rsync_key},
 213                 haltOnFailure = True,
 214                 logEnviron = False
 215         ))
 216
 217         factory.addStep(ShellCommand(
 218                 name = "packageupload",
 219                 description = "Uploading package files",
 220                 workdir = "build/sdk",
 221                 command = ["rsync", "--delete", "--delay-updates", "-avz", "bin/packages/%s/" %(arch[0]), "%s/packages/%s/" %(rsync_url, arch[0])],
 222                 env={'RSYNC_PASSWORD': rsync_key},
 223                 haltOnFailure = True,
 224                 logEnviron = False
 225         ))
 226
 227         factory.addStep(ShellCommand(
 228                 name = "logprepare",
 229                 description = "Preparing log directory",
 230                 workdir = "build/sdk",
 231                 command = ["rsync", "-av", "--include", "/%s/" %(arch[0]), "--exclude", "/*", "--exclude", "/%s/*" %(arch[0]), "bin/packages/", "%s/faillogs/" %(rsync_url)],
 232                 env={'RSYNC_PASSWORD': rsync_key},
 233                 haltOnFailure = True,
 234                 logEnviron = False
 235         ))
 236
 237         factory.addStep(ShellCommand(
 238                 name = "logfind",
 239                 description = "Finding failure logs",
 240                 workdir = "build/sdk/logs/package/feeds",
 241                 command = ["sh", "-c", "sed -ne 's!^ *ERROR: package/feeds/\\([^ ]*\\) .*$!\\1!p' ../error.txt | sort -u | xargs -r find > ../../../logs.txt"],
 242                 haltOnFailure = False
 243         ))
 244
 245         factory.addStep(ShellCommand(
 246                 name = "logcollect",
 247                 description = "Collecting failure logs",
 248                 workdir = "build/sdk",
 249                 command = ["rsync", "-av", "--files-from=logs.txt", "logs/package/feeds/", "faillogs/"],
 250                 haltOnFailure = False
 251         ))
 252
 253         factory.addStep(ShellCommand(
 254                 name = "logupload",
 255                 description = "Uploading failure logs",
 256                 workdir = "build/sdk",
 257                 command = ["rsync", "--delete", "--delay-updates", "-avz", "faillogs/", "%s/faillogs/%s/" %(rsync_url, arch[0])],
 258                 env={'RSYNC_PASSWORD': rsync_key},
 259                 haltOnFailure = False,
 260                 logEnviron = False
 261         ))
 262
 263         from buildbot.config import BuilderConfig
 264
 265         c['builders'].append(BuilderConfig(name=arch[0], slavenames=slaveNames, factory=factory))
 266
 267
 268 ####### STATUS arches
 269
 270 # 'status' is a list of Status arches. The results of each build will be
 271 # pushed to these arches. buildbot/status/*.py has a variety to choose from,
 272 # including web pages, email senders, and IRC bots.
 273
 274 c['status'] = []
 275
 276 from buildbot.status import html
 277 from buildbot.status.web import authz, auth
 278
 279 if ini.has_option("status", "bind"):
 280         if ini.has_option("status", "user") and ini.has_option("status", "password"):
 281                 authz_cfg=authz.Authz(
 282                         # change any of these to True to enable; see the manual for more
 283                         # options
 284                         auth=auth.BasicAuth([(ini.get("status", "user"), ini.get("status", "password"))]),
 285                         gracefulShutdown = False,
 286                         forceBuild = 'auth', # use this to test your slave once it is set up
 287                         forceAllBuilds = 'auth',
 288                         pingBuilder = False,
 289                         stopBuild = 'auth',
 290                         stopAllBuilds = 'auth',
 291                         cancelPendingBuild = 'auth',
 292                 )
 293                 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind"), authz=authz_cfg))
 294         else:
 295                 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind")))
 296
 297 ####### PROJECT IDENTITY
 298
 299 # the 'title' string will appear at the top of this buildbot
 300 # installation's html.WebStatus home page (linked to the
 301 # 'titleURL') and is embedded in the title of the waterfall HTML page.
 302
 303 c['title'] = ini.get("general", "title")
 304 c['titleURL'] = ini.get("general", "title_url")
 305
 306 # the 'buildbotURL' string should point to the location where the buildbot's
 307 # internal web server (usually the html.WebStatus page) is visible. This
 308 # typically uses the port number set in the Waterfall 'status' entry, but
 309 # with an externally-visible host name which the buildbot cannot figure out
 310 # without some help.
 311
 312 c['buildbotURL'] = ini.get("general", "buildbot_url")
 313
 314 ####### DB URL
 315
 316 c['db'] = {
 317         # This specifies what database buildbot uses to store its state.  You can leave
 318         # this at its default for all but the largest installations.
 319         'db_url' : "sqlite:///state.sqlite",
 320 }