phase2/master.cfg

   1 # -*- python -*-
   2 # ex: set syntax=python:
   3
   4 import os
   5 import re
   6 import subprocess
   7 import ConfigParser
   8
   9 from buildbot import locks
  10
  11 ini = ConfigParser.ConfigParser()
  12 ini.read("./config.ini")
  13
  14 # This is a sample buildmaster config file. It must be installed as
  15 # 'master.cfg' in your buildmaster's base directory.
  16
  17 # This is the dictionary that the buildmaster pays attention to. We also use
  18 # a shorter alias to save typing.
  19 c = BuildmasterConfig = {}
  20
  21 ####### BUILDSLAVES
  22
  23 # The 'slaves' list defines the set of recognized buildslaves. Each element is
  24 # a BuildSlave object, specifying a unique slave name and password.  The same
  25 # slave name and password must be configured on the slave.
  26 from buildbot.buildslave import BuildSlave
  27
  28 c['slaves'] = []
  29
  30 for section in ini.sections():
  31         if section.startswith("slave "):
  32                 if ini.has_option(section, "name") and ini.has_option(section, "password"):
  33                         name = ini.get(section, "name")
  34                         password = ini.get(section, "password")
  35                         max_builds = 1
  36                         if ini.has_option(section, "builds"):
  37                                 max_builds = ini.getint(section, "builds")
  38                         c['slaves'].append(BuildSlave(name, password, max_builds = max_builds))
  39
  40 # 'slavePortnum' defines the TCP port to listen on for connections from slaves.
  41 # This must match the value configured into the buildslaves (with their
  42 # --master option)
  43 c['slavePortnum'] = 9990
  44
  45 # coalesce builds
  46 c['mergeRequests'] = True
  47
  48 ####### CHANGESOURCES
  49
  50 home_dir = os.path.abspath(ini.get("general", "homedir"))
  51
  52 repo_url = ini.get("repo", "url")
  53
  54 rsync_bin_url = ini.get("rsync", "binary_url")
  55 rsync_bin_key = ini.get("rsync", "binary_password")
  56
  57 rsync_src_url = None
  58 rsync_src_key = None
  59
  60 if ini.has_option("rsync", "source_url"):
  61         rsync_src_url = ini.get("rsync", "source_url")
  62         rsync_src_key = ini.get("rsync", "source_password")
  63
  64 # find arches
  65 arches = [ ]
  66 archnames = [ ]
  67
  68 findarches = subprocess.Popen([home_dir+'/dumpinfo.pl', 'architectures'],
  69         stdout = subprocess.PIPE, cwd = home_dir+'/source.git')
  70
  71 while True:
  72         line = findarches.stdout.readline()
  73         if not line:
  74                 break
  75         at = line.strip().split()
  76         arches.append(at)
  77         archnames.append(at[0])
  78
  79
  80 # find feeds
  81 feeds = []
  82
  83 from buildbot.changes.gitpoller import GitPoller
  84 c['change_source'] = []
  85
  86 with open(home_dir+'/source.git/feeds.conf.default', 'r') as f:
  87         for line in f:
  88                 parts = line.strip().split()
  89                 if parts[0] == "src-git":
  90                         feeds.append(parts)
  91                         c['change_source'].append(GitPoller(parts[2], workdir='%s/%s.git' %(os.getcwd(), parts[1]), branch='master', pollinterval=300))
  92
  93
  94 ####### SCHEDULERS
  95
  96 # Configure the Schedulers, which decide how to react to incoming changes.  In this
  97 # case, just kick off a 'basebuild' build
  98
  99 from buildbot.schedulers.basic import SingleBranchScheduler
 100 from buildbot.schedulers.forcesched import ForceScheduler
 101 from buildbot.changes import filter
 102 c['schedulers'] = []
 103 c['schedulers'].append(SingleBranchScheduler(
 104         name="all",
 105         change_filter=filter.ChangeFilter(branch='master'),
 106         treeStableTimer=60,
 107         builderNames=archnames))
 108
 109 c['schedulers'].append(ForceScheduler(
 110         name="force",
 111         builderNames=archnames))
 112
 113 ####### BUILDERS
 114
 115 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
 116 # what steps, and which slaves can execute them.  Note that any particular build will
 117 # only take place on one slave.
 118
 119 from buildbot.process.factory import BuildFactory
 120 from buildbot.steps.source import Git
 121 from buildbot.steps.shell import ShellCommand
 122 from buildbot.steps.shell import SetProperty
 123 from buildbot.steps.transfer import FileDownload
 124 from buildbot.process.properties import WithProperties
 125
 126 c['builders'] = []
 127
 128 dlLock = locks.SlaveLock("slave_dl")
 129
 130 slaveNames = [ ]
 131
 132 for slave in c['slaves']:
 133         slaveNames.append(slave.slavename)
 134
 135 for arch in arches:
 136         ts = arch[1].split('/')
 137
 138         factory = BuildFactory()
 139
 140         # find number of cores
 141         factory.addStep(SetProperty(
 142                 name = "nproc",
 143                 property = "nproc",
 144                 description = "Finding number of CPUs",
 145                 command = ["nproc"]))
 146
 147         # prepare workspace
 148         factory.addStep(FileDownload(mastersrc="cleanup.sh", slavedest="cleanup.sh", mode=0755))
 149
 150         factory.addStep(ShellCommand(
 151                 name = "cleanold",
 152                 description = "Cleaning previous builds",
 153                 command = ["./cleanup.sh", WithProperties("%(slavename)s"), WithProperties("%(buildername)s"), "full"],
 154                 haltOnFailure = True,
 155                 timeout = 2400))
 156
 157         factory.addStep(ShellCommand(
 158                 name = "cleanup",
 159                 description = "Cleaning work area",
 160                 command = ["./cleanup.sh", WithProperties("%(slavename)s"), WithProperties("%(buildername)s"), "single"],
 161                 haltOnFailure = True,
 162                 timeout = 2400))
 163
 164         factory.addStep(ShellCommand(
 165                 name = "mksdkdir",
 166                 description = "Preparing SDK directory",
 167                 command = ["mkdir", "sdk"],
 168                 haltOnFailure = True))
 169
 170         factory.addStep(ShellCommand(
 171                 name = "downloadsdk",
 172                 description = "Downloading SDK archive",
 173                 command = ["rsync", "-va", "downloads.lede-project.org::downloads/snapshots/targets/%s/%s/lede-sdk-*.tar.bz2" %(ts[0], ts[1]), "sdk.tar.bz2"],
 174                 haltOnFailure = True))
 175
 176         factory.addStep(ShellCommand(
 177                 name = "unpacksdk",
 178                 description = "Unpacking SDK archive",
 179                 command = ["tar", "--strip-components=1", "-C", "sdk/", "-vxjf", "sdk.tar.bz2"],
 180                 haltOnFailure = True))
 181
 182         factory.addStep(FileDownload(mastersrc=home_dir+'/key-build', slavedest="sdk/key-build", mode=0600))
 183         factory.addStep(FileDownload(mastersrc=home_dir+'/key-build.pub', slavedest="sdk/key-build.pub", mode=0600))
 184
 185         factory.addStep(ShellCommand(
 186                 name = "mkdldir",
 187                 description = "Preparing download directory",
 188                 command = ["sh", "-c", "mkdir -p $HOME/dl && rmdir ./sdk/dl && ln -sf $HOME/dl ./sdk/dl"]))
 189
 190         factory.addStep(ShellCommand(
 191                 name = "mkconf",
 192                 description = "Preparing SDK configuration",
 193                 workdir = "build/sdk",
 194                 command = ["sh", "-c", "rm -f .config && make defconfig"]))
 195
 196         factory.addStep(ShellCommand(
 197                 name = "updatefeeds",
 198                 description = "Updating feeds",
 199                 workdir = "build/sdk",
 200                 command = ["./scripts/feeds", "update"]))
 201
 202         factory.addStep(ShellCommand(
 203                 name = "installfeeds",
 204                 description = "Installing feeds",
 205                 workdir = "build/sdk",
 206                 command = ["./scripts/feeds", "install", "-a"]))
 207
 208         factory.addStep(ShellCommand(
 209                 name = "compile",
 210                 description = "Building packages",
 211                 workdir = "build/sdk",
 212                 command = ["make", WithProperties("-j%(nproc:~4)s"), "V=s", "IGNORE_ERRORS=n m y", "BUILD_LOG=1", "CONFIG_SIGNED_PACKAGES=y"]))
 213
 214         factory.addStep(ShellCommand(
 215                 name = "uploadprepare",
 216                 description = "Preparing package directory",
 217                 workdir = "build/sdk",
 218                 command = ["rsync", "-av", "--include", "/%s/" %(arch[0]), "--exclude", "/*", "--exclude", "/%s/*" %(arch[0]), "bin/packages/", "%s/packages/" %(rsync_bin_url)],
 219                 env={'RSYNC_PASSWORD': rsync_bin_key},
 220                 haltOnFailure = True,
 221                 logEnviron = False
 222         ))
 223
 224         factory.addStep(ShellCommand(
 225                 name = "packageupload",
 226                 description = "Uploading package files",
 227                 workdir = "build/sdk",
 228                 command = ["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-avz", "bin/packages/%s/" %(arch[0]), "%s/packages/%s/" %(rsync_bin_url, arch[0])],
 229                 env={'RSYNC_PASSWORD': rsync_bin_key},
 230                 haltOnFailure = True,
 231                 logEnviron = False
 232         ))
 233
 234         factory.addStep(ShellCommand(
 235                 name = "logprepare",
 236                 description = "Preparing log directory",
 237                 workdir = "build/sdk",
 238                 command = ["rsync", "-av", "--include", "/%s/" %(arch[0]), "--exclude", "/*", "--exclude", "/%s/*" %(arch[0]), "bin/packages/", "%s/faillogs/" %(rsync_bin_url)],
 239                 env={'RSYNC_PASSWORD': rsync_bin_key},
 240                 haltOnFailure = True,
 241                 logEnviron = False
 242         ))
 243
 244         factory.addStep(ShellCommand(
 245                 name = "logfind",
 246                 description = "Finding failure logs",
 247                 workdir = "build/sdk/logs/package/feeds",
 248                 command = ["sh", "-c", "sed -ne 's!^ *ERROR: package/feeds/\\([^ ]*\\) .*$!\\1!p' ../error.txt | sort -u | xargs -r find > ../../../logs.txt"],
 249                 haltOnFailure = False
 250         ))
 251
 252         factory.addStep(ShellCommand(
 253                 name = "logcollect",
 254                 description = "Collecting failure logs",
 255                 workdir = "build/sdk",
 256                 command = ["rsync", "-av", "--files-from=logs.txt", "logs/package/feeds/", "faillogs/"],
 257                 haltOnFailure = False
 258         ))
 259
 260         factory.addStep(ShellCommand(
 261                 name = "logupload",
 262                 description = "Uploading failure logs",
 263                 workdir = "build/sdk",
 264                 command = ["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-avz", "faillogs/", "%s/faillogs/%s/" %(rsync_bin_url, arch[0])],
 265                 env={'RSYNC_PASSWORD': rsync_bin_key},
 266                 haltOnFailure = False,
 267                 logEnviron = False
 268         ))
 269
 270         if rsync_src_url is not None:
 271                 factory.addStep(ShellCommand(
 272                         name = "sourceupload",
 273                         description = "Uploading source archives",
 274                         workdir = "build/sdk",
 275                         command = ["rsync", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-avz", "dl/", "%s/" %(rsync_src_url)],
 276                         env={'RSYNC_PASSWORD': rsync_src_key},
 277                         haltOnFailure = False,
 278                         logEnviron = False
 279                 ))
 280
 281         from buildbot.config import BuilderConfig
 282
 283         c['builders'].append(BuilderConfig(name=arch[0], slavenames=slaveNames, factory=factory))
 284
 285
 286 ####### STATUS arches
 287
 288 # 'status' is a list of Status arches. The results of each build will be
 289 # pushed to these arches. buildbot/status/*.py has a variety to choose from,
 290 # including web pages, email senders, and IRC bots.
 291
 292 c['status'] = []
 293
 294 from buildbot.status import html
 295 from buildbot.status.web import authz, auth
 296
 297 if ini.has_option("status", "bind"):
 298         if ini.has_option("status", "user") and ini.has_option("status", "password"):
 299                 authz_cfg=authz.Authz(
 300                         # change any of these to True to enable; see the manual for more
 301                         # options
 302                         auth=auth.BasicAuth([(ini.get("status", "user"), ini.get("status", "password"))]),
 303                         gracefulShutdown = 'auth',
 304                         forceBuild = 'auth', # use this to test your slave once it is set up
 305                         forceAllBuilds = 'auth',
 306                         pingBuilder = False,
 307                         stopBuild = 'auth',
 308                         stopAllBuilds = 'auth',
 309                         cancelPendingBuild = 'auth',
 310                 )
 311                 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind"), authz=authz_cfg))
 312         else:
 313                 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind")))
 314
 315 ####### PROJECT IDENTITY
 316
 317 # the 'title' string will appear at the top of this buildbot
 318 # installation's html.WebStatus home page (linked to the
 319 # 'titleURL') and is embedded in the title of the waterfall HTML page.
 320
 321 c['title'] = ini.get("general", "title")
 322 c['titleURL'] = ini.get("general", "title_url")
 323
 324 # the 'buildbotURL' string should point to the location where the buildbot's
 325 # internal web server (usually the html.WebStatus page) is visible. This
 326 # typically uses the port number set in the Waterfall 'status' entry, but
 327 # with an externally-visible host name which the buildbot cannot figure out
 328 # without some help.
 329
 330 c['buildbotURL'] = ini.get("general", "buildbot_url")
 331
 332 ####### DB URL
 333
 334 c['db'] = {
 335         # This specifies what database buildbot uses to store its state.  You can leave
 336         # this at its default for all but the largest installations.
 337         'db_url' : "sqlite:///state.sqlite",
 338 }