phase2/master.cfg

   1 # -*- python -*-
   2 # ex: set syntax=python:
   3
   4 import os
   5 import re
   6 import subprocess
   7 import ConfigParser
   8
   9 from buildbot import locks
  10
  11 ini = ConfigParser.ConfigParser()
  12 ini.read("./config.ini")
  13
  14 # This is a sample buildmaster config file. It must be installed as
  15 # 'master.cfg' in your buildmaster's base directory.
  16
  17 # This is the dictionary that the buildmaster pays attention to. We also use
  18 # a shorter alias to save typing.
  19 c = BuildmasterConfig = {}
  20
  21 ####### BUILDSLAVES
  22
  23 # The 'slaves' list defines the set of recognized buildslaves. Each element is
  24 # a BuildSlave object, specifying a unique slave name and password.  The same
  25 # slave name and password must be configured on the slave.
  26 from buildbot.buildslave import BuildSlave
  27
  28 slave_port = 9990
  29
  30 if ini.has_option("general", "port"):
  31         slave_port = ini.getint("general", "port")
  32
  33 c['slaves'] = []
  34
  35 for section in ini.sections():
  36         if section.startswith("slave "):
  37                 if ini.has_option(section, "name") and ini.has_option(section, "password"):
  38                         name = ini.get(section, "name")
  39                         password = ini.get(section, "password")
  40                         max_builds = 1
  41                         if ini.has_option(section, "builds"):
  42                                 max_builds = ini.getint(section, "builds")
  43                         c['slaves'].append(BuildSlave(name, password, max_builds = max_builds))
  44
  45 # 'slavePortnum' defines the TCP port to listen on for connections from slaves.
  46 # This must match the value configured into the buildslaves (with their
  47 # --master option)
  48 c['slavePortnum'] = slave_port
  49
  50 # coalesce builds
  51 c['mergeRequests'] = True
  52
  53 ####### CHANGESOURCES
  54
  55 home_dir = os.path.abspath(ini.get("general", "homedir"))
  56
  57 repo_url = ini.get("repo", "url")
  58
  59 rsync_bin_url = ini.get("rsync", "binary_url")
  60 rsync_bin_key = ini.get("rsync", "binary_password")
  61
  62 rsync_src_url = None
  63 rsync_src_key = None
  64
  65 if ini.has_option("rsync", "source_url"):
  66         rsync_src_url = ini.get("rsync", "source_url")
  67         rsync_src_key = ini.get("rsync", "source_password")
  68
  69 gpg_keyid = None
  70 gpg_comment = "Unattended build signature"
  71 gpg_passfile = "/dev/null"
  72
  73 if ini.has_option("gpg", "keyid"):
  74         gpg_keyid = ini.get("gpg", "keyid")
  75
  76 if ini.has_option("gpg", "comment"):
  77         gpg_comment = ini.get("gpg", "comment")
  78
  79 if ini.has_option("gpg", "passfile"):
  80         gpg_passfile = ini.get("gpg", "passfile")
  81
  82
  83 # find arches
  84 arches = [ ]
  85 archnames = [ ]
  86
  87 findarches = subprocess.Popen([home_dir+'/dumpinfo.pl', 'architectures'],
  88         stdout = subprocess.PIPE, cwd = home_dir+'/source.git')
  89
  90 while True:
  91         line = findarches.stdout.readline()
  92         if not line:
  93                 break
  94         at = line.strip().split()
  95         arches.append(at)
  96         archnames.append(at[0])
  97
  98
  99 # find feeds
 100 feeds = []
 101
 102 from buildbot.changes.gitpoller import GitPoller
 103 c['change_source'] = []
 104
 105 with open(home_dir+'/source.git/feeds.conf.default', 'r') as f:
 106         for line in f:
 107                 parts = line.strip().split()
 108                 if parts[0] == "src-git":
 109                         feeds.append(parts)
 110                         c['change_source'].append(GitPoller(parts[2], workdir='%s/%s.git' %(os.getcwd(), parts[1]), branch='master', pollinterval=300))
 111
 112
 113 ####### SCHEDULERS
 114
 115 # Configure the Schedulers, which decide how to react to incoming changes.  In this
 116 # case, just kick off a 'basebuild' build
 117
 118 from buildbot.schedulers.basic import SingleBranchScheduler
 119 from buildbot.schedulers.forcesched import ForceScheduler
 120 from buildbot.changes import filter
 121 c['schedulers'] = []
 122 c['schedulers'].append(SingleBranchScheduler(
 123         name="all",
 124         change_filter=filter.ChangeFilter(branch='master'),
 125         treeStableTimer=60,
 126         builderNames=archnames))
 127
 128 c['schedulers'].append(ForceScheduler(
 129         name="force",
 130         builderNames=archnames))
 131
 132 ####### BUILDERS
 133
 134 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
 135 # what steps, and which slaves can execute them.  Note that any particular build will
 136 # only take place on one slave.
 137
 138 from buildbot.process.factory import BuildFactory
 139 from buildbot.steps.source import Git
 140 from buildbot.steps.shell import ShellCommand
 141 from buildbot.steps.shell import SetProperty
 142 from buildbot.steps.transfer import FileUpload
 143 from buildbot.steps.transfer import FileDownload
 144 from buildbot.steps.master import MasterShellCommand
 145 from buildbot.process.properties import WithProperties
 146
 147 c['builders'] = []
 148
 149 dlLock = locks.SlaveLock("slave_dl")
 150
 151 slaveNames = [ ]
 152
 153 for slave in c['slaves']:
 154         slaveNames.append(slave.slavename)
 155
 156 for arch in arches:
 157         ts = arch[1].split('/')
 158
 159         factory = BuildFactory()
 160
 161         # find number of cores
 162         factory.addStep(SetProperty(
 163                 name = "nproc",
 164                 property = "nproc",
 165                 description = "Finding number of CPUs",
 166                 command = ["nproc"]))
 167
 168         # prepare workspace
 169         factory.addStep(FileDownload(mastersrc="cleanup.sh", slavedest="cleanup.sh", mode=0755))
 170
 171         factory.addStep(ShellCommand(
 172                 name = "cleanold",
 173                 description = "Cleaning previous builds",
 174                 command = ["./cleanup.sh", WithProperties("%(slavename)s"), WithProperties("%(buildername)s"), "full"],
 175                 haltOnFailure = True,
 176                 timeout = 2400))
 177
 178         factory.addStep(ShellCommand(
 179                 name = "cleanup",
 180                 description = "Cleaning work area",
 181                 command = ["./cleanup.sh", WithProperties("%(slavename)s"), WithProperties("%(buildername)s"), "single"],
 182                 haltOnFailure = True,
 183                 timeout = 2400))
 184
 185         factory.addStep(ShellCommand(
 186                 name = "mksdkdir",
 187                 description = "Preparing SDK directory",
 188                 command = ["mkdir", "sdk"],
 189                 haltOnFailure = True))
 190
 191         factory.addStep(ShellCommand(
 192                 name = "downloadsdk",
 193                 description = "Downloading SDK archive",
 194                 command = ["rsync", "-va", "downloads.lede-project.org::downloads/snapshots/targets/%s/%s/lede-sdk-*.tar.xz" %(ts[0], ts[1]), "sdk.tar.xz"],
 195                 haltOnFailure = True))
 196
 197         factory.addStep(ShellCommand(
 198                 name = "unpacksdk",
 199                 description = "Unpacking SDK archive",
 200                 command = ["tar", "--strip-components=1", "-C", "sdk/", "-vxJf", "sdk.tar.xz"],
 201                 haltOnFailure = True))
 202
 203         factory.addStep(FileDownload(mastersrc=home_dir+'/key-build', slavedest="sdk/key-build", mode=0600))
 204         factory.addStep(FileDownload(mastersrc=home_dir+'/key-build.pub', slavedest="sdk/key-build.pub", mode=0600))
 205
 206         factory.addStep(ShellCommand(
 207                 name = "mkdldir",
 208                 description = "Preparing download directory",
 209                 command = ["sh", "-c", "mkdir -p $HOME/dl && rmdir ./sdk/dl && ln -sf $HOME/dl ./sdk/dl"]))
 210
 211         factory.addStep(ShellCommand(
 212                 name = "mkconf",
 213                 description = "Preparing SDK configuration",
 214                 workdir = "build/sdk",
 215                 command = ["sh", "-c", "rm -f .config && make defconfig"]))
 216
 217         factory.addStep(ShellCommand(
 218                 name = "updatefeeds",
 219                 description = "Updating feeds",
 220                 workdir = "build/sdk",
 221                 command = ["./scripts/feeds", "update"]))
 222
 223         factory.addStep(ShellCommand(
 224                 name = "installfeeds",
 225                 description = "Installing feeds",
 226                 workdir = "build/sdk",
 227                 command = ["./scripts/feeds", "install", "-a"]))
 228
 229         factory.addStep(ShellCommand(
 230                 name = "compile",
 231                 description = "Building packages",
 232                 workdir = "build/sdk",
 233                 command = ["make", WithProperties("-j%(nproc:~4)s"), "V=s", "IGNORE_ERRORS=n m y", "BUILD_LOG=1", "CONFIG_SIGNED_PACKAGES=y"]))
 234
 235         if gpg_keyid is not None:
 236                 factory.addStep(MasterShellCommand(
 237                         name = "signprepare",
 238                         description = "Preparing temporary signing directory",
 239                         command = ["mkdir", "-p", "%s/signing" %(home_dir)],
 240                         haltOnFailure = True
 241                 ))
 242
 243                 factory.addStep(ShellCommand(
 244                         name = "signpack",
 245                         description = "Packing files to sign",
 246                         workdir = "build/sdk",
 247                         command = "find bin/packages/%s/ -mindepth 2 -maxdepth 2 -type f -name Packages -print0 | xargs -0 tar -czf sign.tar.gz" %(arch[0]),
 248                         haltOnFailure = True
 249                 ))
 250
 251                 factory.addStep(FileUpload(
 252                         slavesrc = "sdk/sign.tar.gz",
 253                         masterdest = "%s/signing/%s.tar.gz" %(home_dir, arch[0]),
 254                         haltOnFailure = True
 255                 ))
 256
 257                 factory.addStep(MasterShellCommand(
 258                         name = "signfiles",
 259                         description = "Signing files",
 260                         command = ["%s/signall.sh" %(home_dir), "%s/signing/%s.tar.gz" %(home_dir, arch[0]), gpg_keyid, gpg_passfile, gpg_comment],
 261                         haltOnFailure = True
 262                 ))
 263
 264                 factory.addStep(FileDownload(
 265                         mastersrc = "%s/signing/%s.tar.gz" %(home_dir, arch[0]),
 266                         slavedest = "sdk/sign.tar.gz",
 267                         haltOnFailure = True
 268                 ))
 269
 270                 factory.addStep(ShellCommand(
 271                         name = "signunpack",
 272                         description = "Unpacking signed files",
 273                         workdir = "build/sdk",
 274                         command = ["tar", "-xzf", "sign.tar.gz"],
 275                         haltOnFailure = True
 276                 ))
 277
 278         factory.addStep(ShellCommand(
 279                 name = "uploadprepare",
 280                 description = "Preparing package directory",
 281                 workdir = "build/sdk",
 282                 command = ["rsync", "-av", "--include", "/%s/" %(arch[0]), "--exclude", "/*", "--exclude", "/%s/*" %(arch[0]), "bin/packages/", "%s/packages/" %(rsync_bin_url)],
 283                 env={'RSYNC_PASSWORD': rsync_bin_key},
 284                 haltOnFailure = True,
 285                 logEnviron = False
 286         ))
 287
 288         factory.addStep(ShellCommand(
 289                 name = "packageupload",
 290                 description = "Uploading package files",
 291                 workdir = "build/sdk",
 292                 command = ["rsync", "--delete", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-avz", "bin/packages/%s/" %(arch[0]), "%s/packages/%s/" %(rsync_bin_url, arch[0])],
 293                 env={'RSYNC_PASSWORD': rsync_bin_key},
 294                 haltOnFailure = True,
 295                 logEnviron = False
 296         ))
 297
 298         factory.addStep(ShellCommand(
 299                 name = "logprepare",
 300                 description = "Preparing log directory",
 301                 workdir = "build/sdk",
 302                 command = ["rsync", "-av", "--include", "/%s/" %(arch[0]), "--exclude", "/*", "--exclude", "/%s/*" %(arch[0]), "bin/packages/", "%s/faillogs/" %(rsync_bin_url)],
 303                 env={'RSYNC_PASSWORD': rsync_bin_key},
 304                 haltOnFailure = True,
 305                 logEnviron = False
 306         ))
 307
 308         factory.addStep(ShellCommand(
 309                 name = "logfind",
 310                 description = "Finding failure logs",
 311                 workdir = "build/sdk/logs/package/feeds",
 312                 command = ["sh", "-c", "sed -ne 's!^ *ERROR: package/feeds/\\([^ ]*\\) .*$!\\1!p' ../error.txt | sort -u | xargs -r find > ../../../logs.txt"],
 313                 haltOnFailure = False
 314         ))
 315
 316         factory.addStep(ShellCommand(
 317                 name = "logcollect",
 318                 description = "Collecting failure logs",
 319                 workdir = "build/sdk",
 320                 command = ["rsync", "-av", "--files-from=logs.txt", "logs/package/feeds/", "faillogs/"],
 321                 haltOnFailure = False
 322         ))
 323
 324         factory.addStep(ShellCommand(
 325                 name = "logupload",
 326                 description = "Uploading failure logs",
 327                 workdir = "build/sdk",
 328                 command = ["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-avz", "faillogs/", "%s/faillogs/%s/" %(rsync_bin_url, arch[0])],
 329                 env={'RSYNC_PASSWORD': rsync_bin_key},
 330                 haltOnFailure = False,
 331                 logEnviron = False
 332         ))
 333
 334         if rsync_src_url is not None:
 335                 factory.addStep(ShellCommand(
 336                         name = "sourceupload",
 337                         description = "Uploading source archives",
 338                         workdir = "build/sdk",
 339                         command = ["rsync", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-avz", "dl/", "%s/" %(rsync_src_url)],
 340                         env={'RSYNC_PASSWORD': rsync_src_key},
 341                         haltOnFailure = False,
 342                         logEnviron = False
 343                 ))
 344
 345         from buildbot.config import BuilderConfig
 346
 347         c['builders'].append(BuilderConfig(name=arch[0], slavenames=slaveNames, factory=factory))
 348
 349
 350 ####### STATUS arches
 351
 352 # 'status' is a list of Status arches. The results of each build will be
 353 # pushed to these arches. buildbot/status/*.py has a variety to choose from,
 354 # including web pages, email senders, and IRC bots.
 355
 356 c['status'] = []
 357
 358 from buildbot.status import html
 359 from buildbot.status.web import authz, auth
 360
 361 if ini.has_option("status", "bind"):
 362         if ini.has_option("status", "user") and ini.has_option("status", "password"):
 363                 authz_cfg=authz.Authz(
 364                         # change any of these to True to enable; see the manual for more
 365                         # options
 366                         auth=auth.BasicAuth([(ini.get("status", "user"), ini.get("status", "password"))]),
 367                         gracefulShutdown = 'auth',
 368                         forceBuild = 'auth', # use this to test your slave once it is set up
 369                         forceAllBuilds = 'auth',
 370                         pingBuilder = False,
 371                         stopBuild = 'auth',
 372                         stopAllBuilds = 'auth',
 373                         cancelPendingBuild = 'auth',
 374                 )
 375                 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind"), authz=authz_cfg))
 376         else:
 377                 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind")))
 378
 379 ####### PROJECT IDENTITY
 380
 381 # the 'title' string will appear at the top of this buildbot
 382 # installation's html.WebStatus home page (linked to the
 383 # 'titleURL') and is embedded in the title of the waterfall HTML page.
 384
 385 c['title'] = ini.get("general", "title")
 386 c['titleURL'] = ini.get("general", "title_url")
 387
 388 # the 'buildbotURL' string should point to the location where the buildbot's
 389 # internal web server (usually the html.WebStatus page) is visible. This
 390 # typically uses the port number set in the Waterfall 'status' entry, but
 391 # with an externally-visible host name which the buildbot cannot figure out
 392 # without some help.
 393
 394 c['buildbotURL'] = ini.get("general", "buildbot_url")
 395
 396 ####### DB URL
 397
 398 c['db'] = {
 399         # This specifies what database buildbot uses to store its state.  You can leave
 400         # this at its default for all but the largest installations.
 401         'db_url' : "sqlite:///state.sqlite",
 402 }