phase2/master.cfg

   1 # -*- python -*-
   2 # ex: set syntax=python:
   3
   4 import os
   5 import re
   6 import subprocess
   7 import ConfigParser
   8
   9 from buildbot import locks
  10
  11 ini = ConfigParser.ConfigParser()
  12 ini.read("./config.ini")
  13
  14 # This is a sample buildmaster config file. It must be installed as
  15 # 'master.cfg' in your buildmaster's base directory.
  16
  17 # This is the dictionary that the buildmaster pays attention to. We also use
  18 # a shorter alias to save typing.
  19 c = BuildmasterConfig = {}
  20
  21 ####### BUILDSLAVES
  22
  23 # The 'slaves' list defines the set of recognized buildslaves. Each element is
  24 # a BuildSlave object, specifying a unique slave name and password.  The same
  25 # slave name and password must be configured on the slave.
  26 from buildbot.buildslave import BuildSlave
  27
  28 c['slaves'] = []
  29
  30 for section in ini.sections():
  31         if section.startswith("slave "):
  32                 if ini.has_option(section, "name") and ini.has_option(section, "password"):
  33                         name = ini.get(section, "name")
  34                         password = ini.get(section, "password")
  35                         max_builds = 1
  36                         if ini.has_option(section, "builds"):
  37                                 max_builds = ini.getint(section, "builds")
  38                         c['slaves'].append(BuildSlave(name, password, max_builds = max_builds))
  39
  40 # 'slavePortnum' defines the TCP port to listen on for connections from slaves.
  41 # This must match the value configured into the buildslaves (with their
  42 # --master option)
  43 c['slavePortnum'] = 9990
  44
  45 # coalesce builds
  46 c['mergeRequests'] = True
  47
  48 ####### CHANGESOURCES
  49
  50 home_dir = os.path.abspath(ini.get("general", "homedir"))
  51
  52 repo_url = ini.get("repo", "url")
  53
  54 rsync_bin_url = ini.get("rsync", "binary_url")
  55 rsync_bin_key = ini.get("rsync", "binary_password")
  56
  57 rsync_src_url = None
  58 rsync_src_key = None
  59
  60 if ini.has_option("rsync", "source_url"):
  61         rsync_src_url = ini.get("rsync", "source_url")
  62         rsync_src_key = ini.get("rsync", "source_password")
  63
  64 gpg_keyid = None
  65 gpg_comment = "Unattended build signature"
  66 gpg_passfile = "/dev/null"
  67
  68 if ini.has_option("gpg", "keyid"):
  69         gpg_keyid = ini.get("gpg", "keyid")
  70
  71 if ini.has_option("gpg", "comment"):
  72         gpg_comment = ini.get("gpg", "comment")
  73
  74 if ini.has_option("gpg", "passfile"):
  75         gpg_passfile = ini.get("gpg", "passfile")
  76
  77
  78 # find arches
  79 arches = [ ]
  80 archnames = [ ]
  81
  82 findarches = subprocess.Popen([home_dir+'/dumpinfo.pl', 'architectures'],
  83         stdout = subprocess.PIPE, cwd = home_dir+'/source.git')
  84
  85 while True:
  86         line = findarches.stdout.readline()
  87         if not line:
  88                 break
  89         at = line.strip().split()
  90         arches.append(at)
  91         archnames.append(at[0])
  92
  93
  94 # find feeds
  95 feeds = []
  96
  97 from buildbot.changes.gitpoller import GitPoller
  98 c['change_source'] = []
  99
 100 with open(home_dir+'/source.git/feeds.conf.default', 'r') as f:
 101         for line in f:
 102                 parts = line.strip().split()
 103                 if parts[0] == "src-git":
 104                         feeds.append(parts)
 105                         c['change_source'].append(GitPoller(parts[2], workdir='%s/%s.git' %(os.getcwd(), parts[1]), branch='master', pollinterval=300))
 106
 107
 108 ####### SCHEDULERS
 109
 110 # Configure the Schedulers, which decide how to react to incoming changes.  In this
 111 # case, just kick off a 'basebuild' build
 112
 113 from buildbot.schedulers.basic import SingleBranchScheduler
 114 from buildbot.schedulers.forcesched import ForceScheduler
 115 from buildbot.changes import filter
 116 c['schedulers'] = []
 117 c['schedulers'].append(SingleBranchScheduler(
 118         name="all",
 119         change_filter=filter.ChangeFilter(branch='master'),
 120         treeStableTimer=60,
 121         builderNames=archnames))
 122
 123 c['schedulers'].append(ForceScheduler(
 124         name="force",
 125         builderNames=archnames))
 126
 127 ####### BUILDERS
 128
 129 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
 130 # what steps, and which slaves can execute them.  Note that any particular build will
 131 # only take place on one slave.
 132
 133 from buildbot.process.factory import BuildFactory
 134 from buildbot.steps.source import Git
 135 from buildbot.steps.shell import ShellCommand
 136 from buildbot.steps.shell import SetProperty
 137 from buildbot.steps.transfer import FileUpload
 138 from buildbot.steps.transfer import FileDownload
 139 from buildbot.steps.master import MasterShellCommand
 140 from buildbot.process.properties import WithProperties
 141
 142 c['builders'] = []
 143
 144 dlLock = locks.SlaveLock("slave_dl")
 145
 146 slaveNames = [ ]
 147
 148 for slave in c['slaves']:
 149         slaveNames.append(slave.slavename)
 150
 151 for arch in arches:
 152         ts = arch[1].split('/')
 153
 154         factory = BuildFactory()
 155
 156         # find number of cores
 157         factory.addStep(SetProperty(
 158                 name = "nproc",
 159                 property = "nproc",
 160                 description = "Finding number of CPUs",
 161                 command = ["nproc"]))
 162
 163         # prepare workspace
 164         factory.addStep(FileDownload(mastersrc="cleanup.sh", slavedest="cleanup.sh", mode=0755))
 165
 166         factory.addStep(ShellCommand(
 167                 name = "cleanold",
 168                 description = "Cleaning previous builds",
 169                 command = ["./cleanup.sh", WithProperties("%(slavename)s"), WithProperties("%(buildername)s"), "full"],
 170                 haltOnFailure = True,
 171                 timeout = 2400))
 172
 173         factory.addStep(ShellCommand(
 174                 name = "cleanup",
 175                 description = "Cleaning work area",
 176                 command = ["./cleanup.sh", WithProperties("%(slavename)s"), WithProperties("%(buildername)s"), "single"],
 177                 haltOnFailure = True,
 178                 timeout = 2400))
 179
 180         factory.addStep(ShellCommand(
 181                 name = "mksdkdir",
 182                 description = "Preparing SDK directory",
 183                 command = ["mkdir", "sdk"],
 184                 haltOnFailure = True))
 185
 186         factory.addStep(ShellCommand(
 187                 name = "downloadsdk",
 188                 description = "Downloading SDK archive",
 189                 command = ["rsync", "-va", "downloads.lede-project.org::downloads/snapshots/targets/%s/%s/lede-sdk-*.tar.xz" %(ts[0], ts[1]), "sdk.tar.xz"],
 190                 haltOnFailure = True))
 191
 192         factory.addStep(ShellCommand(
 193                 name = "unpacksdk",
 194                 description = "Unpacking SDK archive",
 195                 command = ["tar", "--strip-components=1", "-C", "sdk/", "-vxJf", "sdk.tar.xz"],
 196                 haltOnFailure = True))
 197
 198         factory.addStep(FileDownload(mastersrc=home_dir+'/key-build', slavedest="sdk/key-build", mode=0600))
 199         factory.addStep(FileDownload(mastersrc=home_dir+'/key-build.pub', slavedest="sdk/key-build.pub", mode=0600))
 200
 201         factory.addStep(ShellCommand(
 202                 name = "mkdldir",
 203                 description = "Preparing download directory",
 204                 command = ["sh", "-c", "mkdir -p $HOME/dl && rmdir ./sdk/dl && ln -sf $HOME/dl ./sdk/dl"]))
 205
 206         factory.addStep(ShellCommand(
 207                 name = "mkconf",
 208                 description = "Preparing SDK configuration",
 209                 workdir = "build/sdk",
 210                 command = ["sh", "-c", "rm -f .config && make defconfig"]))
 211
 212         factory.addStep(ShellCommand(
 213                 name = "updatefeeds",
 214                 description = "Updating feeds",
 215                 workdir = "build/sdk",
 216                 command = ["./scripts/feeds", "update"]))
 217
 218         factory.addStep(ShellCommand(
 219                 name = "installfeeds",
 220                 description = "Installing feeds",
 221                 workdir = "build/sdk",
 222                 command = ["./scripts/feeds", "install", "-a"]))
 223
 224         factory.addStep(ShellCommand(
 225                 name = "compile",
 226                 description = "Building packages",
 227                 workdir = "build/sdk",
 228                 command = ["make", WithProperties("-j%(nproc:~4)s"), "V=s", "IGNORE_ERRORS=n m y", "BUILD_LOG=1", "CONFIG_SIGNED_PACKAGES=y"]))
 229
 230         if gpg_keyid is not None:
 231                 factory.addStep(MasterShellCommand(
 232                         name = "signprepare",
 233                         description = "Preparing temporary signing directory",
 234                         command = ["mkdir", "-p", "%s/signing" %(home_dir)],
 235                         haltOnFailure = True
 236                 ))
 237
 238                 factory.addStep(ShellCommand(
 239                         name = "signpack",
 240                         description = "Packing files to sign",
 241                         workdir = "build/sdk",
 242                         command = "find bin/packages/%s/ -mindepth 2 -maxdepth 2 -type f -name Packages -print0 | xargs -0 tar -czf sign.tar.gz" %(arch[0]),
 243                         haltOnFailure = True
 244                 ))
 245
 246                 factory.addStep(FileUpload(
 247                         slavesrc = "sdk/sign.tar.gz",
 248                         masterdest = "%s/signing/%s.tar.gz" %(home_dir, arch[0]),
 249                         haltOnFailure = True
 250                 ))
 251
 252                 factory.addStep(MasterShellCommand(
 253                         name = "signfiles",
 254                         description = "Signing files",
 255                         command = ["%s/signall.sh" %(home_dir), "%s/signing/%s.tar.gz" %(home_dir, arch[0]), gpg_keyid, gpg_passfile, gpg_comment],
 256                         haltOnFailure = True
 257                 ))
 258
 259                 factory.addStep(FileDownload(
 260                         mastersrc = "%s/signing/%s.tar.gz" %(home_dir, arch[0]),
 261                         slavedest = "sdk/sign.tar.gz",
 262                         haltOnFailure = True
 263                 ))
 264
 265                 factory.addStep(ShellCommand(
 266                         name = "signunpack",
 267                         description = "Unpacking signed files",
 268                         workdir = "build/sdk",
 269                         command = ["tar", "-xzf", "sign.tar.gz"],
 270                         haltOnFailure = True
 271                 ))
 272
 273         factory.addStep(ShellCommand(
 274                 name = "uploadprepare",
 275                 description = "Preparing package directory",
 276                 workdir = "build/sdk",
 277                 command = ["rsync", "-av", "--include", "/%s/" %(arch[0]), "--exclude", "/*", "--exclude", "/%s/*" %(arch[0]), "bin/packages/", "%s/packages/" %(rsync_bin_url)],
 278                 env={'RSYNC_PASSWORD': rsync_bin_key},
 279                 haltOnFailure = True,
 280                 logEnviron = False
 281         ))
 282
 283         factory.addStep(ShellCommand(
 284                 name = "packageupload",
 285                 description = "Uploading package files",
 286                 workdir = "build/sdk",
 287                 command = ["rsync", "--delete", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-avz", "bin/packages/%s/" %(arch[0]), "%s/packages/%s/" %(rsync_bin_url, arch[0])],
 288                 env={'RSYNC_PASSWORD': rsync_bin_key},
 289                 haltOnFailure = True,
 290                 logEnviron = False
 291         ))
 292
 293         factory.addStep(ShellCommand(
 294                 name = "logprepare",
 295                 description = "Preparing log directory",
 296                 workdir = "build/sdk",
 297                 command = ["rsync", "-av", "--include", "/%s/" %(arch[0]), "--exclude", "/*", "--exclude", "/%s/*" %(arch[0]), "bin/packages/", "%s/faillogs/" %(rsync_bin_url)],
 298                 env={'RSYNC_PASSWORD': rsync_bin_key},
 299                 haltOnFailure = True,
 300                 logEnviron = False
 301         ))
 302
 303         factory.addStep(ShellCommand(
 304                 name = "logfind",
 305                 description = "Finding failure logs",
 306                 workdir = "build/sdk/logs/package/feeds",
 307                 command = ["sh", "-c", "sed -ne 's!^ *ERROR: package/feeds/\\([^ ]*\\) .*$!\\1!p' ../error.txt | sort -u | xargs -r find > ../../../logs.txt"],
 308                 haltOnFailure = False
 309         ))
 310
 311         factory.addStep(ShellCommand(
 312                 name = "logcollect",
 313                 description = "Collecting failure logs",
 314                 workdir = "build/sdk",
 315                 command = ["rsync", "-av", "--files-from=logs.txt", "logs/package/feeds/", "faillogs/"],
 316                 haltOnFailure = False
 317         ))
 318
 319         factory.addStep(ShellCommand(
 320                 name = "logupload",
 321                 description = "Uploading failure logs",
 322                 workdir = "build/sdk",
 323                 command = ["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-avz", "faillogs/", "%s/faillogs/%s/" %(rsync_bin_url, arch[0])],
 324                 env={'RSYNC_PASSWORD': rsync_bin_key},
 325                 haltOnFailure = False,
 326                 logEnviron = False
 327         ))
 328
 329         if rsync_src_url is not None:
 330                 factory.addStep(ShellCommand(
 331                         name = "sourceupload",
 332                         description = "Uploading source archives",
 333                         workdir = "build/sdk",
 334                         command = ["rsync", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-avz", "dl/", "%s/" %(rsync_src_url)],
 335                         env={'RSYNC_PASSWORD': rsync_src_key},
 336                         haltOnFailure = False,
 337                         logEnviron = False
 338                 ))
 339
 340         from buildbot.config import BuilderConfig
 341
 342         c['builders'].append(BuilderConfig(name=arch[0], slavenames=slaveNames, factory=factory))
 343
 344
 345 ####### STATUS arches
 346
 347 # 'status' is a list of Status arches. The results of each build will be
 348 # pushed to these arches. buildbot/status/*.py has a variety to choose from,
 349 # including web pages, email senders, and IRC bots.
 350
 351 c['status'] = []
 352
 353 from buildbot.status import html
 354 from buildbot.status.web import authz, auth
 355
 356 if ini.has_option("status", "bind"):
 357         if ini.has_option("status", "user") and ini.has_option("status", "password"):
 358                 authz_cfg=authz.Authz(
 359                         # change any of these to True to enable; see the manual for more
 360                         # options
 361                         auth=auth.BasicAuth([(ini.get("status", "user"), ini.get("status", "password"))]),
 362                         gracefulShutdown = 'auth',
 363                         forceBuild = 'auth', # use this to test your slave once it is set up
 364                         forceAllBuilds = 'auth',
 365                         pingBuilder = False,
 366                         stopBuild = 'auth',
 367                         stopAllBuilds = 'auth',
 368                         cancelPendingBuild = 'auth',
 369                 )
 370                 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind"), authz=authz_cfg))
 371         else:
 372                 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind")))
 373
 374 ####### PROJECT IDENTITY
 375
 376 # the 'title' string will appear at the top of this buildbot
 377 # installation's html.WebStatus home page (linked to the
 378 # 'titleURL') and is embedded in the title of the waterfall HTML page.
 379
 380 c['title'] = ini.get("general", "title")
 381 c['titleURL'] = ini.get("general", "title_url")
 382
 383 # the 'buildbotURL' string should point to the location where the buildbot's
 384 # internal web server (usually the html.WebStatus page) is visible. This
 385 # typically uses the port number set in the Waterfall 'status' entry, but
 386 # with an externally-visible host name which the buildbot cannot figure out
 387 # without some help.
 388
 389 c['buildbotURL'] = ini.get("general", "buildbot_url")
 390
 391 ####### DB URL
 392
 393 c['db'] = {
 394         # This specifies what database buildbot uses to store its state.  You can leave
 395         # this at its default for all but the largest installations.
 396         'db_url' : "sqlite:///state.sqlite",
 397 }