phase2/master.cfg

   1 # -*- python -*-
   2 # ex: set syntax=python:
   3
   4 import os
   5 import re
   6 import subprocess
   7 import ConfigParser
   8
   9 from buildbot import locks
  10
  11 ini = ConfigParser.ConfigParser()
  12 ini.read("./config.ini")
  13
  14 buildbot_url = ini.get("general", "buildbot_url")
  15
  16 # This is a sample buildmaster config file. It must be installed as
  17 # 'master.cfg' in your buildmaster's base directory.
  18
  19 # This is the dictionary that the buildmaster pays attention to. We also use
  20 # a shorter alias to save typing.
  21 c = BuildmasterConfig = {}
  22
  23 ####### BUILDSLAVES
  24
  25 # The 'slaves' list defines the set of recognized buildslaves. Each element is
  26 # a BuildSlave object, specifying a unique slave name and password.  The same
  27 # slave name and password must be configured on the slave.
  28 from buildbot.buildslave import BuildSlave
  29
  30 slave_port = 9990
  31
  32 if ini.has_option("general", "port"):
  33         slave_port = ini.getint("general", "port")
  34
  35 c['slaves'] = []
  36
  37 for section in ini.sections():
  38         if section.startswith("slave "):
  39                 if ini.has_option(section, "name") and ini.has_option(section, "password"):
  40                         name = ini.get(section, "name")
  41                         password = ini.get(section, "password")
  42                         max_builds = 1
  43                         if ini.has_option(section, "builds"):
  44                                 max_builds = ini.getint(section, "builds")
  45                         c['slaves'].append(BuildSlave(name, password, max_builds = max_builds))
  46
  47 # 'slavePortnum' defines the TCP port to listen on for connections from slaves.
  48 # This must match the value configured into the buildslaves (with their
  49 # --master option)
  50 c['slavePortnum'] = slave_port
  51
  52 # coalesce builds
  53 c['mergeRequests'] = True
  54
  55 ####### CHANGESOURCES
  56
  57 home_dir = os.path.abspath(ini.get("general", "homedir"))
  58
  59 rsync_bin_url = ini.get("rsync", "binary_url")
  60 rsync_bin_key = ini.get("rsync", "binary_password")
  61
  62 rsync_src_url = None
  63 rsync_src_key = None
  64
  65 if ini.has_option("rsync", "source_url"):
  66         rsync_src_url = ini.get("rsync", "source_url")
  67         rsync_src_key = ini.get("rsync", "source_password")
  68
  69 rsync_sdk_url = None
  70 rsync_sdk_key = None
  71 rsync_sdk_pat = "lede-sdk-*.tar.xz"
  72
  73 if ini.has_option("rsync", "sdk_url"):
  74         rsync_sdk_url = ini.get("rsync", "sdk_url")
  75         rsync_sdk_key = ini.get("rsync", "sdk_password")
  76
  77 if ini.has_option("rsync", "sdk_pattern"):
  78         rsync_sdk_pat = ini.get("rsync", "sdk_pattern")
  79
  80 gpg_keyid = None
  81 gpg_comment = "Unattended build signature"
  82 gpg_passfile = "/dev/null"
  83
  84 if ini.has_option("gpg", "keyid"):
  85         gpg_keyid = ini.get("gpg", "keyid")
  86
  87 if ini.has_option("gpg", "comment"):
  88         gpg_comment = ini.get("gpg", "comment")
  89
  90 if ini.has_option("gpg", "passfile"):
  91         gpg_passfile = ini.get("gpg", "passfile")
  92
  93
  94 # find arches
  95 arches = [ ]
  96 archnames = [ ]
  97
  98 findarches = subprocess.Popen([home_dir+'/dumpinfo.pl', 'architectures'],
  99         stdout = subprocess.PIPE, cwd = home_dir+'/source.git')
 100
 101 while True:
 102         line = findarches.stdout.readline()
 103         if not line:
 104                 break
 105         at = line.strip().split()
 106         arches.append(at)
 107         archnames.append(at[0])
 108
 109
 110 # find feeds
 111 feeds = []
 112
 113 from buildbot.changes.gitpoller import GitPoller
 114 c['change_source'] = []
 115
 116 with open(home_dir+'/source.git/feeds.conf.default', 'r') as f:
 117         for line in f:
 118                 parts = line.strip().split()
 119                 if parts[0] == "src-git":
 120                         feeds.append(parts)
 121                         c['change_source'].append(GitPoller(parts[2], workdir='%s/%s.git' %(os.getcwd(), parts[1]), branch='master', pollinterval=300))
 122
 123
 124 ####### SCHEDULERS
 125
 126 # Configure the Schedulers, which decide how to react to incoming changes.  In this
 127 # case, just kick off a 'basebuild' build
 128
 129 from buildbot.schedulers.basic import SingleBranchScheduler
 130 from buildbot.schedulers.forcesched import ForceScheduler
 131 from buildbot.changes import filter
 132 c['schedulers'] = []
 133 c['schedulers'].append(SingleBranchScheduler(
 134         name="all",
 135         change_filter=filter.ChangeFilter(branch='master'),
 136         treeStableTimer=60,
 137         builderNames=archnames))
 138
 139 c['schedulers'].append(ForceScheduler(
 140         name="force",
 141         builderNames=archnames))
 142
 143 ####### BUILDERS
 144
 145 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
 146 # what steps, and which slaves can execute them.  Note that any particular build will
 147 # only take place on one slave.
 148
 149 from buildbot.process.factory import BuildFactory
 150 from buildbot.steps.source import Git
 151 from buildbot.steps.shell import ShellCommand
 152 from buildbot.steps.shell import SetProperty
 153 from buildbot.steps.transfer import FileUpload
 154 from buildbot.steps.transfer import FileDownload
 155 from buildbot.steps.master import MasterShellCommand
 156 from buildbot.process.properties import WithProperties
 157
 158 c['builders'] = []
 159
 160 dlLock = locks.SlaveLock("slave_dl")
 161
 162 slaveNames = [ ]
 163
 164 for slave in c['slaves']:
 165         slaveNames.append(slave.slavename)
 166
 167 for arch in arches:
 168         ts = arch[1].split('/')
 169
 170         factory = BuildFactory()
 171
 172         # find number of cores
 173         factory.addStep(SetProperty(
 174                 name = "nproc",
 175                 property = "nproc",
 176                 description = "Finding number of CPUs",
 177                 command = ["nproc"]))
 178
 179         # prepare workspace
 180         factory.addStep(FileDownload(mastersrc="cleanup.sh", slavedest="cleanup.sh", mode=0755))
 181
 182         factory.addStep(ShellCommand(
 183                 name = "cleanold",
 184                 description = "Cleaning previous builds",
 185                 command = ["./cleanup.sh", buildbot_url, WithProperties("%(slavename)s"), WithProperties("%(buildername)s"), "full"],
 186                 haltOnFailure = True,
 187                 timeout = 2400))
 188
 189         factory.addStep(ShellCommand(
 190                 name = "cleanup",
 191                 description = "Cleaning work area",
 192                 command = ["./cleanup.sh", buildbot_url, WithProperties("%(slavename)s"), WithProperties("%(buildername)s"), "single"],
 193                 haltOnFailure = True,
 194                 timeout = 2400))
 195
 196         factory.addStep(ShellCommand(
 197                 name = "mksdkdir",
 198                 description = "Preparing SDK directory",
 199                 command = ["mkdir", "sdk"],
 200                 haltOnFailure = True))
 201
 202         factory.addStep(ShellCommand(
 203                 name = "downloadsdk",
 204                 description = "Downloading SDK archive",
 205                 command = ["rsync", "-va", "%s/%s/%s/%s" %(rsync_sdk_url, ts[0], ts[1], rsync_sdk_pat), "sdk.archive"],
 206                 env={'RSYNC_PASSWORD': rsync_sdk_key},
 207                 haltOnFailure = True,
 208                 logEnviron = False))
 209
 210         factory.addStep(ShellCommand(
 211                 name = "unpacksdk",
 212                 description = "Unpacking SDK archive",
 213                 command = ["tar", "--strip-components=1", "-C", "sdk/", "-vxf", "sdk.archive"],
 214                 haltOnFailure = True))
 215
 216         factory.addStep(FileDownload(mastersrc=home_dir+'/key-build', slavedest="sdk/key-build", mode=0600))
 217         factory.addStep(FileDownload(mastersrc=home_dir+'/key-build.pub', slavedest="sdk/key-build.pub", mode=0600))
 218
 219         factory.addStep(ShellCommand(
 220                 name = "mkdldir",
 221                 description = "Preparing download directory",
 222                 command = ["sh", "-c", "mkdir -p $HOME/dl && rmdir ./sdk/dl && ln -sf $HOME/dl ./sdk/dl"]))
 223
 224         factory.addStep(ShellCommand(
 225                 name = "mkconf",
 226                 description = "Preparing SDK configuration",
 227                 workdir = "build/sdk",
 228                 command = ["sh", "-c", "rm -f .config && make defconfig"]))
 229
 230         factory.addStep(ShellCommand(
 231                 name = "updatefeeds",
 232                 description = "Updating feeds",
 233                 workdir = "build/sdk",
 234                 command = ["./scripts/feeds", "update"]))
 235
 236         factory.addStep(ShellCommand(
 237                 name = "installfeeds",
 238                 description = "Installing feeds",
 239                 workdir = "build/sdk",
 240                 command = ["./scripts/feeds", "install", "-a"]))
 241
 242         factory.addStep(ShellCommand(
 243                 name = "compile",
 244                 description = "Building packages",
 245                 workdir = "build/sdk",
 246                 command = ["make", WithProperties("-j%(nproc:~4)s"), "V=s", "IGNORE_ERRORS=n m y", "BUILD_LOG=1", "CONFIG_SIGNED_PACKAGES=y"]))
 247
 248         if gpg_keyid is not None:
 249                 factory.addStep(MasterShellCommand(
 250                         name = "signprepare",
 251                         description = "Preparing temporary signing directory",
 252                         command = ["mkdir", "-p", "%s/signing" %(home_dir)],
 253                         haltOnFailure = True
 254                 ))
 255
 256                 factory.addStep(ShellCommand(
 257                         name = "signpack",
 258                         description = "Packing files to sign",
 259                         workdir = "build/sdk",
 260                         command = "find bin/packages/%s/ -mindepth 2 -maxdepth 2 -type f -name Packages -print0 | xargs -0 tar -czf sign.tar.gz" %(arch[0]),
 261                         haltOnFailure = True
 262                 ))
 263
 264                 factory.addStep(FileUpload(
 265                         slavesrc = "sdk/sign.tar.gz",
 266                         masterdest = "%s/signing/%s.tar.gz" %(home_dir, arch[0]),
 267                         haltOnFailure = True
 268                 ))
 269
 270                 factory.addStep(MasterShellCommand(
 271                         name = "signfiles",
 272                         description = "Signing files",
 273                         command = ["%s/signall.sh" %(home_dir), "%s/signing/%s.tar.gz" %(home_dir, arch[0]), gpg_keyid, gpg_passfile, gpg_comment],
 274                         haltOnFailure = True
 275                 ))
 276
 277                 factory.addStep(FileDownload(
 278                         mastersrc = "%s/signing/%s.tar.gz" %(home_dir, arch[0]),
 279                         slavedest = "sdk/sign.tar.gz",
 280                         haltOnFailure = True
 281                 ))
 282
 283                 factory.addStep(ShellCommand(
 284                         name = "signunpack",
 285                         description = "Unpacking signed files",
 286                         workdir = "build/sdk",
 287                         command = ["tar", "-xzf", "sign.tar.gz"],
 288                         haltOnFailure = True
 289                 ))
 290
 291         factory.addStep(ShellCommand(
 292                 name = "uploadprepare",
 293                 description = "Preparing package directory",
 294                 workdir = "build/sdk",
 295                 command = ["rsync", "-av", "--include", "/%s/" %(arch[0]), "--exclude", "/*", "--exclude", "/%s/*" %(arch[0]), "bin/packages/", "%s/packages/" %(rsync_bin_url)],
 296                 env={'RSYNC_PASSWORD': rsync_bin_key},
 297                 haltOnFailure = True,
 298                 logEnviron = False
 299         ))
 300
 301         factory.addStep(ShellCommand(
 302                 name = "packageupload",
 303                 description = "Uploading package files",
 304                 workdir = "build/sdk",
 305                 command = ["rsync", "--delete", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-avz", "bin/packages/%s/" %(arch[0]), "%s/packages/%s/" %(rsync_bin_url, arch[0])],
 306                 env={'RSYNC_PASSWORD': rsync_bin_key},
 307                 haltOnFailure = True,
 308                 logEnviron = False
 309         ))
 310
 311         factory.addStep(ShellCommand(
 312                 name = "logprepare",
 313                 description = "Preparing log directory",
 314                 workdir = "build/sdk",
 315                 command = ["rsync", "-av", "--include", "/%s/" %(arch[0]), "--exclude", "/*", "--exclude", "/%s/*" %(arch[0]), "bin/packages/", "%s/faillogs/" %(rsync_bin_url)],
 316                 env={'RSYNC_PASSWORD': rsync_bin_key},
 317                 haltOnFailure = True,
 318                 logEnviron = False
 319         ))
 320
 321         factory.addStep(ShellCommand(
 322                 name = "logfind",
 323                 description = "Finding failure logs",
 324                 workdir = "build/sdk/logs/package/feeds",
 325                 command = ["sh", "-c", "sed -ne 's!^ *ERROR: package/feeds/\\([^ ]*\\) .*$!\\1!p' ../error.txt | sort -u | xargs -r find > ../../../logs.txt"],
 326                 haltOnFailure = False
 327         ))
 328
 329         factory.addStep(ShellCommand(
 330                 name = "logcollect",
 331                 description = "Collecting failure logs",
 332                 workdir = "build/sdk",
 333                 command = ["rsync", "-av", "--files-from=logs.txt", "logs/package/feeds/", "faillogs/"],
 334                 haltOnFailure = False
 335         ))
 336
 337         factory.addStep(ShellCommand(
 338                 name = "logupload",
 339                 description = "Uploading failure logs",
 340                 workdir = "build/sdk",
 341                 command = ["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-avz", "faillogs/", "%s/faillogs/%s/" %(rsync_bin_url, arch[0])],
 342                 env={'RSYNC_PASSWORD': rsync_bin_key},
 343                 haltOnFailure = False,
 344                 logEnviron = False
 345         ))
 346
 347         if rsync_src_url is not None:
 348                 factory.addStep(ShellCommand(
 349                         name = "sourceupload",
 350                         description = "Uploading source archives",
 351                         workdir = "build/sdk",
 352                         command = ["rsync", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-avz", "dl/", "%s/" %(rsync_src_url)],
 353                         env={'RSYNC_PASSWORD': rsync_src_key},
 354                         haltOnFailure = False,
 355                         logEnviron = False
 356                 ))
 357
 358         from buildbot.config import BuilderConfig
 359
 360         c['builders'].append(BuilderConfig(name=arch[0], slavenames=slaveNames, factory=factory))
 361
 362
 363 ####### STATUS arches
 364
 365 # 'status' is a list of Status arches. The results of each build will be
 366 # pushed to these arches. buildbot/status/*.py has a variety to choose from,
 367 # including web pages, email senders, and IRC bots.
 368
 369 c['status'] = []
 370
 371 from buildbot.status import html
 372 from buildbot.status.web import authz, auth
 373
 374 if ini.has_option("status", "bind"):
 375         if ini.has_option("status", "user") and ini.has_option("status", "password"):
 376                 authz_cfg=authz.Authz(
 377                         # change any of these to True to enable; see the manual for more
 378                         # options
 379                         auth=auth.BasicAuth([(ini.get("status", "user"), ini.get("status", "password"))]),
 380                         gracefulShutdown = 'auth',
 381                         forceBuild = 'auth', # use this to test your slave once it is set up
 382                         forceAllBuilds = 'auth',
 383                         pingBuilder = False,
 384                         stopBuild = 'auth',
 385                         stopAllBuilds = 'auth',
 386                         cancelPendingBuild = 'auth',
 387                 )
 388                 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind"), authz=authz_cfg))
 389         else:
 390                 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind")))
 391
 392 ####### PROJECT IDENTITY
 393
 394 # the 'title' string will appear at the top of this buildbot
 395 # installation's html.WebStatus home page (linked to the
 396 # 'titleURL') and is embedded in the title of the waterfall HTML page.
 397
 398 c['title'] = ini.get("general", "title")
 399 c['titleURL'] = ini.get("general", "title_url")
 400
 401 # the 'buildbotURL' string should point to the location where the buildbot's
 402 # internal web server (usually the html.WebStatus page) is visible. This
 403 # typically uses the port number set in the Waterfall 'status' entry, but
 404 # with an externally-visible host name which the buildbot cannot figure out
 405 # without some help.
 406
 407 c['buildbotURL'] = buildbot_url
 408
 409 ####### DB URL
 410
 411 c['db'] = {
 412         # This specifies what database buildbot uses to store its state.  You can leave
 413         # this at its default for all but the largest installations.
 414         'db_url' : "sqlite:///state.sqlite",
 415 }