phase2/master.cfg

   1 # -*- python -*-
   2 # ex: set syntax=python:
   3
   4 import os
   5 import re
   6 import subprocess
   7 import ConfigParser
   8
   9 from buildbot import locks
  10
  11 ini = ConfigParser.ConfigParser()
  12 ini.read("./config.ini")
  13
  14 # This is a sample buildmaster config file. It must be installed as
  15 # 'master.cfg' in your buildmaster's base directory.
  16
  17 # This is the dictionary that the buildmaster pays attention to. We also use
  18 # a shorter alias to save typing.
  19 c = BuildmasterConfig = {}
  20
  21 ####### BUILDSLAVES
  22
  23 # The 'slaves' list defines the set of recognized buildslaves. Each element is
  24 # a BuildSlave object, specifying a unique slave name and password.  The same
  25 # slave name and password must be configured on the slave.
  26 from buildbot.buildslave import BuildSlave
  27
  28 slave_port = 9990
  29
  30 if ini.has_option("general", "port"):
  31         slave_port = ini.getint("general", "port")
  32
  33 c['slaves'] = []
  34
  35 for section in ini.sections():
  36         if section.startswith("slave "):
  37                 if ini.has_option(section, "name") and ini.has_option(section, "password"):
  38                         name = ini.get(section, "name")
  39                         password = ini.get(section, "password")
  40                         max_builds = 1
  41                         if ini.has_option(section, "builds"):
  42                                 max_builds = ini.getint(section, "builds")
  43                         c['slaves'].append(BuildSlave(name, password, max_builds = max_builds))
  44
  45 # 'slavePortnum' defines the TCP port to listen on for connections from slaves.
  46 # This must match the value configured into the buildslaves (with their
  47 # --master option)
  48 c['slavePortnum'] = slave_port
  49
  50 # coalesce builds
  51 c['mergeRequests'] = True
  52
  53 ####### CHANGESOURCES
  54
  55 home_dir = os.path.abspath(ini.get("general", "homedir"))
  56
  57 rsync_bin_url = ini.get("rsync", "binary_url")
  58 rsync_bin_key = ini.get("rsync", "binary_password")
  59
  60 rsync_src_url = None
  61 rsync_src_key = None
  62
  63 if ini.has_option("rsync", "source_url"):
  64         rsync_src_url = ini.get("rsync", "source_url")
  65         rsync_src_key = ini.get("rsync", "source_password")
  66
  67 rsync_sdk_url = None
  68 rsync_sdk_key = None
  69 rsync_sdk_pat = "lede-sdk-*.tar.xz"
  70
  71 if ini.has_option("rsync", "sdk_url"):
  72         rsync_sdk_url = ini.get("rsync", "sdk_url")
  73         rsync_sdk_key = ini.get("rsync", "sdk_password")
  74
  75 if ini.has_option("rsync", "sdk_pattern"):
  76         rsync_sdk_pat = ini.get("rsync", "sdk_pattern")
  77
  78 gpg_keyid = None
  79 gpg_comment = "Unattended build signature"
  80 gpg_passfile = "/dev/null"
  81
  82 if ini.has_option("gpg", "keyid"):
  83         gpg_keyid = ini.get("gpg", "keyid")
  84
  85 if ini.has_option("gpg", "comment"):
  86         gpg_comment = ini.get("gpg", "comment")
  87
  88 if ini.has_option("gpg", "passfile"):
  89         gpg_passfile = ini.get("gpg", "passfile")
  90
  91
  92 # find arches
  93 arches = [ ]
  94 archnames = [ ]
  95
  96 findarches = subprocess.Popen([home_dir+'/dumpinfo.pl', 'architectures'],
  97         stdout = subprocess.PIPE, cwd = home_dir+'/source.git')
  98
  99 while True:
 100         line = findarches.stdout.readline()
 101         if not line:
 102                 break
 103         at = line.strip().split()
 104         arches.append(at)
 105         archnames.append(at[0])
 106
 107
 108 # find feeds
 109 feeds = []
 110
 111 from buildbot.changes.gitpoller import GitPoller
 112 c['change_source'] = []
 113
 114 with open(home_dir+'/source.git/feeds.conf.default', 'r') as f:
 115         for line in f:
 116                 parts = line.strip().split()
 117                 if parts[0] == "src-git":
 118                         feeds.append(parts)
 119                         c['change_source'].append(GitPoller(parts[2], workdir='%s/%s.git' %(os.getcwd(), parts[1]), branch='master', pollinterval=300))
 120
 121
 122 ####### SCHEDULERS
 123
 124 # Configure the Schedulers, which decide how to react to incoming changes.  In this
 125 # case, just kick off a 'basebuild' build
 126
 127 from buildbot.schedulers.basic import SingleBranchScheduler
 128 from buildbot.schedulers.forcesched import ForceScheduler
 129 from buildbot.changes import filter
 130 c['schedulers'] = []
 131 c['schedulers'].append(SingleBranchScheduler(
 132         name="all",
 133         change_filter=filter.ChangeFilter(branch='master'),
 134         treeStableTimer=60,
 135         builderNames=archnames))
 136
 137 c['schedulers'].append(ForceScheduler(
 138         name="force",
 139         builderNames=archnames))
 140
 141 ####### BUILDERS
 142
 143 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
 144 # what steps, and which slaves can execute them.  Note that any particular build will
 145 # only take place on one slave.
 146
 147 from buildbot.process.factory import BuildFactory
 148 from buildbot.steps.source import Git
 149 from buildbot.steps.shell import ShellCommand
 150 from buildbot.steps.shell import SetProperty
 151 from buildbot.steps.transfer import FileUpload
 152 from buildbot.steps.transfer import FileDownload
 153 from buildbot.steps.master import MasterShellCommand
 154 from buildbot.process.properties import WithProperties
 155
 156 c['builders'] = []
 157
 158 dlLock = locks.SlaveLock("slave_dl")
 159
 160 slaveNames = [ ]
 161
 162 for slave in c['slaves']:
 163         slaveNames.append(slave.slavename)
 164
 165 for arch in arches:
 166         ts = arch[1].split('/')
 167
 168         factory = BuildFactory()
 169
 170         # find number of cores
 171         factory.addStep(SetProperty(
 172                 name = "nproc",
 173                 property = "nproc",
 174                 description = "Finding number of CPUs",
 175                 command = ["nproc"]))
 176
 177         # prepare workspace
 178         factory.addStep(FileDownload(mastersrc="cleanup.sh", slavedest="cleanup.sh", mode=0755))
 179
 180         factory.addStep(ShellCommand(
 181                 name = "cleanold",
 182                 description = "Cleaning previous builds",
 183                 command = ["./cleanup.sh", WithProperties("%(slavename)s"), WithProperties("%(buildername)s"), "full"],
 184                 haltOnFailure = True,
 185                 timeout = 2400))
 186
 187         factory.addStep(ShellCommand(
 188                 name = "cleanup",
 189                 description = "Cleaning work area",
 190                 command = ["./cleanup.sh", WithProperties("%(slavename)s"), WithProperties("%(buildername)s"), "single"],
 191                 haltOnFailure = True,
 192                 timeout = 2400))
 193
 194         factory.addStep(ShellCommand(
 195                 name = "mksdkdir",
 196                 description = "Preparing SDK directory",
 197                 command = ["mkdir", "sdk"],
 198                 haltOnFailure = True))
 199
 200         factory.addStep(ShellCommand(
 201                 name = "downloadsdk",
 202                 description = "Downloading SDK archive",
 203                 command = ["rsync", "-va", "%s/%s/%s/%s" %(rsync_sdk_url, ts[0], ts[1], rsync_sdk_pat), "sdk.archive"],
 204                 env={'RSYNC_PASSWORD': rsync_sdk_key},
 205                 haltOnFailure = True,
 206                 logEnviron = False))
 207
 208         factory.addStep(ShellCommand(
 209                 name = "unpacksdk",
 210                 description = "Unpacking SDK archive",
 211                 command = ["tar", "--strip-components=1", "-C", "sdk/", "-vxf", "sdk.archive"],
 212                 haltOnFailure = True))
 213
 214         factory.addStep(FileDownload(mastersrc=home_dir+'/key-build', slavedest="sdk/key-build", mode=0600))
 215         factory.addStep(FileDownload(mastersrc=home_dir+'/key-build.pub', slavedest="sdk/key-build.pub", mode=0600))
 216
 217         factory.addStep(ShellCommand(
 218                 name = "mkdldir",
 219                 description = "Preparing download directory",
 220                 command = ["sh", "-c", "mkdir -p $HOME/dl && rmdir ./sdk/dl && ln -sf $HOME/dl ./sdk/dl"]))
 221
 222         factory.addStep(ShellCommand(
 223                 name = "mkconf",
 224                 description = "Preparing SDK configuration",
 225                 workdir = "build/sdk",
 226                 command = ["sh", "-c", "rm -f .config && make defconfig"]))
 227
 228         factory.addStep(ShellCommand(
 229                 name = "updatefeeds",
 230                 description = "Updating feeds",
 231                 workdir = "build/sdk",
 232                 command = ["./scripts/feeds", "update"]))
 233
 234         factory.addStep(ShellCommand(
 235                 name = "installfeeds",
 236                 description = "Installing feeds",
 237                 workdir = "build/sdk",
 238                 command = ["./scripts/feeds", "install", "-a"]))
 239
 240         factory.addStep(ShellCommand(
 241                 name = "compile",
 242                 description = "Building packages",
 243                 workdir = "build/sdk",
 244                 command = ["make", WithProperties("-j%(nproc:~4)s"), "V=s", "IGNORE_ERRORS=n m y", "BUILD_LOG=1", "CONFIG_SIGNED_PACKAGES=y"]))
 245
 246         if gpg_keyid is not None:
 247                 factory.addStep(MasterShellCommand(
 248                         name = "signprepare",
 249                         description = "Preparing temporary signing directory",
 250                         command = ["mkdir", "-p", "%s/signing" %(home_dir)],
 251                         haltOnFailure = True
 252                 ))
 253
 254                 factory.addStep(ShellCommand(
 255                         name = "signpack",
 256                         description = "Packing files to sign",
 257                         workdir = "build/sdk",
 258                         command = "find bin/packages/%s/ -mindepth 2 -maxdepth 2 -type f -name Packages -print0 | xargs -0 tar -czf sign.tar.gz" %(arch[0]),
 259                         haltOnFailure = True
 260                 ))
 261
 262                 factory.addStep(FileUpload(
 263                         slavesrc = "sdk/sign.tar.gz",
 264                         masterdest = "%s/signing/%s.tar.gz" %(home_dir, arch[0]),
 265                         haltOnFailure = True
 266                 ))
 267
 268                 factory.addStep(MasterShellCommand(
 269                         name = "signfiles",
 270                         description = "Signing files",
 271                         command = ["%s/signall.sh" %(home_dir), "%s/signing/%s.tar.gz" %(home_dir, arch[0]), gpg_keyid, gpg_passfile, gpg_comment],
 272                         haltOnFailure = True
 273                 ))
 274
 275                 factory.addStep(FileDownload(
 276                         mastersrc = "%s/signing/%s.tar.gz" %(home_dir, arch[0]),
 277                         slavedest = "sdk/sign.tar.gz",
 278                         haltOnFailure = True
 279                 ))
 280
 281                 factory.addStep(ShellCommand(
 282                         name = "signunpack",
 283                         description = "Unpacking signed files",
 284                         workdir = "build/sdk",
 285                         command = ["tar", "-xzf", "sign.tar.gz"],
 286                         haltOnFailure = True
 287                 ))
 288
 289         factory.addStep(ShellCommand(
 290                 name = "uploadprepare",
 291                 description = "Preparing package directory",
 292                 workdir = "build/sdk",
 293                 command = ["rsync", "-av", "--include", "/%s/" %(arch[0]), "--exclude", "/*", "--exclude", "/%s/*" %(arch[0]), "bin/packages/", "%s/packages/" %(rsync_bin_url)],
 294                 env={'RSYNC_PASSWORD': rsync_bin_key},
 295                 haltOnFailure = True,
 296                 logEnviron = False
 297         ))
 298
 299         factory.addStep(ShellCommand(
 300                 name = "packageupload",
 301                 description = "Uploading package files",
 302                 workdir = "build/sdk",
 303                 command = ["rsync", "--delete", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-avz", "bin/packages/%s/" %(arch[0]), "%s/packages/%s/" %(rsync_bin_url, arch[0])],
 304                 env={'RSYNC_PASSWORD': rsync_bin_key},
 305                 haltOnFailure = True,
 306                 logEnviron = False
 307         ))
 308
 309         factory.addStep(ShellCommand(
 310                 name = "logprepare",
 311                 description = "Preparing log directory",
 312                 workdir = "build/sdk",
 313                 command = ["rsync", "-av", "--include", "/%s/" %(arch[0]), "--exclude", "/*", "--exclude", "/%s/*" %(arch[0]), "bin/packages/", "%s/faillogs/" %(rsync_bin_url)],
 314                 env={'RSYNC_PASSWORD': rsync_bin_key},
 315                 haltOnFailure = True,
 316                 logEnviron = False
 317         ))
 318
 319         factory.addStep(ShellCommand(
 320                 name = "logfind",
 321                 description = "Finding failure logs",
 322                 workdir = "build/sdk/logs/package/feeds",
 323                 command = ["sh", "-c", "sed -ne 's!^ *ERROR: package/feeds/\\([^ ]*\\) .*$!\\1!p' ../error.txt | sort -u | xargs -r find > ../../../logs.txt"],
 324                 haltOnFailure = False
 325         ))
 326
 327         factory.addStep(ShellCommand(
 328                 name = "logcollect",
 329                 description = "Collecting failure logs",
 330                 workdir = "build/sdk",
 331                 command = ["rsync", "-av", "--files-from=logs.txt", "logs/package/feeds/", "faillogs/"],
 332                 haltOnFailure = False
 333         ))
 334
 335         factory.addStep(ShellCommand(
 336                 name = "logupload",
 337                 description = "Uploading failure logs",
 338                 workdir = "build/sdk",
 339                 command = ["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-avz", "faillogs/", "%s/faillogs/%s/" %(rsync_bin_url, arch[0])],
 340                 env={'RSYNC_PASSWORD': rsync_bin_key},
 341                 haltOnFailure = False,
 342                 logEnviron = False
 343         ))
 344
 345         if rsync_src_url is not None:
 346                 factory.addStep(ShellCommand(
 347                         name = "sourceupload",
 348                         description = "Uploading source archives",
 349                         workdir = "build/sdk",
 350                         command = ["rsync", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-avz", "dl/", "%s/" %(rsync_src_url)],
 351                         env={'RSYNC_PASSWORD': rsync_src_key},
 352                         haltOnFailure = False,
 353                         logEnviron = False
 354                 ))
 355
 356         from buildbot.config import BuilderConfig
 357
 358         c['builders'].append(BuilderConfig(name=arch[0], slavenames=slaveNames, factory=factory))
 359
 360
 361 ####### STATUS arches
 362
 363 # 'status' is a list of Status arches. The results of each build will be
 364 # pushed to these arches. buildbot/status/*.py has a variety to choose from,
 365 # including web pages, email senders, and IRC bots.
 366
 367 c['status'] = []
 368
 369 from buildbot.status import html
 370 from buildbot.status.web import authz, auth
 371
 372 if ini.has_option("status", "bind"):
 373         if ini.has_option("status", "user") and ini.has_option("status", "password"):
 374                 authz_cfg=authz.Authz(
 375                         # change any of these to True to enable; see the manual for more
 376                         # options
 377                         auth=auth.BasicAuth([(ini.get("status", "user"), ini.get("status", "password"))]),
 378                         gracefulShutdown = 'auth',
 379                         forceBuild = 'auth', # use this to test your slave once it is set up
 380                         forceAllBuilds = 'auth',
 381                         pingBuilder = False,
 382                         stopBuild = 'auth',
 383                         stopAllBuilds = 'auth',
 384                         cancelPendingBuild = 'auth',
 385                 )
 386                 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind"), authz=authz_cfg))
 387         else:
 388                 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind")))
 389
 390 ####### PROJECT IDENTITY
 391
 392 # the 'title' string will appear at the top of this buildbot
 393 # installation's html.WebStatus home page (linked to the
 394 # 'titleURL') and is embedded in the title of the waterfall HTML page.
 395
 396 c['title'] = ini.get("general", "title")
 397 c['titleURL'] = ini.get("general", "title_url")
 398
 399 # the 'buildbotURL' string should point to the location where the buildbot's
 400 # internal web server (usually the html.WebStatus page) is visible. This
 401 # typically uses the port number set in the Waterfall 'status' entry, but
 402 # with an externally-visible host name which the buildbot cannot figure out
 403 # without some help.
 404
 405 c['buildbotURL'] = ini.get("general", "buildbot_url")
 406
 407 ####### DB URL
 408
 409 c['db'] = {
 410         # This specifies what database buildbot uses to store its state.  You can leave
 411         # this at its default for all but the largest installations.
 412         'db_url' : "sqlite:///state.sqlite",
 413 }