phase2/master.cfg

   1 # -*- python -*-
   2 # ex: set syntax=python:
   3
   4 import os
   5 import re
   6 import subprocess
   7 import ConfigParser
   8
   9 from buildbot import locks
  10
  11 ini = ConfigParser.ConfigParser()
  12 ini.read("./config.ini")
  13
  14 buildbot_url = ini.get("general", "buildbot_url")
  15
  16 # This is a sample buildmaster config file. It must be installed as
  17 # 'master.cfg' in your buildmaster's base directory.
  18
  19 # This is the dictionary that the buildmaster pays attention to. We also use
  20 # a shorter alias to save typing.
  21 c = BuildmasterConfig = {}
  22
  23 ####### BUILDSLAVES
  24
  25 # The 'slaves' list defines the set of recognized buildslaves. Each element is
  26 # a BuildSlave object, specifying a unique slave name and password.  The same
  27 # slave name and password must be configured on the slave.
  28 from buildbot.buildslave import BuildSlave
  29
  30 slave_port = 9990
  31
  32 if ini.has_option("general", "port"):
  33         slave_port = ini.getint("general", "port")
  34
  35 c['slaves'] = []
  36
  37 for section in ini.sections():
  38         if section.startswith("slave "):
  39                 if ini.has_option(section, "name") and ini.has_option(section, "password"):
  40                         name = ini.get(section, "name")
  41                         password = ini.get(section, "password")
  42                         max_builds = 1
  43                         if ini.has_option(section, "builds"):
  44                                 max_builds = ini.getint(section, "builds")
  45                         c['slaves'].append(BuildSlave(name, password, max_builds = max_builds))
  46
  47 # 'slavePortnum' defines the TCP port to listen on for connections from slaves.
  48 # This must match the value configured into the buildslaves (with their
  49 # --master option)
  50 c['slavePortnum'] = slave_port
  51
  52 # coalesce builds
  53 c['mergeRequests'] = True
  54
  55 ####### CHANGESOURCES
  56
  57 home_dir = os.path.abspath(ini.get("general", "homedir"))
  58
  59 rsync_bin_url = ini.get("rsync", "binary_url")
  60 rsync_bin_key = ini.get("rsync", "binary_password")
  61
  62 rsync_src_url = None
  63 rsync_src_key = None
  64
  65 if ini.has_option("rsync", "source_url"):
  66         rsync_src_url = ini.get("rsync", "source_url")
  67         rsync_src_key = ini.get("rsync", "source_password")
  68
  69 rsync_sdk_url = None
  70 rsync_sdk_key = None
  71 rsync_sdk_pat = "lede-sdk-*.tar.xz"
  72
  73 if ini.has_option("rsync", "sdk_url"):
  74         rsync_sdk_url = ini.get("rsync", "sdk_url")
  75
  76 if ini.has_option("rsync", "sdk_password"):
  77         rsync_sdk_key = ini.get("rsync", "sdk_password")
  78
  79 if ini.has_option("rsync", "sdk_pattern"):
  80         rsync_sdk_pat = ini.get("rsync", "sdk_pattern")
  81
  82 gpg_keyid = None
  83 gpg_comment = "Unattended build signature"
  84 gpg_passfile = "/dev/null"
  85
  86 if ini.has_option("gpg", "keyid"):
  87         gpg_keyid = ini.get("gpg", "keyid")
  88
  89 if ini.has_option("gpg", "comment"):
  90         gpg_comment = ini.get("gpg", "comment")
  91
  92 if ini.has_option("gpg", "passfile"):
  93         gpg_passfile = ini.get("gpg", "passfile")
  94
  95
  96 # find arches
  97 arches = [ ]
  98 archnames = [ ]
  99
 100 findarches = subprocess.Popen([home_dir+'/dumpinfo.pl', 'architectures'],
 101         stdout = subprocess.PIPE, cwd = home_dir+'/source.git')
 102
 103 while True:
 104         line = findarches.stdout.readline()
 105         if not line:
 106                 break
 107         at = line.strip().split()
 108         arches.append(at)
 109         archnames.append(at[0])
 110
 111
 112 # find feeds
 113 feeds = []
 114
 115 from buildbot.changes.gitpoller import GitPoller
 116 c['change_source'] = []
 117
 118 with open(home_dir+'/source.git/feeds.conf.default', 'r') as f:
 119         for line in f:
 120                 parts = line.strip().split()
 121                 if parts[0] == "src-git":
 122                         feeds.append(parts)
 123                         c['change_source'].append(GitPoller(parts[2], workdir='%s/%s.git' %(os.getcwd(), parts[1]), branch='master', pollinterval=300))
 124
 125
 126 ####### SCHEDULERS
 127
 128 # Configure the Schedulers, which decide how to react to incoming changes.  In this
 129 # case, just kick off a 'basebuild' build
 130
 131 from buildbot.schedulers.basic import SingleBranchScheduler
 132 from buildbot.schedulers.forcesched import ForceScheduler
 133 from buildbot.changes import filter
 134 c['schedulers'] = []
 135 c['schedulers'].append(SingleBranchScheduler(
 136         name="all",
 137         change_filter=filter.ChangeFilter(branch='master'),
 138         treeStableTimer=60,
 139         builderNames=archnames))
 140
 141 c['schedulers'].append(ForceScheduler(
 142         name="force",
 143         builderNames=archnames))
 144
 145 ####### BUILDERS
 146
 147 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
 148 # what steps, and which slaves can execute them.  Note that any particular build will
 149 # only take place on one slave.
 150
 151 from buildbot.process.factory import BuildFactory
 152 from buildbot.steps.source import Git
 153 from buildbot.steps.shell import ShellCommand
 154 from buildbot.steps.shell import SetProperty
 155 from buildbot.steps.transfer import FileUpload
 156 from buildbot.steps.transfer import FileDownload
 157 from buildbot.steps.master import MasterShellCommand
 158 from buildbot.process.properties import WithProperties
 159
 160 c['builders'] = []
 161
 162 dlLock = locks.SlaveLock("slave_dl")
 163
 164 slaveNames = [ ]
 165
 166 for slave in c['slaves']:
 167         slaveNames.append(slave.slavename)
 168
 169 for arch in arches:
 170         ts = arch[1].split('/')
 171
 172         factory = BuildFactory()
 173
 174         # find number of cores
 175         factory.addStep(SetProperty(
 176                 name = "nproc",
 177                 property = "nproc",
 178                 description = "Finding number of CPUs",
 179                 command = ["nproc"]))
 180
 181         # prepare workspace
 182         factory.addStep(FileDownload(mastersrc="cleanup.sh", slavedest="cleanup.sh", mode=0755))
 183
 184         factory.addStep(ShellCommand(
 185                 name = "cleanold",
 186                 description = "Cleaning previous builds",
 187                 command = ["./cleanup.sh", buildbot_url, WithProperties("%(slavename)s"), WithProperties("%(buildername)s"), "full"],
 188                 haltOnFailure = True,
 189                 timeout = 2400))
 190
 191         factory.addStep(ShellCommand(
 192                 name = "cleanup",
 193                 description = "Cleaning work area",
 194                 command = ["./cleanup.sh", buildbot_url, WithProperties("%(slavename)s"), WithProperties("%(buildername)s"), "single"],
 195                 haltOnFailure = True,
 196                 timeout = 2400))
 197
 198         factory.addStep(ShellCommand(
 199                 name = "mksdkdir",
 200                 description = "Preparing SDK directory",
 201                 command = ["mkdir", "sdk"],
 202                 haltOnFailure = True))
 203
 204         factory.addStep(ShellCommand(
 205                 name = "downloadsdk",
 206                 description = "Downloading SDK archive",
 207                 command = ["rsync", "-va", "%s/%s/%s/%s" %(rsync_sdk_url, ts[0], ts[1], rsync_sdk_pat), "sdk.archive"],
 208                 env={'RSYNC_PASSWORD': rsync_sdk_key},
 209                 haltOnFailure = True,
 210                 logEnviron = False))
 211
 212         factory.addStep(ShellCommand(
 213                 name = "unpacksdk",
 214                 description = "Unpacking SDK archive",
 215                 command = ["tar", "--strip-components=1", "-C", "sdk/", "-vxf", "sdk.archive"],
 216                 haltOnFailure = True))
 217
 218         factory.addStep(FileDownload(mastersrc=home_dir+'/key-build', slavedest="sdk/key-build", mode=0600))
 219         factory.addStep(FileDownload(mastersrc=home_dir+'/key-build.pub', slavedest="sdk/key-build.pub", mode=0600))
 220
 221         factory.addStep(ShellCommand(
 222                 name = "mkdldir",
 223                 description = "Preparing download directory",
 224                 command = ["sh", "-c", "mkdir -p $HOME/dl && rm -rf ./sdk/dl && ln -sf $HOME/dl ./sdk/dl"]))
 225
 226         factory.addStep(ShellCommand(
 227                 name = "mkconf",
 228                 description = "Preparing SDK configuration",
 229                 workdir = "build/sdk",
 230                 command = ["sh", "-c", "rm -f .config && make defconfig"]))
 231
 232         factory.addStep(ShellCommand(
 233                 name = "updatefeeds",
 234                 description = "Updating feeds",
 235                 workdir = "build/sdk",
 236                 command = ["./scripts/feeds", "update"]))
 237
 238         factory.addStep(ShellCommand(
 239                 name = "installfeeds",
 240                 description = "Installing feeds",
 241                 workdir = "build/sdk",
 242                 command = ["./scripts/feeds", "install", "-a"]))
 243
 244         factory.addStep(ShellCommand(
 245                 name = "compile",
 246                 description = "Building packages",
 247                 workdir = "build/sdk",
 248                 command = ["make", WithProperties("-j%(nproc:~4)s"), "V=s", "IGNORE_ERRORS=n m y", "BUILD_LOG=1", "CONFIG_SIGNED_PACKAGES=y"]))
 249
 250         if gpg_keyid is not None:
 251                 factory.addStep(MasterShellCommand(
 252                         name = "signprepare",
 253                         description = "Preparing temporary signing directory",
 254                         command = ["mkdir", "-p", "%s/signing" %(home_dir)],
 255                         haltOnFailure = True
 256                 ))
 257
 258                 factory.addStep(ShellCommand(
 259                         name = "signpack",
 260                         description = "Packing files to sign",
 261                         workdir = "build/sdk",
 262                         command = "find bin/packages/%s/ -mindepth 2 -maxdepth 2 -type f -name Packages -print0 | xargs -0 tar -czf sign.tar.gz" %(arch[0]),
 263                         haltOnFailure = True
 264                 ))
 265
 266                 factory.addStep(FileUpload(
 267                         slavesrc = "sdk/sign.tar.gz",
 268                         masterdest = "%s/signing/%s.tar.gz" %(home_dir, arch[0]),
 269                         haltOnFailure = True
 270                 ))
 271
 272                 factory.addStep(MasterShellCommand(
 273                         name = "signfiles",
 274                         description = "Signing files",
 275                         command = ["%s/signall.sh" %(home_dir), "%s/signing/%s.tar.gz" %(home_dir, arch[0]), gpg_keyid, gpg_passfile, gpg_comment],
 276                         haltOnFailure = True
 277                 ))
 278
 279                 factory.addStep(FileDownload(
 280                         mastersrc = "%s/signing/%s.tar.gz" %(home_dir, arch[0]),
 281                         slavedest = "sdk/sign.tar.gz",
 282                         haltOnFailure = True
 283                 ))
 284
 285                 factory.addStep(ShellCommand(
 286                         name = "signunpack",
 287                         description = "Unpacking signed files",
 288                         workdir = "build/sdk",
 289                         command = ["tar", "-xzf", "sign.tar.gz"],
 290                         haltOnFailure = True
 291                 ))
 292
 293         factory.addStep(ShellCommand(
 294                 name = "uploadprepare",
 295                 description = "Preparing package directory",
 296                 workdir = "build/sdk",
 297                 command = ["rsync", "-av", "--include", "/%s/" %(arch[0]), "--exclude", "/*", "--exclude", "/%s/*" %(arch[0]), "bin/packages/", "%s/packages/" %(rsync_bin_url)],
 298                 env={'RSYNC_PASSWORD': rsync_bin_key},
 299                 haltOnFailure = True,
 300                 logEnviron = False
 301         ))
 302
 303         factory.addStep(ShellCommand(
 304                 name = "packageupload",
 305                 description = "Uploading package files",
 306                 workdir = "build/sdk",
 307                 command = ["rsync", "--delete", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-avz", "bin/packages/%s/" %(arch[0]), "%s/packages/%s/" %(rsync_bin_url, arch[0])],
 308                 env={'RSYNC_PASSWORD': rsync_bin_key},
 309                 haltOnFailure = True,
 310                 logEnviron = False
 311         ))
 312
 313         factory.addStep(ShellCommand(
 314                 name = "logprepare",
 315                 description = "Preparing log directory",
 316                 workdir = "build/sdk",
 317                 command = ["rsync", "-av", "--include", "/%s/" %(arch[0]), "--exclude", "/*", "--exclude", "/%s/*" %(arch[0]), "bin/packages/", "%s/faillogs/" %(rsync_bin_url)],
 318                 env={'RSYNC_PASSWORD': rsync_bin_key},
 319                 haltOnFailure = True,
 320                 logEnviron = False
 321         ))
 322
 323         factory.addStep(ShellCommand(
 324                 name = "logfind",
 325                 description = "Finding failure logs",
 326                 workdir = "build/sdk/logs/package/feeds",
 327                 command = ["sh", "-c", "sed -ne 's!^ *ERROR: package/feeds/\\([^ ]*\\) .*$!\\1!p' ../error.txt | sort -u | xargs -r find > ../../../logs.txt"],
 328                 haltOnFailure = False
 329         ))
 330
 331         factory.addStep(ShellCommand(
 332                 name = "logcollect",
 333                 description = "Collecting failure logs",
 334                 workdir = "build/sdk",
 335                 command = ["rsync", "-av", "--files-from=logs.txt", "logs/package/feeds/", "faillogs/"],
 336                 haltOnFailure = False
 337         ))
 338
 339         factory.addStep(ShellCommand(
 340                 name = "logupload",
 341                 description = "Uploading failure logs",
 342                 workdir = "build/sdk",
 343                 command = ["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-avz", "faillogs/", "%s/faillogs/%s/" %(rsync_bin_url, arch[0])],
 344                 env={'RSYNC_PASSWORD': rsync_bin_key},
 345                 haltOnFailure = False,
 346                 logEnviron = False
 347         ))
 348
 349         if rsync_src_url is not None:
 350                 factory.addStep(ShellCommand(
 351                         name = "sourceupload",
 352                         description = "Uploading source archives",
 353                         workdir = "build/sdk",
 354                         command = ["rsync", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-avz", "dl/", "%s/" %(rsync_src_url)],
 355                         env={'RSYNC_PASSWORD': rsync_src_key},
 356                         haltOnFailure = False,
 357                         logEnviron = False
 358                 ))
 359
 360         from buildbot.config import BuilderConfig
 361
 362         c['builders'].append(BuilderConfig(name=arch[0], slavenames=slaveNames, factory=factory))
 363
 364
 365 ####### STATUS arches
 366
 367 # 'status' is a list of Status arches. The results of each build will be
 368 # pushed to these arches. buildbot/status/*.py has a variety to choose from,
 369 # including web pages, email senders, and IRC bots.
 370
 371 c['status'] = []
 372
 373 from buildbot.status import html
 374 from buildbot.status.web import authz, auth
 375
 376 if ini.has_option("status", "bind"):
 377         if ini.has_option("status", "user") and ini.has_option("status", "password"):
 378                 authz_cfg=authz.Authz(
 379                         # change any of these to True to enable; see the manual for more
 380                         # options
 381                         auth=auth.BasicAuth([(ini.get("status", "user"), ini.get("status", "password"))]),
 382                         gracefulShutdown = 'auth',
 383                         forceBuild = 'auth', # use this to test your slave once it is set up
 384                         forceAllBuilds = 'auth',
 385                         pingBuilder = False,
 386                         stopBuild = 'auth',
 387                         stopAllBuilds = 'auth',
 388                         cancelPendingBuild = 'auth',
 389                 )
 390                 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind"), authz=authz_cfg))
 391         else:
 392                 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind")))
 393
 394 ####### PROJECT IDENTITY
 395
 396 # the 'title' string will appear at the top of this buildbot
 397 # installation's html.WebStatus home page (linked to the
 398 # 'titleURL') and is embedded in the title of the waterfall HTML page.
 399
 400 c['title'] = ini.get("general", "title")
 401 c['titleURL'] = ini.get("general", "title_url")
 402
 403 # the 'buildbotURL' string should point to the location where the buildbot's
 404 # internal web server (usually the html.WebStatus page) is visible. This
 405 # typically uses the port number set in the Waterfall 'status' entry, but
 406 # with an externally-visible host name which the buildbot cannot figure out
 407 # without some help.
 408
 409 c['buildbotURL'] = buildbot_url
 410
 411 ####### DB URL
 412
 413 c['db'] = {
 414         # This specifies what database buildbot uses to store its state.  You can leave
 415         # this at its default for all but the largest installations.
 416         'db_url' : "sqlite:///state.sqlite",
 417 }