phase2/master.cfg

   1 # -*- python -*-
   2 # ex: set syntax=python:
   3
   4 import os
   5 import re
   6 import subprocess
   7 import ConfigParser
   8
   9 from buildbot import locks
  10
  11 ini = ConfigParser.ConfigParser()
  12 ini.read("./config.ini")
  13
  14 buildbot_url = ini.get("general", "buildbot_url")
  15
  16 # This is a sample buildmaster config file. It must be installed as
  17 # 'master.cfg' in your buildmaster's base directory.
  18
  19 # This is the dictionary that the buildmaster pays attention to. We also use
  20 # a shorter alias to save typing.
  21 c = BuildmasterConfig = {}
  22
  23 ####### BUILDSLAVES
  24
  25 # The 'slaves' list defines the set of recognized buildslaves. Each element is
  26 # a BuildSlave object, specifying a unique slave name and password.  The same
  27 # slave name and password must be configured on the slave.
  28 from buildbot.buildslave import BuildSlave
  29
  30 slave_port = 9990
  31 persistent = False
  32
  33 if ini.has_option("general", "port"):
  34         slave_port = ini.getint("general", "port")
  35
  36 if ini.has_option("general", "persistent"):
  37         persistent = ini.getboolean("general", "persistent")
  38
  39 c['slaves'] = []
  40
  41 for section in ini.sections():
  42         if section.startswith("slave "):
  43                 if ini.has_option(section, "name") and ini.has_option(section, "password"):
  44                         name = ini.get(section, "name")
  45                         password = ini.get(section, "password")
  46                         max_builds = 1
  47                         if ini.has_option(section, "builds"):
  48                                 max_builds = ini.getint(section, "builds")
  49                         c['slaves'].append(BuildSlave(name, password, max_builds = max_builds))
  50
  51 # 'slavePortnum' defines the TCP port to listen on for connections from slaves.
  52 # This must match the value configured into the buildslaves (with their
  53 # --master option)
  54 c['slavePortnum'] = slave_port
  55
  56 # coalesce builds
  57 c['mergeRequests'] = True
  58
  59 ####### CHANGESOURCES
  60
  61 home_dir = os.path.abspath(ini.get("general", "homedir"))
  62
  63 rsync_bin_url = ini.get("rsync", "binary_url")
  64 rsync_bin_key = ini.get("rsync", "binary_password")
  65
  66 rsync_src_url = None
  67 rsync_src_key = None
  68
  69 if ini.has_option("rsync", "source_url"):
  70         rsync_src_url = ini.get("rsync", "source_url")
  71         rsync_src_key = ini.get("rsync", "source_password")
  72
  73 rsync_sdk_url = None
  74 rsync_sdk_key = None
  75 rsync_sdk_pat = "lede-sdk-*.tar.xz"
  76
  77 if ini.has_option("rsync", "sdk_url"):
  78         rsync_sdk_url = ini.get("rsync", "sdk_url")
  79
  80 if ini.has_option("rsync", "sdk_password"):
  81         rsync_sdk_key = ini.get("rsync", "sdk_password")
  82
  83 if ini.has_option("rsync", "sdk_pattern"):
  84         rsync_sdk_pat = ini.get("rsync", "sdk_pattern")
  85
  86 gpg_home = "~/.gnupg"
  87 gpg_keyid = None
  88 gpg_comment = "Unattended build signature"
  89 gpg_passfile = "/dev/null"
  90
  91 if ini.has_option("gpg", "home"):
  92         gpg_home = ini.get("gpg", "home")
  93
  94 if ini.has_option("gpg", "keyid"):
  95         gpg_keyid = ini.get("gpg", "keyid")
  96
  97 if ini.has_option("gpg", "comment"):
  98         gpg_comment = ini.get("gpg", "comment")
  99
 100 if ini.has_option("gpg", "passfile"):
 101         gpg_passfile = ini.get("gpg", "passfile")
 102
 103
 104 # find arches
 105 arches = [ ]
 106 archnames = [ ]
 107
 108 findarches = subprocess.Popen([home_dir+'/dumpinfo.pl', 'architectures'],
 109         stdout = subprocess.PIPE, cwd = home_dir+'/source.git')
 110
 111 while True:
 112         line = findarches.stdout.readline()
 113         if not line:
 114                 break
 115         at = line.strip().split()
 116         arches.append(at)
 117         archnames.append(at[0])
 118
 119
 120 # find feeds
 121 feeds = []
 122
 123 from buildbot.changes.gitpoller import GitPoller
 124 c['change_source'] = []
 125
 126 with open(home_dir+'/source.git/feeds.conf.default', 'r') as f:
 127         for line in f:
 128                 parts = line.strip().split()
 129                 if parts[0] == "src-git":
 130                         feeds.append(parts)
 131                         url = parts[2].strip().split(';')
 132                         branch = url[1] if len(url) > 1 else 'master'
 133                         c['change_source'].append(GitPoller(url[0], branch=branch, workdir='%s/%s.git' %(os.getcwd(), parts[1]), pollinterval=300))
 134
 135
 136 ####### SCHEDULERS
 137
 138 # Configure the Schedulers, which decide how to react to incoming changes.  In this
 139 # case, just kick off a 'basebuild' build
 140
 141 from buildbot.schedulers.basic import SingleBranchScheduler
 142 from buildbot.schedulers.forcesched import ForceScheduler
 143 from buildbot.changes import filter
 144 c['schedulers'] = []
 145 c['schedulers'].append(SingleBranchScheduler(
 146         name="all",
 147         change_filter=filter.ChangeFilter(branch='master'),
 148         treeStableTimer=60,
 149         builderNames=archnames))
 150
 151 c['schedulers'].append(ForceScheduler(
 152         name="force",
 153         builderNames=archnames))
 154
 155 ####### BUILDERS
 156
 157 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
 158 # what steps, and which slaves can execute them.  Note that any particular build will
 159 # only take place on one slave.
 160
 161 from buildbot.process.factory import BuildFactory
 162 from buildbot.steps.source import Git
 163 from buildbot.steps.shell import ShellCommand
 164 from buildbot.steps.shell import SetProperty
 165 from buildbot.steps.transfer import FileUpload
 166 from buildbot.steps.transfer import FileDownload
 167 from buildbot.steps.master import MasterShellCommand
 168 from buildbot.process.properties import WithProperties
 169
 170
 171 def GetDirectorySuffix(props):
 172         if props.hasProperty("slavename") and re.match("^[^-]+-[0-9]+\.[0-9]+-[^-]+$", props["slavename"]):
 173                 return "-%s" % props["slavename"].split('-')[1]
 174         else:
 175                 return ""
 176
 177
 178 c['builders'] = []
 179
 180 dlLock = locks.SlaveLock("slave_dl")
 181
 182 slaveNames = [ ]
 183
 184 for slave in c['slaves']:
 185         slaveNames.append(slave.slavename)
 186
 187 for arch in arches:
 188         ts = arch[1].split('/')
 189
 190         factory = BuildFactory()
 191
 192         # find number of cores
 193         factory.addStep(SetProperty(
 194                 name = "nproc",
 195                 property = "nproc",
 196                 description = "Finding number of CPUs",
 197                 command = ["nproc"]))
 198
 199         # prepare workspace
 200         factory.addStep(FileDownload(mastersrc="cleanup.sh", slavedest="cleanup.sh", mode=0755))
 201
 202         if not persistent:
 203                 factory.addStep(ShellCommand(
 204                         name = "cleanold",
 205                         description = "Cleaning previous builds",
 206                         command = ["./cleanup.sh", buildbot_url, WithProperties("%(slavename)s"), WithProperties("%(buildername)s"), "full"],
 207                         haltOnFailure = True,
 208                         timeout = 2400))
 209
 210                 factory.addStep(ShellCommand(
 211                         name = "cleanup",
 212                         description = "Cleaning work area",
 213                         command = ["./cleanup.sh", buildbot_url, WithProperties("%(slavename)s"), WithProperties("%(buildername)s"), "single"],
 214                         haltOnFailure = True,
 215                         timeout = 2400))
 216
 217         factory.addStep(ShellCommand(
 218                 name = "mksdkdir",
 219                 description = "Preparing SDK directory",
 220                 command = ["mkdir", "-p", "sdk"],
 221                 haltOnFailure = True))
 222
 223         factory.addStep(ShellCommand(
 224                 name = "downloadsdk",
 225                 description = "Downloading SDK archive",
 226                 command = ["rsync", "-va", "%s/%s/%s/%s" %(rsync_sdk_url, ts[0], ts[1], rsync_sdk_pat), "sdk.archive"],
 227                 env={'RSYNC_PASSWORD': rsync_sdk_key},
 228                 haltOnFailure = True,
 229                 logEnviron = False))
 230
 231         factory.addStep(ShellCommand(
 232                 name = "unpacksdk",
 233                 description = "Unpacking SDK archive",
 234                 command = ["tar", "--strip-components=1", "-C", "sdk/", "-vxf", "sdk.archive"],
 235                 haltOnFailure = True))
 236
 237         factory.addStep(FileDownload(mastersrc=home_dir+'/key-build', slavedest="sdk/key-build", mode=0600))
 238         factory.addStep(FileDownload(mastersrc=home_dir+'/key-build.pub', slavedest="sdk/key-build.pub", mode=0600))
 239
 240         factory.addStep(ShellCommand(
 241                 name = "mkdldir",
 242                 description = "Preparing download directory",
 243                 command = ["sh", "-c", "mkdir -p $HOME/dl && rm -rf ./sdk/dl && ln -sf $HOME/dl ./sdk/dl"]))
 244
 245         factory.addStep(ShellCommand(
 246                 name = "mkconf",
 247                 description = "Preparing SDK configuration",
 248                 workdir = "build/sdk",
 249                 command = ["sh", "-c", "rm -f .config && make defconfig"]))
 250
 251         factory.addStep(ShellCommand(
 252                 name = "updatefeeds",
 253                 description = "Updating feeds",
 254                 workdir = "build/sdk",
 255                 command = ["./scripts/feeds", "update"]))
 256
 257         factory.addStep(ShellCommand(
 258                 name = "installfeeds",
 259                 description = "Installing feeds",
 260                 workdir = "build/sdk",
 261                 command = ["./scripts/feeds", "install", "-a"]))
 262
 263         factory.addStep(ShellCommand(
 264                 name = "compile",
 265                 description = "Building packages",
 266                 workdir = "build/sdk",
 267                 command = ["make", WithProperties("-j%(nproc:~4)s"), "V=s", "IGNORE_ERRORS=n m y", "BUILD_LOG=1", "CONFIG_SIGNED_PACKAGES=y"]))
 268
 269         if gpg_keyid is not None:
 270                 factory.addStep(MasterShellCommand(
 271                         name = "signprepare",
 272                         description = "Preparing temporary signing directory",
 273                         command = ["mkdir", "-p", "%s/signing" %(home_dir)],
 274                         haltOnFailure = True
 275                 ))
 276
 277                 factory.addStep(ShellCommand(
 278                         name = "signpack",
 279                         description = "Packing files to sign",
 280                         workdir = "build/sdk",
 281                         command = "find bin/packages/%s/ -mindepth 2 -maxdepth 2 -type f -name Packages -print0 | xargs -0 tar -czf sign.tar.gz" %(arch[0]),
 282                         haltOnFailure = True
 283                 ))
 284
 285                 factory.addStep(FileUpload(
 286                         slavesrc = "sdk/sign.tar.gz",
 287                         masterdest = "%s/signing/%s.tar.gz" %(home_dir, arch[0]),
 288                         haltOnFailure = True
 289                 ))
 290
 291                 factory.addStep(MasterShellCommand(
 292                         name = "signfiles",
 293                         description = "Signing files",
 294                         command = ["%s/signall.sh" %(home_dir), "%s/signing/%s.tar.gz" %(home_dir, arch[0]), gpg_keyid, gpg_comment],
 295                         env = {'GNUPGHOME': gpg_home, 'PASSFILE': gpg_passfile},
 296                         haltOnFailure = True
 297                 ))
 298
 299                 factory.addStep(FileDownload(
 300                         mastersrc = "%s/signing/%s.tar.gz" %(home_dir, arch[0]),
 301                         slavedest = "sdk/sign.tar.gz",
 302                         haltOnFailure = True
 303                 ))
 304
 305                 factory.addStep(ShellCommand(
 306                         name = "signunpack",
 307                         description = "Unpacking signed files",
 308                         workdir = "build/sdk",
 309                         command = ["tar", "-xzf", "sign.tar.gz"],
 310                         haltOnFailure = True
 311                 ))
 312
 313         factory.addStep(ShellCommand(
 314                 name = "uploadprepare",
 315                 description = "Preparing package directory",
 316                 workdir = "build/sdk",
 317                 command = ["rsync", "-av", "--include", "/%s/" %(arch[0]), "--exclude", "/*", "--exclude", "/%s/*" %(arch[0]), "bin/packages/", WithProperties("%s/packages%%(suffix)s/" %(rsync_bin_url), suffix=GetDirectorySuffix)],
 318                 env={'RSYNC_PASSWORD': rsync_bin_key},
 319                 haltOnFailure = True,
 320                 logEnviron = False
 321         ))
 322
 323         factory.addStep(ShellCommand(
 324                 name = "packageupload",
 325                 description = "Uploading package files",
 326                 workdir = "build/sdk",
 327                 command = ["rsync", "--delete", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-avz", "bin/packages/%s/" %(arch[0]), WithProperties("%s/packages%%(suffix)s/%s/" %(rsync_bin_url, arch[0]), suffix=GetDirectorySuffix)],
 328                 env={'RSYNC_PASSWORD': rsync_bin_key},
 329                 haltOnFailure = True,
 330                 logEnviron = False
 331         ))
 332
 333         factory.addStep(ShellCommand(
 334                 name = "logprepare",
 335                 description = "Preparing log directory",
 336                 workdir = "build/sdk",
 337                 command = ["rsync", "-av", "--include", "/%s/" %(arch[0]), "--exclude", "/*", "--exclude", "/%s/*" %(arch[0]), "bin/packages/", "%s/faillogs/" %(rsync_bin_url)],
 338                 env={'RSYNC_PASSWORD': rsync_bin_key},
 339                 haltOnFailure = True,
 340                 logEnviron = False
 341         ))
 342
 343         factory.addStep(ShellCommand(
 344                 name = "logfind",
 345                 description = "Finding failure logs",
 346                 workdir = "build/sdk/logs/package/feeds",
 347                 command = ["sh", "-c", "sed -ne 's!^ *ERROR: package/feeds/\\([^ ]*\\) .*$!\\1!p' ../error.txt | sort -u | xargs -r find > ../../../logs.txt"],
 348                 haltOnFailure = False
 349         ))
 350
 351         factory.addStep(ShellCommand(
 352                 name = "logcollect",
 353                 description = "Collecting failure logs",
 354                 workdir = "build/sdk",
 355                 command = ["rsync", "-av", "--files-from=logs.txt", "logs/package/feeds/", "faillogs/"],
 356                 haltOnFailure = False
 357         ))
 358
 359         factory.addStep(ShellCommand(
 360                 name = "logupload",
 361                 description = "Uploading failure logs",
 362                 workdir = "build/sdk",
 363                 command = ["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-avz", "faillogs/", "%s/faillogs/%s/" %(rsync_bin_url, arch[0])],
 364                 env={'RSYNC_PASSWORD': rsync_bin_key},
 365                 haltOnFailure = False,
 366                 logEnviron = False
 367         ))
 368
 369         if rsync_src_url is not None:
 370                 factory.addStep(ShellCommand(
 371                         name = "sourceupload",
 372                         description = "Uploading source archives",
 373                         workdir = "build/sdk",
 374                         command = ["rsync", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-avz", "dl/", "%s/" %(rsync_src_url)],
 375                         env={'RSYNC_PASSWORD': rsync_src_key},
 376                         haltOnFailure = False,
 377                         logEnviron = False
 378                 ))
 379
 380         from buildbot.config import BuilderConfig
 381
 382         c['builders'].append(BuilderConfig(name=arch[0], slavenames=slaveNames, factory=factory))
 383
 384
 385 ####### STATUS arches
 386
 387 # 'status' is a list of Status arches. The results of each build will be
 388 # pushed to these arches. buildbot/status/*.py has a variety to choose from,
 389 # including web pages, email senders, and IRC bots.
 390
 391 c['status'] = []
 392
 393 from buildbot.status import html
 394 from buildbot.status.web import authz, auth
 395
 396 if ini.has_option("status", "bind"):
 397         if ini.has_option("status", "user") and ini.has_option("status", "password"):
 398                 authz_cfg=authz.Authz(
 399                         # change any of these to True to enable; see the manual for more
 400                         # options
 401                         auth=auth.BasicAuth([(ini.get("status", "user"), ini.get("status", "password"))]),
 402                         gracefulShutdown = 'auth',
 403                         forceBuild = 'auth', # use this to test your slave once it is set up
 404                         forceAllBuilds = 'auth',
 405                         pingBuilder = False,
 406                         stopBuild = 'auth',
 407                         stopAllBuilds = 'auth',
 408                         cancelPendingBuild = 'auth',
 409                 )
 410                 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind"), authz=authz_cfg))
 411         else:
 412                 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind")))
 413
 414 ####### PROJECT IDENTITY
 415
 416 # the 'title' string will appear at the top of this buildbot
 417 # installation's html.WebStatus home page (linked to the
 418 # 'titleURL') and is embedded in the title of the waterfall HTML page.
 419
 420 c['title'] = ini.get("general", "title")
 421 c['titleURL'] = ini.get("general", "title_url")
 422
 423 # the 'buildbotURL' string should point to the location where the buildbot's
 424 # internal web server (usually the html.WebStatus page) is visible. This
 425 # typically uses the port number set in the Waterfall 'status' entry, but
 426 # with an externally-visible host name which the buildbot cannot figure out
 427 # without some help.
 428
 429 c['buildbotURL'] = buildbot_url
 430
 431 ####### DB URL
 432
 433 c['db'] = {
 434         # This specifies what database buildbot uses to store its state.  You can leave
 435         # this at its default for all but the largest installations.
 436         'db_url' : "sqlite:///state.sqlite",
 437 }