2 # ex: set syntax=python:
9 from buildbot import locks
11 ini = ConfigParser.ConfigParser()
12 ini.read("./config.ini")
14 buildbot_url = ini.get("general", "buildbot_url")
16 # This is a sample buildmaster config file. It must be installed as
17 # 'master.cfg' in your buildmaster's base directory.
19 # This is the dictionary that the buildmaster pays attention to. We also use
20 # a shorter alias to save typing.
21 c = BuildmasterConfig = {}
25 # The 'slaves' list defines the set of recognized buildslaves. Each element is
26 # a BuildSlave object, specifying a unique slave name and password. The same
27 # slave name and password must be configured on the slave.
28 from buildbot.buildslave import BuildSlave
33 if ini.has_option("general", "port"):
34 slave_port = ini.getint("general", "port")
36 if ini.has_option("general", "persistent"):
37 persistent = ini.getboolean("general", "persistent")
41 for section in ini.sections():
42 if section.startswith("slave "):
43 if ini.has_option(section, "name") and ini.has_option(section, "password"):
44 name = ini.get(section, "name")
45 password = ini.get(section, "password")
47 if ini.has_option(section, "builds"):
48 max_builds = ini.getint(section, "builds")
49 c['slaves'].append(BuildSlave(name, password, max_builds = max_builds))
51 # 'slavePortnum' defines the TCP port to listen on for connections from slaves.
52 # This must match the value configured into the buildslaves (with their
54 c['slavePortnum'] = slave_port
57 c['mergeRequests'] = True
61 home_dir = os.path.abspath(ini.get("general", "homedir"))
63 rsync_bin_url = ini.get("rsync", "binary_url")
64 rsync_bin_key = ini.get("rsync", "binary_password")
69 if ini.has_option("rsync", "source_url"):
70 rsync_src_url = ini.get("rsync", "source_url")
71 rsync_src_key = ini.get("rsync", "source_password")
75 rsync_sdk_pat = "lede-sdk-*.tar.xz"
77 if ini.has_option("rsync", "sdk_url"):
78 rsync_sdk_url = ini.get("rsync", "sdk_url")
80 if ini.has_option("rsync", "sdk_password"):
81 rsync_sdk_key = ini.get("rsync", "sdk_password")
83 if ini.has_option("rsync", "sdk_pattern"):
84 rsync_sdk_pat = ini.get("rsync", "sdk_pattern")
88 gpg_comment = "Unattended build signature"
89 gpg_passfile = "/dev/null"
91 if ini.has_option("gpg", "home"):
92 gpg_home = ini.get("gpg", "home")
94 if ini.has_option("gpg", "keyid"):
95 gpg_keyid = ini.get("gpg", "keyid")
97 if ini.has_option("gpg", "comment"):
98 gpg_comment = ini.get("gpg", "comment")
100 if ini.has_option("gpg", "passfile"):
101 gpg_passfile = ini.get("gpg", "passfile")
108 findarches = subprocess.Popen([home_dir+'/dumpinfo.pl', 'architectures'],
109 stdout = subprocess.PIPE, cwd = home_dir+'/source.git')
112 line = findarches.stdout.readline()
115 at = line.strip().split()
117 archnames.append(at[0])
122 feedbranches = dict()
124 from buildbot.changes.gitpoller import GitPoller
125 c['change_source'] = []
127 with open(home_dir+'/source.git/feeds.conf.default', 'r') as f:
129 parts = line.strip().split()
130 if parts[0] == "src-git":
132 url = parts[2].strip().split(';')
133 branch = url[1] if len(url) > 1 else 'master'
134 feedbranches[url[0]] = branch
135 c['change_source'].append(GitPoller(url[0], branch=branch, workdir='%s/%s.git' %(os.getcwd(), parts[1]), pollinterval=300))
140 # Configure the Schedulers, which decide how to react to incoming changes. In this
141 # case, just kick off a 'basebuild' build
143 def branch_change_filter(change):
144 return change.branch == feedbranches[change.repository]
146 from buildbot.schedulers.basic import SingleBranchScheduler
147 from buildbot.schedulers.forcesched import ForceScheduler
148 from buildbot.changes import filter
150 c['schedulers'].append(SingleBranchScheduler(
152 change_filter=filter.ChangeFilter(filter_fn=branch_change_filter),
154 builderNames=archnames))
156 c['schedulers'].append(ForceScheduler(
158 builderNames=archnames))
162 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
163 # what steps, and which slaves can execute them. Note that any particular build will
164 # only take place on one slave.
166 from buildbot.process.factory import BuildFactory
167 from buildbot.steps.source import Git
168 from buildbot.steps.shell import ShellCommand
169 from buildbot.steps.shell import SetProperty
170 from buildbot.steps.transfer import FileUpload
171 from buildbot.steps.transfer import FileDownload
172 from buildbot.steps.master import MasterShellCommand
173 from buildbot.process.properties import WithProperties
176 def GetDirectorySuffix(props):
177 if props.hasProperty("slavename") and re.match("^[^-]+-[0-9]+\.[0-9]+-[^-]+$", props["slavename"]):
178 return "-%s" % props["slavename"].split('-')[1]
185 dlLock = locks.SlaveLock("slave_dl")
189 for slave in c['slaves']:
190 slaveNames.append(slave.slavename)
193 ts = arch[1].split('/')
195 factory = BuildFactory()
197 # find number of cores
198 factory.addStep(SetProperty(
201 description = "Finding number of CPUs",
202 command = ["nproc"]))
205 factory.addStep(FileDownload(mastersrc="cleanup.sh", slavedest="cleanup.sh", mode=0755))
208 factory.addStep(ShellCommand(
210 description = "Cleaning previous builds",
211 command = ["./cleanup.sh", buildbot_url, WithProperties("%(slavename)s"), WithProperties("%(buildername)s"), "full"],
212 haltOnFailure = True,
215 factory.addStep(ShellCommand(
217 description = "Cleaning work area",
218 command = ["./cleanup.sh", buildbot_url, WithProperties("%(slavename)s"), WithProperties("%(buildername)s"), "single"],
219 haltOnFailure = True,
222 factory.addStep(ShellCommand(
224 description = "Preparing SDK directory",
225 command = ["mkdir", "-p", "sdk"],
226 haltOnFailure = True))
228 factory.addStep(ShellCommand(
229 name = "downloadsdk",
230 description = "Downloading SDK archive",
231 command = ["rsync", "-va", "%s/%s/%s/%s" %(rsync_sdk_url, ts[0], ts[1], rsync_sdk_pat), "sdk.archive"],
232 env={'RSYNC_PASSWORD': rsync_sdk_key},
233 haltOnFailure = True,
236 factory.addStep(ShellCommand(
238 description = "Unpacking SDK archive",
239 command = ["tar", "--strip-components=1", "-C", "sdk/", "-vxf", "sdk.archive"],
240 haltOnFailure = True))
242 factory.addStep(FileDownload(mastersrc=home_dir+'/key-build', slavedest="sdk/key-build", mode=0600))
243 factory.addStep(FileDownload(mastersrc=home_dir+'/key-build.pub', slavedest="sdk/key-build.pub", mode=0600))
245 factory.addStep(ShellCommand(
247 description = "Preparing download directory",
248 command = ["sh", "-c", "mkdir -p $HOME/dl && rm -rf ./sdk/dl && ln -sf $HOME/dl ./sdk/dl"]))
250 factory.addStep(ShellCommand(
252 description = "Preparing SDK configuration",
253 workdir = "build/sdk",
254 command = ["sh", "-c", "rm -f .config && make defconfig"]))
256 factory.addStep(ShellCommand(
257 name = "updatefeeds",
258 description = "Updating feeds",
259 workdir = "build/sdk",
260 command = ["./scripts/feeds", "update"]))
262 factory.addStep(ShellCommand(
263 name = "installfeeds",
264 description = "Installing feeds",
265 workdir = "build/sdk",
266 command = ["./scripts/feeds", "install", "-a"]))
268 factory.addStep(ShellCommand(
270 description = "Building packages",
271 workdir = "build/sdk",
272 command = ["make", WithProperties("-j%(nproc:~4)s"), "V=s", "IGNORE_ERRORS=n m y", "BUILD_LOG=1", "CONFIG_SIGNED_PACKAGES=y"]))
274 if gpg_keyid is not None:
275 factory.addStep(MasterShellCommand(
276 name = "signprepare",
277 description = "Preparing temporary signing directory",
278 command = ["mkdir", "-p", "%s/signing" %(home_dir)],
282 factory.addStep(ShellCommand(
284 description = "Packing files to sign",
285 workdir = "build/sdk",
286 command = "find bin/packages/%s/ -mindepth 2 -maxdepth 2 -type f -name Packages -print0 | xargs -0 tar -czf sign.tar.gz" %(arch[0]),
290 factory.addStep(FileUpload(
291 slavesrc = "sdk/sign.tar.gz",
292 masterdest = "%s/signing/%s.tar.gz" %(home_dir, arch[0]),
296 factory.addStep(MasterShellCommand(
298 description = "Signing files",
299 command = ["%s/signall.sh" %(home_dir), "%s/signing/%s.tar.gz" %(home_dir, arch[0]), gpg_keyid, gpg_comment],
300 env = {'GNUPGHOME': gpg_home, 'PASSFILE': gpg_passfile},
304 factory.addStep(FileDownload(
305 mastersrc = "%s/signing/%s.tar.gz" %(home_dir, arch[0]),
306 slavedest = "sdk/sign.tar.gz",
310 factory.addStep(ShellCommand(
312 description = "Unpacking signed files",
313 workdir = "build/sdk",
314 command = ["tar", "-xzf", "sign.tar.gz"],
318 factory.addStep(ShellCommand(
319 name = "uploadprepare",
320 description = "Preparing package directory",
321 workdir = "build/sdk",
322 command = ["rsync", "-av", "--include", "/%s/" %(arch[0]), "--exclude", "/*", "--exclude", "/%s/*" %(arch[0]), "bin/packages/", WithProperties("%s/packages%%(suffix)s/" %(rsync_bin_url), suffix=GetDirectorySuffix)],
323 env={'RSYNC_PASSWORD': rsync_bin_key},
324 haltOnFailure = True,
328 factory.addStep(ShellCommand(
329 name = "packageupload",
330 description = "Uploading package files",
331 workdir = "build/sdk",
332 command = ["rsync", "--delete", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-avz", "bin/packages/%s/" %(arch[0]), WithProperties("%s/packages%%(suffix)s/%s/" %(rsync_bin_url, arch[0]), suffix=GetDirectorySuffix)],
333 env={'RSYNC_PASSWORD': rsync_bin_key},
334 haltOnFailure = True,
338 factory.addStep(ShellCommand(
340 description = "Preparing log directory",
341 workdir = "build/sdk",
342 command = ["rsync", "-av", "--include", "/%s/" %(arch[0]), "--exclude", "/*", "--exclude", "/%s/*" %(arch[0]), "bin/packages/", "%s/faillogs/" %(rsync_bin_url)],
343 env={'RSYNC_PASSWORD': rsync_bin_key},
344 haltOnFailure = True,
348 factory.addStep(ShellCommand(
350 description = "Finding failure logs",
351 workdir = "build/sdk/logs/package/feeds",
352 command = ["sh", "-c", "sed -ne 's!^ *ERROR: package/feeds/\\([^ ]*\\) .*$!\\1!p' ../error.txt | sort -u | xargs -r find > ../../../logs.txt"],
353 haltOnFailure = False
356 factory.addStep(ShellCommand(
358 description = "Collecting failure logs",
359 workdir = "build/sdk",
360 command = ["rsync", "-av", "--files-from=logs.txt", "logs/package/feeds/", "faillogs/"],
361 haltOnFailure = False
364 factory.addStep(ShellCommand(
366 description = "Uploading failure logs",
367 workdir = "build/sdk",
368 command = ["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-avz", "faillogs/", "%s/faillogs/%s/" %(rsync_bin_url, arch[0])],
369 env={'RSYNC_PASSWORD': rsync_bin_key},
370 haltOnFailure = False,
374 if rsync_src_url is not None:
375 factory.addStep(ShellCommand(
376 name = "sourceupload",
377 description = "Uploading source archives",
378 workdir = "build/sdk",
379 command = ["rsync", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-avz", "dl/", "%s/" %(rsync_src_url)],
380 env={'RSYNC_PASSWORD': rsync_src_key},
381 haltOnFailure = False,
385 from buildbot.config import BuilderConfig
387 c['builders'].append(BuilderConfig(name=arch[0], slavenames=slaveNames, factory=factory))
390 ####### STATUS arches
392 # 'status' is a list of Status arches. The results of each build will be
393 # pushed to these arches. buildbot/status/*.py has a variety to choose from,
394 # including web pages, email senders, and IRC bots.
398 from buildbot.status import html
399 from buildbot.status.web import authz, auth
401 if ini.has_option("status", "bind"):
402 if ini.has_option("status", "user") and ini.has_option("status", "password"):
403 authz_cfg=authz.Authz(
404 # change any of these to True to enable; see the manual for more
406 auth=auth.BasicAuth([(ini.get("status", "user"), ini.get("status", "password"))]),
407 gracefulShutdown = 'auth',
408 forceBuild = 'auth', # use this to test your slave once it is set up
409 forceAllBuilds = 'auth',
412 stopAllBuilds = 'auth',
413 cancelPendingBuild = 'auth',
415 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind"), authz=authz_cfg))
417 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind")))
419 ####### PROJECT IDENTITY
421 # the 'title' string will appear at the top of this buildbot
422 # installation's html.WebStatus home page (linked to the
423 # 'titleURL') and is embedded in the title of the waterfall HTML page.
425 c['title'] = ini.get("general", "title")
426 c['titleURL'] = ini.get("general", "title_url")
428 # the 'buildbotURL' string should point to the location where the buildbot's
429 # internal web server (usually the html.WebStatus page) is visible. This
430 # typically uses the port number set in the Waterfall 'status' entry, but
431 # with an externally-visible host name which the buildbot cannot figure out
434 c['buildbotURL'] = buildbot_url
439 # This specifies what database buildbot uses to store its state. You can leave
440 # this at its default for all but the largest installations.
441 'db_url' : "sqlite:///state.sqlite",