2 # ex: set syntax=python:
9 from buildbot import locks
11 ini = ConfigParser.ConfigParser()
12 ini.read("./config.ini")
14 buildbot_url = ini.get("general", "buildbot_url")
16 # This is a sample buildmaster config file. It must be installed as
17 # 'master.cfg' in your buildmaster's base directory.
19 # This is the dictionary that the buildmaster pays attention to. We also use
20 # a shorter alias to save typing.
21 c = BuildmasterConfig = {}
25 # The 'slaves' list defines the set of recognized buildslaves. Each element is
26 # a BuildSlave object, specifying a unique slave name and password. The same
27 # slave name and password must be configured on the slave.
28 from buildbot.buildslave import BuildSlave
34 if ini.has_option("general", "port"):
35 slave_port = ini.getint("general", "port")
37 if ini.has_option("general", "persistent"):
38 persistent = ini.getboolean("general", "persistent")
40 if ini.has_option("general", "other_builds"):
41 other_builds = ini.getint("general", "other_builds")
46 for section in ini.sections():
47 if section.startswith("slave "):
48 if ini.has_option(section, "name") and ini.has_option(section, "password"):
49 name = ini.get(section, "name")
50 password = ini.get(section, "password")
52 if ini.has_option(section, "builds"):
53 max_builds[name] = ini.getint(section, "builds")
54 c['slaves'].append(BuildSlave(name, password, max_builds = max_builds[name]))
56 # 'slavePortnum' defines the TCP port to listen on for connections from slaves.
57 # This must match the value configured into the buildslaves (with their
59 c['slavePortnum'] = slave_port
62 c['mergeRequests'] = True
66 home_dir = os.path.abspath(ini.get("general", "homedir"))
68 rsync_bin_url = ini.get("rsync", "binary_url")
69 rsync_bin_key = ini.get("rsync", "binary_password")
74 if ini.has_option("rsync", "source_url"):
75 rsync_src_url = ini.get("rsync", "source_url")
76 rsync_src_key = ini.get("rsync", "source_password")
80 rsync_sdk_pat = "lede-sdk-*.tar.xz"
82 if ini.has_option("rsync", "sdk_url"):
83 rsync_sdk_url = ini.get("rsync", "sdk_url")
85 if ini.has_option("rsync", "sdk_password"):
86 rsync_sdk_key = ini.get("rsync", "sdk_password")
88 if ini.has_option("rsync", "sdk_pattern"):
89 rsync_sdk_pat = ini.get("rsync", "sdk_pattern")
93 gpg_comment = "Unattended build signature"
94 gpg_passfile = "/dev/null"
96 if ini.has_option("gpg", "home"):
97 gpg_home = ini.get("gpg", "home")
99 if ini.has_option("gpg", "keyid"):
100 gpg_keyid = ini.get("gpg", "keyid")
102 if ini.has_option("gpg", "comment"):
103 gpg_comment = ini.get("gpg", "comment")
105 if ini.has_option("gpg", "passfile"):
106 gpg_passfile = ini.get("gpg", "passfile")
113 findarches = subprocess.Popen([home_dir+'/dumpinfo.pl', 'architectures'],
114 stdout = subprocess.PIPE, cwd = home_dir+'/source.git')
117 line = findarches.stdout.readline()
120 at = line.strip().split()
122 archnames.append(at[0])
127 feedbranches = dict()
129 from buildbot.changes.gitpoller import GitPoller
130 c['change_source'] = []
132 with open(home_dir+'/source.git/feeds.conf.default', 'r') as f:
134 parts = line.strip().split()
135 if parts[0] == "src-git":
137 url = parts[2].strip().split(';')
138 branch = url[1] if len(url) > 1 else 'master'
139 feedbranches[url[0]] = branch
140 c['change_source'].append(GitPoller(url[0], branch=branch, workdir='%s/%s.git' %(os.getcwd(), parts[1]), pollinterval=300))
145 # Configure the Schedulers, which decide how to react to incoming changes. In this
146 # case, just kick off a 'basebuild' build
148 def branch_change_filter(change):
149 return change.branch == feedbranches[change.repository]
151 from buildbot.schedulers.basic import SingleBranchScheduler
152 from buildbot.schedulers.forcesched import ForceScheduler
153 from buildbot.changes import filter
155 c['schedulers'].append(SingleBranchScheduler(
157 change_filter=filter.ChangeFilter(filter_fn=branch_change_filter),
159 builderNames=archnames))
161 c['schedulers'].append(ForceScheduler(
163 builderNames=archnames))
167 # The 'builders' list defines the Builders, which tell Buildbot how to perform a build:
168 # what steps, and which slaves can execute them. Note that any particular build will
169 # only take place on one slave.
171 from buildbot.process.factory import BuildFactory
172 from buildbot.steps.source import Git
173 from buildbot.steps.shell import ShellCommand
174 from buildbot.steps.shell import SetProperty
175 from buildbot.steps.transfer import FileUpload
176 from buildbot.steps.transfer import FileDownload
177 from buildbot.steps.master import MasterShellCommand
178 from buildbot.process.properties import WithProperties
181 def GetDirectorySuffix(props):
182 if props.hasProperty("slavename") and re.match("^[^-]+-[0-9]+\.[0-9]+-[^-]+$", props["slavename"]):
183 return "-%s" % props["slavename"].split('-')[1]
187 def GetNumJobs(props):
188 if props.hasProperty("slavename") and props.hasProperty("nproc"):
189 return ((int(props["nproc"]) / (max_builds[props["slavename"]] + other_builds)) + 1)
196 dlLock = locks.SlaveLock("slave_dl")
200 for slave in c['slaves']:
201 slaveNames.append(slave.slavename)
204 ts = arch[1].split('/')
206 factory = BuildFactory()
208 # find number of cores
209 factory.addStep(SetProperty(
212 description = "Finding number of CPUs",
213 command = ["nproc"]))
216 factory.addStep(FileDownload(mastersrc="cleanup.sh", slavedest="cleanup.sh", mode=0755))
219 factory.addStep(ShellCommand(
221 description = "Cleaning previous builds",
222 command = ["./cleanup.sh", buildbot_url, WithProperties("%(slavename)s"), WithProperties("%(buildername)s"), "full"],
223 haltOnFailure = True,
226 factory.addStep(ShellCommand(
228 description = "Cleaning work area",
229 command = ["./cleanup.sh", buildbot_url, WithProperties("%(slavename)s"), WithProperties("%(buildername)s"), "single"],
230 haltOnFailure = True,
233 factory.addStep(ShellCommand(
235 description = "Preparing SDK directory",
236 command = ["mkdir", "-p", "sdk"],
237 haltOnFailure = True))
239 factory.addStep(ShellCommand(
240 name = "downloadsdk",
241 description = "Downloading SDK archive",
242 command = ["rsync", "-va", "%s/%s/%s/%s" %(rsync_sdk_url, ts[0], ts[1], rsync_sdk_pat), "sdk.archive"],
243 env={'RSYNC_PASSWORD': rsync_sdk_key},
244 haltOnFailure = True,
247 factory.addStep(ShellCommand(
249 description = "Unpacking SDK archive",
250 command = ["tar", "--strip-components=1", "-C", "sdk/", "-vxf", "sdk.archive"],
251 haltOnFailure = True))
253 factory.addStep(FileDownload(mastersrc=home_dir+'/key-build', slavedest="sdk/key-build", mode=0600))
254 factory.addStep(FileDownload(mastersrc=home_dir+'/key-build.pub', slavedest="sdk/key-build.pub", mode=0600))
256 factory.addStep(ShellCommand(
258 description = "Preparing download directory",
259 command = ["sh", "-c", "mkdir -p $HOME/dl && rm -rf ./sdk/dl && ln -sf $HOME/dl ./sdk/dl"]))
261 factory.addStep(ShellCommand(
263 description = "Preparing SDK configuration",
264 workdir = "build/sdk",
265 command = ["sh", "-c", "rm -f .config && make defconfig"]))
267 factory.addStep(ShellCommand(
268 name = "updatefeeds",
269 description = "Updating feeds",
270 workdir = "build/sdk",
271 command = ["./scripts/feeds", "update"]))
273 factory.addStep(ShellCommand(
274 name = "installfeeds",
275 description = "Installing feeds",
276 workdir = "build/sdk",
277 command = ["./scripts/feeds", "install", "-a"]))
279 factory.addStep(ShellCommand(
281 description = "Building packages",
282 workdir = "build/sdk",
283 command = ["make", WithProperties("-j%(jobs)d", jobs=GetNumJobs), "V=s", "IGNORE_ERRORS=n m y", "BUILD_LOG=1", "CONFIG_SIGNED_PACKAGES=y"]))
285 factory.addStep(ShellCommand(
286 name = "mkfeedsconf",
287 description = "Generating pinned feeds.conf",
288 workdir = "build/sdk",
289 command = "./scripts/feeds list -s -f > bin/packages/%s/feeds.conf" %(arch[0])))
291 if gpg_keyid is not None:
292 factory.addStep(MasterShellCommand(
293 name = "signprepare",
294 description = "Preparing temporary signing directory",
295 command = ["mkdir", "-p", "%s/signing" %(home_dir)],
299 factory.addStep(ShellCommand(
301 description = "Packing files to sign",
302 workdir = "build/sdk",
303 command = "find bin/packages/%s/ -mindepth 2 -maxdepth 2 -type f -name Packages -print0 | xargs -0 tar -czf sign.tar.gz" %(arch[0]),
307 factory.addStep(FileUpload(
308 slavesrc = "sdk/sign.tar.gz",
309 masterdest = "%s/signing/%s.tar.gz" %(home_dir, arch[0]),
313 factory.addStep(MasterShellCommand(
315 description = "Signing files",
316 command = ["%s/signall.sh" %(home_dir), "%s/signing/%s.tar.gz" %(home_dir, arch[0]), gpg_keyid, gpg_comment],
317 env = {'GNUPGHOME': gpg_home, 'PASSFILE': gpg_passfile},
321 factory.addStep(FileDownload(
322 mastersrc = "%s/signing/%s.tar.gz" %(home_dir, arch[0]),
323 slavedest = "sdk/sign.tar.gz",
327 factory.addStep(ShellCommand(
329 description = "Unpacking signed files",
330 workdir = "build/sdk",
331 command = ["tar", "-xzf", "sign.tar.gz"],
335 factory.addStep(ShellCommand(
336 name = "uploadprepare",
337 description = "Preparing package directory",
338 workdir = "build/sdk",
339 command = ["rsync", "-av", "--include", "/%s/" %(arch[0]), "--exclude", "/*", "--exclude", "/%s/*" %(arch[0]), "bin/packages/", WithProperties("%s/packages%%(suffix)s/" %(rsync_bin_url), suffix=GetDirectorySuffix)],
340 env={'RSYNC_PASSWORD': rsync_bin_key},
341 haltOnFailure = True,
345 factory.addStep(ShellCommand(
346 name = "packageupload",
347 description = "Uploading package files",
348 workdir = "build/sdk",
349 command = ["rsync", "--delete", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-avz", "bin/packages/%s/" %(arch[0]), WithProperties("%s/packages%%(suffix)s/%s/" %(rsync_bin_url, arch[0]), suffix=GetDirectorySuffix)],
350 env={'RSYNC_PASSWORD': rsync_bin_key},
351 haltOnFailure = True,
355 factory.addStep(ShellCommand(
357 description = "Preparing log directory",
358 workdir = "build/sdk",
359 command = ["rsync", "-av", "--include", "/%s/" %(arch[0]), "--exclude", "/*", "--exclude", "/%s/*" %(arch[0]), "bin/packages/", "%s/faillogs/" %(rsync_bin_url)],
360 env={'RSYNC_PASSWORD': rsync_bin_key},
361 haltOnFailure = True,
365 factory.addStep(ShellCommand(
367 description = "Finding failure logs",
368 workdir = "build/sdk/logs/package/feeds",
369 command = ["sh", "-c", "sed -ne 's!^ *ERROR: package/feeds/\\([^ ]*\\) .*$!\\1!p' ../error.txt | sort -u | xargs -r find > ../../../logs.txt"],
370 haltOnFailure = False
373 factory.addStep(ShellCommand(
375 description = "Collecting failure logs",
376 workdir = "build/sdk",
377 command = ["rsync", "-av", "--files-from=logs.txt", "logs/package/feeds/", "faillogs/"],
378 haltOnFailure = False
381 factory.addStep(ShellCommand(
383 description = "Uploading failure logs",
384 workdir = "build/sdk",
385 command = ["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-avz", "faillogs/", "%s/faillogs/%s/" %(rsync_bin_url, arch[0])],
386 env={'RSYNC_PASSWORD': rsync_bin_key},
387 haltOnFailure = False,
391 if rsync_src_url is not None:
392 factory.addStep(ShellCommand(
393 name = "sourceupload",
394 description = "Uploading source archives",
395 workdir = "build/sdk",
396 command = ["rsync", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s" %(arch[0]), "-avz", "dl/", "%s/" %(rsync_src_url)],
397 env={'RSYNC_PASSWORD': rsync_src_key},
398 haltOnFailure = False,
402 from buildbot.config import BuilderConfig
404 c['builders'].append(BuilderConfig(name=arch[0], slavenames=slaveNames, factory=factory))
407 ####### STATUS arches
409 # 'status' is a list of Status arches. The results of each build will be
410 # pushed to these arches. buildbot/status/*.py has a variety to choose from,
411 # including web pages, email senders, and IRC bots.
415 from buildbot.status import html
416 from buildbot.status.web import authz, auth
418 if ini.has_option("status", "bind"):
419 if ini.has_option("status", "user") and ini.has_option("status", "password"):
420 authz_cfg=authz.Authz(
421 # change any of these to True to enable; see the manual for more
423 auth=auth.BasicAuth([(ini.get("status", "user"), ini.get("status", "password"))]),
424 gracefulShutdown = 'auth',
425 forceBuild = 'auth', # use this to test your slave once it is set up
426 forceAllBuilds = 'auth',
429 stopAllBuilds = 'auth',
430 cancelPendingBuild = 'auth',
432 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind"), authz=authz_cfg))
434 c['status'].append(html.WebStatus(http_port=ini.get("status", "bind")))
436 ####### PROJECT IDENTITY
438 # the 'title' string will appear at the top of this buildbot
439 # installation's html.WebStatus home page (linked to the
440 # 'titleURL') and is embedded in the title of the waterfall HTML page.
442 c['title'] = ini.get("general", "title")
443 c['titleURL'] = ini.get("general", "title_url")
445 # the 'buildbotURL' string should point to the location where the buildbot's
446 # internal web server (usually the html.WebStatus page) is visible. This
447 # typically uses the port number set in the Waterfall 'status' entry, but
448 # with an externally-visible host name which the buildbot cannot figure out
451 c['buildbotURL'] = buildbot_url
456 # This specifies what database buildbot uses to store its state. You can leave
457 # this at its default for all but the largest installations.
458 'db_url' : "sqlite:///state.sqlite",