X-Git-Url: http://git.openwrt.org/?a=blobdiff_plain;f=phase1%2Fmaster.cfg;h=7e74bd39f7a0c438ff1d8b5bed547e67a1d99065;hb=58d6f62aafdd86f2626fc6c71f0bd0fbe2565329;hp=c6c7d47e868dbe2441e5cb889041466f6f242bf0;hpb=85d1c7075b0206c878e0c574129e47057eefa72f;p=buildbot.git diff --git a/phase1/master.cfg b/phase1/master.cfg index c6c7d47..7e74bd3 100644 --- a/phase1/master.cfg +++ b/phase1/master.cfg @@ -48,15 +48,43 @@ c['mergeRequests'] = True ####### CHANGESOURCES home_dir = os.path.abspath(ini.get("general", "homedir")) +tree_expire = 0 + +if ini.has_option("general", "expire"): + tree_expire = ini.getint("general", "expire") repo_url = ini.get("repo", "url") -rsync_url = ini.get("rsync", "url") -rsync_key = ini.get("rsync", "password") +rsync_bin_url = ini.get("rsync", "binary_url") +rsync_bin_key = ini.get("rsync", "binary_password") + +rsync_src_url = None +rsync_src_key = None + +if ini.has_option("rsync", "source_url"): + rsync_src_url = ini.get("rsync", "source_url") + rsync_src_key = ini.get("rsync", "source_password") + +gpg_keyid = None +gpg_comment = "Unattended build signature" +gpg_passfile = "/dev/null" + +if ini.has_option("gpg", "keyid"): + gpg_keyid = ini.get("gpg", "keyid") + +if ini.has_option("gpg", "comment"): + gpg_comment = ini.get("gpg", "comment") + +if ini.has_option("gpg", "passfile"): + gpg_passfile = ini.get("gpg", "passfile") + # find targets targets = [ ] +if not os.path.isdir(home_dir+'/source.git'): + subprocess.call(["git", "clone", "--depth=1", repo_url, home_dir+'/source.git']) + findtargets = subprocess.Popen([home_dir+'/dumpinfo.pl', 'targets'], stdout = subprocess.PIPE, cwd = home_dir+'/source.git') @@ -106,70 +134,30 @@ c['schedulers'].append(ForceScheduler( from buildbot.process.factory import BuildFactory from buildbot.steps.source import Git from buildbot.steps.shell import ShellCommand +from buildbot.steps.shell import SetProperty +from buildbot.steps.transfer import FileUpload from buildbot.steps.transfer import FileDownload - - -MakeTargetMap = { - "^tools/": "tools/clean", - "^toolchain/": "toolchain/clean", - "^target/linux/": "target/linux/clean", - "^(config|include)/": "dirclean" -} - -def IsAffected(pattern): - def CheckAffected(change): - for request in change.build.requests: - for source in request.sources: - for change in source.changes: - for file in change.files: - if re.match(pattern, file): - return True - return False - return CheckAffected - -def isPathBuiltin(path): - incl = {} - pkgs = {} - conf = open(".config", "r") - - while True: - line = conf.readline() - if line == '': - break - m = re.match("^(CONFIG_PACKAGE_.+?)=y", line) - if m: - incl[m.group(1)] = True - - conf.close() - - deps = open("tmp/.packagedeps", "r") - - while True: - line = deps.readline() - if line == '': - break - m = re.match("^package-\$\((CONFIG_PACKAGE_.+?)\) \+= (\S+)", line) - if m and incl.get(m.group(1)) == True: - pkgs["package/%s" % m.group(2)] = True - - deps.close() - - while path != '': - if pkgs.get(path) == True: +from buildbot.steps.master import MasterShellCommand +from buildbot.process.properties import WithProperties + + +CleanTargetMap = [ + [ "tools", "tools/clean" ], + [ "chain", "toolchain/clean" ], + [ "linux", "target/linux/clean" ], + [ "dir", "dirclean" ], + [ "dist", "distclean" ] +] + +def IsCleanRequested(pattern): + def CheckCleanProperty(step): + val = step.getProperty("clean") + if val and re.match(pattern, val): return True - path = os.path.dirname(path) - - return False + else: + return False -def isChangeBuiltin(change): - return True -# for request in change.build.requests: -# for source in request.sources: -# for change in source.changes: -# for file in change.files: -# if isPathBuiltin(file): -# return True -# return False + return CheckCleanProperty c['builders'] = [] @@ -239,6 +227,37 @@ for target in targets: factory = BuildFactory() + # find number of cores + factory.addStep(SetProperty( + name = "nproc", + property = "nproc", + description = "Finding number of CPUs", + command = ["nproc"])) + + # expire tree if needed + if tree_expire > 0: + factory.addStep(FileDownload( + mastersrc = "expire.sh", + slavedest = "../expire.sh", + mode = 0755)) + + factory.addStep(ShellCommand( + name = "expire", + description = "Checking for build tree expiry", + command = ["./expire.sh", str(tree_expire)], + workdir = ".", + haltOnFailure = True, + timeout = 2400)) + + # user-requested clean targets + for tuple in CleanTargetMap: + factory.addStep(ShellCommand( + name = tuple[1], + description = 'User-requested "make %s"' % tuple[1], + command = ["make", tuple[1], "V=s"], + doStepIf = IsCleanRequested(tuple[0]) + )) + # check out the source factory.addStep(Git(repourl=repo_url, mode='update')) @@ -253,6 +272,12 @@ for target in targets: # description = "Copy the feeds.conf", # command='''cp ~/feeds.conf ./feeds.conf''' )) + # feed + factory.addStep(ShellCommand( + name = "rmfeedlinks", + description = "Remove feed symlinks", + command=["rm", "-rf", "package/feeds/"])) + # feed factory.addStep(ShellCommand( name = "updatefeeds", @@ -280,8 +305,11 @@ CONFIG_DEVEL=y CONFIG_CCACHE=y CONFIG_SIGNED_PACKAGES=y # CONFIG_PER_FEED_REPO_ADD_COMMENTED is not set -CONFIG_KERNEL_KALLSYMS=y +CONFIG_KERNEL_KALLSYMS=y CONFIG_COLLECT_KERNEL_DEBUG=y +CONFIG_TARGET_ALL_PROFILES=y +CONFIG_TARGET_MULTI_PROFILE=y +CONFIG_TARGET_PER_DEVICE_ROOTFS=y EOT''' %(ts[0], ts[0], ts[1]) )) factory.addStep(ShellCommand( @@ -307,6 +335,13 @@ EOT''' %(ts[0], ts[0], ts[1]) )) haltOnFailure = True )) + # find libc suffix + factory.addStep(SetProperty( + name = "libc", + property = "libc", + description = "Finding libc suffix", + command = ["sed", "-ne", '/^CONFIG_LIBC=/ { s!^CONFIG_LIBC="\\(.*\\)"!\\1!; s!^musl$!!; s!.\\+!-&!p }', ".config"])) + # install build key factory.addStep(FileDownload(mastersrc=home_dir+'/key-build', slavedest="key-build", mode=0600)) factory.addStep(FileDownload(mastersrc=home_dir+'/key-build.pub', slavedest="key-build.pub", mode=0600)) @@ -320,11 +355,19 @@ EOT''' %(ts[0], ts[0], ts[1]) )) want_stdout = False )) + # prepare tar + factory.addStep(ShellCommand( + name = "dltar", + description = "Building GNU tar", + command = ["make", WithProperties("-j%(nproc:~4)s"), "tools/tar/install", "V=s"], + haltOnFailure = True + )) + # populate dl factory.addStep(ShellCommand( name = "dlrun", description = "Populating dl/", - command = ["make", "-j4", "download", "V=s"], + command = ["make", WithProperties("-j%(nproc:~4)s"), "download", "V=s"], logEnviron = False, locks = [dlLock.access('exclusive')] )) @@ -335,33 +378,25 @@ EOT''' %(ts[0], ts[0], ts[1]) )) command=["make", "package/base-files/clean", "V=s"] )) - # optional clean steps - for pattern, maketarget in MakeTargetMap.items(): - factory.addStep(ShellCommand( - name = maketarget, - description = maketarget, - command=["make", maketarget, "V=s"], doStepIf=IsAffected(pattern) - )) - # build factory.addStep(ShellCommand( name = "tools", description = "Building tools", - command = ["make", "-j4", "tools/install", "V=s"], + command = ["make", WithProperties("-j%(nproc:~4)s"), "tools/install", "V=s"], haltOnFailure = True )) factory.addStep(ShellCommand( name = "toolchain", description = "Building toolchain", - command=["make", "-j4", "toolchain/install", "V=s"], + command=["make", WithProperties("-j%(nproc:~4)s"), "toolchain/install", "V=s"], haltOnFailure = True )) factory.addStep(ShellCommand( name = "kmods", description = "Building kmods", - command=["make", "-j4", "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"], + command=["make", WithProperties("-j%(nproc:~4)s"), "target/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"], #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])}, haltOnFailure = True )) @@ -369,7 +404,7 @@ EOT''' %(ts[0], ts[0], ts[1]) )) factory.addStep(ShellCommand( name = "pkgbuild", description = "Building packages", - command=["make", "-j4", "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"], + command=["make", WithProperties("-j%(nproc:~4)s"), "package/compile", "V=s", "IGNORE_ERRORS=n m", "BUILD_LOG=1"], #env={'BUILD_LOG_DIR': 'bin/%s' %(ts[0])}, haltOnFailure = True )) @@ -378,15 +413,14 @@ EOT''' %(ts[0], ts[0], ts[1]) )) factory.addStep(ShellCommand( name = "pkginstall", description = "Installing packages", - command=["make", "-j4", "package/install", "V=s"], - doStepIf = isChangeBuiltin, + command=["make", WithProperties("-j%(nproc:~4)s"), "package/install", "V=s"], haltOnFailure = True )) factory.addStep(ShellCommand( name = "pkgindex", description = "Indexing packages", - command=["make", "-j4", "package/index", "V=s"], + command=["make", WithProperties("-j%(nproc:~4)s"), "package/index", "V=s"], haltOnFailure = True )) @@ -394,17 +428,65 @@ EOT''' %(ts[0], ts[0], ts[1]) )) factory.addStep(ShellCommand( name = "images", description = "Building images", - command=["make", "-j1", "target/install", "V=s"], - doStepIf = isChangeBuiltin, + command=["make", WithProperties("-j%(nproc:~4)s"), "target/install", "V=s"], + haltOnFailure = True + )) + + factory.addStep(ShellCommand( + name = "checksums", + description = "Calculating checksums", + command=["make", "-j1", "checksum", "V=s"], haltOnFailure = True )) + # sign + if gpg_keyid is not None: + factory.addStep(MasterShellCommand( + name = "signprepare", + description = "Preparing temporary signing directory", + command = ["mkdir", "-p", "%s/signing" %(home_dir)], + haltOnFailure = True + )) + + factory.addStep(ShellCommand( + name = "signpack", + description = "Packing files to sign", + command = WithProperties("find bin/targets/%s/%s%%(libc)s/ -mindepth 1 -maxdepth 2 -type f -name sha256sums -print0 -or -name Packages -print0 | xargs -0 tar -czf sign.tar.gz" %(ts[0], ts[1])), + haltOnFailure = True + )) + + factory.addStep(FileUpload( + slavesrc = "sign.tar.gz", + masterdest = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]), + haltOnFailure = True + )) + + factory.addStep(MasterShellCommand( + name = "signfiles", + description = "Signing files", + command = ["%s/signall.sh" %(home_dir), "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]), gpg_keyid, gpg_passfile, gpg_comment], + haltOnFailure = True + )) + + factory.addStep(FileDownload( + mastersrc = "%s/signing/%s.%s.tar.gz" %(home_dir, ts[0], ts[1]), + slavedest = "sign.tar.gz", + haltOnFailure = True + )) + + factory.addStep(ShellCommand( + name = "signunpack", + description = "Unpacking signed files", + command = ["tar", "-xzf", "sign.tar.gz"], + haltOnFailure = True + )) + # upload factory.addStep(ShellCommand( name = "uploadprepare", description = "Preparing target directory", - command=["rsync", "-av", "--include", "/%s/" %(ts[0]), "--include", "/%s/%s/" %(ts[0], ts[1]), "--exclude", "/*", "--exclude", "/*/*", "--exclude", "/%s/%s/*" %(ts[0], ts[1]), "bin/targets/", "%s/targets/" %(rsync_url)], - env={'RSYNC_PASSWORD': rsync_key}, + command=["rsync", "-av", "--include", "/%s/" %(ts[0]), "--include", "/%s/%s/" %(ts[0], ts[1]), "--exclude", "/*", "--exclude", "/*/*", "--exclude", "/%s/%s/*" %(ts[0], ts[1]), "bin/targets/", "%s/targets/" %(rsync_bin_url)], + env={'RSYNC_PASSWORD': rsync_bin_key}, haltOnFailure = True, logEnviron = False )) @@ -412,18 +494,28 @@ EOT''' %(ts[0], ts[0], ts[1]) )) factory.addStep(ShellCommand( name = "targetupload", description = "Uploading target files", - command=["rsync", "--delete", "--delay-updates", "-avz", "bin/targets/%s/%s/" %(ts[0], ts[1]), "%s/targets/%s/%s/" %(rsync_url, ts[0], ts[1])], - env={'RSYNC_PASSWORD': rsync_key}, + command=["rsync", "--delete", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", WithProperties("bin/targets/%s/%s%%(libc)s/" %(ts[0], ts[1])), "%s/targets/%s/%s/" %(rsync_bin_url, ts[0], ts[1])], + env={'RSYNC_PASSWORD': rsync_bin_key}, haltOnFailure = True, logEnviron = False )) + if rsync_src_url is not None: + factory.addStep(ShellCommand( + name = "sourceupload", + description = "Uploading source archives", + command=["rsync", "--checksum", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "dl/", "%s/" %(rsync_src_url)], + env={'RSYNC_PASSWORD': rsync_src_key}, + haltOnFailure = True, + logEnviron = False + )) + if False: factory.addStep(ShellCommand( name = "packageupload", description = "Uploading package files", - command=["rsync", "--delete", "--delay-updates", "-avz", "bin/packages/", "%s/packages/" %(rsync_url)], - env={'RSYNC_PASSWORD': rsync_key}, + command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "bin/packages/", "%s/packages/" %(rsync_bin_url)], + env={'RSYNC_PASSWORD': rsync_bin_key}, haltOnFailure = False, logEnviron = False )) @@ -433,8 +525,8 @@ EOT''' %(ts[0], ts[0], ts[1]) )) factory.addStep(ShellCommand( name = "upload", description = "Uploading logs", - command=["rsync", "--delete", "--delay-updates", "-avz", "logs/", "%s/logs/%s/%s/" %(rsync_url, ts[0], ts[1])], - env={'RSYNC_PASSWORD': rsync_key}, + command=["rsync", "--delete", "--delay-updates", "--partial-dir=.~tmp~%s~%s" %(ts[0], ts[1]), "-avz", "logs/", "%s/logs/%s/%s/" %(rsync_bin_url, ts[0], ts[1])], + env={'RSYNC_PASSWORD': rsync_bin_key}, haltOnFailure = False, alwaysRun = True, logEnviron = False @@ -462,7 +554,7 @@ if ini.has_option("status", "bind"): # change any of these to True to enable; see the manual for more # options auth=auth.BasicAuth([(ini.get("status", "user"), ini.get("status", "password"))]), - gracefulShutdown = False, + gracefulShutdown = 'auth', forceBuild = 'auth', # use this to test your slave once it is set up forceAllBuilds = 'auth', pingBuilder = False, @@ -474,6 +566,34 @@ if ini.has_option("status", "bind"): else: c['status'].append(html.WebStatus(http_port=ini.get("status", "bind"))) + +from buildbot.status import words + +if ini.has_option("irc", "host") and ini.has_option("irc", "nickname") and ini.has_option("irc", "channel"): + irc_host = ini.get("irc", "host") + irc_port = 6667 + irc_chan = ini.get("irc", "channel") + irc_nick = ini.get("irc", "nickname") + irc_pass = None + + if ini.has_option("irc", "port"): + irc_port = ini.getint("irc", "port") + + if ini.has_option("irc", "password"): + irc_pass = ini.get("irc", "password") + + irc = words.IRC(irc_host, irc_nick, port = irc_port, password = irc_pass, + channels = [{ "channel": irc_chan }], + notify_events = { + 'exception': 1, + 'successToFailure': 1, + 'failureToSuccess': 1 + } + ) + + c['status'].append(irc) + + ####### PROJECT IDENTITY # the 'title' string will appear at the top of this buildbot @@ -498,4 +618,3 @@ c['db'] = { # this at its default for all but the largest installations. 'db_url' : "sqlite:///state.sqlite", } -