phase2: s/SetProperty/SetPropertyFromCommand/

[buildbot.git] / phase2 / master.cfg

diff --git a/phase2/master.cfg b/phase2/master.cfg
index b5050e578cf2c8909e5d2eba1ef09b3ce225e312..9a0211f49997ccf0428ca2c073360ce161998009 100644 (file)
--- a/phase2/master.cfg
+++ b/phase2/master.cfg
@@ -3,8 +3,8 @@
 
 import os
 import re
+import sys
 import base64
-import random
 import subprocess
 import configparser
 
@@ -29,7 +29,7 @@ from buildbot.process.properties import WithProperties
 from buildbot.schedulers.basic import SingleBranchScheduler
 from buildbot.schedulers.forcesched import ForceScheduler
 from buildbot.steps.master import MasterShellCommand
-from buildbot.steps.shell import SetProperty
+from buildbot.steps.shell import SetPropertyFromCommand
 from buildbot.steps.shell import ShellCommand
 from buildbot.steps.transfer import FileDownload
 from buildbot.steps.transfer import FileUpload
@@ -61,7 +61,6 @@ c = BuildmasterConfig = {}
 
 worker_port = 9990
 persistent = False
-other_builds = 0
 tree_expire = 0
 git_ssh = False
 git_ssh_key = None
@@ -72,9 +71,6 @@ if ini.has_option("phase2", "port"):
 if ini.has_option("phase2", "persistent"):
        persistent = ini.getboolean("phase2", "persistent")
 
-if ini.has_option("phase2", "other_builds"):
-       other_builds = ini.getint("phase2", "other_builds")
-
 if ini.has_option("phase2", "expire"):
        tree_expire = ini.getint("phase2", "expire")
 
@@ -92,7 +88,7 @@ max_builds = dict()
 for section in ini.sections():
        if section.startswith("worker "):
                if ini.has_option(section, "name") and ini.has_option(section, "password") and \
-                  ini.has_option(section, "phase") and ini.getint(section, "phase") == 2:
+                       ini.has_option(section, "phase") and ini.getint(section, "phase") == 2:
                        name = ini.get(section, "name")
                        password = ini.get(section, "password")
                        sl_props = { 'shared_wd': False }
@@ -199,7 +195,7 @@ c['change_source'] = []
 
 def parse_feed_entry(line):
        parts = line.strip().split()
-       if parts[0] == "src-git":
+       if parts[0].startswith("src-git"):
                feeds.append(parts)
                url = parts[2].strip().split(';')
                branch = url[1] if len(url) > 1 else 'master'
@@ -211,12 +207,15 @@ make = subprocess.Popen(['make', '--no-print-directory', '-C', work_dir+'/source
 
 line = make.stdout.readline()
 if line:
-       parse_feed_entry(line)
+       parse_feed_entry(str(line, 'utf-8'))
 
-with open(work_dir+'/source.git/feeds.conf.default', 'r') as f:
+with open(work_dir+'/source.git/feeds.conf.default', 'r', encoding='utf-8') as f:
        for line in f:
                parse_feed_entry(line)
 
+if len(c['change_source']) == 0:
+       log.err("FATAL ERROR: no change_sources defined, aborting!")
+       sys.exit(-1)
 
 ####### SCHEDULERS
 
@@ -290,7 +289,7 @@ def GetDirectorySuffix(props):
 
 def GetNumJobs(props):
        if props.hasProperty("workername") and props.hasProperty("nproc"):
-               return ((int(props["nproc"]) / (max_builds[props["workername"]] + other_builds)) + 1)
+               return (int(props["nproc"]) / max_builds[props["workername"]])
        else:
                return 1
 
@@ -320,7 +319,7 @@ def IsArchitectureSelected(target):
 def UsignSec2Pub(seckey, comment="untrusted comment: secret key"):
        try:
                seckey = base64.b64decode(seckey)
-       except:
+       except Exception:
                return None
 
        return "{}\n{}".format(re.sub(r"\bsecret key$", "public key", comment),
@@ -351,7 +350,21 @@ def getNewestCompleteTime(bldr):
        if not completed:
                return
 
-       return completed[0]['complete_at']
+       complete_at = completed[0]['complete_at']
+
+       last_build = yield bldr.master.data.get(
+                       ('builds', ),
+                       [
+                               resultspec.Filter('builderid', 'eq', [bldrid]),
+                       ],
+                       order=['-started_at'], limit=1)
+
+       if last_build and last_build[0]:
+               last_complete_at = last_build[0]['complete_at']
+               if last_complete_at and (last_complete_at > complete_at):
+                       return last_complete_at
+
+       return complete_at
 
 @defer.inlineCallbacks
 def prioritizeBuilders(master, builders):
@@ -422,7 +435,7 @@ for arch in arches:
                doStepIf = IsSharedWorkdir))
 
        # find number of cores
-       factory.addStep(SetProperty(
+       factory.addStep(SetPropertyFromCommand(
                name = "nproc",
                property = "nproc",
                description = "Finding number of CPUs",
@@ -504,7 +517,7 @@ for arch in arches:
                workerdest = "sdk/getversion.mk",
                mode = 0o755))
 
-       factory.addStep(SetProperty(
+       factory.addStep(SetPropertyFromCommand(
                name = "getversion",
                property = "release_version",
                description = "Finding SDK release version",
@@ -744,7 +757,7 @@ for arch in arches:
                        description = "Uploading source archives",
                        workdir = "build/sdk",
                        command = ["rsync", "--files-from=sourcelist", "-4", "--progress", "--checksum", "--delay-updates",
-                                  WithProperties("--partial-dir=.~tmp~%s~%%(workername)s" %(arch[0])), "-avz", "dl/", "%s/" %(rsync_src_url)],
+                                       WithProperties("--partial-dir=.~tmp~%s~%%(workername)s" %(arch[0])), "-avz", "dl/", "%s/" %(rsync_src_url)],
                        env={'RSYNC_PASSWORD': rsync_src_key},
                        haltOnFailure = False,
                        flunkOnFailure = False,
@@ -763,6 +776,17 @@ for arch in arches:
                alwaysRun = True
        ))
 
+       factory.addStep(ShellCommand(
+               name = "du",
+               description = "Reporting estimated file space usage",
+               command=["du", "-sh", "."],
+               env={'LC_ALL': 'C'},
+               haltOnFailure = False,
+               flunkOnFailure = False,
+               warnOnFailure = False,
+               alwaysRun = True
+       ))
+
        c['builders'].append(BuilderConfig(name=arch[0], workernames=workerNames, factory=factory))
 
        c['schedulers'].append(schedulers.Triggerable(name="trigger_%s" % arch[0], builderNames=[ arch[0] ]))