From 34b4378d9a6f3cad8e728e9e91ba163a716c451e Mon Sep 17 00:00:00 2001 From: =?utf8?q?Thibaut=20VAR=C3=88NE?= Date: Mon, 13 Nov 2023 15:23:49 +0100 Subject: [PATCH] phase1: raise priority of tag builds MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit Currently the buildmaster would only order tag builds within their own branch, meaning that if e.g. a higher priority branch has normal buildrequests (i.e. buildrequests comming from the AnyBranchScheduler), and a lower priority branch has "tag" buildrequests (i.e. from the Triggerable scheduler), the former builderequests would be served first, deferring the build of e.g. release tags. We want forced builds (release tag) to have maximum priority, regardless of branch priority. This commit attempts to solve this problem by leveraging the newly (as of buildbot 3.9.0) introduced "priority" scheduler parameter, by raising the Triggerable scheduler buildrequests priority, and then considering this higher priority in Builders' order. The net result is that Builders are now prioritized if they have pending higher priority buildrequest, still preserving the branch order. In other words, tag requests are front run while preserving branch order, meaning that if two branches have tag buildrequests, the higher priority branch is still served first. This requires buildbot 3.9.0 Signed-off-by: Thibaut VARÈNE --- .github/workflows/build-push.yml | 2 +- phase1/master.cfg | 30 +++++++++++++++++++----------- 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml index 199b739..9a2eed4 100644 --- a/.github/workflows/build-push.yml +++ b/.github/workflows/build-push.yml @@ -8,7 +8,7 @@ on: pull_request: env: - BUILDBOT_VERSION: 3.8.0 + BUILDBOT_VERSION: 3.9.0 GITHUB_SHA_LEN: 8 concurrency: diff --git a/phase1/master.cfg b/phase1/master.cfg index 6f6c650..9b749b4 100644 --- a/phase1/master.cfg +++ b/phase1/master.cfg @@ -191,16 +191,20 @@ c["configurators"] = [ @defer.inlineCallbacks -def getNewestCompleteTime(bldr): - """Returns the complete_at of the latest completed and not SKIPPED +def getNewestCompleteTimePrio(bldr): + """Returns the priority and the complete_at of the latest completed and not SKIPPED build request for this builder, or None if there are no such build requests. We need to filter out SKIPPED requests because we're using collapseRequests=True which is unfortunately marking all previous requests as complete when new buildset is created. - @returns: datetime instance or None, via Deferred + @returns: (priority, datetime instance or None), via Deferred """ + prio = yield bldr.get_highest_priority() + if prio is None: + prio = 0 + bldrid = yield bldr.getBuilderId() completed = yield bldr.master.data.get( ("builders", bldrid, "buildrequests"), @@ -212,7 +216,7 @@ def getNewestCompleteTime(bldr): limit=1, ) if not completed: - return + return (prio, None) complete_at = completed[0]["complete_at"] @@ -228,9 +232,9 @@ def getNewestCompleteTime(bldr): if last_build and last_build[0]: last_complete_at = last_build[0]["complete_at"] if last_complete_at and (last_complete_at > complete_at): - return last_complete_at + return (prio, last_complete_at) - return complete_at + return (prio, complete_at) @defer.inlineCallbacks @@ -251,19 +255,23 @@ def prioritizeBuilders(master, builders): return bool(bldr.building) or bool(bldr.old_building) def bldr_info(bldr): - d = defer.maybeDeferred(getNewestCompleteTime, bldr) - d.addCallback(lambda complete_at: (complete_at, bldr)) + d = defer.maybeDeferred(getNewestCompleteTimePrio, bldr) + d.addCallback(lambda retval: (retval, bldr)) return d def bldr_sort(item): - (complete_at, bldr) = item + ((hiprio, complete_at), bldr) = item + # check if we have some high prio build requests pending (i.e. tag builds), + # if so, front-run these builders, while preserving the per-branch static priority pos = 99 for name, prio in bldrNamePrio.items(): if bldr.name.startswith(name): - pos = prio + pos = prio + 50 - min(hiprio, 50) # higher priority (larger positive number) raises position break + # pos order: janitor/local (0), tag builds per branch order if any [1..50], !tag builds per branch order [51...] + if not complete_at: date = datetime.min complete_at = date.replace(tzinfo=tzutc()) @@ -531,7 +539,7 @@ c["schedulers"].append( ) c["schedulers"].append( - schedulers.Triggerable(name="trigger", builderNames=builderNames) + schedulers.Triggerable(name="trigger", builderNames=builderNames, priority=20) ) ####### BUILDERS -- 2.30.2