From e22dfe0f8f2a6e1ae306eb92227a48f5e035e4d1 Mon Sep 17 00:00:00 2001 From: David Spickett Date: Tue, 12 Nov 2024 12:04:01 +0000 Subject: [PATCH] Collapse clean and non clean build requests on clean builders (#266) Fixes #250 Where build requests were piling up on slower builders because they were clean/don't clean/clean/ don't clean etc. and we could not merge those requests because of that difference. Galina suggested that if we knew that the builder was going to clean the build first anyway, we could merge them. This commit implements that. We do this by first setting a "clean" attribute on the builder's factory object, that we can find later via in the collapse callback. So far only ClangBuilder passes this on but any builder can opt in by copying what it does. In the collapse function when comparing properties we delete the "clean_obj" key from both sets of properties if we know that the builder's factory is always clean. With some logging (not included in this commit) we can see it now collapses a clean/non-clean pair of requests: ``` collapseRequests selfBuildSetProperties: {'scheduler': ('main:clang,clang-tools-extra,compiler-rt,flang,lld,llvm,mlir', 'Scheduler')} otherBuildSetProperties: {'scheduler': ('main:clang,clang-tools-extra,compiler-rt,flang,lld,llvm,mlir', 'Scheduler'), 'clean_obj': (True, 'Change')} Removing clean_obj from build set properties. Build set properties were the same. ``` This has beeen tested by my colleague Omair Javaid on a test build master. Of course we only tested it with one of Linaro's builders, so it's possible we see other side effects in production. --- zorg/buildbot/builders/ClangBuilder.py | 3 ++- zorg/buildbot/process/buildrequest.py | 8 ++++++++ zorg/buildbot/process/factory.py | 2 ++ 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/zorg/buildbot/builders/ClangBuilder.py b/zorg/buildbot/builders/ClangBuilder.py index 001d824a3..2c375d211 100644 --- a/zorg/buildbot/builders/ClangBuilder.py +++ b/zorg/buildbot/builders/ClangBuilder.py @@ -278,7 +278,8 @@ def _getClangCMakeBuildFactory( f = LLVMBuildFactory( depends_on_projects=depends_on_projects, llvm_srcdir='llvm', - enable_runtimes=enable_runtimes) + enable_runtimes=enable_runtimes, + clean=clean) # Checkout the latest code for LNT # and the test-suite separately. Le's do this first, diff --git a/zorg/buildbot/process/buildrequest.py b/zorg/buildbot/process/buildrequest.py index 594a09d25..ebd6371ef 100644 --- a/zorg/buildbot/process/buildrequest.py +++ b/zorg/buildbot/process/buildrequest.py @@ -31,6 +31,14 @@ def collapseRequests(master, builder, req1, req2): str(req2['buildsetid']) ) + # If the build is going to be a clean build anyway, we can collapse a clean + # build and a non-clean build. + if getattr(builder.config.factory, "clean", False): + if 'clean_obj' in selfBuildsetPoperties: + del selfBuildsetPoperties["clean_obj"] + if 'clean_obj' in otherBuildsetPoperties: + del otherBuildsetPoperties["clean_obj"] + # Check buildsets properties and do not collapse # if properties do not match. This includes the check # for different schedulers. diff --git a/zorg/buildbot/process/factory.py b/zorg/buildbot/process/factory.py index b15bbaa19..a2ac70ca8 100644 --- a/zorg/buildbot/process/factory.py +++ b/zorg/buildbot/process/factory.py @@ -40,6 +40,8 @@ def __init__(self, steps=None, depends_on_projects=None, hint=None, **kwargs): BuildFactory.__init__(self, steps) self.hint = hint + + self.clean = kwargs.pop('clean', False) # Handle the dependencies. if depends_on_projects is None: