From d6622b474bca4a7446a286a79d52f2d799d59eb8 Mon Sep 17 00:00:00 2001 From: Marcos Amorim Date: Fri, 8 May 2026 10:37:56 -0400 Subject: [PATCH] fix: retry loop in remove_run_from_status exits without retrying on 422 The break statement was outside the try block, causing the while loop to exit after a single iteration regardless of success or 422 conflict. On a 422, the subject was refreshed but the removal was never retried, leaving a ghost run reference permanently in status.runs.active. This blocked the FIFO run queue, preventing subsequent runs from being promoted to pending and ultimately preventing subject deletion. Changed to a bounded for loop (matching add_run_to_status pattern) with break inside the try block, so 422 conflicts trigger refresh + retry. --- operator/anarchysubject.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/operator/anarchysubject.py b/operator/anarchysubject.py index 5d68a24..35bdb79 100644 --- a/operator/anarchysubject.py +++ b/operator/anarchysubject.py @@ -651,8 +651,8 @@ async def remove_run_from_status(self, anarchy_run) -> None: # Suspicious if AnarchyRun is not in status, refresh object from API to be sure. if not self.has_run_in_status(anarchy_run): await self.refresh() - # Loop until consistent - while True: + max_retries = 10 + for attempt in range(max_retries): patch = [] removed_active_run = False # Loop over status runs entries building patch to remove run. @@ -676,16 +676,20 @@ async def remove_run_from_status(self, anarchy_run) -> None: return try: await self.json_patch_status(patch) + break except kubernetes_asyncio.client.rest.ApiException as e: # 404 indicates run was being removed because subject was also being deleted. if e.status == 404: return if e.status == 422: - # Patch failed test condition, must be out of sync + # Refresh AnarchySubject state and retry await self.refresh() else: raise - break + else: + raise kopf.TemporaryError( + f"Failed to remove {anarchy_run} from {self} status after {max_retries} retries", + ) if removed_active_run: logging.info("Removed active %s from %s status", anarchy_run, self)