diff --git a/pkg/scheduler/framework/session.go b/pkg/scheduler/framework/session.go index ede8d69efe1..af13ed3715a 100644 --- a/pkg/scheduler/framework/session.go +++ b/pkg/scheduler/framework/session.go @@ -376,6 +376,28 @@ func closeSession(ssn *Session) { } func getPodGroupPhase(jobInfo *api.JobInfo, unschedulable bool) scheduling.PodGroupPhase { + // Latch the terminal phase: once a PodGroup reached Completed, keep it + // Completed while every remaining member task is itself in a terminal + // state. Pod deletion (TTL/GC/workload controller) is not atomic and + // usually spans multiple sessions; without this latch a partially or + // fully deleted group is recomputed below minMember, downgraded back to + // Pending/Inqueue, re-admitted by the enqueue action, and the + // capacity/proportion plugins reserve its MinResources queue quota again + // even though it can never run again. A remaining non-completed task + // (e.g. a fresh Pending pod) still falls through to the normal recompute. + if jobInfo.PodGroup.Status.Phase == scheduling.PodGroupCompleted { + allCompleted := true + for _, task := range jobInfo.Tasks { + if !api.CompletedStatus(task.Status) { + allCompleted = false + break + } + } + if allCompleted { + return scheduling.PodGroupCompleted + } + } + // If running tasks && unschedulable, unknown phase if len(jobInfo.TaskStatusIndex[api.Running]) != 0 && unschedulable { return scheduling.PodGroupUnknown diff --git a/pkg/scheduler/framework/session_test.go b/pkg/scheduler/framework/session_test.go index fa8d9fb02a6..fda88664c4c 100644 --- a/pkg/scheduler/framework/session_test.go +++ b/pkg/scheduler/framework/session_test.go @@ -91,6 +91,38 @@ func TestGetPodGroupPhase(t *testing.T) { newTask("task-2", api.Succeeded, "node-2")), expected: scheduling.PodGroupCompleted, }, + { + name: "completed podgroup with no tasks left keeps Completed", + job: newJob(2, scheduling.PodGroupCompleted), + expected: scheduling.PodGroupCompleted, + }, + { + name: "partially deleted completed podgroup keeps Completed", + job: newJob(2, scheduling.PodGroupCompleted, + newTask("task-1", api.Succeeded, "node-1")), + expected: scheduling.PodGroupCompleted, + }, + { + name: "completed podgroup with failed remaining task keeps Completed", + job: newJob(2, scheduling.PodGroupCompleted, + newTask("task-1", api.Succeeded, "node-1"), + newTask("task-2", api.Failed, "node-2")), + expected: scheduling.PodGroupCompleted, + }, + { + name: "completed podgroup with remaining running task recomputes", + job: newJob(2, scheduling.PodGroupCompleted, + newTask("task-1", api.Succeeded, "node-1"), + newTask("task-2", api.Running, "node-2")), + expected: scheduling.PodGroupRunning, + }, + { + name: "completed podgroup with new pending task recomputes", + job: newJob(2, scheduling.PodGroupCompleted, + newTask("task-1", api.Succeeded, "node-1"), + newTask("task-2", api.Pending, "")), + expected: scheduling.PodGroupPending, + }, { name: "scheduled releasing tasks below minMember fall to Pending", job: newJob(2, scheduling.PodGroupRunning,