Only surface live pods in manage command output

src/stack/deploy/k8s/deploy_k8s.py

@@ -32,6 +32,7 @@
 )
 from stack.deploy.k8s.helpers import install_ingress_for_kind, wait_for_ingress_in_kind
 from stack.deploy.k8s.helpers import (
+    live_pods,
     pods_in_deployment,
     containers_in_pod,
     log_stream_from_string,
@@ -419,7 +420,7 @@ def status(self):
             label_selector=f"app={self.cluster_info.app_name}",
             watch=False,
         )
-        pods = pod_response.items if pod_response.items else []
+        pods = live_pods(pod_response.items) if pod_response.items else []
 
         if not pods:
             return
@@ -504,7 +505,7 @@ def ps(self):
 
         ret = []
 
-        for p in pod_response.items:
+        for p in live_pods(pod_response.items):
             pod_ip = p.status.pod_ip
             ports = AttrDict()
             for c in p.spec.containers:

src/stack/deploy/k8s/helpers.py

@@ -88,11 +88,24 @@ def load_images_into_kind(kind_cluster_name: str, image_set: Set[str]):
             raise DeployerException(f"kind create cluster failed: {result}")
 
 
+def live_pods(pod_items):
+    """The pods that are still part of the deployment, dropping any being deleted.
+
+    A deleted pod keeps being listed until its containers have actually stopped,
+    which on a real cluster is a graceful shutdown lasting tens of seconds. It is
+    on its way out, and nothing the user asks about the deployment should count
+    it: "ps" and "status" would report a container that has already been told to
+    go, "logs" would replay output from before a restart, and "exec" could land
+    in a pod that is about to disappear underneath it.
+    """
+    return [pod for pod in pod_items if pod.metadata.deletion_timestamp is None]
+
+
 def pods_in_deployment(core_api: client.CoreV1Api, deployment_name: str, namespace: str = DEFAULT_K8S_NAMESPACE):
     pods = []
     pod_response = core_api.list_namespaced_pod(namespace=namespace, label_selector=f"app={deployment_name}")
     log_debug(f"pod_response: {pod_response}")
-    for pod_info in pod_response.items:
+    for pod_info in live_pods(pod_response.items):
         pod_name = pod_info.metadata.name
         pods.append(pod_name)
     return pods

tests/app-deploy/run-test.sh

@@ -103,6 +103,7 @@ fi
 # data directory -- stopping a kind deployment deletes the cluster -- and a real
 # PVC).
 $TEST_TARGET_STACK manage --dir $test_deployment_dir stop
+wait_for_stopped
 $TEST_TARGET_STACK manage --dir $test_deployment_dir start
 wait_for_running 3 $TEST_START_CHECK_LIMIT
 

tests/database/run-test.sh

@@ -68,10 +68,12 @@ else
     fail "Create database content test: FAILED"
 fi
 
-# Stop then start again and check the volume was preserved
+# Stop then start again and check the volume was preserved.  The wait for the
+# containers to be gone is what makes the log check below mean anything: a pod
+# still terminating carries this test's own output from before the restart, and
+# the wait for "test complete" would match that instead of the new run.
 $TEST_TARGET_STACK manage --dir $test_deployment_dir stop
-# Sleep a bit just in case
-sleep 20
+wait_for_stopped
 $TEST_TARGET_STACK manage --dir $test_deployment_dir start
 wait_for_containers_started
 wait_for_log_content "Database test client: test complete"

tests/lib/common.sh

@@ -312,6 +312,31 @@ wait_for_containers_started () {
     fail "waiting for containers to start: FAILED"
 }
 
+# Wait until the deployment's containers are gone, after a stop.  $1 overrides
+# the number of 5-second checks (default 24, so two minutes).
+#
+# `stop` returns once it has deleted the objects, not once they are gone. On a
+# real cluster a pod terminates gracefully and lingers for a while, and both `ps`
+# and `logs` keep reporting it -- so a test that stops, starts, and then waits
+# for a log line can match the *previous* run's output from a pod that is on its
+# way out, and carry on before the new one has done anything at all. That is a
+# race the local targets usually win and a real cluster usually loses.
+wait_for_stopped () {
+    local check_limit=${1:-24}
+    local check=0
+    local ps_output
+    while [ $check -lt $check_limit ]; do
+        check=$((check + 1))
+        ps_output=$( $TEST_TARGET_STACK manage --dir "$TEST_DEPLOYMENT_DIR" ps ) || true
+        if [[ "$ps_output" != *"id:"* ]]; then
+            return
+        fi
+        echo "waiting for containers to stop..."
+        sleep 5
+    done
+    fail "waiting for containers to stop: FAILED"
+}
+
 # Wait until `manage logs` output contains $1 -- or, with no argument, until it
 # produces any output at all.  $2 overrides the number of 5-second checks
 # (default 50).

tests/unit/test_k8s_pod_listing.py

@@ -0,0 +1,66 @@
+# Copyright © 2026 Bozeman Pass, Inc.
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http:#www.gnu.org/licenses/>.
+
+"""Which pods the deployment commands report.
+
+A pod that has been deleted is still listed by the API until its containers have
+stopped, which on a real cluster is a graceful shutdown lasting tens of seconds.
+Reporting it means "ps" and "status" describe a container that is on its way out,
+and "logs" replays output from before a restart -- which is exactly how a stopped
+and restarted deployment came to look like it had already finished its work.
+"""
+
+from types import SimpleNamespace
+
+from stack.deploy.k8s.helpers import live_pods, pods_in_deployment
+
+NAMESPACE = "stack-test"
+
+
+def pod(name, terminating=False):
+    """Stands in for a V1Pod; only the metadata is read here."""
+    return SimpleNamespace(
+        metadata=SimpleNamespace(name=name, namespace=NAMESPACE, deletion_timestamp="2026-08-13T23:54:05Z" if terminating else None)
+    )
+
+
+def fake_api(pods):
+    return SimpleNamespace(list_namespaced_pod=lambda **kwargs: SimpleNamespace(items=pods))
+
+
+def test_live_pods_keeps_pods_that_are_not_being_deleted():
+    pods = [pod("deploy-web-new"), pod("deploy-web-old", terminating=True)]
+
+    assert [p.metadata.name for p in live_pods(pods)] == ["deploy-web-new"]
+
+
+def test_live_pods_of_nothing_is_nothing():
+    assert live_pods([]) == []
+
+
+def test_all_pods_terminating_reports_none():
+    # Mid-stop every pod is on its way out, and the deployment has nothing to
+    # report rather than a list of containers that are already going away.
+    pods = [pod("deploy-web", terminating=True), pod("deploy-db", terminating=True)]
+
+    assert live_pods(pods) == []
+
+
+def test_pods_in_deployment_skips_a_terminating_pod():
+    # The listing that logs and exec work from: a restart leaves the old pod
+    # present for a while, and its log still holds everything from before.
+    api = fake_api([pod("deploy-web-old", terminating=True), pod("deploy-web-new")])
+
+    assert pods_in_deployment(api, "deploy", NAMESPACE) == ["deploy-web-new"]