From 2a048b864b8c4d41c4bcf52534f04042ec4343ca Mon Sep 17 00:00:00 2001 From: Kyle Squizzato Date: Thu, 3 Sep 2026 14:49:08 -0700 Subject: [PATCH 1/2] feat(e2e): various fixes for openshift tests * We now patch the controller with GC test values and then revert that patch when e2e concludes for both Kind and OpenShift rather than having it live in kustomization.yaml * Remove Kind-specific hacks from e2e scripts since OpenShift is now in the hood. Assisted-by: Claude Sonnet 5 Signed-off-by: Kyle Squizzato --- deploy/kind/kustomization.yaml | 7 -- specs/platform/e2e-testing.spec.md | 5 +- tests/e2e/drivers/kind.sh | 71 ++++++++++++ tests/e2e/drivers/openshift.sh | 38 +++++++ tests/e2e/e2e-openshell.sh | 177 +++++++++++++++-------------- tests/e2e/e2e-performance.sh | 3 + tests/e2e/lib.sh | 17 +++ tests/e2e/openshift_driver_test.sh | 23 ++++ 8 files changed, 246 insertions(+), 95 deletions(-) diff --git a/deploy/kind/kustomization.yaml b/deploy/kind/kustomization.yaml index 64f5d60d..6d7bfe89 100644 --- a/deploy/kind/kustomization.yaml +++ b/deploy/kind/kustomization.yaml @@ -58,13 +58,6 @@ patches: # production/OpenShift leaves this unset to keep them enforced. - name: GATEWAY_SKIP_NETWORK_POLICIES value: "true" - # Shorten periodic namespace GC for e2e (production defaults 5m/10m). - # Go time.ParseDuration accepts seconds too (e.g. 30s, 1m30s). - # tests/e2e seeds a synthetic orphan after gateway provisioning (after step 2); step 11a asserts the reaper deleted it. - - name: GATEWAY_NAMESPACE_GC_INTERVAL - value: "30s" - - name: GATEWAY_NAMESPACE_GC_GRACE_PERIOD - value: "30s" - name: OIDC_ISSUER value: "http://keycloak-service.keycloak.svc.cluster.local:8080/realms/hypershell" - name: OIDC_CLIENT_ID diff --git a/specs/platform/e2e-testing.spec.md b/specs/platform/e2e-testing.spec.md index 7297ebae..f2ffa907 100644 --- a/specs/platform/e2e-testing.spec.md +++ b/specs/platform/e2e-testing.spec.md @@ -49,6 +49,7 @@ Each driver exports shell functions that abstract infrastructure-specific operat | Function | Purpose | Kind Implementation | OpenShift Implementation | |----------|---------|---------------------|--------------------------| | `discover_api_host` | Find the HyperShell API server URL | HTTPRoute hostname `api.hypershell.localhost` or port-forward to `svc/hypershell-api-server` | `oc get route hypershell-api -o jsonpath='{.spec.host}'` | +| `discover_console_host` | Find the HyperShell web console (BFF) URL | HTTPRoute hostname `console.hypershell.localhost` | `oc get route hypershell-web-console -o jsonpath='{.spec.host}'`. Not derivable from `discover_api_host`'s result by string substitution -- Route hostnames are cluster-generated and unrelated to each other | | `discover_gateway_endpoint` | Find the gateway gRPC endpoint | GRPCRoute hostname `.gw.localhost` via Gateway status address | GRPCRoute hostname via shared Gateway `Programmed=True` (Gateway API, not a per-gateway Route) | | `get_cluster_domain` | Get the base domain for constructing gateway DNS names | `gw.localhost` (static, matching `GATEWAY_API_BASE_DOMAIN` in `deploy/kind/`) | Gateway base domain derived from the shared Gateway listener hostname -- the same value `make openshift-up` sets on the control plane. Not a developer-supplied `GATEWAY_API_BASE_DOMAIN`, and not the cluster apps domain | | `get_cli_binary` | Return the Kubernetes CLI binary path | `kubectl` | `oc` | @@ -58,6 +59,8 @@ Each driver exports shell functions that abstract infrastructure-specific operat | `assign_gateway_client_role` | Grant a user a role on a gateway's per-gateway OIDC client (mirrors the `gateway:viewer` RoleBinding); idempotent | Keycloak admin API assigns the client role in the `hypershell` realm | HyperShell Keycloak admin API (at its Route in the `${OPENSHIFT_NAMESPACE}-keycloak` namespace) assigns the client role in the `hypershell` realm | | `assign_realm_role` | Grant a user a platform-wide realm role, for example `platform:admin`; idempotent | Keycloak admin API assigns the realm role | HyperShell Keycloak admin API (at its Route in the `${OPENSHIFT_NAMESPACE}-keycloak` namespace) assigns the realm role in the `hypershell` realm | | `acquire_gateway_token_with_role` | Acquire a per-gateway OIDC token and block until the named role lands in it (roles reconcile asynchronously after gateway create); sets `_OIDC_ACCESS_TOKEN` | Password grant against the per-gateway client, polling until the role appears | Password grant against the per-gateway client on the HyperShell Keycloak at its Route, polling until the role appears | +| `configure_namespace_gc_timing` | Temporarily shorten the controller's namespace-GC interval/grace period for the duration of a long-mode run, so the orphan-GC assertion (area 11a) doesn't have to wait out production timing; blocks until the resulting rollout completes | `kubectl set env deployment/hypershell-controller` in the Kind namespace, then wait for rollout | `oc set env deployment/hypershell-controller` in `OPENSHIFT_NAMESPACE`, then wait for rollout | +| `restore_namespace_gc_timing` | Revert the override applied by `configure_namespace_gc_timing`, restoring the deployment's configured (production) defaults; a no-op if never patched; called unconditionally from the suite's cleanup trap, even on failure | Same mechanism as `configure_namespace_gc_timing`, in reverse | Same mechanism as `configure_namespace_gc_timing`, in reverse | ### CI Pipeline @@ -224,7 +227,7 @@ Each target SHALL auto-detect the driver from the current KUBECONFIG context (se **Driver behavior needed for parity.** For the shared suite to pass on OpenShift, the OpenShift driver SHALL use the current `oc` project when `OPENSHIFT_NAMESPACE` is unset (and fail clearly when neither is available), matching `make openshift-up`; derive the OIDC issuer from the Keycloak Route in `${OPENSHIFT_NAMESPACE}-keycloak` (not the Kind default `keycloak.hypershell.localhost`); return `get_cluster_domain` from the same shared-Gateway listener hostname `make openshift-up` used; and provide the same Keycloak admin and role-assignment helpers the Kind driver provides, so the RBAC areas (developer and platform-admin) run unchanged. The OpenShift deployment SHALL enforce RBAC (`RBAC_ENFORCE=true`) and SHALL keep the OpenShift SCC posture (per-namespace privileged SCC for sandbox pods), so the sandbox and RBAC areas behave the same as on Kind. These behaviors are specified in `openshift-development.spec.md`; this spec only depends on them. -**Namespace GC timing.** Area 11 exercises the periodic namespace reaper. To make it pass on OpenShift without waiting the production GC defaults (5m sweep / 10m grace), the OpenShift deployment SHOULD set shortened `GATEWAY_NAMESPACE_GC_INTERVAL` and `GATEWAY_NAMESPACE_GC_GRACE_PERIOD` (as the Kind overlay does), or the user SHOULD raise `E2E_ORPHAN_GC_TIMEOUT` and `E2E_GC_TIMEOUT` to fit the cluster's configured timing. +**Namespace GC timing.** Area 11 exercises the periodic namespace reaper. Every deploy target (Kind included) runs with the production `GATEWAY_NAMESPACE_GC_INTERVAL`/`GATEWAY_NAMESPACE_GC_GRACE_PERIOD` defaults (5m sweep / 10m grace) -- no overlay bakes in shortened e2e timing, so Kind stays representative of a vanilla deployment. Instead, a long-mode run SHALL call `configure_namespace_gc_timing` once, before any gateway is created, to patch the controller deployment to a short interval/grace period for the duration of the run, and SHALL call `restore_namespace_gc_timing` from the suite's cleanup path so the deployment's production defaults are always restored, pass or fail. **Not in CI.** OpenShift e2e and performance runs SHALL NOT be wired into CI in this iteration (see [Design Decisions](#design-decisions)). CI runs the Kind e2e workflow only. diff --git a/tests/e2e/drivers/kind.sh b/tests/e2e/drivers/kind.sh index 375bb652..5662d9c3 100755 --- a/tests/e2e/drivers/kind.sh +++ b/tests/e2e/drivers/kind.sh @@ -46,6 +46,30 @@ discover_api_host() { _DISCOVER_API_HOST="${url}" } +# discover_console_host - find the HyperShell web console (BFF) base URL. +# Sets _DISCOVER_CONSOLE_HOST to the gateway HTTPS route for the web console. +discover_console_host() { + _DISCOVER_CONSOLE_HOST="" + local host + host=$(kubectl get httproute -A -o jsonpath='{range .items[*]}{.spec.hostnames[0]}{"\n"}{end}' 2>/dev/null \ + | grep -m1 'console\.hypershell\.localhost' || true) + if [[ -z "$host" ]]; then + red " No HTTPRoute with hostname console.hypershell.localhost found" + return 1 + fi + + local url="https://${host}" + local code + code=$(_driver_curl --connect-timeout 5 -o /dev/null -w '%{http_code}' \ + "${url}/auth/session" 2>/dev/null || true) + if [[ -z "$code" || "$code" == "000" ]]; then + red " Console route ${url} is not reachable (no HTTP response)" + return 1 + fi + + _DISCOVER_CONSOLE_HOST="${url}" +} + # discover_gateway_endpoint - find the gateway gRPC endpoint. # Sets _DISCOVER_GW_ENDPOINT from the GRPCRoute hostname once the # parent Gateway is Programmed. @@ -120,6 +144,53 @@ acquire_oidc_token() { _driver_acquire_oidc_token "$@" } +: "${E2E_GATEWAY_NAMESPACE_GC_INTERVAL:=30s}" +: "${E2E_GATEWAY_NAMESPACE_GC_GRACE_PERIOD:=30s}" +_GC_TIMING_PATCHED="" + +# _patch_namespace_gc_timing / _restore_namespace_gc_timing - shared +# implementation for configure_namespace_gc_timing / restore_namespace_gc_timing. +# Kind deploys the same manifests as production (deploy/base/), so it also runs +# with production namespace-GC defaults (5m sweep / 10m grace) -- too slow for +# the e2e orphan-GC assertion (area 11a) to wait out. Rather than bake e2e-only +# timing into any deploy overlay, every driver patches the controller +# deployment to a short interval/grace period for the duration of the run and +# restores it afterward. Drivers that need to resolve their own namespace +# first (e.g. OpenShift) override the public functions and delegate here. +_patch_namespace_gc_timing() { + local cli="$1" namespace="$2" + dim " Shortening controller namespace GC timing for e2e (interval=${E2E_GATEWAY_NAMESPACE_GC_INTERVAL}, grace=${E2E_GATEWAY_NAMESPACE_GC_GRACE_PERIOD})..." + if ! "$cli" set env deployment/hypershell-controller -n "$namespace" -c controller \ + "GATEWAY_NAMESPACE_GC_INTERVAL=${E2E_GATEWAY_NAMESPACE_GC_INTERVAL}" \ + "GATEWAY_NAMESPACE_GC_GRACE_PERIOD=${E2E_GATEWAY_NAMESPACE_GC_GRACE_PERIOD}" >/dev/null; then + red " Failed to patch hypershell-controller namespace GC timing" + return 1 + fi + _GC_TIMING_PATCHED=1 + if ! "$cli" rollout status deployment/hypershell-controller -n "$namespace" --timeout=120s >/dev/null; then + red " hypershell-controller did not roll out after GC timing patch" + return 1 + fi +} + +_restore_namespace_gc_timing() { + local cli="$1" namespace="$2" + [[ -n "$_GC_TIMING_PATCHED" ]] || return 0 + dim " Restoring controller namespace GC timing to deployment defaults..." + "$cli" set env deployment/hypershell-controller -n "$namespace" -c controller \ + GATEWAY_NAMESPACE_GC_INTERVAL- GATEWAY_NAMESPACE_GC_GRACE_PERIOD- >/dev/null 2>&1 || true + "$cli" rollout status deployment/hypershell-controller -n "$namespace" --timeout=120s >/dev/null 2>&1 || true + _GC_TIMING_PATCHED="" +} + +configure_namespace_gc_timing() { + _patch_namespace_gc_timing kubectl "${E2E_HS_NAMESPACE}" +} + +restore_namespace_gc_timing() { + _restore_namespace_gc_timing kubectl "${E2E_HS_NAMESPACE}" +} + # _kc_base / _kc_realm - derive the Keycloak base URL and realm from the issuer. # E2E_OIDC_ISSUER is "/realms/". _kc_base() { echo "${E2E_OIDC_ISSUER%/realms/*}"; } diff --git a/tests/e2e/drivers/openshift.sh b/tests/e2e/drivers/openshift.sh index b648a40b..b7ea7ba3 100644 --- a/tests/e2e/drivers/openshift.sh +++ b/tests/e2e/drivers/openshift.sh @@ -94,6 +94,32 @@ discover_api_host() { fi } +# discover_console_host - find the HyperShell web console (BFF) base URL. +# The console Route hostname is cluster-generated and unrelated to the API +# Route hostname, so it must be discovered independently rather than derived +# by string substitution on discover_api_host's result. +discover_console_host() { + _DISCOVER_CONSOLE_HOST="" + _openshift_require_config || return 1 + + local host code + host=$(oc get route hypershell-web-console -n "${OPENSHIFT_NAMESPACE}" \ + -o jsonpath='{.spec.host}' 2>/dev/null || true) + if [[ -z "$host" ]]; then + red " HyperShell web console Route 'hypershell-web-console' not found in ${OPENSHIFT_NAMESPACE}" + return 1 + fi + + _DISCOVER_CONSOLE_HOST="https://${host}" + code=$(_driver_curl --connect-timeout 5 -o /dev/null -w '%{http_code}' \ + "${_DISCOVER_CONSOLE_HOST}/auth/session" 2>/dev/null || true) + if [[ -z "$code" || "$code" == "000" ]]; then + red " HyperShell web console Route ${_DISCOVER_CONSOLE_HOST} returned no HTTP response" + _DISCOVER_CONSOLE_HOST="" + return 1 + fi +} + discover_gateway_endpoint() { _DISCOVER_GW_ENDPOINT="" local gw_name="${1:?gateway name required}" @@ -169,3 +195,15 @@ acquire_oidc_token() { _openshift_configure_tls || return 1 _driver_acquire_oidc_token "$@" } + +# configure_namespace_gc_timing / restore_namespace_gc_timing - resolve the +# OpenShift namespace, then delegate to the shared implementation in kind.sh. +configure_namespace_gc_timing() { + _openshift_require_config || return 1 + _patch_namespace_gc_timing oc "${OPENSHIFT_NAMESPACE}" +} + +restore_namespace_gc_timing() { + _openshift_require_config || return 1 + _restore_namespace_gc_timing oc "${OPENSHIFT_NAMESPACE}" +} diff --git a/tests/e2e/e2e-openshell.sh b/tests/e2e/e2e-openshell.sh index ecaf4c8c..e29f2090 100755 --- a/tests/e2e/e2e-openshell.sh +++ b/tests/e2e/e2e-openshell.sh @@ -52,7 +52,13 @@ e2e_validate_mode # the route.openshift.io API group, an API only OpenShift clusters expose. # Any other cluster is assumed to be Kind. detect_infra_driver() { - if kubectl api-versions 2>/dev/null | grep -q '^route\.openshift\.io/'; then + local api_versions + if ! api_versions=$(kubectl api-versions 2>&1); then + red "ERROR: 'kubectl api-versions' failed against the current KUBECONFIG context" >&2 + red "$api_versions" >&2 + exit 1 + fi + if echo "$api_versions" | grep -q '^route\.openshift\.io/'; then echo "openshift" else echo "kind" @@ -72,7 +78,7 @@ fi # shellcheck source=drivers/kind.sh source "$DRIVER_FILE" -REQUIRED_FUNCTIONS=(discover_api_host discover_gateway_endpoint get_cluster_domain get_cli_binary wait_for_gateway_route acquire_oidc_token api_curl) +REQUIRED_FUNCTIONS=(discover_api_host discover_console_host discover_gateway_endpoint get_cluster_domain get_cli_binary wait_for_gateway_route acquire_oidc_token api_curl configure_namespace_gc_timing restore_namespace_gc_timing) for fn in "${REQUIRED_FUNCTIONS[@]}"; do if ! declare -f "$fn" >/dev/null 2>&1; then red "ERROR: Driver '${E2E_INFRA_DRIVER}' does not implement required function: ${fn}" @@ -101,6 +107,7 @@ fi # --- Cleanup trap --- cleanup() { + restore_namespace_gc_timing if [[ -n "${SB_CREATE_PID:-}" ]]; then kill "$SB_CREATE_PID" 2>/dev/null || true wait "$SB_CREATE_PID" 2>/dev/null || true @@ -126,9 +133,26 @@ cleanup() { acquire_oidc_token 2>/dev/null || true api_curl -X DELETE "${API_HOST}/api/hypershell/v1/gateways/${GW_ID}" &>/dev/null || true fi + # Runs on every exit path -- a fatal exit 1 mid-run included -- so the + # summary always prints, and print_results itself notes when E2E_COMPLETED + # was never set (i.e. the run aborted before reaching the results section). + print_results } trap cleanup EXIT +# --- Namespace GC timing --- +# Long mode seeds a synthetic orphan namespace later and waits for the periodic +# reaper to collect it (see area 11a); on drivers whose deployment runs with +# production GC defaults, that wait can't complete in time unless shortened +# first. Done once up front, before any gateway is created, so the controller +# restart this can trigger doesn't land mid-reconciliation. +if e2e_step long; then + if ! configure_namespace_gc_timing; then + red "ERROR: Could not configure namespace GC timing for the e2e run" + exit 1 + fi +fi + # --- Discover API host via driver --- if ! discover_api_host; then @@ -171,7 +195,7 @@ sep # ── 1. infrastructure validation + OIDC verification ───────────────────── echo "" -bold "1. Infrastructure Validation + OIDC Verification" +e2e_area "1. Infrastructure Validation + OIDC Verification" echo "" # Acquire a token for authenticated API calls @@ -204,7 +228,11 @@ else fi # Verify: BFF /auth/session returns unauthenticated -CONSOLE_HOST="${API_HOST/api./console.}" +if ! discover_console_host; then + fail_test "Could not discover HyperShell web console host" + exit 1 +fi +CONSOLE_HOST="${_DISCOVER_CONSOLE_HOST}" show_cmd "curl -s ${CONSOLE_HOST}/auth/session (driver TLS policy)" SESSION_RESP=$(_driver_curl "${CONSOLE_HOST}/auth/session" 2>/dev/null || true) SESSION_AUTH=$(echo "${SESSION_RESP}" | python3 -c "import json,sys; print(json.load(sys.stdin).get('authenticated',''))" 2>/dev/null || true) @@ -330,7 +358,7 @@ sep # ── 2. gateway provisioning ──────────────────────────────────────────────── echo "" -bold "2. Gateway Provisioning via HyperShell API" +e2e_area "2. Gateway Provisioning via HyperShell API" echo "" # JWT enforcement means every gateway CRUD call below needs a bearer token. @@ -514,22 +542,20 @@ EOF ORPHAN_GC_DEADLINE=$(($(date +%s) + E2E_ORPHAN_GC_TIMEOUT)) fi -# Per-gateway Keycloak client id. When Keycloak provisioning is enabled (the Kind -# path), the control-plane reconciler creates a dedicated public client named -# "${gw.Name}-${gatewayID}" with an audience mapper and overrides the gateway's -# OIDC config to require aud == this client. Gateway and CLI tokens must therefore -# be minted against this client, not the shared frontend client, or Envoy rejects -# them with InvalidAudience. gatewayID is the API resource id (GW_ID). +# Per-gateway Keycloak client id. The control-plane reconciler creates a +# dedicated public client named "${gw.Name}-${gatewayID}" with an audience +# mapper and overrides the gateway's OIDC config to require aud == this +# client, on every infra target. Gateway and CLI tokens must therefore be +# minted against this client, not the shared frontend client, or Envoy +# rejects them with InvalidAudience. gatewayID is the API resource id (GW_ID). GW_KC_CLIENT_ID="${GW_NAME}-${GW_ID}" -if [[ "${E2E_INFRA_DRIVER}" == "kind" ]]; then - dim " Per-gateway OIDC client: ${GW_KC_CLIENT_ID}" -fi +dim " Per-gateway OIDC client: ${GW_KC_CLIENT_ID}" sep # ── 3. gateway infrastructure ────────────────────────────────────────────── echo "" -bold "3. Gateway Infrastructure" +e2e_area "3. Gateway Infrastructure" echo "" show_cmd "$CLI get deployment openshell-gateway -n $GW_NAMESPACE" @@ -709,17 +735,16 @@ sep # ── 4. OIDC token acquisition + CA certificate setup ───────────────────── echo "" -bold "4. OIDC Token Acquisition + CA Certificate Setup" +e2e_area "4. OIDC Token Acquisition + CA Certificate Setup" echo "" -# The client the admin's gateway/CLI tokens are minted against. On Kind the -# reconciler forces a per-gateway audience, so we use the per-gateway client and -# wait for the async owner-binding -> openshell-admin role to land in the token. -OIDC_CLIENT_ID_EFFECTIVE="${E2E_OIDC_CLIENT_ID}" -if [[ "${E2E_INFRA_DRIVER}" == "kind" ]]; then - OIDC_CLIENT_ID_EFFECTIVE="${GW_KC_CLIENT_ID}" +# The client the admin's gateway/CLI tokens are minted against. The +# reconciler forces a per-gateway audience on every infra target, so we +# always use the per-gateway client and wait for the async owner-binding -> +# openshell-admin role to land in the token. +OIDC_CLIENT_ID_EFFECTIVE="${GW_KC_CLIENT_ID}" - if e2e_step long; then +if [[ "${E2E_INFRA_DRIVER}" == "kind" ]] && e2e_step long; then # Exercise the real Keycloak device authorization endpoint for the client # provisioned by the control plane. A successful authorization response proves # that oauth2.device.authorization.grant.enabled reached Keycloak; polling once @@ -772,26 +797,15 @@ if [[ "${E2E_INFRA_DRIVER}" == "kind" ]]; then fail_test "Device code poll did not return authorization_pending: ${DEVICE_TOKEN_DESCRIPTION}" exit 1 fi - fi +fi - show_cmd "# resource-owner password grant → ${E2E_OIDC_ISSUER} (client: ${GW_KC_CLIENT_ID}, await role: openshell-admin)" - if acquire_gateway_token_with_role "$E2E_OIDC_USERNAME" "$E2E_OIDC_PASSWORD" "$GW_KC_CLIENT_ID" openshell-admin; then - OIDC_TOKEN="${_OIDC_ACCESS_TOKEN}" - pass "OIDC token acquired with openshell-admin (user: ${E2E_OIDC_USERNAME}, client: ${GW_KC_CLIENT_ID})" - else - fail_test "Failed to acquire per-gateway OIDC token with openshell-admin role" - exit 1 - fi -else - show_cmd "# resource-owner password grant → ${E2E_OIDC_ISSUER}" - acquire_oidc_token +show_cmd "# resource-owner password grant → ${E2E_OIDC_ISSUER} (client: ${GW_KC_CLIENT_ID}, await role: openshell-admin)" +if acquire_gateway_token_with_role "$E2E_OIDC_USERNAME" "$E2E_OIDC_PASSWORD" "$GW_KC_CLIENT_ID" openshell-admin; then OIDC_TOKEN="${_OIDC_ACCESS_TOKEN}" - if [[ -n "$OIDC_TOKEN" ]]; then - pass "OIDC token acquired (user: ${E2E_OIDC_USERNAME})" - else - fail_test "Failed to acquire OIDC token from Keycloak" - exit 1 - fi + pass "OIDC token acquired with openshell-admin (user: ${E2E_OIDC_USERNAME}, client: ${GW_KC_CLIENT_ID})" +else + fail_test "Failed to acquire per-gateway OIDC token with openshell-admin role" + exit 1 fi @@ -814,7 +828,7 @@ sep # ── 5. route discovery + CLI registration ───────────────────────────────── echo "" -bold "5. Route Discovery + CLI Registration" +e2e_area "5. Route Discovery + CLI Registration" echo "" GW_LOCAL_NAME="${GW_NAMESPACE}-openshell" @@ -881,7 +895,7 @@ sep # ── 6. gateway connectivity ─────────────────────────────────────────────── echo "" -bold "6. Gateway Connectivity" +e2e_area "6. Gateway Connectivity" echo "" show_cmd "${OPENSHELL_BIN} -g ${GW_LOCAL_NAME} status" @@ -918,7 +932,7 @@ sep # ── 7. sandbox lifecycle ────────────────────────────────────────────────── echo "" -bold "7. Sandbox Lifecycle" +e2e_area "7. Sandbox Lifecycle" echo "" RUN_ID=$(date +%s | tail -c5) @@ -985,7 +999,7 @@ sep # ── 8. sandbox interaction + active sandbox count ───────────────────────── echo "" -bold "8. Sandbox Interaction + Active Sandbox Count" +e2e_area "8. Sandbox Interaction + Active Sandbox Count" echo "" GW_FLAG="-g ${GW_LOCAL_NAME}" @@ -1166,43 +1180,32 @@ sep # ── 9. developer user RBAC verification ────────────────────────────────── echo "" -bold "9. Developer User RBAC Verification" +e2e_area "9. Developer User RBAC Verification" echo "" # The developer's gateway/CLI token, like the admin's, must be minted against the -# per-gateway client on Kind. The gateway requires user_role (openshell-user) on -# that client or it rejects the developer outright ("role 'openshell-user' -# required"). In production the RoleBinding reconciler assigns this when a -# gateway:viewer binding is created, but that grant is not expressible through the -# API for a non-owner (no user_id discovery path), so we provision the same end -# state directly in Keycloak -- a test-setup shortcut, not a product change. -DEV_OIDC_CLIENT_ID_EFFECTIVE="${E2E_OIDC_CLIENT_ID}" -if [[ "${E2E_INFRA_DRIVER}" == "kind" ]]; then - DEV_OIDC_CLIENT_ID_EFFECTIVE="${GW_KC_CLIENT_ID}" - show_cmd "# grant developer openshell-user on ${GW_KC_CLIENT_ID} (mirrors gateway:viewer RoleBinding)" - if assign_gateway_client_role "$E2E_DEV_USERNAME" "$GW_KC_CLIENT_ID" openshell-user; then - pass "Developer granted openshell-user on per-gateway client" - else - fail_test "Failed to grant developer openshell-user on per-gateway client" - fi - - show_cmd "# acquire per-gateway OIDC token for developer (client: ${GW_KC_CLIENT_ID}, await role: openshell-user)" - if acquire_gateway_token_with_role "$E2E_DEV_USERNAME" "$E2E_DEV_PASSWORD" "$GW_KC_CLIENT_ID" openshell-user; then - DEV_TOKEN="${_OIDC_ACCESS_TOKEN}" - pass "Developer OIDC token acquired with openshell-user (user: ${E2E_DEV_USERNAME})" - else - DEV_TOKEN="" - fail_test "Failed to acquire developer per-gateway OIDC token with openshell-user role" - fi +# per-gateway client on every infra target. The gateway requires user_role +# (openshell-user) on that client or it rejects the developer outright ("role +# 'openshell-user' required"). In production the RoleBinding reconciler assigns +# this when a gateway:viewer binding is created, but that grant is not +# expressible through the API for a non-owner (no user_id discovery path), so we +# provision the same end state directly in Keycloak -- a test-setup shortcut, +# not a product change. +DEV_OIDC_CLIENT_ID_EFFECTIVE="${GW_KC_CLIENT_ID}" +show_cmd "# grant developer openshell-user on ${GW_KC_CLIENT_ID} (mirrors gateway:viewer RoleBinding)" +if assign_gateway_client_role "$E2E_DEV_USERNAME" "$GW_KC_CLIENT_ID" openshell-user; then + pass "Developer granted openshell-user on per-gateway client" else - show_cmd "# acquire OIDC token for developer user" - acquire_oidc_token "$E2E_DEV_USERNAME" "$E2E_DEV_PASSWORD" + fail_test "Failed to grant developer openshell-user on per-gateway client" +fi + +show_cmd "# acquire per-gateway OIDC token for developer (client: ${GW_KC_CLIENT_ID}, await role: openshell-user)" +if acquire_gateway_token_with_role "$E2E_DEV_USERNAME" "$E2E_DEV_PASSWORD" "$GW_KC_CLIENT_ID" openshell-user; then DEV_TOKEN="${_OIDC_ACCESS_TOKEN}" - if [[ -n "$DEV_TOKEN" ]]; then - pass "Developer OIDC token acquired (user: ${E2E_DEV_USERNAME})" - else - fail_test "Failed to acquire developer OIDC token" - fi + pass "Developer OIDC token acquired with openshell-user (user: ${E2E_DEV_USERNAME})" +else + DEV_TOKEN="" + fail_test "Failed to acquire developer per-gateway OIDC token with openshell-user role" fi if [[ -n "$DEV_TOKEN" ]]; then @@ -1424,7 +1427,7 @@ sep # ── 10. platform admin RBAC verification ───────────────────────────────── echo "" -bold "10. Platform Admin RBAC Verification" +e2e_area "10. Platform Admin RBAC Verification" echo "" if ! e2e_step long; then @@ -1436,13 +1439,11 @@ else # Assign platform:admin realm role to the platform admin user (best-effort; user may # already have the role from Keycloak realm import) -if [[ "${E2E_INFRA_DRIVER}" == "kind" ]]; then - show_cmd "# verify/assign platform:admin realm role to ${E2E_PLATFORM_ADMIN_USERNAME}" - if assign_realm_role "$E2E_PLATFORM_ADMIN_USERNAME" "platform:admin"; then - pass "Platform admin has platform:admin realm role" - else - dim " Note: Could not verify platform:admin role assignment (user may already have it from realm import)" - fi +show_cmd "# verify/assign platform:admin realm role to ${E2E_PLATFORM_ADMIN_USERNAME}" +if assign_realm_role "$E2E_PLATFORM_ADMIN_USERNAME" "platform:admin"; then + pass "Platform admin has platform:admin realm role" +else + dim " Note: Could not verify platform:admin role assignment (user may already have it from realm import)" fi # Acquire OIDC token for platform admin @@ -1575,7 +1576,7 @@ sep # ── 11. gateway deletion + namespace garbage collection ──────────────────── echo "" -bold "11. Gateway Deletion + Namespace Garbage Collection" +e2e_area "11. Gateway Deletion + Namespace Garbage Collection" echo "" if [[ "$E2E_MODE" == "short" ]]; then @@ -1790,7 +1791,9 @@ sep # ── results ─────────────────────────────────────────────────────────────── -print_results +# Reached every planned area without a fatal abort; cleanup's EXIT trap prints +# the results (see cleanup()), so print_results itself is not called here. +E2E_COMPLETED=1 if [[ $E2E_FAIL -gt 0 ]]; then exit 1 diff --git a/tests/e2e/e2e-performance.sh b/tests/e2e/e2e-performance.sh index 268c0ac0..6ddf367e 100755 --- a/tests/e2e/e2e-performance.sh +++ b/tests/e2e/e2e-performance.sh @@ -678,6 +678,9 @@ bold "Performance summary" sep perf_print_summary +# Reached the summary without a fatal abort; print_results (lib.sh) notes when +# this was never set, which does not apply to this script's own cleanup path. +E2E_COMPLETED=1 print_results if [[ "${PERF_RUN_RESULT}" != "pass" ]]; then diff --git a/tests/e2e/lib.sh b/tests/e2e/lib.sh index 709423c4..61bb6e3d 100755 --- a/tests/e2e/lib.sh +++ b/tests/e2e/lib.sh @@ -41,6 +41,8 @@ sep() { printf "${_DIM}────────────────── E2E_PASS=0 E2E_FAIL=0 E2E_TESTS=() +E2E_CURRENT_AREA="" +E2E_COMPLETED="" pass() { E2E_PASS=$((E2E_PASS + 1)) @@ -59,9 +61,24 @@ show_cmd() { sleep "${E2E_PAUSE:-1}" } +# e2e_area - announce a numbered test area and record it as the current one, +# so print_results can name where the run stopped if it never reaches the end. +e2e_area() { + E2E_CURRENT_AREA="$1" + bold "$1" +} + print_results() { echo "" bold "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + if [[ -z "$E2E_COMPLETED" ]]; then + if [[ -n "$E2E_CURRENT_AREA" ]]; then + red "⚠ Run aborted during Area ${E2E_CURRENT_AREA} -- later areas did not run and are not reflected below." + else + red "⚠ Run aborted before any test area started -- no checks ran." + fi + echo "" + fi bold "Results: $E2E_PASS passed, $E2E_FAIL failed" echo "" for t in "${E2E_TESTS[@]}"; do diff --git a/tests/e2e/openshift_driver_test.sh b/tests/e2e/openshift_driver_test.sh index 1e0828ea..e8792672 100644 --- a/tests/e2e/openshift_driver_test.sh +++ b/tests/e2e/openshift_driver_test.sh @@ -27,6 +27,7 @@ oc() { case "$args" in "project -q") printf '%s' "${OC_PROJECT:-}" ;; *"get route hypershell-api -n test-team"*) printf '%s' 'api-test.apps.example.com' ;; + *"get route hypershell-web-console -n test-team"*) printf '%s' 'console-test.apps.example.com' ;; *"get route keycloak -n test-team-keycloak"*) printf '%s' 'sso-test.apps.example.com' ;; *"get deployment hypershell-controller -n test-team"*) printf '%s' 'gw.test.example.com' ;; *"get grpcroute openshell-gateway -n tenant-a -o jsonpath={.spec.hostnames[0]}"*) printf '%s' 'gw-a.gw.test.example.com' ;; @@ -34,6 +35,9 @@ oc() { *"get grpcroute openshell-gateway -n tenant-a -o jsonpath={.spec.parentRefs[0].namespace}"*) printf '%s' 'openshift-ingress' ;; *"get gateway shared-gateway -n openshift-ingress"*) printf '%s\n' 'Programmed=True' ;; *"get grpcroute openshell-gateway -n tenant-a"*) printf '%s\n' 'Accepted=True' ;; + *"set env deployment/hypershell-controller -n test-team -c controller GATEWAY_NAMESPACE_GC_INTERVAL=30s GATEWAY_NAMESPACE_GC_GRACE_PERIOD=30s"*) : ;; + *"rollout status deployment/hypershell-controller -n test-team --timeout=120s"*) : ;; + *"set env deployment/hypershell-controller -n test-team -c controller GATEWAY_NAMESPACE_GC_INTERVAL- GATEWAY_NAMESPACE_GC_GRACE_PERIOD-"*) : ;; *) return 1 ;; esac } @@ -51,6 +55,9 @@ assert_eq 'test-team-keycloak' "${E2E_KEYCLOAK_NAMESPACE}" 'Keycloak namespace d assert_eq 'gw.test.example.com' "$(get_cluster_domain)" 'configured gateway domain' assert_eq 'oc' "$(get_cli_binary)" 'OpenShift CLI' +discover_console_host +assert_eq 'https://console-test.apps.example.com' "${_DISCOVER_CONSOLE_HOST}" 'Console Route discovery' + discover_gateway_endpoint gw-a tenant-a assert_eq 'https://gw-a.gw.test.example.com:443' "${_DISCOVER_GW_ENDPOINT}" 'Gateway API endpoint discovery' @@ -75,5 +82,21 @@ else PASS=$((PASS + 1)) fi +if configure_namespace_gc_timing >/dev/null; then + PASS=$((PASS + 1)) +else + FAIL=$((FAIL + 1)) + echo 'FAIL: configure_namespace_gc_timing did not succeed' +fi +assert_eq '1' "${_GC_TIMING_PATCHED}" 'namespace GC timing marked patched' + +if restore_namespace_gc_timing >/dev/null; then + PASS=$((PASS + 1)) +else + FAIL=$((FAIL + 1)) + echo 'FAIL: restore_namespace_gc_timing did not succeed' +fi +assert_eq '' "${_GC_TIMING_PATCHED}" 'namespace GC timing patch cleared after restore' + printf 'OpenShift driver tests: %d passed, %d failed\n' "$PASS" "$FAIL" [[ "$FAIL" -eq 0 ]] From 1c16712f64f10ba80ac09bb32b4a8b1ce15f645f Mon Sep 17 00:00:00 2001 From: Kyle Squizzato Date: Fri, 4 Sep 2026 16:29:30 -0700 Subject: [PATCH 2/2] fix(e2e): Various fixes for e2e against OpenShift, Kind Bug fix pass for OpenShift and Kind E2E tests. Assisted-by: Claude Sonnet 5 Signed-off-by: Kyle Squizzato --- Makefile | 12 +- .../control-plane/internal/gateway/console.go | 13 +- .../internal/gateway/reconciler.go | 2 +- deploy/openshift/keycloak-networkpolicy.yaml | 16 +- scripts/cluster/drivers/openshift.sh | 167 ++++++++++++++++-- scripts/cluster/lib_test.sh | 26 ++- scripts/kind/lib.sh | 84 ++++++++- scripts/kind/status.sh | 11 +- scripts/kind/swap-component.sh | 4 +- scripts/kind/up.sh | 12 +- specs/platform/local-development.spec.md | 2 +- specs/platform/openshift-development.spec.md | 39 +++- tests/e2e/e2e-openshell.sh | 23 +-- tests/e2e/e2e-performance.sh | 12 +- tests/e2e/lib.sh | 34 +++- tests/e2e/perf/lib_test.sh | 90 +++++++++- 16 files changed, 466 insertions(+), 81 deletions(-) diff --git a/Makefile b/Makefile index 9329fbc9..318165cd 100644 --- a/Makefile +++ b/Makefile @@ -130,7 +130,7 @@ help: @echo " Test & Lint" @echo " test-all Run all test suites" @echo " e2e Run E2E tests against target KUBECONFIG cluster" - @echo " e2e-performance Run the performance harness (requires a running cluster)" + @echo " e2e-performance Run the performance harness (modify with E2E_PERF_GATEWAY_COUNT, E2E_PERF_BATCH_SIZE)" @echo " e2e-performance-report Tabulate recent local performance runs" @echo " lint Run all linters (Go + JS/TS)" @echo " lint-api-server Lint API server (gofmt, go vet, golangci-lint)" @@ -521,8 +521,9 @@ generate-sdk-go: # E2E Tests # ============================================================================ -# Default driver is kind; honor E2E_INFRA_DRIVER=openshift make e2e on the CLI. -E2E_INFRA_DRIVER ?= kind +# Driver is auto-detected from the current KUBECONFIG context +# (route.openshift.io => openshift, otherwise kind). Set E2E_INFRA_DRIVER +# on the command line to override. .PHONY: e2e e2e: @@ -536,10 +537,9 @@ e2e: .PHONY: e2e-performance e2e-performance: @echo "" - @echo "==> Running E2E performance harness ($(E2E_INFRA_DRIVER))" + @echo "==> Running E2E performance harness" @echo "" - @E2E_INFRA_DRIVER=$(E2E_INFRA_DRIVER) \ - bash tests/e2e/e2e-performance.sh + @bash tests/e2e/e2e-performance.sh .PHONY: e2e-performance-report e2e-performance-report: diff --git a/components/control-plane/internal/gateway/console.go b/components/control-plane/internal/gateway/console.go index cc9689cd..fd02aa83 100644 --- a/components/control-plane/internal/gateway/console.go +++ b/components/control-plane/internal/gateway/console.go @@ -241,10 +241,13 @@ func routeConditionState(conditions []interface{}, condType string) (isTrue bool return false, "" } -// consoleListenerName returns the sectionName of the shared Gateway HTTP -// listener that console HTTPRoutes attach to (GATEWAY_API_HTTP_LISTENER_NAME, -// default "https"). -func consoleListenerName() string { +// sharedGatewayListenerName returns the sectionName of the shared Gateway +// listener that both the gateway's GRPCRoute and the per-gateway console's +// HTTPRoute attach to (GATEWAY_API_HTTP_LISTENER_NAME, default "https"). Both +// route kinds must resolve to the same listener, so this is the single source +// of truth for both call sites -- a mismatch here reproduces as GRPCRoute or +// HTTPRoute status NoMatchingParent even when the Gateway itself is Programmed. +func sharedGatewayListenerName() string { if n := os.Getenv("GATEWAY_API_HTTP_LISTENER_NAME"); n != "" { return n } @@ -749,7 +752,7 @@ func buildConsoleHTTPRoute(namespace, host string) *unstructured.Unstructured { parentRef := map[string]interface{}{ "name": gatewayIngressName(), "namespace": gatewayIngressNamespace(), - "sectionName": consoleListenerName(), + "sectionName": sharedGatewayListenerName(), } return &unstructured.Unstructured{ Object: map[string]interface{}{ diff --git a/components/control-plane/internal/gateway/reconciler.go b/components/control-plane/internal/gateway/reconciler.go index 8e60dcfd..11d399ad 100644 --- a/components/control-plane/internal/gateway/reconciler.go +++ b/components/control-plane/internal/gateway/reconciler.go @@ -2249,7 +2249,7 @@ func reconcileGatewayAPIResources(ctx context.Context, dynamicClient dynamic.Int parentRef := map[string]interface{}{ "name": gwName, "namespace": gwNS, - "sectionName": "grpc", + "sectionName": sharedGatewayListenerName(), } grpcRoute := &unstructured.Unstructured{ diff --git a/deploy/openshift/keycloak-networkpolicy.yaml b/deploy/openshift/keycloak-networkpolicy.yaml index 3b5d4725..7ab3e308 100644 --- a/deploy/openshift/keycloak-networkpolicy.yaml +++ b/deploy/openshift/keycloak-networkpolicy.yaml @@ -1,7 +1,12 @@ -# oc new-project installs default-deny Ingress policies (same-namespace + -# openshift-ingress only). Platform pods live in a different namespace, so -# without this policy the API server cannot load JWKS and the control plane -# cannot reach the Keycloak Admin API. +# This namespace is created from a raw Namespace manifest, not `oc new-project`, +# so it gets none of the default same-namespace/openshift-ingress NetworkPolicies +# a project-request template would install. Selecting the Keycloak pod here +# makes it default-deny for Ingress, so every legitimate path needs its own +# rule: platform pods load JWKS and reach the Keycloak Admin API directly +# (in-cluster), and the router needs its own rule to proxy the public Keycloak +# Route -- used by browser logins and by the web console's server-side OIDC +# discovery call, both of which hit the public issuer URL, not the in-cluster +# Service DNS name. apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: @@ -21,6 +26,9 @@ spec: - namespaceSelector: matchLabels: kubernetes.io/metadata.name: hypershell-system + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: openshift-ingress ports: - protocol: TCP port: 8080 diff --git a/scripts/cluster/drivers/openshift.sh b/scripts/cluster/drivers/openshift.sh index 6731bbaf..9e970265 100755 --- a/scripts/cluster/drivers/openshift.sh +++ b/scripts/cluster/drivers/openshift.sh @@ -25,7 +25,7 @@ require_openshift_cluster() { error "Log in with 'oc login' (or set KUBECONFIG) and retry. 'make openshift-up' does not create a cluster." exit 1 fi - if ! oc_cli api-resources --api-group=route.openshift.io --no-headers 2>/dev/null | grep -q .; then + if ! api_group_available route.openshift.io; then error "The current kubeconfig context is not an OpenShift cluster (route.openshift.io is missing)." error "Provide an OpenShift cluster target before running 'make openshift-up'." exit 1 @@ -216,23 +216,38 @@ ensure_namespace_group() { discover_gateway_base_domain() { local gw_name="$1" local gw_ns="$2" - local host="" + local host="" name="" host="$(oc_cli get "gateway.gateway.networking.k8s.io/${gw_name}" -n "${gw_ns}" \ -o jsonpath='{.spec.listeners[?(@.name=="grpc")].hostname}' 2>/dev/null || true)" - if [[ -z "${host}" ]]; then + if [[ -n "${host}" ]]; then + name="grpc" + else + # No listener is literally named "grpc" -- a multi-hub shared Gateway + # names each listener after its hub (e.g. grpc-hyp4, grpc-hyp5), so fall + # back to the first listener and use ITS name, not the literal "grpc". + # GATEWAY_API_HTTP_LISTENER_NAME must match whichever listener supplied + # the hostname below, or GRPCRoutes fail sectionName resolution + # (NoMatchingParent) even though the domain looks right. host="$(oc_cli get "gateway.gateway.networking.k8s.io/${gw_name}" -n "${gw_ns}" \ -o jsonpath='{.spec.listeners[0].hostname}' 2>/dev/null || true)" + name="$(oc_cli get "gateway.gateway.networking.k8s.io/${gw_name}" -n "${gw_ns}" \ + -o jsonpath='{.spec.listeners[0].name}' 2>/dev/null || true)" fi if [[ -z "${host}" ]]; then error "Shared Gateway '${gw_ns}/${gw_name}' has no listener hostname." error "The gateway wildcard hostname is the base domain; 'make openshift-up' does not take GATEWAY_API_BASE_DOMAIN." exit 1 fi + if [[ -z "${name}" ]]; then + error "Shared Gateway '${gw_ns}/${gw_name}' listener has no name." + exit 1 + fi GATEWAY_API_BASE_DOMAIN="$(gateway_base_domain_from_hostname "${host}")" if [[ -z "${GATEWAY_API_BASE_DOMAIN}" ]]; then error "Could not derive a base domain from Gateway listener hostname '${host}'." exit 1 fi + GATEWAY_API_HTTP_LISTENER_NAME="${name}" } check_infrastructure() { @@ -263,7 +278,7 @@ check_infrastructure() { fi discover_gateway_base_domain "${gw_name}" "${gw_ns}" success "Shared Gateway ${gw_ns}/${gw_name} (GatewayClass ${gw_class}) is ready" - info "Gateway base domain: ${GATEWAY_API_BASE_DOMAIN} (from ${gw_ns}/${gw_name} listener)" + info "Gateway base domain: ${GATEWAY_API_BASE_DOMAIN} (from ${gw_ns}/${gw_name} listener '${GATEWAY_API_HTTP_LISTENER_NAME}')" } warn_skipped_cluster_rbac() { @@ -505,6 +520,8 @@ patches: value: "${GATEWAY_API_GATEWAY_NAMESPACE}" - name: GATEWAY_API_BASE_DOMAIN value: "${GATEWAY_API_BASE_DOMAIN}" + - name: GATEWAY_API_HTTP_LISTENER_NAME + value: "${GATEWAY_API_HTTP_LISTENER_NAME}" - name: GATEWAY_OIDC_ISSUER_URL value: "https://keycloak.pending.invalid/realms/hypershell" EOF @@ -531,13 +548,124 @@ EOF } api_group_available() { - oc_cli api-resources --api-group="$1" --no-headers 2>/dev/null | grep -q . + # Capture into a variable rather than piping into `grep -q`: under + # pipefail, grep -q exits the instant it sees a match, and if oc + # api-resources is still writing more output at that moment it gets + # SIGPIPE'd and the pipeline reports a spurious non-zero (141) status -- + # a real, observed race that intermittently (and cluster-dependently) + # makes an installed API group look unavailable. + local out + out="$(oc_cli api-resources --api-group="$1" --no-headers 2>/dev/null)" + [[ -n "${out}" ]] } cnpg_available() { api_group_available postgresql.cnpg.io } +# effective_database_provider - the DB provider this run of openshift-up will +# deploy: DATABASE_PROVIDER if explicitly set (validated against what the +# cluster can actually run), otherwise auto-detected from CNPG operator +# availability. This is "what should run"; cutover_database_provider +# reconciles live cluster state toward it. +effective_database_provider() { + case "${DATABASE_PROVIDER:-}" in + cnpg) + if ! cnpg_available; then + error "DATABASE_PROVIDER=cnpg requested but the CNPG operator (postgresql.cnpg.io) is not installed on this cluster." + exit 1 + fi + printf 'cnpg' + ;; + deployment) + printf 'deployment' + ;; + "") + if cnpg_available; then printf 'cnpg'; else printf 'deployment'; fi + ;; + *) + error "Unknown DATABASE_PROVIDER '${DATABASE_PROVIDER}': expected 'cnpg' or 'deployment'." + exit 1 + ;; + esac +} + +# database_provider_key - the Secret key that proves hypershell-db-app was +# shaped for the given provider (deployment: user; CNPG: username). +database_provider_key() { + case "$1" in + cnpg) printf 'username' ;; + deployment) printf 'user' ;; + esac +} + +# secret_shaped_for_provider - true if hypershell-db-app either does not exist +# yet (the target provider will create its own) or already carries the key +# that provider's connection info requires. +secret_shaped_for_provider() { + local provider="$1" key + key="$(database_provider_key "${provider}")" + if ! oc_cli get secret hypershell-db-app -n "${OPENSHIFT_NAMESPACE}" >/dev/null 2>&1; then + return 0 + fi + local val + val="$(oc_cli get secret hypershell-db-app -n "${OPENSHIFT_NAMESPACE}" \ + -o jsonpath="{.data.${key}}" 2>/dev/null || true)" + [[ -n "${val}" ]] +} + +DATABASE_PROVIDER_CUTOVER_PERFORMED="" + +# cutover_database_provider - reconcile live database resources toward the +# target provider, rather than trusting a single "current provider" read: a +# prior interrupted or pre-fix run can leave BOTH a CNPG Cluster and the +# bundled Deployment present at once (e.g. a Cluster created after CNPG +# detection flipped true, sitting alongside a Deployment an earlier run never +# tore down), which a first-match "current" check would misreport as already +# matching the target and skip entirely. Instead: remove whichever provider's +# resources are NOT the target, unconditionally, and separately validate the +# shared hypershell-db-app Secret is actually shaped for the target -- it is +# not safe to infer the Secret's shape from which provider's resources exist, +# since either can adopt a Secret the other left behind. Both providers reuse +# that Secret name with incompatible key shapes (deployment: +# user/password/host/port/dbname; CNPG: username/password, auto-generated +# only if the Secret doesn't already exist) -- an unnoticed mismatch leaves +# the primary instance stuck in CreateContainerConfigError waiting on a key +# that will never appear. +# DESTRUCTIVE: deletes the outgoing provider's database and its data. This +# namespace group is ephemeral dev/e2e infrastructure, not production. +cutover_database_provider() { + local target="$1" + local changed="" + + if [[ "${target}" != "cnpg" ]] \ + && oc_cli get cluster.postgresql.cnpg.io hypershell-db -n "${OPENSHIFT_NAMESPACE}" >/dev/null 2>&1; then + warn "Removing CNPG database in ${OPENSHIFT_NAMESPACE} (target provider: ${target}); its data will be lost." + oc_cli delete cluster.postgresql.cnpg.io hypershell-db -n "${OPENSHIFT_NAMESPACE}" --wait=true --timeout=120s 2>/dev/null || true + oc_cli delete pvc -n "${OPENSHIFT_NAMESPACE}" -l cnpg.io/cluster=hypershell-db --ignore-not-found=true + changed=true + fi + + if [[ "${target}" != "deployment" ]] \ + && oc_cli get deployment hypershell-postgres -n "${OPENSHIFT_NAMESPACE}" >/dev/null 2>&1; then + warn "Removing bundled PostgreSQL Deployment in ${OPENSHIFT_NAMESPACE} (target provider: ${target}); its data will be lost." + oc_cli delete deployment hypershell-postgres -n "${OPENSHIFT_NAMESPACE}" --ignore-not-found=true + oc_cli delete service hypershell-postgres -n "${OPENSHIFT_NAMESPACE}" --ignore-not-found=true + changed=true + fi + + if ! secret_shaped_for_provider "${target}"; then + warn "hypershell-db-app in ${OPENSHIFT_NAMESPACE} is not shaped for ${target}; clearing it so ${target} can create its own." + oc_cli delete secret hypershell-db-app -n "${OPENSHIFT_NAMESPACE}" --ignore-not-found=true + changed=true + fi + + if [[ -n "${changed}" ]]; then + DATABASE_PROVIDER_CUTOVER_PERFORMED=true + success "Database provider reconciled to ${target} in ${OPENSHIFT_NAMESPACE}" + fi +} + apply_rendered_overlay() { local rendered="$1" local prune_args=( @@ -603,7 +731,7 @@ configure_postgres_fallback_ssl() { developer_omit_kinds() { local kinds="ClusterRole,ClusterRoleBinding" - if ! cnpg_available; then + if [[ "$(effective_database_provider)" != "cnpg" ]]; then kinds+=",Cluster" fi if ! api_group_available cert-manager.io; then @@ -626,7 +754,7 @@ apply_overlay() { info "Applying HyperShell in project ${OPENSHIFT_NAMESPACE}..." use_project "${OPENSHIFT_NAMESPACE}" - if ! cnpg_available; then + if [[ "$(effective_database_provider)" != "cnpg" ]]; then apply_postgres_fallback fi if ! rendered="$(render_openshift_manifests \ @@ -636,7 +764,7 @@ apply_overlay() { exit 1 fi apply_rendered_overlay "${rendered}" - if ! cnpg_available; then + if [[ "$(effective_database_provider)" != "cnpg" ]]; then configure_postgres_fallback_ssl fi @@ -763,6 +891,21 @@ wait_for_deployments() { wait_for_named_rollout hypershell-web-console "${OPENSHIFT_NAMESPACE}" } +# restart_after_database_cutover - api-server and controller mount +# hypershell-db-app and read it at process start, so if cutover_database_provider +# recreated that Secret with a different shape, a pod that was already running +# against the old provider keeps its stale connection until restarted -- +# reapplying the (unchanged) Deployment spec does not trigger a new rollout. +# No-op unless a cutover actually happened this run. +restart_after_database_cutover() { + [[ -n "${DATABASE_PROVIDER_CUTOVER_PERFORMED}" ]] || return 0 + info "Restarting api-server and controller to pick up the cut-over database..." + oc_cli rollout restart deployment/hypershell-api-server -n "${OPENSHIFT_NAMESPACE}" + oc_cli rollout restart deployment/hypershell-controller -n "${OPENSHIFT_NAMESPACE}" + wait_for_named_rollout hypershell-api-server "${OPENSHIFT_NAMESPACE}" + wait_for_named_rollout hypershell-controller "${OPENSHIFT_NAMESPACE}" +} + # Talk to OpenShift Routes from the developer machine. The API server image has # no curl, so oc exec cannot reach Keycloak or the API. -k matches kind-up: # cluster default certs are not always in the local trust store. @@ -932,10 +1075,8 @@ seed_via_api() { fi if [[ -z "${seed_failed}" ]]; then - local db_provider="cnpg" - if ! cnpg_available; then - db_provider="deployment" - fi + local db_provider + db_provider="$(effective_database_provider)" raw="$(api_exec GET /api/hypershell/v1/managed_databases)" http="$(printf '%s' "${raw}" | tail -1)" body="$(printf '%s' "${raw}" | sed '$d')" @@ -1039,10 +1180,12 @@ cluster_up() { ensure_namespace_group create_bootstrap_secrets apply_cluster_rbac + cutover_database_provider "$(effective_database_provider)" apply_overlay restore_swaps_after_reconcile configure_oidc_from_routes wait_for_deployments + restart_after_database_cutover add_keycloak_redirect_uri || true if skip_seed; then info "SKIP_SEED=true - skipping platform seeding" diff --git a/scripts/cluster/lib_test.sh b/scripts/cluster/lib_test.sh index 2a30596c..11b77d62 100755 --- a/scripts/cluster/lib_test.sh +++ b/scripts/cluster/lib_test.sh @@ -256,13 +256,37 @@ else FAIL=$((FAIL + 1)) echo 'FAIL: OpenShift cluster_up does not honor SKIP_SEED' fi -if grep -A8 'db_provider="cnpg"' "${SCRIPT_DIR}/drivers/openshift.sh" | grep -q 'cnpg_available' \ +if grep -B2 'db_provider="\$(effective_database_provider)"' "${SCRIPT_DIR}/drivers/openshift.sh" >/dev/null \ && grep -A20 'Creating ManagedDatabase' "${SCRIPT_DIR}/drivers/openshift.sh" | grep -q 'provider='; then PASS=$((PASS + 1)) else FAIL=$((FAIL + 1)) echo 'FAIL: OpenShift seed still hardcodes ManagedDatabase provider=cnpg' fi +if grep -A20 '^cluster_up()' "${SCRIPT_DIR}/drivers/openshift.sh" | grep -q 'cutover_database_provider'; then + PASS=$((PASS + 1)) +else + FAIL=$((FAIL + 1)) + echo 'FAIL: OpenShift cluster_up does not reconcile the database provider on cutover' +fi +if grep -A20 '^cluster_up()' "${SCRIPT_DIR}/drivers/openshift.sh" | grep -q 'restart_after_database_cutover'; then + PASS=$((PASS + 1)) +else + FAIL=$((FAIL + 1)) + echo 'FAIL: OpenShift cluster_up does not restart components after a database cutover' +fi +if grep -A20 '^effective_database_provider()' "${SCRIPT_DIR}/drivers/openshift.sh" | grep -q 'DATABASE_PROVIDER'; then + PASS=$((PASS + 1)) +else + FAIL=$((FAIL + 1)) + echo 'FAIL: OpenShift effective_database_provider does not honor DATABASE_PROVIDER override' +fi +if grep -A25 '^cutover_database_provider()' "${SCRIPT_DIR}/drivers/openshift.sh" | grep -q 'delete secret hypershell-db-app'; then + PASS=$((PASS + 1)) +else + FAIL=$((FAIL + 1)) + echo 'FAIL: OpenShift cutover_database_provider does not clear the provider-shaped Secret' +fi if grep -A30 '^wait_for_deployments()' "${SCRIPT_DIR}/drivers/openshift.sh" | grep -q 'is_openshift_swapped'; then FAIL=$((FAIL + 1)) echo 'FAIL: wait_for_deployments skips swapped components' diff --git a/scripts/kind/lib.sh b/scripts/kind/lib.sh index ad5cb927..0ca5e417 100755 --- a/scripts/kind/lib.sh +++ b/scripts/kind/lib.sh @@ -82,25 +82,101 @@ kube() { } # --- Swap tracking (.kind-swaps) --- +# Format matches the OpenShift driver's per-namespace ledger +# (openshift_swap_file in scripts/cluster/lib.sh): one "componentimage" +# line per swapped component, so both drivers can record and restore the +# exact working-tree image identity, not just the fact that a swap happened. +# The web console's hot-reload mode (KIND_HOT_RELOAD=true, the default) has no +# image of its own -- it redirects the Service to a host-run dev server -- so +# it is tracked with the sentinel image "hot-reload". SWAP_FILE=".kind-swaps" track_swap() { local component="$1" - grep -q "^${component}$" "${SWAP_FILE}" 2>/dev/null || echo "${component}" >> "${SWAP_FILE}" + local image="$2" + touch "${SWAP_FILE}" + if grep -q "^${component}[[:space:]]" "${SWAP_FILE}" 2>/dev/null; then + local tmp + tmp="$(mktemp)" + sed "/^${component}[[:space:]]/d" "${SWAP_FILE}" > "${tmp}" + mv "${tmp}" "${SWAP_FILE}" + fi + printf '%s\t%s\n' "${component}" "${image}" >> "${SWAP_FILE}" } clear_swap() { local component="$1" if [[ -f "${SWAP_FILE}" ]]; then - sed -i.bak "/^${component}$/d" "${SWAP_FILE}" 2>/dev/null - rm -f "${SWAP_FILE}.bak" + local tmp + tmp="$(mktemp)" + sed "/^${component}[[:space:]]/d" "${SWAP_FILE}" > "${tmp}" + mv "${tmp}" "${SWAP_FILE}" + [[ -s "${SWAP_FILE}" ]] || rm -f "${SWAP_FILE}" fi } is_swapped() { local component="$1" - grep -q "^${component}$" "${SWAP_FILE}" 2>/dev/null + [[ -f "${SWAP_FILE}" ]] && grep -q "^${component}[[:space:]]" "${SWAP_FILE}" 2>/dev/null +} + +swap_image() { + local component="$1" + [[ -f "${SWAP_FILE}" ]] || return 0 + awk -F '\t' -v c="${component}" '$1 == c { print $2; exit }' "${SWAP_FILE}" +} + +# Deployment/container mapping for the three swappable components, kept local +# to Kind's swap ledger so restore_swaps_after_reconcile does not need to pull +# in scripts/cluster/lib.sh's component_spec (which also carries build/push +# fields Kind's restore path does not need). +kind_swap_deployment() { + case "$1" in + api-server) printf 'hypershell-api-server' ;; + control-plane) printf 'hypershell-controller' ;; + web-console) printf 'hypershell-web-console' ;; + esac +} + +kind_swap_containers() { + case "$1" in + api-server) printf 'api-server migrate' ;; + control-plane) printf 'controller' ;; + web-console) printf 'web-console' ;; + esac +} + +# Mirrors the OpenShift driver's restore_swaps_after_reconcile +# (scripts/cluster/drivers/openshift.sh): `kind-up` re-applies the full +# manifest set on every run, which resets any swapped Deployment's image back +# to the overlay baseline. Call this right after that apply so a swapped +# component's working-tree image is restored immediately, the same +# apply-then-restore sequencing OpenShift uses. Hot-reload web console has no +# image to restore -- its Service/EndpointSlice redirect is handled by the +# scale-to-zero guard in up.sh -- so it is skipped here. +restore_swaps_after_reconcile() { + local component image deployment containers args c + for component in api-server control-plane web-console; do + is_swapped "${component}" || continue + image="$(swap_image "${component}")" + if [[ -z "${image}" ]]; then + warn "Swap state for ${component} is empty; clearing stale entry" + clear_swap "${component}" + continue + fi + if [[ "${component}" == "web-console" && "${image}" == "hot-reload" ]]; then + continue + fi + info "Preserving ${component} working-tree image ${image}" + deployment="$(kind_swap_deployment "${component}")" + containers="$(kind_swap_containers "${component}")" + args=() + for c in ${containers}; do + args+=("${c}=${image}") + done + kube set image "deployment/${deployment}" "${args[@]}" -n "${KIND_NAMESPACE}" + done } # --- DNS (CoreDNS container) --- diff --git a/scripts/kind/status.sh b/scripts/kind/status.sh index ca8d5099..2f3f686a 100755 --- a/scripts/kind/status.sh +++ b/scripts/kind/status.sh @@ -32,12 +32,17 @@ echo "" header "Component Swap Status" if [[ -f "${SWAP_FILE}" ]] && [[ -s "${SWAP_FILE}" ]]; then info "Swapped components:" - while IFS= read -r comp; do - info " - ${comp} (local build)" + while IFS=$'\t' read -r comp image; do + [[ -n "${comp}" ]] || continue + if [[ "${image}" == "hot-reload" ]]; then + info " - ${comp} (hot reload via npm)" + else + info " - ${comp} (working-tree ${image})" + fi done < "${SWAP_FILE}" info "Baseline components:" for comp in api-server control-plane web-console; do - if ! grep -q "^${comp}$" "${SWAP_FILE}" 2>/dev/null; then + if ! is_swapped "${comp}"; then info " - ${comp} (registry image)" fi done diff --git a/scripts/kind/swap-component.sh b/scripts/kind/swap-component.sh index 226c6ffe..a33bfa8e 100755 --- a/scripts/kind/swap-component.sh +++ b/scripts/kind/swap-component.sh @@ -205,7 +205,7 @@ EOF ) & API_PF_PID=$! - track_swap "${COMPONENT}" + track_swap "${COMPONENT}" "hot-reload" _cleaned=false cleanup_hot_reload() { @@ -305,7 +305,7 @@ EOF # `rollout status` requires updated replicas to actually become available, so # it fails (rather than passing trivially) if no pods come up. kube rollout status "deployment/${DEPLOYMENT}" -n "${KIND_NAMESPACE}" --timeout=120s - track_swap "${COMPONENT}" + track_swap "${COMPONENT}" "${LOCAL_IMAGE}" success "${COMPONENT} swapped to local build." } diff --git a/scripts/kind/up.sh b/scripts/kind/up.sh index 6002f339..95295de7 100755 --- a/scripts/kind/up.sh +++ b/scripts/kind/up.sh @@ -334,6 +334,14 @@ EOF rm -rf "${_kustomize_dir}" fi +# The apply above re-renders every Deployment from the overlay baseline, +# which resets any previously swapped component's image. Restore swapped +# images immediately, the same apply-then-restore sequencing the OpenShift +# driver uses (restore_swaps_after_reconcile in +# scripts/cluster/drivers/openshift.sh), so `kind-up` and `openshift-up` +# preserve swap state the same way. +restore_swaps_after_reconcile + if [[ "${DB_PROVIDER}" == "deployment" ]]; then info "Waiting for PostgreSQL deployment..." kube wait --for=condition=available deployment/hypershell-postgres -n "${KIND_NAMESPACE}" --timeout=120s @@ -568,8 +576,8 @@ if ! is_swapped control-plane; then kube wait --for=condition=available deployment/hypershell-controller -n "${KIND_NAMESPACE}" --timeout=120s fi -if is_swapped web-console; then - warn "Web console is swapped -- scaling to zero (runs locally via npm)" +if is_swapped web-console && [[ "$(swap_image web-console)" == "hot-reload" ]]; then + warn "Web console is swapped (hot reload) -- scaling to zero (runs locally via npm)" kube scale deployment/hypershell-web-console -n "${KIND_NAMESPACE}" --replicas=0 fi diff --git a/specs/platform/local-development.spec.md b/specs/platform/local-development.spec.md index b9e3cd88..ead9f6c7 100644 --- a/specs/platform/local-development.spec.md +++ b/specs/platform/local-development.spec.md @@ -567,7 +567,7 @@ All containers in the Kind deployment manifests SHALL set restricted security co ### Requirement: Swap Tracking -The system SHALL track which components have been swapped to local builds using a `.kind-swaps` file at the repository root. This file SHALL be listed in `.gitignore`. The file records the set of currently swapped components so that `make kind-status` can report this information. Running `make kind-up` SHALL preserve existing swap state: for non-swapped components, it pulls the latest baseline images and reapplies manifests normally; for swapped components, it skips manifest reapplication to avoid overwriting the locally-built image. Swap tracking is not cleared by `kind-up`. +The system SHALL track which components have been swapped to local builds using a `.kind-swaps` file at the repository root. This file SHALL be listed in `.gitignore`. The file records, per swapped component, the exact working-tree image identity that is deployed (the same shape as the OpenShift driver's per-namespace ledger), so that `make kind-status` can report which components run a working-tree build, which run the baseline image, and the exact image each one runs. Running `make kind-up` SHALL reapply the full manifest set unconditionally (as it does for a fresh cluster) and SHALL then restore each swapped component's working-tree image immediately afterward, the same apply-then-restore sequencing `make openshift-up` uses -- rather than skipping manifest reapplication for swapped components. Swap tracking is not cleared by `kind-up`. The web console's hot-reload mode (`KIND_HOT_RELOAD=true`) has no image of its own; it is tracked with a sentinel value and restored by re-establishing its Service/EndpointSlice redirect and scaling its Deployment to zero, not by restoring an image. #### Scenario: Swap Reported in Status - GIVEN a developer has run `make kind-api-server-up` diff --git a/specs/platform/openshift-development.spec.md b/specs/platform/openshift-development.spec.md index 90bdac2f..bbbdd104 100644 --- a/specs/platform/openshift-development.spec.md +++ b/specs/platform/openshift-development.spec.md @@ -194,15 +194,36 @@ Like `make kind-up`, `make openshift-up` SHALL seed the domain resources a developer needs for a working gateway -- a ManagedCluster, a GatewayRelease, a ManagedDatabase, and a Gateway -- with the OpenShift Route and OIDC values for the environment, so that one command produces a working gateway and -the OpenShift workflow matches the Kind workflow. When the CloudNativePG operator -is not on the cluster, `make openshift-up` already falls back to the bundled -PostgreSQL Deployment for the API server; seeding SHALL create the ManagedDatabase -with `provider=deployment` in that case, not `provider=cnpg`. When CNPG is -present, seeding MAY use `provider=cnpg`. The OpenShift overlay SHALL set -`GATEWAY_API_HTTP_LISTENER_NAME=grpc` so console HTTPRoutes attach to the shared -Gateway listener of that name. The default `https` sectionName SHALL NOT be used -on this overlay: the shared Gateway has no `https` listener, and that mismatch -reports `NoMatchingParent` and does not self-heal. +the OpenShift workflow matches the Kind workflow. + +The platform's own database provider (CNPG `Cluster` vs. the bundled PostgreSQL +Deployment) SHALL be selectable with `DATABASE_PROVIDER=cnpg|deployment`, mirroring +`make kind-up`. When unset, `make openshift-up` SHALL auto-detect: CNPG if the +CloudNativePG operator is on the cluster, the bundled Deployment otherwise. +Seeding SHALL create the ManagedDatabase with `provider=deployment` when the +bundled Deployment is what's actually running, and `provider=cnpg` when CNPG is. +Both providers persist their connection info in a Secret of the same name +(`hypershell-db-app`) with different, incompatible key shapes (deployment: +`user`/`password`/`host`/`port`/`dbname`; CNPG: `username`/`password`, +auto-generated only if that Secret does not already exist). When the effective +provider for a run differs from what is actually deployed in the namespace group +(detected directly from live cluster state, not from `DATABASE_PROVIDER`), +`make openshift-up` SHALL cut over automatically: delete the outgoing provider's +resources (its Secret, and its Deployment/Service or its CNPG `Cluster`/PVCs) before +applying the target provider, and restart `hypershell-api-server` and +`hypershell-controller` afterward so they pick up the new connection info. This +cutover is destructive to the outgoing provider's data, acceptable because this +namespace group is ephemeral dev/e2e infrastructure, not production. + +The OpenShift overlay SHALL derive `GATEWAY_API_HTTP_LISTENER_NAME` from the +shared Gateway's actual listener (preferring one literally named `grpc` for +single-hub clusters, otherwise the listener that supplied `GATEWAY_API_BASE_DOMAIN`) +rather than hardcoding a static name, so console HTTPRoutes and gateway GRPCRoutes +attach to a listener that actually exists. A multi-hub shared Gateway names each +listener after its hub (e.g. `grpc-hyp4`, `grpc-hyp5`), not literally `grpc`; a +static `grpc` default, or the `https` default `sharedGatewayListenerName()` falls +back to when the override is unset, both reproduce as GRPCRoute or HTTPRoute status +`NoMatchingParent` that does not self-heal on such a cluster. The `make openshift-down` command SHALL delete the applied manifests and SHALL remove every project in the environment namespace group: the platform project diff --git a/tests/e2e/e2e-openshell.sh b/tests/e2e/e2e-openshell.sh index e29f2090..9266f171 100755 --- a/tests/e2e/e2e-openshell.sh +++ b/tests/e2e/e2e-openshell.sh @@ -47,28 +47,7 @@ DB_PROVIDER="${DATABASE_PROVIDER:-deployment}" # --- Driver selection and validation --- e2e_validate_mode - -# Detects OpenShift by checking whether the current KUBECONFIG context serves -# the route.openshift.io API group, an API only OpenShift clusters expose. -# Any other cluster is assumed to be Kind. -detect_infra_driver() { - local api_versions - if ! api_versions=$(kubectl api-versions 2>&1); then - red "ERROR: 'kubectl api-versions' failed against the current KUBECONFIG context" >&2 - red "$api_versions" >&2 - exit 1 - fi - if echo "$api_versions" | grep -q '^route\.openshift\.io/'; then - echo "openshift" - else - echo "kind" - fi -} - -if [[ -z "${E2E_INFRA_DRIVER:-}" ]]; then - E2E_INFRA_DRIVER="$(detect_infra_driver)" - dim " Detected infra driver: ${E2E_INFRA_DRIVER} (from KUBECONFIG context; set E2E_INFRA_DRIVER to override)" -fi +e2e_select_infra_driver DRIVER_FILE="${SCRIPT_DIR}/drivers/${E2E_INFRA_DRIVER}.sh" if [[ ! -f "$DRIVER_FILE" ]]; then diff --git a/tests/e2e/e2e-performance.sh b/tests/e2e/e2e-performance.sh index 6ddf367e..3f33874e 100755 --- a/tests/e2e/e2e-performance.sh +++ b/tests/e2e/e2e-performance.sh @@ -6,10 +6,14 @@ # functional gate. Reuses the e2e driver interface: no kubectl/oc/kind # commands appear in this file. # +# The infrastructure driver is auto-detected from the current KUBECONFIG +# context, the same as e2e-openshell.sh. Set E2E_INFRA_DRIVER to override. +# # Usage: -# E2E_INFRA_DRIVER=kind bash tests/e2e/e2e-performance.sh +# bash tests/e2e/e2e-performance.sh # make e2e-performance -# OPENSHIFT_NAMESPACE=my-env E2E_INFRA_DRIVER=openshift make e2e-performance +# OPENSHIFT_NAMESPACE=my-env E2E_INFRA_DRIVER=openshift \ +# make e2e-performance # override detection # # See specs/platform/e2e-testing.spec.md "Performance Testing". set -euo pipefail @@ -50,9 +54,7 @@ E2E_HS_NAMESPACE="${E2E_HS_NAMESPACE:-hypershell-system}" # --- Driver selection --- -if [[ -z "${E2E_INFRA_DRIVER:-}" ]]; then - e2e_die_unknown_driver "E2E_INFRA_DRIVER is not set." -fi +e2e_select_infra_driver DRIVER_FILE="${SCRIPT_DIR}/drivers/${E2E_INFRA_DRIVER}.sh" if [[ ! -f "$DRIVER_FILE" ]]; then diff --git a/tests/e2e/lib.sh b/tests/e2e/lib.sh index 61bb6e3d..b1060962 100755 --- a/tests/e2e/lib.sh +++ b/tests/e2e/lib.sh @@ -1,8 +1,9 @@ #!/usr/bin/env bash # lib.sh - shared e2e test utilities. # -# Provides pass/fail tracking, colored output, retry helpers, and common -# environment defaults. Sourced by e2e-openshell.sh. +# Provides pass/fail tracking, colored output, retry helpers, driver +# selection, and common environment defaults. Sourced by e2e-openshell.sh +# and e2e-performance.sh. set -euo pipefail @@ -203,7 +204,8 @@ e2e_list_available_drivers() { fi } -# Print available drivers and exit 1. Used when E2E_INFRA_DRIVER is unset or unknown. +# Print available drivers and exit 1. Used when E2E_INFRA_DRIVER names a +# missing driver file. e2e_die_unknown_driver() { local reason="$1" red "ERROR: ${reason}" @@ -213,6 +215,32 @@ e2e_die_unknown_driver() { exit 1 } +# Detects OpenShift by checking whether the current KUBECONFIG context serves +# the route.openshift.io API group, an API only OpenShift clusters expose. +# Any other cluster is assumed to be Kind. Prints the driver name to stdout. +e2e_detect_infra_driver() { + local api_versions + if ! api_versions=$(kubectl api-versions 2>&1); then + red "ERROR: 'kubectl api-versions' failed against the current KUBECONFIG context" >&2 + red "$api_versions" >&2 + exit 1 + fi + if echo "$api_versions" | grep -q '^route\.openshift\.io/'; then + echo "openshift" + else + echo "kind" + fi +} + +# Auto-detect E2E_INFRA_DRIVER from the current KUBECONFIG context when unset. +# An explicit E2E_INFRA_DRIVER value is left unchanged. +e2e_select_infra_driver() { + if [[ -z "${E2E_INFRA_DRIVER:-}" ]]; then + E2E_INFRA_DRIVER="$(e2e_detect_infra_driver)" + dim " Detected infra driver: ${E2E_INFRA_DRIVER} (from KUBECONFIG context; set E2E_INFRA_DRIVER to override)" + fi +} + # First item id from a HyperShell list JSON on stdin. Optional name match. # Usage: echo "$json" | e2e_json_first_id [name] e2e_json_first_id() { diff --git a/tests/e2e/perf/lib_test.sh b/tests/e2e/perf/lib_test.sh index 9532e488..68f29893 100755 --- a/tests/e2e/perf/lib_test.sh +++ b/tests/e2e/perf/lib_test.sh @@ -326,7 +326,7 @@ else fail_u "SIGINT was swallowed (rc=${signal_rc}, cleanup=$([[ -f "$signal_cleanup" ]] && echo yes || echo no), continued=$([[ -f "$signal_continued" ]] && echo yes || echo no))" fi -# --- Driver-not-set message (harness + suite share e2e_die_unknown_driver) --- +# --- Driver listing + auto-detection (shared by e2e and e2e-performance) --- if e2e_list_available_drivers | grep -qx kind; then pass_u "kind driver is listed" @@ -334,6 +334,94 @@ else fail_u "kind driver should be listed in tests/e2e/drivers" fi +_fake_kubectl_versions="" +_fake_kubectl_rc=0 +kubectl() { + if [[ "$*" == "api-versions" ]]; then + printf '%s\n' "${_fake_kubectl_versions}" + return "${_fake_kubectl_rc}" + fi + return 1 +} + +_fake_kubectl_versions=$'apps/v1\nroute.openshift.io/v1' +if [[ "$(e2e_detect_infra_driver)" == "openshift" ]]; then + pass_u "route.openshift.io auto-detects the openshift driver" +else + fail_u "route.openshift.io should auto-detect openshift" +fi + +_fake_kubectl_versions=$'apps/v1\nv1' +if [[ "$(e2e_detect_infra_driver)" == "kind" ]]; then + pass_u "clusters without route.openshift.io auto-detect kind" +else + fail_u "clusters without route.openshift.io should auto-detect kind" +fi + +_fake_kubectl_rc=1 +_fake_kubectl_versions="The connection to the server was refused" +if (e2e_detect_infra_driver) &>/dev/null; then + fail_u "detect should fail when kubectl api-versions fails" +else + pass_u "detect fails when kubectl cannot reach the cluster" +fi +_fake_kubectl_rc=0 + +_saved_driver="${E2E_INFRA_DRIVER:-}" +E2E_INFRA_DRIVER=kind +_fake_kubectl_versions=$'route.openshift.io/v1' +e2e_select_infra_driver >/dev/null +if [[ "$E2E_INFRA_DRIVER" == "kind" ]]; then + pass_u "explicit E2E_INFRA_DRIVER overrides auto-detection" +else + fail_u "explicit E2E_INFRA_DRIVER was overwritten: ${E2E_INFRA_DRIVER}" +fi + +unset E2E_INFRA_DRIVER +e2e_select_infra_driver >/dev/null +if [[ "$E2E_INFRA_DRIVER" == "openshift" ]]; then + pass_u "unset E2E_INFRA_DRIVER auto-detects from KUBECONFIG" +else + fail_u "unset E2E_INFRA_DRIVER did not auto-detect openshift: ${E2E_INFRA_DRIVER:-}" +fi +if [[ -n "$_saved_driver" ]]; then + E2E_INFRA_DRIVER="$_saved_driver" +else + unset E2E_INFRA_DRIVER +fi +unset _saved_driver +unset -f kubectl +unset _fake_kubectl_versions _fake_kubectl_rc + +if grep -q 'e2e_select_infra_driver' "${SCRIPT_DIR}/../e2e-performance.sh" \ + && grep -q 'e2e_select_infra_driver' "${SCRIPT_DIR}/../e2e-openshell.sh"; then + pass_u "e2e and e2e-performance share e2e_select_infra_driver" +else + fail_u "e2e-performance.sh or e2e-openshell.sh does not call e2e_select_infra_driver" +fi + +if grep -q 'E2E_INFRA_DRIVER is not set' "${SCRIPT_DIR}/../e2e-performance.sh"; then + fail_u "e2e-performance.sh still requires E2E_INFRA_DRIVER to be set" +else + pass_u "e2e-performance.sh no longer requires E2E_INFRA_DRIVER to be set" +fi + +makefile="${SCRIPT_DIR}/../../../Makefile" +if grep -q 'E2E_INFRA_DRIVER ?=' "$makefile" \ + || grep -q 'E2E_INFRA_DRIVER=\$(E2E_INFRA_DRIVER)' "$makefile"; then + fail_u "Makefile still forces E2E_INFRA_DRIVER for e2e-performance" +else + pass_u "make e2e-performance does not force E2E_INFRA_DRIVER" +fi + +help_out="$(make -s -C "${SCRIPT_DIR}/../../.." help)" +if echo "$help_out" | grep -q 'E2E_PERF_GATEWAY_COUNT' \ + && echo "$help_out" | grep -q 'E2E_PERF_BATCH_SIZE'; then + pass_u "make help lists E2E_PERF_GATEWAY_COUNT and E2E_PERF_BATCH_SIZE" +else + fail_u "make help missing E2E_PERF_GATEWAY_COUNT or E2E_PERF_BATCH_SIZE" +fi + # --- Harness is infra-agnostic --- hits=$(grep -nE '\b(kubectl|oc)\b' "${SCRIPT_DIR}/../e2e-performance.sh" | grep -v '#' || true)