feat(probes): liveness probes for worker and ui pods, overridable per component

CHANGELOG.md

@@ -5,6 +5,7 @@ All notable changes to this project are documented in this file.
 ## Unreleased
 
 Improvements:
+- Add liveness probes to the worker, UI and admin tools containers (the other services already had one), and expose `livenessProbe` on every service, the UI and the admin tools so a generated probe can be replaced or disabled (`disabled: true`). The worker has no gRPC endpoint, so its default probe targets the membership port with deliberately conservative thresholds. See [Liveness probes](https://temporal-operator.pages.dev/features/liveness-probes/).
 - Add support for Temporal Server v1.29.x. Temporal v1.29 introduces only dynamic-config changes (task-queue fairness, task-queue config API), which are already supported through the cluster `dynamicConfig` field.
 - Add support for Temporal Server v1.30.x. (The defaults and supported range moved on again with v1.31 support below; see that entry for the values this release actually ships.)
   - Temporal v1.30 removed `dockerize`/`auto-setup` from the `temporalio/server` image and moved config-template rendering into the server binary (embedded sprig engine). For clusters running `>= 1.30`, the operator now emits config templates with the `# enable-template` header and sprig `{{ env "NAME" }}` placeholders (instead of the dockerize `{{ .Env.NAME }}` syntax), sets `TEMPORAL_SERVER_CONFIG_FILE_PATH`, and selects the service to start through the new `TEMPORAL_SERVICES` environment variable (the legacy `SERVICES` variable is still set for backward compatibility).

api/v1beta1/livenessprobe_test.go

@@ -0,0 +1,81 @@
+// Licensed to Alexandre VILAIN under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Alexandre VILAIN licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package v1beta1_test
+
+import (
+	"testing"
+
+	"github.com/alexandrevilain/temporal-operator/api/v1beta1"
+	"github.com/stretchr/testify/assert"
+	corev1 "k8s.io/api/core/v1"
+)
+
+func TestLivenessProbeSpecResolve(t *testing.T) {
+	defaultProbe := &corev1.Probe{
+		ProbeHandler: corev1.ProbeHandler{
+			TCPSocket: &corev1.TCPSocketAction{},
+		},
+	}
+	customProbe := &corev1.Probe{
+		ProbeHandler: corev1.ProbeHandler{
+			Exec: &corev1.ExecAction{Command: []string{"true"}},
+		},
+	}
+
+	tests := map[string]struct {
+		spec     *v1beta1.LivenessProbeSpec
+		expected *corev1.Probe
+	}{
+		"unset spec keeps the default": {
+			spec:     nil,
+			expected: defaultProbe,
+		},
+		"empty spec keeps the default": {
+			spec:     &v1beta1.LivenessProbeSpec{},
+			expected: defaultProbe,
+		},
+		"disabled removes the probe": {
+			spec:     &v1beta1.LivenessProbeSpec{Disabled: true},
+			expected: nil,
+		},
+		"custom probe replaces the default": {
+			spec:     &v1beta1.LivenessProbeSpec{Probe: customProbe},
+			expected: customProbe,
+		},
+		"disabled wins over a custom probe": {
+			spec:     &v1beta1.LivenessProbeSpec{Disabled: true, Probe: customProbe},
+			expected: nil,
+		},
+	}
+
+	for name, test := range tests {
+		t.Run(name, func(t *testing.T) {
+			assert.Equal(t, test.expected, test.spec.Resolve(defaultProbe))
+		})
+	}
+}
+
+// TestLivenessProbeSpecResolveNilSafe asserts the accessor can be called on a nil
+// spec, which is what the deployment builders do when the field is unset.
+func TestLivenessProbeSpecResolveNilSafe(t *testing.T) {
+	var spec *v1beta1.LivenessProbeSpec
+
+	defaultProbe := &corev1.Probe{}
+	assert.Equal(t, defaultProbe, spec.Resolve(defaultProbe))
+	assert.Nil(t, spec.Resolve(nil))
+}

api/v1beta1/temporalcluster_types.go

@@ -62,6 +62,37 @@ type LogSpec struct {
 	Development bool `json:"development"`
 }
 
+// LivenessProbeSpec customizes the liveness probe the operator sets on a
+// component's container. When left empty the operator's default probe for that
+// component is used.
+type LivenessProbeSpec struct {
+	// Disabled removes the liveness probe from the container.
+	// Use it when the default probe doesn't fit and no replacement is wanted.
+	// +optional
+	// +kubebuilder:default:=false
+	Disabled bool `json:"disabled,omitempty"`
+	// Probe replaces the operator's default liveness probe.
+	// It is ignored when disabled is set.
+	// +optional
+	Probe *corev1.Probe `json:"probe,omitempty"`
+}
+
+// Resolve returns the liveness probe to set on the container, given the
+// operator's default probe for the component. It is nil-safe: an unset spec
+// keeps the default.
+func (s *LivenessProbeSpec) Resolve(defaultProbe *corev1.Probe) *corev1.Probe {
+	if s == nil {
+		return defaultProbe
+	}
+	if s.Disabled {
+		return nil
+	}
+	if s.Probe != nil {
+		return s.Probe
+	}
+	return defaultProbe
+}
+
 // ServiceSpec contains a temporal service specifications.
 type ServiceSpec struct {
 	// Port defines a custom gRPC port for the service.
@@ -93,6 +124,10 @@ type ServiceSpec struct {
 	// More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
 	// +optional
 	Resources corev1.ResourceRequirements `json:"resources,omitempty"`
+	// LivenessProbe customizes the liveness probe the operator sets on the
+	// service's container. Left empty, the operator's default probe is used.
+	// +optional
+	LivenessProbe *LivenessProbeSpec `json:"livenessProbe,omitempty"`
 	// Overrides adds some overrides to the resources deployed for the service.
 	// Those overrides takes precedence over spec.services.overrides.
 	// +optional
@@ -587,6 +622,10 @@ type TemporalUISpec struct {
 	// More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
 	// +optional
 	Resources corev1.ResourceRequirements `json:"resources,omitempty"`
+	// LivenessProbe customizes the liveness probe the operator sets on the ui's
+	// container. Left empty, the operator's default probe is used.
+	// +optional
+	LivenessProbe *LivenessProbeSpec `json:"livenessProbe,omitempty"`
 	// Overrides adds some overrides to the resources deployed for the ui.
 	// +optional
 	Overrides *ServiceSpecOverride `json:"overrides,omitempty"`
@@ -615,6 +654,10 @@ type TemporalAdminToolsSpec struct {
 	// More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
 	// +optional
 	Resources corev1.ResourceRequirements `json:"resources,omitempty"`
+	// LivenessProbe customizes the liveness probe the operator sets on the admin
+	// tools container. Left empty, the operator's default probe is used.
+	// +optional
+	LivenessProbe *LivenessProbeSpec `json:"livenessProbe,omitempty"`
 	// Overrides adds some overrides to the resources deployed for the ui.
 	// +optional
 	Overrides *ServiceSpecOverride `json:"overrides,omitempty"`