Skip to content

Commit 5dc8ff3

Browse files
patrickleetpatrickleet
authored
feat: add knative resource defaults and nodepool (#14)
1 parent 164568b commit 5dc8ff3

7 files changed

Lines changed: 475 additions & 16 deletions

File tree

‎Makefile‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ generate-configuration:
2525
# Format: example_path::observed_resources_path (observed_resources_path is optional)
2626
EXAMPLES := \
2727
examples/knativestacks/minimal.yaml:: \
28+
examples/knativestacks/nodepool.yaml:: \
2829
examples/knativestacks/standard.yaml::
2930

3031
# Render all examples (parallel execution, output shown per-job when complete)

‎apis/knativestacks/definition.yaml‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,34 @@ spec:
102102
enum:
103103
- ProviderConfig
104104
- ClusterProviderConfig
105+
nodePool:
106+
description: Optional dedicated Karpenter NodePool for Knative and NATS workloads. When enabled, non-DaemonSet pods are scheduled with workload-type=knative and tolerate knative=true:NoSchedule.
107+
type: object
108+
properties:
109+
enabled:
110+
description: Whether to create a dedicated NodePool. Defaults to false.
111+
type: boolean
112+
default: false
113+
name:
114+
description: NodePool name on the target cluster. Defaults to "hops-knative".
115+
type: string
116+
nodeClassName:
117+
description: EKS Auto Mode NodeClass to reference. Defaults to "hops-default".
118+
type: string
119+
limits:
120+
description: Karpenter NodePool resource limits. Defaults to nodes=10.
121+
type: object
122+
x-kubernetes-preserve-unknown-fields: true
123+
requirements:
124+
description: Karpenter scheduling requirements. Defaults to amd64/linux spot or on-demand nodes.
125+
type: array
126+
items:
127+
type: object
128+
x-kubernetes-preserve-unknown-fields: true
129+
disruption:
130+
description: Karpenter disruption settings. Defaults to WhenEmptyOrUnderutilized after 60s.
131+
type: object
132+
x-kubernetes-preserve-unknown-fields: true
105133
knativeOperator:
106134
description: Configuration for the Knative Operator component.
107135
type: object

‎examples/knativestacks/nodepool.yaml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
apiVersion: hops.ops.com.ai/v1alpha1
2+
kind: KnativeStack
3+
metadata:
4+
name: knative
5+
namespace: default
6+
spec:
7+
clusterName: default
8+
labels:
9+
team: platform
10+
nodePool:
11+
enabled: true

‎functions/render/000-state-init.yaml.gotmpl‎

Lines changed: 125 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,38 @@
5858
"kind" ($k8sProviderConfigRef.kind | default "ProviderConfig")
5959
}}
6060

61+
# ==============================================================================
62+
# NodePool configuration
63+
# ==============================================================================
64+
{{- $nodePoolSpec := $spec.nodePool | default dict }}
65+
{{- $nodePoolEnabled := false }}
66+
{{- if hasKey $nodePoolSpec "enabled" }}
67+
{{- $nodePoolEnabled = $nodePoolSpec.enabled }}
68+
{{- end }}
69+
{{- $nodePoolName := $nodePoolSpec.name | default "hops-knative" }}
70+
{{- $nodePoolNodeClassName := $nodePoolSpec.nodeClassName | default "hops-default" }}
71+
{{- $nodePoolLimits := $nodePoolSpec.limits | default (dict "nodes" 10) }}
72+
{{- $nodePoolRequirements := $nodePoolSpec.requirements | default (list
73+
(dict "key" "karpenter.sh/capacity-type" "operator" "In" "values" (list "spot" "on-demand"))
74+
(dict "key" "eks.amazonaws.com/instance-category" "operator" "In" "values" (list "c" "m" "r"))
75+
(dict "key" "eks.amazonaws.com/instance-generation" "operator" "Gt" "values" (list "4"))
76+
(dict "key" "eks.amazonaws.com/instance-memory" "operator" "Gt" "values" (list "7999"))
77+
(dict "key" "eks.amazonaws.com/instance-cpu" "operator" "Gt" "values" (list "1"))
78+
(dict "key" "kubernetes.io/arch" "operator" "In" "values" (list "amd64"))
79+
(dict "key" "kubernetes.io/os" "operator" "In" "values" (list "linux"))
80+
) }}
81+
{{- $nodePoolDisruption := $nodePoolSpec.disruption | default (dict "consolidationPolicy" "WhenEmptyOrUnderutilized" "consolidateAfter" "60s") }}
82+
{{- $nodePoolTaintKey := "knative" }}
83+
{{- $nodePoolTaintValue := "true" }}
84+
{{- $nodePoolNodeSelector := dict }}
85+
{{- $nodePoolTolerations := list }}
86+
{{- $nodePoolAffinity := dict }}
87+
{{- if $nodePoolEnabled }}
88+
{{- $nodePoolNodeSelector = dict "workload-type" "knative" }}
89+
{{- $nodePoolTolerations = list (dict "key" $nodePoolTaintKey "operator" "Equal" "value" $nodePoolTaintValue "effect" "NoSchedule") }}
90+
{{- $nodePoolAffinity = dict "nodeAffinity" (dict "requiredDuringSchedulingIgnoredDuringExecution" (dict "nodeSelectorTerms" (list (dict "matchExpressions" (list (dict "key" "workload-type" "operator" "In" "values" (list "knative"))))))) }}
91+
{{- end }}
92+
6193
# ==============================================================================
6294
# Per-component defaults
6395
# ==============================================================================
@@ -149,6 +181,35 @@
149181
(dict "URL" "https://github.com/knative-extensions/net-gateway-api/releases/download/knative-v1.21.0/release.yaml")
150182
)
151183
}}
184+
{{- $servingDeployments := list }}
185+
{{- range $deploymentDefaults := list
186+
(dict "name" "activator" "container" "activator" "cpu" "15m" "memory" "100Mi")
187+
(dict "name" "autoscaler" "container" "autoscaler" "cpu" "15m" "memory" "100Mi")
188+
(dict "name" "autoscaler-hpa" "container" "autoscaler-hpa" "cpu" "15m" "memory" "100Mi")
189+
(dict "name" "controller" "container" "controller" "cpu" "15m" "memory" "100Mi")
190+
(dict "name" "webhook" "container" "webhook" "cpu" "15m" "memory" "100Mi")
191+
(dict "name" "net-gateway-api-controller" "container" "controller" "cpu" "15m" "memory" "100Mi")
192+
(dict "name" "net-gateway-api-webhook" "container" "webhook" "cpu" "15m" "memory" "100Mi")
193+
(dict "name" "net-istio-controller" "container" "controller" "cpu" "15m" "memory" "100Mi")
194+
(dict "name" "net-istio-webhook" "container" "webhook" "cpu" "15m" "memory" "100Mi")
195+
}}
196+
{{- $deployment := dict
197+
"name" $deploymentDefaults.name
198+
"resources" (list
199+
(dict
200+
"container" $deploymentDefaults.container
201+
"requests" (dict "cpu" $deploymentDefaults.cpu "memory" $deploymentDefaults.memory)
202+
"limits" (dict "memory" "256Mi")
203+
)
204+
)
205+
}}
206+
{{- if $nodePoolEnabled }}
207+
{{- $_ := set $deployment "nodeSelector" $nodePoolNodeSelector }}
208+
{{- $_ := set $deployment "tolerations" $nodePoolTolerations }}
209+
{{- end }}
210+
{{- $servingDeployments = append $servingDeployments $deployment }}
211+
{{- end }}
212+
{{- $_ := set $natsKnServingDefaults "deployments" $servingDeployments }}
152213
{{- $servingSpec := merge ($knServing.spec | default dict) $natsKnServingDefaults }}
153214

154215
# ==============================================================================
@@ -193,20 +254,40 @@
193254
"container" (dict
194255
"merge" (dict
195256
"resources" (dict
196-
"requests" (dict "cpu" "100m" "memory" "256Mi")
257+
"requests" (dict "cpu" "11m" "memory" "50Mi")
197258
"limits" (dict "cpu" "500m" "memory" "512Mi")
198259
)
199260
)
200261
)
201262
"reloader" (dict
202263
"merge" (dict
203264
"resources" (dict
204-
"requests" (dict "cpu" "10m" "memory" "32Mi")
265+
"requests" (dict "cpu" "11m" "memory" "50Mi")
205266
"limits" (dict "cpu" "50m" "memory" "64Mi")
206267
)
207268
)
208269
)
270+
"natsBox" (dict
271+
"container" (dict
272+
"merge" (dict
273+
"resources" (dict
274+
"requests" (dict "cpu" "15m" "memory" "100Mi")
275+
"limits" (dict "cpu" "100m" "memory" "256Mi")
276+
)
277+
)
278+
)
279+
)
209280
}}
281+
{{- if $nodePoolEnabled }}
282+
{{- $_ := set $natsDefaultValues "podTemplate" (dict
283+
"merge" (dict
284+
"spec" (dict
285+
"nodeSelector" $nodePoolNodeSelector
286+
"tolerations" $nodePoolTolerations
287+
)
288+
)
289+
) }}
290+
{{- end }}
210291
{{- $natsValues := mergeOverwrite (deepCopy $natsDefaultValues) ($nats.values | default dict) }}
211292

212293
# ==============================================================================
@@ -216,20 +297,37 @@
216297
# mTLS at the node level — eventing doesn't need to know about the mesh.
217298
{{- /* eventing-webhook ships with a 200Mi limit that OOMKills under modest
218299
ApiServerSource / Trigger reconcile load. Bump to 512Mi so it stays up. */}}
219-
{{- $eventingDefaults := dict
220-
"config" dict
221-
"deployments" (list
222-
(dict
223-
"name" "eventing-webhook"
224-
"resources" (list
225-
(dict
226-
"container" "eventing-webhook"
227-
"requests" (dict "memory" "100Mi")
228-
"limits" (dict "memory" "512Mi")
229-
)
300+
{{- $eventingDeployments := list }}
301+
{{- range $deployment := list
302+
(dict "name" "eventing-controller" "container" "eventing-controller" "cpu" "15m" "memory" "284Mi" "limitMemory" "512Mi")
303+
(dict "name" "eventing-webhook" "container" "eventing-webhook" "cpu" "15m" "memory" "309Mi" "limitMemory" "512Mi")
304+
(dict "name" "imc-controller" "container" "controller" "cpu" "23m" "memory" "100Mi" "limitMemory" "256Mi")
305+
(dict "name" "imc-dispatcher" "container" "dispatcher" "cpu" "15m" "memory" "100Mi" "limitMemory" "256Mi")
306+
(dict "name" "job-sink" "container" "job-sink" "cpu" "15m" "memory" "100Mi" "limitMemory" "256Mi")
307+
(dict "name" "mt-broker-controller" "container" "mt-broker-controller" "cpu" "15m" "memory" "100Mi" "limitMemory" "256Mi")
308+
(dict "name" "mt-broker-filter" "container" "filter" "cpu" "15m" "memory" "100Mi" "limitMemory" "256Mi")
309+
(dict "name" "mt-broker-ingress" "container" "ingress" "cpu" "15m" "memory" "100Mi" "limitMemory" "256Mi")
310+
(dict "name" "request-reply" "container" "request-reply" "cpu" "15m" "memory" "100Mi" "limitMemory" "256Mi")
311+
}}
312+
{{- $eventingDeployment := dict
313+
"name" $deployment.name
314+
"resources" (list
315+
(dict
316+
"container" $deployment.container
317+
"requests" (dict "cpu" $deployment.cpu "memory" $deployment.memory)
318+
"limits" (dict "memory" $deployment.limitMemory)
230319
)
231320
)
232-
)
321+
}}
322+
{{- if $nodePoolEnabled }}
323+
{{- $_ := set $eventingDeployment "nodeSelector" $nodePoolNodeSelector }}
324+
{{- $_ := set $eventingDeployment "tolerations" $nodePoolTolerations }}
325+
{{- end }}
326+
{{- $eventingDeployments = append $eventingDeployments $eventingDeployment }}
327+
{{- end }}
328+
{{- $eventingDefaults := dict
329+
"config" dict
330+
"deployments" $eventingDeployments
233331
}}
234332
{{- if $natsEnabled }}
235333
{{- $natsNs := $nats.namespace | default "nats" }}
@@ -257,6 +355,19 @@
257355
"autoTls" (dict "enabled" $autoTlsEnabled "namespaceSelector" $autoTlsSelector)
258356
"helmProviderConfigRef" $helmProviderConfigRef
259357
"kubernetesProviderConfigRef" $k8sProviderConfigRef
358+
"nodePool" (dict
359+
"enabled" $nodePoolEnabled
360+
"name" $nodePoolName
361+
"nodeClassName" $nodePoolNodeClassName
362+
"limits" $nodePoolLimits
363+
"requirements" $nodePoolRequirements
364+
"disruption" $nodePoolDisruption
365+
"taintKey" $nodePoolTaintKey
366+
"taintValue" $nodePoolTaintValue
367+
"nodeSelector" $nodePoolNodeSelector
368+
"tolerations" $nodePoolTolerations
369+
"affinity" $nodePoolAffinity
370+
)
260371
"knativeOperator" (dict
261372
"name" ($knOp.name | default "knative-operator")
262373
"namespace" ($knOp.namespace | default "knative-operator")

‎functions/render/155-nodepool.yaml.gotmpl‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
# code: language=yaml
2+
#
3+
# Optional Karpenter NodePool for Knative and NATS workloads.
4+
#
5+
6+
{{- if $state.nodePool.enabled }}
7+
---
8+
apiVersion: kubernetes.m.crossplane.io/v1alpha1
9+
kind: Object
10+
metadata:
11+
name: {{ $state.name }}-nodepool-knative
12+
annotations:
13+
{{ setResourceNameAnnotation "nodepool-knative" }}
14+
labels: {{ $state.labels | toJson }}
15+
spec:
16+
managementPolicies: {{ $state.managementPolicies | toJson }}
17+
forProvider:
18+
manifest:
19+
apiVersion: karpenter.sh/v1
20+
kind: NodePool
21+
metadata:
22+
name: {{ $state.nodePool.name }}
23+
spec:
24+
template:
25+
metadata:
26+
labels:
27+
workload-type: knative
28+
spec:
29+
nodeClassRef:
30+
group: eks.amazonaws.com
31+
kind: NodeClass
32+
name: {{ $state.nodePool.nodeClassName }}
33+
taints:
34+
- key: {{ $state.nodePool.taintKey }}
35+
value: {{ $state.nodePool.taintValue | quote }}
36+
effect: NoSchedule
37+
requirements: {{ $state.nodePool.requirements | toJson }}
38+
limits: {{ $state.nodePool.limits | toJson }}
39+
disruption: {{ $state.nodePool.disruption | toJson }}
40+
providerConfigRef:
41+
name: {{ $state.kubernetesProviderConfigRef.name }}
42+
kind: {{ $state.kubernetesProviderConfigRef.kind }}
43+
44+
---
45+
apiVersion: protection.crossplane.io/v1beta1
46+
kind: Usage
47+
metadata:
48+
name: {{ $state.name }}-delete-operator-before-nodepool
49+
annotations:
50+
{{ setResourceNameAnnotation "usage-operator-before-nodepool" }}
51+
labels: {{ $state.labels | toJson }}
52+
spec:
53+
replayDeletion: true
54+
of:
55+
apiVersion: kubernetes.m.crossplane.io/v1alpha1
56+
kind: Object
57+
resourceRef:
58+
name: {{ $state.name }}-nodepool-knative
59+
by:
60+
apiVersion: helm.m.crossplane.io/v1beta1
61+
kind: Release
62+
resourceRef:
63+
name: {{ $state.name }}-{{ $state.knativeOperator.name }}
64+
65+
{{- $natsNodePoolUsageEnabled := $state.nats.enabled }}
66+
{{- if and $state.nats.enabled $state.nats.storageClass.enabled (not $state.observed.natsStorageClass.ready) }}
67+
{{- $natsNodePoolUsageEnabled = false }}
68+
{{- end }}
69+
{{- if $natsNodePoolUsageEnabled }}
70+
---
71+
apiVersion: protection.crossplane.io/v1beta1
72+
kind: Usage
73+
metadata:
74+
name: {{ $state.name }}-delete-nats-before-nodepool
75+
annotations:
76+
{{ setResourceNameAnnotation "usage-nats-before-nodepool" }}
77+
labels: {{ $state.labels | toJson }}
78+
spec:
79+
replayDeletion: true
80+
of:
81+
apiVersion: kubernetes.m.crossplane.io/v1alpha1
82+
kind: Object
83+
resourceRef:
84+
name: {{ $state.name }}-nodepool-knative
85+
by:
86+
apiVersion: helm.m.crossplane.io/v1beta1
87+
kind: Release
88+
resourceRef:
89+
name: {{ $state.name }}-{{ $state.nats.name }}
90+
{{- end }}
91+
{{- end }}

‎functions/render/210-knative-operator.yaml.gotmpl‎

Lines changed: 39 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,42 @@
44
#
55

66
{{- $knOp := $state.knativeOperator }}
7+
{{- $operatorComponent := dict
8+
"resources" (dict
9+
"requests" (dict "cpu" "15m" "memory" "100Mi")
10+
"limits" (dict "cpu" "1000m" "memory" "512Mi")
11+
)
12+
}}
13+
{{- $webhookComponent := dict
14+
"resources" (dict
15+
"requests" (dict "cpu" "15m" "memory" "284Mi")
16+
"limits" (dict "cpu" "500m" "memory" "512Mi")
17+
)
18+
}}
19+
{{- if $state.nodePool.enabled }}
20+
{{- $_ := set $operatorComponent "affinity" $state.nodePool.affinity }}
21+
{{- $_ := set $operatorComponent "tolerations" $state.nodePool.tolerations }}
22+
{{- $webhookAffinity := deepCopy $state.nodePool.affinity }}
23+
{{- $_ := set $webhookAffinity "podAntiAffinity" (dict
24+
"preferredDuringSchedulingIgnoredDuringExecution" (list
25+
(dict
26+
"weight" 100
27+
"podAffinityTerm" (dict
28+
"labelSelector" (dict "matchLabels" (dict "app" "operator-webhook"))
29+
"topologyKey" "kubernetes.io/hostname"
30+
)
31+
)
32+
)
33+
) }}
34+
{{- $_ := set $webhookComponent "affinity" $webhookAffinity }}
35+
{{- $_ := set $webhookComponent "tolerations" $state.nodePool.tolerations }}
36+
{{- end }}
37+
{{- $operatorDefaults := dict
38+
"knative_operator" (dict
39+
"knative_operator" $operatorComponent
40+
"operator_webhook" $webhookComponent
41+
)
42+
}}
743

844
---
945
apiVersion: helm.m.crossplane.io/v1beta1
@@ -26,9 +62,10 @@ spec:
2662
{{- if $knOp.overrideAllValues }}
2763
values:
2864
{{- toYaml $knOp.overrideAllValues | nindent 6 }}
29-
{{- else if $knOp.values }}
65+
{{- else }}
66+
{{- $mergedValues := mergeOverwrite $operatorDefaults ($knOp.values | default dict) }}
3067
values:
31-
{{- toYaml $knOp.values | nindent 6 }}
68+
{{- toYaml $mergedValues | nindent 6 }}
3269
{{- end }}
3370
rollbackLimit: 3
3471
providerConfigRef:

0 commit comments

Comments
 (0)