diff --git a/kubernetes/helm/otoroshi/templates/NOTES.txt b/kubernetes/helm/otoroshi/templates/NOTES.txt index 62557c39dc..985b79b423 100644 --- a/kubernetes/helm/otoroshi/templates/NOTES.txt +++ b/kubernetes/helm/otoroshi/templates/NOTES.txt @@ -119,6 +119,27 @@ Otoroshi is running in cluster mode: Workers connect to the leader API service at: https://{{ .Values.cluster.leaderApiServiceName }}.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.service.https }} {{- end }} +{{- if or .Values.webhooks.validation .Values.webhooks.sidecarInject }} + +============================================================ +7. Admission webhooks (upgrades) +============================================================ +{{ if .Values.webhooks.validation }}- ValidatingWebhookConfiguration "otoroshi-admission-webhook-validation" (CRD validation) +{{ end }}{{ if .Values.webhooks.sidecarInject }}- MutatingWebhookConfiguration "otoroshi-admission-webhook-injector" (sidecar injection) +{{ end }} +They are answered by request sinks registered in the Otoroshi global config through +OTOROSHI_INITIAL_CUSTOMIZATION, which only applies on the FIRST boot, while the datastore +is still empty. If you upgraded from an earlier chart version, or pointed this release at a +pre-existing Redis, check that Danger zone -> Global plugins contains +"Request sink plugin wrapper" entries for: + + cp:otoroshi.plugins.jobs.kubernetes.KubernetesAdmissionWebhookCRDValidator + cp:otoroshi.plugins.jobs.kubernetes.KubernetesAdmissionWebhookSidecarInjector + +Without them Otoroshi answers 404, and once the CRDs controller job has flipped failurePolicy +from Ignore to Fail (first sync loop, <= 60s), every `kubectl apply` of a proxy.otoroshi.io +resource fails with "the server could not find the requested resource". +{{- end }} ============================================================ More diff --git a/kubernetes/helm/otoroshi/templates/deployment-cluster.yaml b/kubernetes/helm/otoroshi/templates/deployment-cluster.yaml index 29a7f19ce3..6df1754616 100644 --- a/kubernetes/helm/otoroshi/templates/deployment-cluster.yaml +++ b/kubernetes/helm/otoroshi/templates/deployment-cluster.yaml @@ -136,6 +136,16 @@ spec: value: "{{ .Values.loadbalancer.nodePort.http }}" - name: OTOROSHI_EXPOSED_PORTS_HTTPS value: "{{ .Values.loadbalancer.nodePort.https }}" + # Only applied on the very first boot, when the datastore is still empty. On an + # upgrade (or a pre-existing Redis) Otoroshi keeps the global config it already has, + # so any change here must also be applied by hand on existing installs. + # + # The admission-webhook request sinks are declared TWICE on purpose: + # - under `plugins.config.ng` (wrapped in RequestSinkWrapper) because the next-gen + # proxy engine — the default — only looks at that list; + # - under `scripts.sinkRefs` for the legacy engine. + # Note that `plugins.config.ng` REPLACES the built-in list (deepMerge does not merge + # arrays), hence the NgClientCredentials entry below which is an Otoroshi default. - name: OTOROSHI_INITIAL_CUSTOMIZATION value: > { @@ -144,6 +154,46 @@ spec: "defaultDomain": "www.{{ .Values.env.domain }}", "randomIfNotFound": false }, + "plugins": { + "enabled": true, + "config": { + "ng": [ + { + "enabled": true, + "debug": false, + "include": [], + "exclude": [], + "plugin": "cp:otoroshi.next.plugins.NgClientCredentials", + "config": { + "expiration": 3600000, + "default_key_pair": "otoroshi-jwt-signing", + "domain": "*", + "secure": true + } + }, + { + "enabled": true, + "debug": false, + "include": [], + "exclude": [], + "plugin": "cp:otoroshi.next.plugins.wrappers.RequestSinkWrapper", + "config": { + "plugin": "cp:otoroshi.plugins.jobs.kubernetes.KubernetesAdmissionWebhookCRDValidator" + } + }, + { + "enabled": true, + "debug": false, + "include": [], + "exclude": [], + "plugin": "cp:otoroshi.next.plugins.wrappers.RequestSinkWrapper", + "config": { + "plugin": "cp:otoroshi.plugins.jobs.kubernetes.KubernetesAdmissionWebhookSidecarInjector" + } + } + ] + } + }, "scripts":{ "enabled":true, "sinkRefs":[ diff --git a/kubernetes/helm/otoroshi/templates/deployment.yaml b/kubernetes/helm/otoroshi/templates/deployment.yaml index 8f556dd8bf..abb6d1f6dc 100644 --- a/kubernetes/helm/otoroshi/templates/deployment.yaml +++ b/kubernetes/helm/otoroshi/templates/deployment.yaml @@ -108,6 +108,16 @@ spec: value: "{{ .Values.loadbalancer.nodePort.http }}" - name: OTOROSHI_EXPOSED_PORTS_HTTPS value: "{{ .Values.loadbalancer.nodePort.https }}" + # Only applied on the very first boot, when the datastore is still empty. On an + # upgrade (or a pre-existing Redis) Otoroshi keeps the global config it already has, + # so any change here must also be applied by hand on existing installs. + # + # The admission-webhook request sinks are declared TWICE on purpose: + # - under `plugins.config.ng` (wrapped in RequestSinkWrapper) because the next-gen + # proxy engine — the default — only looks at that list; + # - under `scripts.sinkRefs` for the legacy engine. + # Note that `plugins.config.ng` REPLACES the built-in list (deepMerge does not merge + # arrays), hence the NgClientCredentials entry below which is an Otoroshi default. - name: OTOROSHI_INITIAL_CUSTOMIZATION value: > { @@ -116,6 +126,46 @@ spec: "defaultDomain": "www.{{ .Values.env.domain }}", "randomIfNotFound": false }, + "plugins": { + "enabled": true, + "config": { + "ng": [ + { + "enabled": true, + "debug": false, + "include": [], + "exclude": [], + "plugin": "cp:otoroshi.next.plugins.NgClientCredentials", + "config": { + "expiration": 3600000, + "default_key_pair": "otoroshi-jwt-signing", + "domain": "*", + "secure": true + } + }, + { + "enabled": true, + "debug": false, + "include": [], + "exclude": [], + "plugin": "cp:otoroshi.next.plugins.wrappers.RequestSinkWrapper", + "config": { + "plugin": "cp:otoroshi.plugins.jobs.kubernetes.KubernetesAdmissionWebhookCRDValidator" + } + }, + { + "enabled": true, + "debug": false, + "include": [], + "exclude": [], + "plugin": "cp:otoroshi.next.plugins.wrappers.RequestSinkWrapper", + "config": { + "plugin": "cp:otoroshi.plugins.jobs.kubernetes.KubernetesAdmissionWebhookSidecarInjector" + } + } + ] + } + }, "scripts":{ "enabled":true, "sinkRefs":[ diff --git a/kubernetes/helm/otoroshi/templates/webhooks.yaml b/kubernetes/helm/otoroshi/templates/webhooks.yaml index c077f2068f..f4fc68524e 100644 --- a/kubernetes/helm/otoroshi/templates/webhooks.yaml +++ b/kubernetes/helm/otoroshi/templates/webhooks.yaml @@ -1,4 +1,11 @@ -{{- $webhookServiceName := ternary .Values.cluster.leaderApiServiceName .Values.service.name .Values.cluster.enabled -}} +{{/* +The webhook MUST target the very service passed as `otoroshiServiceName` in the +KubernetesConfig of the deployment templates: the sinks that answer these calls only +match a Host of `..svc[.cluster.local]` (see +KubernetesAdmissionWebhookCRDValidator.matches), and the certificate Otoroshi generates +for that endpoint is issued for those two SANs only (see createWebhookCerts). +*/}} +{{- $webhookServiceName := ternary .Values.cluster.workerServiceName .Values.service.name .Values.cluster.enabled -}} {{- if .Values.webhooks.validation }} --- apiVersion: admissionregistration.k8s.io/v1 diff --git a/kubernetes/helm/otoroshi/values.yaml b/kubernetes/helm/otoroshi/values.yaml index a44c6da518..c500f2f47c 100644 --- a/kubernetes/helm/otoroshi/values.yaml +++ b/kubernetes/helm/otoroshi/values.yaml @@ -209,8 +209,17 @@ autoscaling: # HPA scaling behavior (v2). Empty means use Kubernetes defaults. behavior: {} +# Admission webhooks. They are answered by request sinks registered in the Otoroshi +# global config by OTOROSHI_INITIAL_CUSTOMIZATION — which only applies on the FIRST boot, +# while the datastore is still empty. On an upgrade, or against a pre-existing Redis, +# register them by hand (Danger zone -> Global plugins) before turning these on, +# otherwise Otoroshi answers 404 and, once the CRDs controller job has flipped +# failurePolicy from Ignore to Fail, every `kubectl apply` of a proxy.otoroshi.io +# resource fails with "the server could not find the requested resource". webhooks: + # kube-apiserver validates every proxy.otoroshi.io CR against Otoroshi before persisting it. validation: true + # Injects an otoroshi-sidecar container into pods labeled `otoroshi.io/sidecar: inject`. sidecarInject: false # PodDisruptionBudget. diff --git a/kubernetes/kustomize/components/initial-customization-cluster/initial-customization.json b/kubernetes/kustomize/components/initial-customization-cluster/initial-customization.json index 799e49830b..5e0e803cc0 100644 --- a/kubernetes/kustomize/components/initial-customization-cluster/initial-customization.json +++ b/kubernetes/kustomize/components/initial-customization-cluster/initial-customization.json @@ -1,63 +1,108 @@ { - "config": {}, - "scripts": { - "enabled": true, - "sinkRefs": [ - "cp:otoroshi.plugins.jobs.kubernetes.KubernetesAdmissionWebhookCRDValidator", - "cp:otoroshi.plugins.jobs.kubernetes.KubernetesAdmissionWebhookSidecarInjector" - ], - "sinkConfig": {}, - "jobRefs": [ - "cp:otoroshi.plugins.jobs.kubernetes.KubernetesOtoroshiCRDsControllerJob" - ], - "jobConfig": { - "KubernetesConfig": { - "trust": false, - "namespaces": ["*"], - "labels": {}, - "namespacesLabels": {}, - "ingressClasses": ["otoroshi"], - "defaultGroup": "default", - "ingresses": false, - "crds": true, - "coreDnsIntegration": false, - "coreDnsIntegrationDryRun": false, - "kubeLeader": false, - "restartDependantDeployments": false, - "watch": false, - "syncDaikokuApikeysOnly": false, - "kubeSystemNamespace": "kube-system", - "coreDnsConfigMapName": "coredns", - "coreDnsDeploymentName": "coredns", - "corednsPort": 53, - "otoroshiServiceName": "otoroshi-worker-service", - "otoroshiNamespace": "otoroshi", - "clusterDomain": "cluster.local", - "syncIntervalSeconds": 60, - "coreDnsEnv": null, - "watchTimeoutSeconds": 60, - "watchGracePeriodSeconds": 5, - "mutatingWebhookName": "otoroshi-admission-webhook-injector", - "validatingWebhookName": "otoroshi-admission-webhook-validation", - "templates": { - "service-group": {}, - "service-descriptor": {}, - "apikeys": {}, - "global-config": {}, - "jwt-verifier": {}, - "tcp-service": {}, - "certificate": {}, - "auth-module": {}, - "script": {}, - "organizations": {}, - "teams": {}, - "webhooks": { - "flags": { - "requestCert": true, - "originCheck": true, - "tokensCheck": true, - "displayEnv": false, - "tlsTrace": false + "config": { + "plugins": { + "enabled": true, + "config": { + "ng": [ + { + "enabled": true, + "debug": false, + "include": [], + "exclude": [], + "plugin": "cp:otoroshi.next.plugins.NgClientCredentials", + "config": { + "expiration": 3600000, + "default_key_pair": "otoroshi-jwt-signing", + "domain": "*", + "secure": true + } + }, + { + "enabled": true, + "debug": false, + "include": [], + "exclude": [], + "plugin": "cp:otoroshi.next.plugins.wrappers.RequestSinkWrapper", + "config": { + "plugin": "cp:otoroshi.plugins.jobs.kubernetes.KubernetesAdmissionWebhookCRDValidator" + } + }, + { + "enabled": true, + "debug": false, + "include": [], + "exclude": [], + "plugin": "cp:otoroshi.next.plugins.wrappers.RequestSinkWrapper", + "config": { + "plugin": "cp:otoroshi.plugins.jobs.kubernetes.KubernetesAdmissionWebhookSidecarInjector" + } + } + ] + } + }, + "scripts": { + "enabled": true, + "sinkRefs": [ + "cp:otoroshi.plugins.jobs.kubernetes.KubernetesAdmissionWebhookCRDValidator", + "cp:otoroshi.plugins.jobs.kubernetes.KubernetesAdmissionWebhookSidecarInjector" + ], + "sinkConfig": {}, + "jobRefs": [ + "cp:otoroshi.plugins.jobs.kubernetes.KubernetesOtoroshiCRDsControllerJob" + ], + "jobConfig": { + "KubernetesConfig": { + "trust": false, + "namespaces": [ + "*" + ], + "labels": {}, + "namespacesLabels": {}, + "ingressClasses": [ + "otoroshi" + ], + "defaultGroup": "default", + "ingresses": false, + "crds": true, + "coreDnsIntegration": false, + "coreDnsIntegrationDryRun": false, + "kubeLeader": false, + "restartDependantDeployments": false, + "watch": false, + "syncDaikokuApikeysOnly": false, + "kubeSystemNamespace": "kube-system", + "coreDnsConfigMapName": "coredns", + "coreDnsDeploymentName": "coredns", + "corednsPort": 53, + "otoroshiServiceName": "otoroshi-worker-service", + "otoroshiNamespace": "otoroshi", + "clusterDomain": "cluster.local", + "syncIntervalSeconds": 60, + "coreDnsEnv": null, + "watchTimeoutSeconds": 60, + "watchGracePeriodSeconds": 5, + "mutatingWebhookName": "otoroshi-admission-webhook-injector", + "validatingWebhookName": "otoroshi-admission-webhook-validation", + "templates": { + "service-group": {}, + "service-descriptor": {}, + "apikeys": {}, + "global-config": {}, + "jwt-verifier": {}, + "tcp-service": {}, + "certificate": {}, + "auth-module": {}, + "script": {}, + "organizations": {}, + "teams": {}, + "webhooks": { + "flags": { + "requestCert": true, + "originCheck": true, + "tokensCheck": true, + "displayEnv": false, + "tlsTrace": false + } } } } diff --git a/kubernetes/kustomize/components/initial-customization-cluster/readme.md b/kubernetes/kustomize/components/initial-customization-cluster/readme.md index 94e11119b7..548a3115be 100644 --- a/kubernetes/kustomize/components/initial-customization-cluster/readme.md +++ b/kubernetes/kustomize/components/initial-customization-cluster/readme.md @@ -9,6 +9,31 @@ setups, use [`initial-customization-single`](../initial-customization-single/) i This component is enabled by default in the `cluster*` overlays. +## Shape of the file + +The whole payload lives under a top-level `config` key — that is the only key +Otoroshi merges into the global config (a sibling top-level `scripts` key means +something else entirely: a list of Script *entities* to import). + +The admission-webhook request sinks are declared twice on purpose: + +- under `config.plugins.config.ng`, wrapped in `RequestSinkWrapper`, because the + next-gen proxy engine — the default — only reads its request sinks from there; +- under `config.scripts.sinkRefs`, for the legacy engine. + +`config.plugins.config.ng` *replaces* Otoroshi's built-in list rather than +merging into it (a JSON deep merge does not merge arrays), which is why the +default `NgClientCredentials` entry is repeated in the file. + +This JSON is only applied on the very first boot, while the datastore is still +empty. On an upgrade — or against a pre-existing Redis — Otoroshi keeps the +global config it already has, so changes here must also be applied by hand on +existing installs (Danger zone → Global plugins). + +Because `otoroshiServiceName` is `otoroshi-worker-service` here, the +[`webhooks`](../webhooks/) component needs its `clientConfig.service.name` +patched to that same service — see its readme. + ## Customize Edit `initial-customization.json` next to this `kustomization.yaml` to tweak diff --git a/kubernetes/kustomize/components/initial-customization-single/initial-customization.json b/kubernetes/kustomize/components/initial-customization-single/initial-customization.json index cbb919fb27..bd4b213351 100644 --- a/kubernetes/kustomize/components/initial-customization-single/initial-customization.json +++ b/kubernetes/kustomize/components/initial-customization-single/initial-customization.json @@ -1,63 +1,108 @@ { - "config": {}, - "scripts": { - "enabled": true, - "sinkRefs": [ - "cp:otoroshi.plugins.jobs.kubernetes.KubernetesAdmissionWebhookCRDValidator", - "cp:otoroshi.plugins.jobs.kubernetes.KubernetesAdmissionWebhookSidecarInjector" - ], - "sinkConfig": {}, - "jobRefs": [ - "cp:otoroshi.plugins.jobs.kubernetes.KubernetesOtoroshiCRDsControllerJob" - ], - "jobConfig": { - "KubernetesConfig": { - "trust": false, - "namespaces": ["*"], - "labels": {}, - "namespacesLabels": {}, - "ingressClasses": ["otoroshi"], - "defaultGroup": "default", - "ingresses": false, - "crds": true, - "coreDnsIntegration": false, - "coreDnsIntegrationDryRun": false, - "kubeLeader": false, - "restartDependantDeployments": false, - "watch": false, - "syncDaikokuApikeysOnly": false, - "kubeSystemNamespace": "kube-system", - "coreDnsConfigMapName": "coredns", - "coreDnsDeploymentName": "coredns", - "corednsPort": 53, - "otoroshiServiceName": "otoroshi-service", - "otoroshiNamespace": "otoroshi", - "clusterDomain": "cluster.local", - "syncIntervalSeconds": 60, - "coreDnsEnv": null, - "watchTimeoutSeconds": 60, - "watchGracePeriodSeconds": 5, - "mutatingWebhookName": "otoroshi-admission-webhook-injector", - "validatingWebhookName": "otoroshi-admission-webhook-validation", - "templates": { - "service-group": {}, - "service-descriptor": {}, - "apikeys": {}, - "global-config": {}, - "jwt-verifier": {}, - "tcp-service": {}, - "certificate": {}, - "auth-module": {}, - "script": {}, - "organizations": {}, - "teams": {}, - "webhooks": { - "flags": { - "requestCert": true, - "originCheck": true, - "tokensCheck": true, - "displayEnv": false, - "tlsTrace": false + "config": { + "plugins": { + "enabled": true, + "config": { + "ng": [ + { + "enabled": true, + "debug": false, + "include": [], + "exclude": [], + "plugin": "cp:otoroshi.next.plugins.NgClientCredentials", + "config": { + "expiration": 3600000, + "default_key_pair": "otoroshi-jwt-signing", + "domain": "*", + "secure": true + } + }, + { + "enabled": true, + "debug": false, + "include": [], + "exclude": [], + "plugin": "cp:otoroshi.next.plugins.wrappers.RequestSinkWrapper", + "config": { + "plugin": "cp:otoroshi.plugins.jobs.kubernetes.KubernetesAdmissionWebhookCRDValidator" + } + }, + { + "enabled": true, + "debug": false, + "include": [], + "exclude": [], + "plugin": "cp:otoroshi.next.plugins.wrappers.RequestSinkWrapper", + "config": { + "plugin": "cp:otoroshi.plugins.jobs.kubernetes.KubernetesAdmissionWebhookSidecarInjector" + } + } + ] + } + }, + "scripts": { + "enabled": true, + "sinkRefs": [ + "cp:otoroshi.plugins.jobs.kubernetes.KubernetesAdmissionWebhookCRDValidator", + "cp:otoroshi.plugins.jobs.kubernetes.KubernetesAdmissionWebhookSidecarInjector" + ], + "sinkConfig": {}, + "jobRefs": [ + "cp:otoroshi.plugins.jobs.kubernetes.KubernetesOtoroshiCRDsControllerJob" + ], + "jobConfig": { + "KubernetesConfig": { + "trust": false, + "namespaces": [ + "*" + ], + "labels": {}, + "namespacesLabels": {}, + "ingressClasses": [ + "otoroshi" + ], + "defaultGroup": "default", + "ingresses": false, + "crds": true, + "coreDnsIntegration": false, + "coreDnsIntegrationDryRun": false, + "kubeLeader": false, + "restartDependantDeployments": false, + "watch": false, + "syncDaikokuApikeysOnly": false, + "kubeSystemNamespace": "kube-system", + "coreDnsConfigMapName": "coredns", + "coreDnsDeploymentName": "coredns", + "corednsPort": 53, + "otoroshiServiceName": "otoroshi-service", + "otoroshiNamespace": "otoroshi", + "clusterDomain": "cluster.local", + "syncIntervalSeconds": 60, + "coreDnsEnv": null, + "watchTimeoutSeconds": 60, + "watchGracePeriodSeconds": 5, + "mutatingWebhookName": "otoroshi-admission-webhook-injector", + "validatingWebhookName": "otoroshi-admission-webhook-validation", + "templates": { + "service-group": {}, + "service-descriptor": {}, + "apikeys": {}, + "global-config": {}, + "jwt-verifier": {}, + "tcp-service": {}, + "certificate": {}, + "auth-module": {}, + "script": {}, + "organizations": {}, + "teams": {}, + "webhooks": { + "flags": { + "requestCert": true, + "originCheck": true, + "tokensCheck": true, + "displayEnv": false, + "tlsTrace": false + } } } } diff --git a/kubernetes/kustomize/components/initial-customization-single/readme.md b/kubernetes/kustomize/components/initial-customization-single/readme.md index 7dab897695..ac94c3b73a 100644 --- a/kubernetes/kustomize/components/initial-customization-single/readme.md +++ b/kubernetes/kustomize/components/initial-customization-single/readme.md @@ -8,6 +8,27 @@ Leader+Worker setups, use [`initial-customization-cluster`](../initial-customiza This component is enabled by default in the `simple*` overlays. +## Shape of the file + +The whole payload lives under a top-level `config` key — that is the only key +Otoroshi merges into the global config (a sibling top-level `scripts` key means +something else entirely: a list of Script *entities* to import). + +The admission-webhook request sinks are declared twice on purpose: + +- under `config.plugins.config.ng`, wrapped in `RequestSinkWrapper`, because the + next-gen proxy engine — the default — only reads its request sinks from there; +- under `config.scripts.sinkRefs`, for the legacy engine. + +`config.plugins.config.ng` *replaces* Otoroshi's built-in list rather than +merging into it (a JSON deep merge does not merge arrays), which is why the +default `NgClientCredentials` entry is repeated in the file. + +This JSON is only applied on the very first boot, while the datastore is still +empty. On an upgrade — or against a pre-existing Redis — Otoroshi keeps the +global config it already has, so changes here must also be applied by hand on +existing installs (Danger zone → Global plugins). + ## Customize Edit `initial-customization.json` next to this `kustomization.yaml` to tweak diff --git a/kubernetes/kustomize/components/webhooks/readme.md b/kubernetes/kustomize/components/webhooks/readme.md index 5729c05969..ae8712925c 100644 --- a/kubernetes/kustomize/components/webhooks/readme.md +++ b/kubernetes/kustomize/components/webhooks/readme.md @@ -5,10 +5,33 @@ Validating + Mutating admission webhooks for Otoroshi. - **Validating** (`otoroshi-admission-webhook-validation`) — kube-apiserver calls Otoroshi on every CREATE/UPDATE of a `proxy.otoroshi.io/v*` resource so Otoroshi can reject malformed CRs before they hit etcd. - **Mutating** (`otoroshi-admission-webhook-injector`) — injects an otoroshi-sidecar container into pods labeled `otoroshi.io/sidecar: inject`. -Both `caBundle` and `failurePolicy` are left as placeholders and patched at runtime by Otoroshi (`KubernetesAdmissionWebhookCRDValidator` + `KubernetesAdmissionWebhookSidecarInjector` sinks — enabled in `initial-customization.json` by default). +Both `caBundle` and `failurePolicy` are left as placeholders and patched at runtime by Otoroshi (`KubernetesAdmissionWebhookCRDValidator` + `KubernetesAdmissionWebhookSidecarInjector` sinks — enabled in `initial-customization.json` by default, both as next-gen `plugins.config.ng` entries and as legacy `scripts.sinkRefs`). + +Note that `failurePolicy` is flipped from `Ignore` to `Fail` by the CRDs controller job on its first sync loop. Until that happens a broken webhook goes unnoticed (the apiserver swallows the error); afterwards every `kubectl apply` of a `proxy.otoroshi.io` resource fails hard. So always validate the setup right after enabling this component, not before. Both webhooks point at the in-cluster `otoroshi-service`. The namespace is automatically rewritten from the overlay's top-level `namespace:` field via a custom `kustomizeconfig.yaml` (kustomize's built-in NamespaceTransformer doesn't cover `webhooks[].clientConfig.service.namespace`). +## The service name must match `otoroshiServiceName` + +The sinks answering these calls only match a Host of `..svc[.cluster.local]`, and the certificate Otoroshi generates for the endpoint is issued for exactly those two SANs. `webhooks[].clientConfig.service.name` must therefore be the same service as `KubernetesConfig.otoroshiServiceName` in `initial-customization.json`. + +That holds out of the box for the `simple*` overlays (`otoroshi-service` on both sides). The `cluster*` overlays use `otoroshi-worker-service`, so add a patch alongside the component: + +```yaml +# overlays/cluster/kustomization.yaml +components: + - ../../components/webhooks + +patches: + - target: + group: admissionregistration.k8s.io + kind: (Validating|Mutating)WebhookConfiguration + patch: |- + - op: replace + path: /webhooks/0/clientConfig/service/name + value: otoroshi-worker-service +``` + ## Enable ```yaml diff --git a/kubernetes/kustomize/components/webhooks/webhooks.yaml b/kubernetes/kustomize/components/webhooks/webhooks.yaml index 7d4b685c20..641abb3366 100644 --- a/kubernetes/kustomize/components/webhooks/webhooks.yaml +++ b/kubernetes/kustomize/components/webhooks/webhooks.yaml @@ -4,6 +4,12 @@ # and gets injected at runtime by the KubernetesAdmissionWebhookCRDValidator # sink, which also patches failurePolicy. Make sure that sink is enabled in # `initial-customization.json` (it is by default). +# +# `clientConfig.service.name` MUST be the service declared as +# `KubernetesConfig.otoroshiServiceName` in `initial-customization.json` — the sink +# only matches a Host of `..svc[.cluster.local]` and +# the generated certificate carries only those SANs. In cluster mode that is +# `otoroshi-worker-service`, not `otoroshi-service` — see readme.md. apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingWebhookConfiguration metadata: diff --git a/manual/next/docs/deploy/kubernetes.mdx b/manual/next/docs/deploy/kubernetes.mdx index f88b3fd690..9a5b5da689 100644 --- a/manual/next/docs/deploy/kubernetes.mdx +++ b/manual/next/docs/deploy/kubernetes.mdx @@ -1483,14 +1483,97 @@ don't forget to update the otoroshi `ClusterRole` ## CRD validation in kubectl -In order to get CRD validation before manifest deployments right inside kubectl, you can deploy a validation webhook that will do the trick. Also check that you have `otoroshi.plugins.jobs.kubernetes.KubernetesAdmissionWebhookCRDValidator` request sink enabled. +In order to get CRD validation before manifest deployments right inside kubectl, you can deploy a validation webhook that will do the trick. Also check that you have `otoroshi.plugins.jobs.kubernetes.KubernetesAdmissionWebhookCRDValidator` request sink enabled (see the [enabling the admission webhook sinks](#enabling-the-admission-webhook-sinks) section below). ```yaml include=../snippets/kubernetes/kustomize/base/validation-webhook.yaml ``` +### Enabling the admission webhook sinks + +Both admission webhooks are served by Otoroshi *request sinks* — plugins that answer requests no route matched. The next-gen proxy engine, which is the default, only picks up request sinks declared in the global config under `plugins.config.ng`; the older `scripts.sinkRefs` list is read by the legacy engine only. Declaring them in both places covers either engine: + +```json +{ + "plugins": { + "enabled": true, + "config": { + "ng": [ + { + "enabled": true, + "debug": false, + "include": [], + "exclude": [], + "plugin": "cp:otoroshi.next.plugins.NgClientCredentials", + "config": { + "expiration": 3600000, + "default_key_pair": "otoroshi-jwt-signing", + "domain": "*", + "secure": true + } + }, + { + "enabled": true, + "debug": false, + "include": [], + "exclude": [], + "plugin": "cp:otoroshi.next.plugins.wrappers.RequestSinkWrapper", + "config": { + "plugin": "cp:otoroshi.plugins.jobs.kubernetes.KubernetesAdmissionWebhookCRDValidator" + } + }, + { + "enabled": true, + "debug": false, + "include": [], + "exclude": [], + "plugin": "cp:otoroshi.next.plugins.wrappers.RequestSinkWrapper", + "config": { + "plugin": "cp:otoroshi.plugins.jobs.kubernetes.KubernetesAdmissionWebhookSidecarInjector" + } + } + ] + } + }, + "scripts": { + "enabled": true, + "sinkRefs": [ + "cp:otoroshi.plugins.jobs.kubernetes.KubernetesAdmissionWebhookCRDValidator", + "cp:otoroshi.plugins.jobs.kubernetes.KubernetesAdmissionWebhookSidecarInjector" + ], + "sinkConfig": {} + } +} +``` + +`RequestSinkWrapper` is what lets the next-gen engine run these plugins, which are written against the legacy request-sink API. They need no configuration of their own: the `KubernetesConfig` block used by the sinks falls back to `scripts.jobConfig.KubernetesConfig`, the very one the CRDs controller job uses. + +:::warning +`plugins.config.ng` replaces Otoroshi's built-in plugin list rather than merging into it — a JSON deep merge does not merge arrays. Keep the entries that were already there, such as the default `NgClientCredentials` above. +::: + +:::warning +When this configuration is passed through `OTOROSHI_INITIAL_CUSTOMIZATION`, it is only applied on the very first boot, while the datastore is still empty. On an upgrade — or against a pre-existing Redis — Otoroshi keeps the global config it already has, and the change must be applied by hand from the admin UI (Danger zone → Global plugins) or through the admin API. +::: + +### Troubleshooting + +`kubectl apply` failing with: + +``` +Internal error occurred: failed calling webhook "otoroshi-admission-webhook.otoroshi.io": +failed to call webhook: the server could not find the requested resource +``` + +means Otoroshi answered `404` on `/apis/webhooks/validation`, i.e. no request sink handled the call. The two usual causes are: + +- the sinks are not registered for the running engine — see the section above; +- the `Host` the apiserver used does not match what the sinks accept. They only match `..svc` and `..svc.`, taken from `KubernetesConfig`. So `webhooks[].clientConfig.service.name` in the webhook manifest must be the very service named by `otoroshiServiceName` — in a Leader/Worker setup, that is the worker service. This is also the name the certificate Otoroshi generates for the endpoint is issued for, so a mismatch breaks the TLS handshake as well. + +Note that the failure can stay invisible for a while: the manifests ship with `failurePolicy: Ignore`, and the CRDs controller job flips it to `Fail` on its first sync loop. Before the flip, the apiserver silently swallows the `404` and manifests are accepted unvalidated; after it, every apply fails. + ## Easier integration with otoroshi-sidecar -Otoroshi can help you to easily use existing services without modifications while gettings all the perks of otoroshi like apikeys, mTLS, exchange protocol, etc. To do so, otoroshi will inject a sidecar container in the pod of your deployment that will handle call coming from otoroshi and going to otoroshi. To enable otoroshi-sidecar, you need to deploy the following admission webhook. Also check that you have `otoroshi.plugins.jobs.kubernetes.KubernetesAdmissionWebhookSidecarInjector` request sink enabled. +Otoroshi can help you to easily use existing services without modifications while gettings all the perks of otoroshi like apikeys, mTLS, exchange protocol, etc. To do so, otoroshi will inject a sidecar container in the pod of your deployment that will handle call coming from otoroshi and going to otoroshi. To enable otoroshi-sidecar, you need to deploy the following admission webhook. Also check that you have `otoroshi.plugins.jobs.kubernetes.KubernetesAdmissionWebhookSidecarInjector` request sink enabled (see the [enabling the admission webhook sinks](#enabling-the-admission-webhook-sinks) section above). ```yaml include=../snippets/kubernetes/kustomize/base/sidecar-webhook.yaml ``` diff --git a/manual/next/docs/snippets/kubernetes/helm/otoroshi/templates/deployment.yaml b/manual/next/docs/snippets/kubernetes/helm/otoroshi/templates/deployment.yaml index a48b77d5e6..f0e3130481 100644 --- a/manual/next/docs/snippets/kubernetes/helm/otoroshi/templates/deployment.yaml +++ b/manual/next/docs/snippets/kubernetes/helm/otoroshi/templates/deployment.yaml @@ -92,6 +92,46 @@ spec: "defaultDomain": "www.{{ .Values.env.domain }}", "randomIfNotFound": false }, + "plugins": { + "enabled": true, + "config": { + "ng": [ + { + "enabled": true, + "debug": false, + "include": [], + "exclude": [], + "plugin": "cp:otoroshi.next.plugins.NgClientCredentials", + "config": { + "expiration": 3600000, + "default_key_pair": "otoroshi-jwt-signing", + "domain": "*", + "secure": true + } + }, + { + "enabled": true, + "debug": false, + "include": [], + "exclude": [], + "plugin": "cp:otoroshi.next.plugins.wrappers.RequestSinkWrapper", + "config": { + "plugin": "cp:otoroshi.plugins.jobs.kubernetes.KubernetesAdmissionWebhookCRDValidator" + } + }, + { + "enabled": true, + "debug": false, + "include": [], + "exclude": [], + "plugin": "cp:otoroshi.next.plugins.wrappers.RequestSinkWrapper", + "config": { + "plugin": "cp:otoroshi.plugins.jobs.kubernetes.KubernetesAdmissionWebhookSidecarInjector" + } + } + ] + } + }, "scripts":{ "enabled":true, "sinkRefs":[ diff --git a/manual/next/docs/snippets/kubernetes/kustomize/base/sidecar-webhook.yaml b/manual/next/docs/snippets/kubernetes/kustomize/base/sidecar-webhook.yaml index 1c53316a9d..2759c9ccff 100644 --- a/manual/next/docs/snippets/kubernetes/kustomize/base/sidecar-webhook.yaml +++ b/manual/next/docs/snippets/kubernetes/kustomize/base/sidecar-webhook.yaml @@ -1,3 +1,9 @@ +# NOTE: `clientConfig.service.name` below must be the service declared as +# `KubernetesConfig.otoroshiServiceName` in the Otoroshi configuration — the sink +# answering these calls only matches a Host of +# `..svc[.cluster.local]`, and the certificate +# Otoroshi generates for the endpoint carries only those SANs. In a Leader/Worker +# setup that is `otoroshi-worker-service`, not `otoroshi-service`. apiVersion: admissionregistration.k8s.io/v1 kind: MutatingWebhookConfiguration metadata: diff --git a/manual/next/docs/snippets/kubernetes/kustomize/base/validation-webhook.yaml b/manual/next/docs/snippets/kubernetes/kustomize/base/validation-webhook.yaml index d5712aa3ec..3aeacbe9cf 100644 --- a/manual/next/docs/snippets/kubernetes/kustomize/base/validation-webhook.yaml +++ b/manual/next/docs/snippets/kubernetes/kustomize/base/validation-webhook.yaml @@ -1,3 +1,9 @@ +# NOTE: `clientConfig.service.name` below must be the service declared as +# `KubernetesConfig.otoroshiServiceName` in the Otoroshi configuration — the sink +# answering these calls only matches a Host of +# `..svc[.cluster.local]`, and the certificate +# Otoroshi generates for the endpoint carries only those SANs. In a Leader/Worker +# setup that is `otoroshi-worker-service`, not `otoroshi-service`. apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingWebhookConfiguration metadata: diff --git a/manual/next/docs/snippets/kubernetes/kustomize/overlays/cluster-baremetal-daemonset/deployment.yaml b/manual/next/docs/snippets/kubernetes/kustomize/overlays/cluster-baremetal-daemonset/deployment.yaml index 7069d0af09..af1fdb754f 100644 --- a/manual/next/docs/snippets/kubernetes/kustomize/overlays/cluster-baremetal-daemonset/deployment.yaml +++ b/manual/next/docs/snippets/kubernetes/kustomize/overlays/cluster-baremetal-daemonset/deployment.yaml @@ -98,6 +98,46 @@ spec: "defaultDomain": "www.${domain}", "randomIfNotFound": false }, + "plugins": { + "enabled": true, + "config": { + "ng": [ + { + "enabled": true, + "debug": false, + "include": [], + "exclude": [], + "plugin": "cp:otoroshi.next.plugins.NgClientCredentials", + "config": { + "expiration": 3600000, + "default_key_pair": "otoroshi-jwt-signing", + "domain": "*", + "secure": true + } + }, + { + "enabled": true, + "debug": false, + "include": [], + "exclude": [], + "plugin": "cp:otoroshi.next.plugins.wrappers.RequestSinkWrapper", + "config": { + "plugin": "cp:otoroshi.plugins.jobs.kubernetes.KubernetesAdmissionWebhookCRDValidator" + } + }, + { + "enabled": true, + "debug": false, + "include": [], + "exclude": [], + "plugin": "cp:otoroshi.next.plugins.wrappers.RequestSinkWrapper", + "config": { + "plugin": "cp:otoroshi.plugins.jobs.kubernetes.KubernetesAdmissionWebhookSidecarInjector" + } + } + ] + } + }, "scripts":{ "enabled":true, "sinkRefs":[ diff --git a/manual/next/docs/snippets/kubernetes/kustomize/overlays/cluster-baremetal/deployment.yaml b/manual/next/docs/snippets/kubernetes/kustomize/overlays/cluster-baremetal/deployment.yaml index ddffb3554e..e00c8cf22e 100644 --- a/manual/next/docs/snippets/kubernetes/kustomize/overlays/cluster-baremetal/deployment.yaml +++ b/manual/next/docs/snippets/kubernetes/kustomize/overlays/cluster-baremetal/deployment.yaml @@ -85,6 +85,46 @@ spec: "defaultDomain": "www.${domain}", "randomIfNotFound": false }, + "plugins": { + "enabled": true, + "config": { + "ng": [ + { + "enabled": true, + "debug": false, + "include": [], + "exclude": [], + "plugin": "cp:otoroshi.next.plugins.NgClientCredentials", + "config": { + "expiration": 3600000, + "default_key_pair": "otoroshi-jwt-signing", + "domain": "*", + "secure": true + } + }, + { + "enabled": true, + "debug": false, + "include": [], + "exclude": [], + "plugin": "cp:otoroshi.next.plugins.wrappers.RequestSinkWrapper", + "config": { + "plugin": "cp:otoroshi.plugins.jobs.kubernetes.KubernetesAdmissionWebhookCRDValidator" + } + }, + { + "enabled": true, + "debug": false, + "include": [], + "exclude": [], + "plugin": "cp:otoroshi.next.plugins.wrappers.RequestSinkWrapper", + "config": { + "plugin": "cp:otoroshi.plugins.jobs.kubernetes.KubernetesAdmissionWebhookSidecarInjector" + } + } + ] + } + }, "scripts":{ "enabled":true, "sinkRefs":[ diff --git a/manual/next/docs/snippets/kubernetes/kustomize/overlays/cluster/deployment.yaml b/manual/next/docs/snippets/kubernetes/kustomize/overlays/cluster/deployment.yaml index 9257c8d147..00cb47b561 100644 --- a/manual/next/docs/snippets/kubernetes/kustomize/overlays/cluster/deployment.yaml +++ b/manual/next/docs/snippets/kubernetes/kustomize/overlays/cluster/deployment.yaml @@ -85,6 +85,46 @@ spec: "defaultDomain": "www.${domain}", "randomIfNotFound": false }, + "plugins": { + "enabled": true, + "config": { + "ng": [ + { + "enabled": true, + "debug": false, + "include": [], + "exclude": [], + "plugin": "cp:otoroshi.next.plugins.NgClientCredentials", + "config": { + "expiration": 3600000, + "default_key_pair": "otoroshi-jwt-signing", + "domain": "*", + "secure": true + } + }, + { + "enabled": true, + "debug": false, + "include": [], + "exclude": [], + "plugin": "cp:otoroshi.next.plugins.wrappers.RequestSinkWrapper", + "config": { + "plugin": "cp:otoroshi.plugins.jobs.kubernetes.KubernetesAdmissionWebhookCRDValidator" + } + }, + { + "enabled": true, + "debug": false, + "include": [], + "exclude": [], + "plugin": "cp:otoroshi.next.plugins.wrappers.RequestSinkWrapper", + "config": { + "plugin": "cp:otoroshi.plugins.jobs.kubernetes.KubernetesAdmissionWebhookSidecarInjector" + } + } + ] + } + }, "scripts":{ "enabled":true, "sinkRefs":[ diff --git a/manual/next/docs/snippets/kubernetes/kustomize/overlays/simple-baremetal-daemonset/deployment.yaml b/manual/next/docs/snippets/kubernetes/kustomize/overlays/simple-baremetal-daemonset/deployment.yaml index 72cab3cd07..9363ead2c2 100644 --- a/manual/next/docs/snippets/kubernetes/kustomize/overlays/simple-baremetal-daemonset/deployment.yaml +++ b/manual/next/docs/snippets/kubernetes/kustomize/overlays/simple-baremetal-daemonset/deployment.yaml @@ -78,6 +78,46 @@ spec: "defaultDomain": "www.${domain}", "randomIfNotFound": false }, + "plugins": { + "enabled": true, + "config": { + "ng": [ + { + "enabled": true, + "debug": false, + "include": [], + "exclude": [], + "plugin": "cp:otoroshi.next.plugins.NgClientCredentials", + "config": { + "expiration": 3600000, + "default_key_pair": "otoroshi-jwt-signing", + "domain": "*", + "secure": true + } + }, + { + "enabled": true, + "debug": false, + "include": [], + "exclude": [], + "plugin": "cp:otoroshi.next.plugins.wrappers.RequestSinkWrapper", + "config": { + "plugin": "cp:otoroshi.plugins.jobs.kubernetes.KubernetesAdmissionWebhookCRDValidator" + } + }, + { + "enabled": true, + "debug": false, + "include": [], + "exclude": [], + "plugin": "cp:otoroshi.next.plugins.wrappers.RequestSinkWrapper", + "config": { + "plugin": "cp:otoroshi.plugins.jobs.kubernetes.KubernetesAdmissionWebhookSidecarInjector" + } + } + ] + } + }, "scripts":{ "enabled":true, "sinkRefs":[ diff --git a/manual/next/docs/snippets/kubernetes/kustomize/overlays/simple-baremetal/deployment.yaml b/manual/next/docs/snippets/kubernetes/kustomize/overlays/simple-baremetal/deployment.yaml index 5c90ea7dd4..d0d1c21ddc 100644 --- a/manual/next/docs/snippets/kubernetes/kustomize/overlays/simple-baremetal/deployment.yaml +++ b/manual/next/docs/snippets/kubernetes/kustomize/overlays/simple-baremetal/deployment.yaml @@ -69,6 +69,46 @@ spec: "defaultDomain": "www.${domain}", "randomIfNotFound": false }, + "plugins": { + "enabled": true, + "config": { + "ng": [ + { + "enabled": true, + "debug": false, + "include": [], + "exclude": [], + "plugin": "cp:otoroshi.next.plugins.NgClientCredentials", + "config": { + "expiration": 3600000, + "default_key_pair": "otoroshi-jwt-signing", + "domain": "*", + "secure": true + } + }, + { + "enabled": true, + "debug": false, + "include": [], + "exclude": [], + "plugin": "cp:otoroshi.next.plugins.wrappers.RequestSinkWrapper", + "config": { + "plugin": "cp:otoroshi.plugins.jobs.kubernetes.KubernetesAdmissionWebhookCRDValidator" + } + }, + { + "enabled": true, + "debug": false, + "include": [], + "exclude": [], + "plugin": "cp:otoroshi.next.plugins.wrappers.RequestSinkWrapper", + "config": { + "plugin": "cp:otoroshi.plugins.jobs.kubernetes.KubernetesAdmissionWebhookSidecarInjector" + } + } + ] + } + }, "scripts":{ "enabled":true, "sinkRefs":[ diff --git a/manual/next/docs/snippets/kubernetes/kustomize/overlays/simple/deployment.yaml b/manual/next/docs/snippets/kubernetes/kustomize/overlays/simple/deployment.yaml index f882a17933..790548bebc 100644 --- a/manual/next/docs/snippets/kubernetes/kustomize/overlays/simple/deployment.yaml +++ b/manual/next/docs/snippets/kubernetes/kustomize/overlays/simple/deployment.yaml @@ -69,6 +69,46 @@ spec: "defaultDomain": "www.${domain}", "randomIfNotFound": false }, + "plugins": { + "enabled": true, + "config": { + "ng": [ + { + "enabled": true, + "debug": false, + "include": [], + "exclude": [], + "plugin": "cp:otoroshi.next.plugins.NgClientCredentials", + "config": { + "expiration": 3600000, + "default_key_pair": "otoroshi-jwt-signing", + "domain": "*", + "secure": true + } + }, + { + "enabled": true, + "debug": false, + "include": [], + "exclude": [], + "plugin": "cp:otoroshi.next.plugins.wrappers.RequestSinkWrapper", + "config": { + "plugin": "cp:otoroshi.plugins.jobs.kubernetes.KubernetesAdmissionWebhookCRDValidator" + } + }, + { + "enabled": true, + "debug": false, + "include": [], + "exclude": [], + "plugin": "cp:otoroshi.next.plugins.wrappers.RequestSinkWrapper", + "config": { + "plugin": "cp:otoroshi.plugins.jobs.kubernetes.KubernetesAdmissionWebhookSidecarInjector" + } + } + ] + } + }, "scripts":{ "enabled":true, "sinkRefs":[