From 9ef3fe3722f87a07f52f856abd9e8d9acd8f8591 Mon Sep 17 00:00:00 2001 From: Andrei Kvapil Date: Mon, 22 Jun 2026 21:39:48 +0200 Subject: [PATCH 01/69] fix(kubevirt-instancetypes): drop persistent EFI/TPM to unblock live-migration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TEMPORARY revert. Restoring persistent EFI/TPM state (4cc60e3) makes windows.11/2k22/2k25 VMs get a backend-storage PVC that is ReadWriteOnce on the default StorageClass, which pins the VM to its node and blocks live-migration and node drains (with evictionStrategy: LiveMigrate this can stall cluster upgrades). See #3005. Strip persistent EFI/TPM from the preferences again, but with `yq del` so an emptied preferredTPM stays `{}` (the v1.8 CRD rejects null) — the bug the original sed strip caused. Secure Boot and the vTPM stay present, just non-persistent. Stopgap until vm-state can request an RWX backend (RWX-Block via the upstream KubeVirt change, or an RWX-Filesystem vmStateStorageClass). Co-Authored-By: Claude Signed-off-by: Andrei Kvapil (cherry picked from commit 6ac2601a68852a0bb238ab68f9088ab10f17e41d) --- .../system/kubevirt-instancetypes/Makefile | 6 +++++ .../templates/preferences.yaml | 24 +++++-------------- 2 files changed, 12 insertions(+), 18 deletions(-) diff --git a/packages/system/kubevirt-instancetypes/Makefile b/packages/system/kubevirt-instancetypes/Makefile index 99771e976c..15ac774d50 100644 --- a/packages/system/kubevirt-instancetypes/Makefile +++ b/packages/system/kubevirt-instancetypes/Makefile @@ -16,6 +16,12 @@ update: mkdir templates kustomize build https://github.com/kubevirt/common-instancetypes/preferences?ref=$(COMMON_INSTANCETYPES_REF) > templates/preferences.yaml yq -i 'select(.kind != "VirtualMachinePreference")' templates/preferences.yaml + # TEMPORARY (see #3005): drop persistent EFI/TPM state. With persistence on, + # KubeVirt creates a backend-storage PVC that is RWO on the default StorageClass, + # which pins the VM to its node and blocks live-migration / drain. Use `del` (not + # `sed -d`) so an emptied preferredTPM stays `{}` rather than null, which the + # KubeVirt v1.8 CRD structural schema rejects. Revert once vm-state can request RWX. + yq -i 'del(.spec.firmware.preferredEfi.persistent) | del(.spec.devices.preferredTPM.persistent)' templates/preferences.yaml kustomize build https://github.com/kubevirt/common-instancetypes/instancetypes?ref=$(COMMON_INSTANCETYPES_REF) > templates/instancetypes.yaml yq -i 'select(.kind != "VirtualMachineInstancetype")' templates/instancetypes.yaml # Re-append Cozystack-local instancetypes the pinned catalog no longer ships diff --git a/packages/system/kubevirt-instancetypes/templates/preferences.yaml b/packages/system/kubevirt-instancetypes/templates/preferences.yaml index 458d8bd7b3..6a42e6304f 100644 --- a/packages/system/kubevirt-instancetypes/templates/preferences.yaml +++ b/packages/system/kubevirt-instancetypes/templates/preferences.yaml @@ -1352,8 +1352,7 @@ spec: preferredAutoattachInputDevice: true preferredDiskBus: sata preferredInterfaceModel: e1000e - preferredTPM: - persistent: true + preferredTPM: {} features: preferredAcpi: {} preferredApic: {} @@ -1375,7 +1374,6 @@ spec: preferredSmm: {} firmware: preferredEfi: - persistent: true secureBoot: true preferredTerminationGracePeriodSeconds: 3600 requirements: @@ -1424,8 +1422,7 @@ spec: preferredInputBus: virtio preferredInputType: tablet preferredInterfaceModel: virtio - preferredTPM: - persistent: true + preferredTPM: {} features: preferredAcpi: {} preferredApic: {} @@ -1447,7 +1444,6 @@ spec: preferredSmm: {} firmware: preferredEfi: - persistent: true secureBoot: true preferredTerminationGracePeriodSeconds: 3600 requirements: @@ -1894,8 +1890,7 @@ spec: preferredAutoattachInputDevice: true preferredDiskBus: sata preferredInterfaceModel: e1000e - preferredTPM: - persistent: true + preferredTPM: {} features: preferredAcpi: {} preferredApic: {} @@ -1917,7 +1912,6 @@ spec: preferredSmm: {} firmware: preferredEfi: - persistent: true secureBoot: true preferredTerminationGracePeriodSeconds: 3600 requirements: @@ -1966,8 +1960,7 @@ spec: preferredInputBus: virtio preferredInputType: tablet preferredInterfaceModel: virtio - preferredTPM: - persistent: true + preferredTPM: {} features: preferredAcpi: {} preferredApic: {} @@ -1989,7 +1982,6 @@ spec: preferredSmm: {} firmware: preferredEfi: - persistent: true secureBoot: true preferredTerminationGracePeriodSeconds: 3600 requirements: @@ -2036,8 +2028,7 @@ spec: preferredAutoattachInputDevice: true preferredDiskBus: sata preferredInterfaceModel: e1000e - preferredTPM: - persistent: true + preferredTPM: {} features: preferredAcpi: {} preferredApic: {} @@ -2059,7 +2050,6 @@ spec: preferredSmm: {} firmware: preferredEfi: - persistent: true secureBoot: true preferredTerminationGracePeriodSeconds: 3600 requirements: @@ -2108,8 +2098,7 @@ spec: preferredInputBus: virtio preferredInputType: tablet preferredInterfaceModel: virtio - preferredTPM: - persistent: true + preferredTPM: {} features: preferredAcpi: {} preferredApic: {} @@ -2131,7 +2120,6 @@ spec: preferredSmm: {} firmware: preferredEfi: - persistent: true secureBoot: true preferredTerminationGracePeriodSeconds: 3600 requirements: From 22d9f224f9af780c9413e9f4e683741dc82fb93b Mon Sep 17 00:00:00 2001 From: "cozystack-ci[bot]" <274107086+cozystack-ci[bot]@users.noreply.github.com> Date: Wed, 24 Jun 2026 01:42:54 +0000 Subject: [PATCH 02/69] Prepare release v1.5.1 Signed-off-by: cozystack-ci[bot] <274107086+cozystack-ci[bot]@users.noreply.github.com> --- .../apps/clickhouse/images/altinity-clickhouse-backup.tag | 2 +- packages/apps/clickhouse/images/clickhouse-backup.tag | 2 +- packages/apps/http-cache/images/nginx-cache.tag | 2 +- packages/apps/kubernetes/images/cluster-autoscaler.tag | 2 +- packages/apps/kubernetes/images/kubevirt-cloud-provider.tag | 2 +- packages/apps/kubernetes/images/kubevirt-csi-driver.tag | 2 +- .../apps/kubernetes/images/ubuntu-container-disk-v1.30.tag | 2 +- .../apps/kubernetes/images/ubuntu-container-disk-v1.31.tag | 2 +- .../apps/kubernetes/images/ubuntu-container-disk-v1.32.tag | 2 +- .../apps/kubernetes/images/ubuntu-container-disk-v1.33.tag | 2 +- .../apps/kubernetes/images/ubuntu-container-disk-v1.34.tag | 2 +- .../apps/kubernetes/images/ubuntu-container-disk-v1.35.tag | 2 +- packages/apps/mariadb/images/mariadb-backup.tag | 2 +- packages/core/installer/values.yaml | 4 ++-- packages/core/platform/values.yaml | 2 +- packages/core/testing/values.yaml | 2 +- packages/extra/bootbox/images/matchbox.tag | 2 +- packages/extra/seaweedfs/images/objectstorage-sidecar.tag | 2 +- packages/system/backup-controller/values.yaml | 2 +- packages/system/backupstrategy-controller/values.yaml | 2 +- packages/system/bucket/images/s3manager.tag | 2 +- packages/system/cilium/values.yaml | 2 +- packages/system/cozystack-api/values.yaml | 2 +- packages/system/cozystack-controller/values.yaml | 2 +- packages/system/dashboard/values.yaml | 4 ++-- packages/system/flux-shard-operator/values.yaml | 2 +- .../system/grafana-operator/images/grafana-dashboards.tag | 2 +- packages/system/kamaji/values.yaml | 4 ++-- packages/system/kubeovn-plunger/values.yaml | 2 +- packages/system/kubeovn-webhook/values.yaml | 2 +- packages/system/kubevirt-csi-node/values.yaml | 2 +- packages/system/lineage-controller-webhook/values.yaml | 2 +- packages/system/linstor-gui/values.yaml | 2 +- packages/system/linstor/values.yaml | 4 ++-- packages/system/metallb/values.yaml | 4 ++-- packages/system/monitoring/images/grafana.tag | 2 +- packages/system/multus/templates/multus-daemonset-thick.yml | 4 ++-- packages/system/objectstorage-controller/values.yaml | 2 +- packages/system/seaweedfs/values.yaml | 2 +- 39 files changed, 45 insertions(+), 45 deletions(-) diff --git a/packages/apps/clickhouse/images/altinity-clickhouse-backup.tag b/packages/apps/clickhouse/images/altinity-clickhouse-backup.tag index 238996cb2b..d183e11dd1 100644 --- a/packages/apps/clickhouse/images/altinity-clickhouse-backup.tag +++ b/packages/apps/clickhouse/images/altinity-clickhouse-backup.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/altinity-clickhouse-backup:v1.5.0@sha256:65d15779d062680ddb69210c1dfaf8920ea4be2beb069d028bcb5934870a4d72 +ghcr.io/cozystack/cozystack/altinity-clickhouse-backup:v1.5.1@sha256:65d15779d062680ddb69210c1dfaf8920ea4be2beb069d028bcb5934870a4d72 diff --git a/packages/apps/clickhouse/images/clickhouse-backup.tag b/packages/apps/clickhouse/images/clickhouse-backup.tag index 1a935dd581..4c59cafde7 100644 --- a/packages/apps/clickhouse/images/clickhouse-backup.tag +++ b/packages/apps/clickhouse/images/clickhouse-backup.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/clickhouse-backup:v1.5.0@sha256:8efef62d59fb82d39544823f81a797c2b9ac130a50241fee48e701161ce2670d +ghcr.io/cozystack/cozystack/clickhouse-backup:v1.5.1@sha256:8efef62d59fb82d39544823f81a797c2b9ac130a50241fee48e701161ce2670d diff --git a/packages/apps/http-cache/images/nginx-cache.tag b/packages/apps/http-cache/images/nginx-cache.tag index 2439140c8a..ee7c322612 100644 --- a/packages/apps/http-cache/images/nginx-cache.tag +++ b/packages/apps/http-cache/images/nginx-cache.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/nginx-cache:v1.5.0@sha256:6468799a55dacc6026198db15fa0c8add516cc481e89c47d107427b0f2ea1626 +ghcr.io/cozystack/cozystack/nginx-cache:v1.5.1@sha256:6468799a55dacc6026198db15fa0c8add516cc481e89c47d107427b0f2ea1626 diff --git a/packages/apps/kubernetes/images/cluster-autoscaler.tag b/packages/apps/kubernetes/images/cluster-autoscaler.tag index c7c07c5705..bd6ceb49d7 100644 --- a/packages/apps/kubernetes/images/cluster-autoscaler.tag +++ b/packages/apps/kubernetes/images/cluster-autoscaler.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/cluster-autoscaler:v1.5.0@sha256:1767a51cf78d6700cb0b39ea1562b1443421afa416cb6d85e724225499859616 +ghcr.io/cozystack/cozystack/cluster-autoscaler:v1.5.1@sha256:1767a51cf78d6700cb0b39ea1562b1443421afa416cb6d85e724225499859616 diff --git a/packages/apps/kubernetes/images/kubevirt-cloud-provider.tag b/packages/apps/kubernetes/images/kubevirt-cloud-provider.tag index b0c55db51d..1f46fbc300 100644 --- a/packages/apps/kubernetes/images/kubevirt-cloud-provider.tag +++ b/packages/apps/kubernetes/images/kubevirt-cloud-provider.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/kubevirt-cloud-provider:v1.5.0@sha256:c7224442a074a6a5e7af7ddec14be6cabf8ae9dd2569bfa38794a0dce9188ef9 +ghcr.io/cozystack/cozystack/kubevirt-cloud-provider:v1.5.1@sha256:c7224442a074a6a5e7af7ddec14be6cabf8ae9dd2569bfa38794a0dce9188ef9 diff --git a/packages/apps/kubernetes/images/kubevirt-csi-driver.tag b/packages/apps/kubernetes/images/kubevirt-csi-driver.tag index 3aa37f76bd..3f3fc4c683 100644 --- a/packages/apps/kubernetes/images/kubevirt-csi-driver.tag +++ b/packages/apps/kubernetes/images/kubevirt-csi-driver.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/kubevirt-csi-driver:v1.5.0@sha256:dad088fc04cc54af6d5f487ca9296669331a5345f143ed64cd9027b1bb20b5de +ghcr.io/cozystack/cozystack/kubevirt-csi-driver:v1.5.1@sha256:d01f1c0377ce00c8bb46f1576fa9348da3b226679130bf7b46a8bc50d684c38d diff --git a/packages/apps/kubernetes/images/ubuntu-container-disk-v1.30.tag b/packages/apps/kubernetes/images/ubuntu-container-disk-v1.30.tag index f47b874a15..1bc5ea9a8d 100644 --- a/packages/apps/kubernetes/images/ubuntu-container-disk-v1.30.tag +++ b/packages/apps/kubernetes/images/ubuntu-container-disk-v1.30.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/ubuntu-container-disk:v1.30-v1.5.0@sha256:98b80b2d18cb1384c96b95e1c8b869618f1a6e21962dcc1ab2dc874e5dff71dd +ghcr.io/cozystack/cozystack/ubuntu-container-disk:v1.30-v1.5.1@sha256:98b80b2d18cb1384c96b95e1c8b869618f1a6e21962dcc1ab2dc874e5dff71dd diff --git a/packages/apps/kubernetes/images/ubuntu-container-disk-v1.31.tag b/packages/apps/kubernetes/images/ubuntu-container-disk-v1.31.tag index a6fae51679..a44c35d429 100644 --- a/packages/apps/kubernetes/images/ubuntu-container-disk-v1.31.tag +++ b/packages/apps/kubernetes/images/ubuntu-container-disk-v1.31.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/ubuntu-container-disk:v1.31-v1.5.0@sha256:dd0094f5b43e00538ca236110f89792cb5aafa234626518f3bdae353b0060f70 +ghcr.io/cozystack/cozystack/ubuntu-container-disk:v1.31-v1.5.1@sha256:dd0094f5b43e00538ca236110f89792cb5aafa234626518f3bdae353b0060f70 diff --git a/packages/apps/kubernetes/images/ubuntu-container-disk-v1.32.tag b/packages/apps/kubernetes/images/ubuntu-container-disk-v1.32.tag index 01e417d554..a028858058 100644 --- a/packages/apps/kubernetes/images/ubuntu-container-disk-v1.32.tag +++ b/packages/apps/kubernetes/images/ubuntu-container-disk-v1.32.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/ubuntu-container-disk:v1.32-v1.5.0@sha256:99f400892d464afc66289c7ac7f9db3e50ef664376dba262fd181252498622a9 +ghcr.io/cozystack/cozystack/ubuntu-container-disk:v1.32-v1.5.1@sha256:99f400892d464afc66289c7ac7f9db3e50ef664376dba262fd181252498622a9 diff --git a/packages/apps/kubernetes/images/ubuntu-container-disk-v1.33.tag b/packages/apps/kubernetes/images/ubuntu-container-disk-v1.33.tag index f4603e64fa..10673b8bd1 100644 --- a/packages/apps/kubernetes/images/ubuntu-container-disk-v1.33.tag +++ b/packages/apps/kubernetes/images/ubuntu-container-disk-v1.33.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/ubuntu-container-disk:v1.33-v1.5.0@sha256:57ca7239396346ff55293588f8f3b10c11a8e8bd0a9f0198c05343582db6c184 +ghcr.io/cozystack/cozystack/ubuntu-container-disk:v1.33-v1.5.1@sha256:57ca7239396346ff55293588f8f3b10c11a8e8bd0a9f0198c05343582db6c184 diff --git a/packages/apps/kubernetes/images/ubuntu-container-disk-v1.34.tag b/packages/apps/kubernetes/images/ubuntu-container-disk-v1.34.tag index 23761242e6..3ded9e92ca 100644 --- a/packages/apps/kubernetes/images/ubuntu-container-disk-v1.34.tag +++ b/packages/apps/kubernetes/images/ubuntu-container-disk-v1.34.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/ubuntu-container-disk:v1.34-v1.5.0@sha256:1f0af945d8c42af3bef9e17149e1530e3afc4d95ff79434dc81ba0fe729d7574 +ghcr.io/cozystack/cozystack/ubuntu-container-disk:v1.34-v1.5.1@sha256:1f0af945d8c42af3bef9e17149e1530e3afc4d95ff79434dc81ba0fe729d7574 diff --git a/packages/apps/kubernetes/images/ubuntu-container-disk-v1.35.tag b/packages/apps/kubernetes/images/ubuntu-container-disk-v1.35.tag index 325da2bdea..d9e7c51932 100644 --- a/packages/apps/kubernetes/images/ubuntu-container-disk-v1.35.tag +++ b/packages/apps/kubernetes/images/ubuntu-container-disk-v1.35.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/ubuntu-container-disk:v1.35-v1.5.0@sha256:aaf3993143e6f1f60a5e758dcb8e134a2c56c0bef3b44763dd097a5134b12cf2 +ghcr.io/cozystack/cozystack/ubuntu-container-disk:v1.35-v1.5.1@sha256:aaf3993143e6f1f60a5e758dcb8e134a2c56c0bef3b44763dd097a5134b12cf2 diff --git a/packages/apps/mariadb/images/mariadb-backup.tag b/packages/apps/mariadb/images/mariadb-backup.tag index 3559372269..4b8eb06e58 100644 --- a/packages/apps/mariadb/images/mariadb-backup.tag +++ b/packages/apps/mariadb/images/mariadb-backup.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/mariadb-backup:v1.5.0@sha256:3cdaed3f19531586756902516feb099a5bece89c2c6685754d945486f0e99999 +ghcr.io/cozystack/cozystack/mariadb-backup:v1.5.1@sha256:3cdaed3f19531586756902516feb099a5bece89c2c6685754d945486f0e99999 diff --git a/packages/core/installer/values.yaml b/packages/core/installer/values.yaml index c0aadf19ad..0615a6359e 100644 --- a/packages/core/installer/values.yaml +++ b/packages/core/installer/values.yaml @@ -7,9 +7,9 @@ bareNamespace: false cozystackOperator: # Deployment variant: talos, generic, hosted variant: talos - image: ghcr.io/cozystack/cozystack/cozystack-operator:v1.5.0@sha256:05027a381a8a97276ec426dabb87e1b7df753d4d76925b68e2dd8819f7eb22df + image: ghcr.io/cozystack/cozystack/cozystack-operator:v1.5.1@sha256:8f0a898c1e1f75b432b8224bcedf1fdb17c8961e7da64b2919ae24fb4393c883 platformSourceUrl: 'oci://ghcr.io/cozystack/cozystack/cozystack-packages' - platformSourceRef: 'digest=sha256:e2e7bc7664a9799332dc8d59ae312879ba3675929edfc0fd6ce005ec41cbeb7b' + platformSourceRef: 'digest=sha256:978596eada21f88a78e5654bfa9a6da97937242e69259a64dc0e85f0b766d980' # When non-empty, overrides the operator's --helmrelease-interval flag # (operator default: 5m). E2E sets this to 30s; production should leave empty. helmReleaseInterval: "" diff --git a/packages/core/platform/values.yaml b/packages/core/platform/values.yaml index 6d1f095a8d..e31130b81d 100644 --- a/packages/core/platform/values.yaml +++ b/packages/core/platform/values.yaml @@ -5,7 +5,7 @@ sourceRef: path: / migrations: enabled: false - image: ghcr.io/cozystack/cozystack/platform-migrations:v1.5.0@sha256:8bf61f17e99eb4a3365394e3d97ac52a1d68cc84aa104894ecb09de25721dc1d + image: ghcr.io/cozystack/cozystack/platform-migrations:v1.5.1@sha256:5d601b0709c578945f2e04d655b77d29008c94666a4350e67960c9f6bf9d2bde targetVersion: 45 # Bundle deployment configuration bundles: diff --git a/packages/core/testing/values.yaml b/packages/core/testing/values.yaml index 7bfaf4fa97..94610c9fc5 100644 --- a/packages/core/testing/values.yaml +++ b/packages/core/testing/values.yaml @@ -1,2 +1,2 @@ e2e: - image: ghcr.io/cozystack/cozystack/e2e-sandbox:v1.5.0@sha256:91b9a2985bcadea4e74cf9cd644a6d4c19442a7270fe14ac52e766b4ec0c93d8 + image: ghcr.io/cozystack/cozystack/e2e-sandbox:v1.5.1@sha256:91b9a2985bcadea4e74cf9cd644a6d4c19442a7270fe14ac52e766b4ec0c93d8 diff --git a/packages/extra/bootbox/images/matchbox.tag b/packages/extra/bootbox/images/matchbox.tag index 2dd0ecebcc..bff481c4ee 100644 --- a/packages/extra/bootbox/images/matchbox.tag +++ b/packages/extra/bootbox/images/matchbox.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/matchbox:v1.5.0@sha256:69e89917f3213484c77b1e1948309c340be871245ec5e5cf9850db4f88202e13 +ghcr.io/cozystack/cozystack/matchbox:v1.5.1@sha256:2fca5e260f5d441bd045292b7e81ee2a95bcd09904ae429388f51cbd1059c361 diff --git a/packages/extra/seaweedfs/images/objectstorage-sidecar.tag b/packages/extra/seaweedfs/images/objectstorage-sidecar.tag index 3648ad696e..7619994d92 100644 --- a/packages/extra/seaweedfs/images/objectstorage-sidecar.tag +++ b/packages/extra/seaweedfs/images/objectstorage-sidecar.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/objectstorage-sidecar:v1.5.0@sha256:fdc09634688d5579a329b48aabf6706cc55bcd9503b0331fa47764706f0d6f62 +ghcr.io/cozystack/cozystack/objectstorage-sidecar:v1.5.1@sha256:fdc09634688d5579a329b48aabf6706cc55bcd9503b0331fa47764706f0d6f62 diff --git a/packages/system/backup-controller/values.yaml b/packages/system/backup-controller/values.yaml index b57ccc124d..f000bf7c60 100644 --- a/packages/system/backup-controller/values.yaml +++ b/packages/system/backup-controller/values.yaml @@ -1,5 +1,5 @@ backupController: - image: "ghcr.io/cozystack/cozystack/backup-controller:v1.5.0@sha256:11d55b79cf136f9ec161410e2357dde7bc063246f905d5ac962ad8432af04962" + image: "ghcr.io/cozystack/cozystack/backup-controller:v1.5.1@sha256:11d55b79cf136f9ec161410e2357dde7bc063246f905d5ac962ad8432af04962" replicas: 2 debug: false metrics: diff --git a/packages/system/backupstrategy-controller/values.yaml b/packages/system/backupstrategy-controller/values.yaml index d36635507a..b34e081a67 100644 --- a/packages/system/backupstrategy-controller/values.yaml +++ b/packages/system/backupstrategy-controller/values.yaml @@ -1,5 +1,5 @@ backupStrategyController: - image: "ghcr.io/cozystack/cozystack/backupstrategy-controller:v1.5.0@sha256:d965283f935e4b24b655bc76e37dcb3b731bec832bb5c2e3e407b0284c7526a3" + image: "ghcr.io/cozystack/cozystack/backupstrategy-controller:v1.5.1@sha256:d965283f935e4b24b655bc76e37dcb3b731bec832bb5c2e3e407b0284c7526a3" # chBackupClientImage is the image rendered into the Altinity strategy # Pod: it drives clickhouse-backup's HTTP API via curl + jq. We reuse # platform-migrations (already pre-built and digested by release CI in diff --git a/packages/system/bucket/images/s3manager.tag b/packages/system/bucket/images/s3manager.tag index 82cda22748..655c9e78e6 100644 --- a/packages/system/bucket/images/s3manager.tag +++ b/packages/system/bucket/images/s3manager.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/s3manager:v1.5.0@sha256:8b36ab67904e34b2fb843b964c20d4822f3a6673f6745afb6a834efd02e017e2 +ghcr.io/cozystack/cozystack/s3manager:v1.5.1@sha256:8b36ab67904e34b2fb843b964c20d4822f3a6673f6745afb6a834efd02e017e2 diff --git a/packages/system/cilium/values.yaml b/packages/system/cilium/values.yaml index 8a6ab3df66..fc702c0805 100644 --- a/packages/system/cilium/values.yaml +++ b/packages/system/cilium/values.yaml @@ -15,7 +15,7 @@ cilium: mode: "kubernetes" image: repository: ghcr.io/cozystack/cozystack/cilium - tag: v1.5.0 + tag: v1.5.1 digest: "sha256:94dc74c9301f0da5695c80e77b77ee0486ad0086b5be60a94136a2cafd410b73" envoy: enabled: true diff --git a/packages/system/cozystack-api/values.yaml b/packages/system/cozystack-api/values.yaml index db6102a35c..42d4d9d5f0 100644 --- a/packages/system/cozystack-api/values.yaml +++ b/packages/system/cozystack-api/values.yaml @@ -1,3 +1,3 @@ cozystackAPI: - image: ghcr.io/cozystack/cozystack/cozystack-api:v1.5.0@sha256:ef50b155e419cf8f0b4d57af9fa385ecbacd5c1248f13591ffcca75c2f9f2a07 + image: ghcr.io/cozystack/cozystack/cozystack-api:v1.5.1@sha256:ef50b155e419cf8f0b4d57af9fa385ecbacd5c1248f13591ffcca75c2f9f2a07 replicas: 2 diff --git a/packages/system/cozystack-controller/values.yaml b/packages/system/cozystack-controller/values.yaml index 2d4adaa88c..355daf70ce 100644 --- a/packages/system/cozystack-controller/values.yaml +++ b/packages/system/cozystack-controller/values.yaml @@ -1,4 +1,4 @@ cozystackController: - image: ghcr.io/cozystack/cozystack/cozystack-controller:v1.5.0@sha256:d3be63579fcabfa802a61ff2f4c2f54d5d6d52c1cbcdff52a3f14d101ca85afc + image: ghcr.io/cozystack/cozystack/cozystack-controller:v1.5.1@sha256:c27aa360d8081d8bf3fa3d4ec833d38a96b33bda10b3a41183b46389ce091c07 debug: false disableTelemetry: false diff --git a/packages/system/dashboard/values.yaml b/packages/system/dashboard/values.yaml index c5ff6b4b30..36aab09226 100644 --- a/packages/system/dashboard/values.yaml +++ b/packages/system/dashboard/values.yaml @@ -1,4 +1,4 @@ console: - image: ghcr.io/cozystack/cozystack/cozystack-ui:v1.5.0@sha256:f0329ac5689447d3aec0e4d420196f42bda466a7789312ac5ce9b11405f1948d + image: ghcr.io/cozystack/cozystack/cozystack-ui:v1.5.1@sha256:932b11c6f3faf42cb1ec8828412ac417727e2c3baef8af48c667822f692fd9bb tokenProxy: - image: ghcr.io/cozystack/cozystack/token-proxy:v1.5.0@sha256:0382e8a9014a842d2eddf1bac5b598f42a4ada7a9be673d9272f1610d47631c4 + image: ghcr.io/cozystack/cozystack/token-proxy:v1.5.1@sha256:0382e8a9014a842d2eddf1bac5b598f42a4ada7a9be673d9272f1610d47631c4 diff --git a/packages/system/flux-shard-operator/values.yaml b/packages/system/flux-shard-operator/values.yaml index 18bddbbba3..17c6762973 100644 --- a/packages/system/flux-shard-operator/values.yaml +++ b/packages/system/flux-shard-operator/values.yaml @@ -1,5 +1,5 @@ fluxShardOperator: - image: ghcr.io/cozystack/cozystack/flux-shard-operator:v1.5.0@sha256:22f99597b1d21a913cb7b81f2faa3cdb983f2121164d220955d4d6e120ef4cd3 + image: ghcr.io/cozystack/cozystack/flux-shard-operator:v1.5.1@sha256:56f5ef64a853a9efa5270f73f2794edcd993d3472ba482d7ef9630dce8b8df4a debug: false replicas: 2 ## Number of helm-controller shards to provision and distribute tenants diff --git a/packages/system/grafana-operator/images/grafana-dashboards.tag b/packages/system/grafana-operator/images/grafana-dashboards.tag index 7a4976500d..ae75a6387c 100644 --- a/packages/system/grafana-operator/images/grafana-dashboards.tag +++ b/packages/system/grafana-operator/images/grafana-dashboards.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/grafana-dashboards:v1.5.0@sha256:f56dc47b970ebf4720c555468aabfb0f6bad4e15efa5bbd6644bf3e5ae9d843e +ghcr.io/cozystack/cozystack/grafana-dashboards:v1.5.1@sha256:ec97b9ee9aa591b5737da595ed7772b22714b9517f9a365d67e7977dd45dd77c diff --git a/packages/system/kamaji/values.yaml b/packages/system/kamaji/values.yaml index 0acb625ef8..7e99299964 100644 --- a/packages/system/kamaji/values.yaml +++ b/packages/system/kamaji/values.yaml @@ -3,7 +3,7 @@ kamaji: deploy: false image: pullPolicy: IfNotPresent - tag: v1.5.0@sha256:130d5013a9b35be0013b96efefc656210cc30c6fe92a8c39c9d2ab8d47f791e0 + tag: v1.5.1@sha256:130d5013a9b35be0013b96efefc656210cc30c6fe92a8c39c9d2ab8d47f791e0 repository: ghcr.io/cozystack/cozystack/kamaji resources: limits: @@ -36,4 +36,4 @@ kamaji: periodSeconds: 10 timeoutSeconds: 1 extraArgs: - - --migrate-image=ghcr.io/cozystack/cozystack/kamaji:v1.5.0@sha256:130d5013a9b35be0013b96efefc656210cc30c6fe92a8c39c9d2ab8d47f791e0 + - --migrate-image=ghcr.io/cozystack/cozystack/kamaji:v1.5.1@sha256:130d5013a9b35be0013b96efefc656210cc30c6fe92a8c39c9d2ab8d47f791e0 diff --git a/packages/system/kubeovn-plunger/values.yaml b/packages/system/kubeovn-plunger/values.yaml index aa48658363..88f56ea1b6 100644 --- a/packages/system/kubeovn-plunger/values.yaml +++ b/packages/system/kubeovn-plunger/values.yaml @@ -1,4 +1,4 @@ portSecurity: true routes: "" -image: ghcr.io/cozystack/cozystack/kubeovn-plunger:v1.5.0@sha256:908b5903c014f6fa4348f57b1fced929a2ba823aadfbcda7ad9b76134bcf3fce +image: ghcr.io/cozystack/cozystack/kubeovn-plunger:v1.5.1@sha256:908b5903c014f6fa4348f57b1fced929a2ba823aadfbcda7ad9b76134bcf3fce ovnCentralName: ovn-central diff --git a/packages/system/kubeovn-webhook/values.yaml b/packages/system/kubeovn-webhook/values.yaml index 6f5fecb0f8..5ed0d24baf 100644 --- a/packages/system/kubeovn-webhook/values.yaml +++ b/packages/system/kubeovn-webhook/values.yaml @@ -1,3 +1,3 @@ portSecurity: true routes: "" -image: ghcr.io/cozystack/cozystack/kubeovn-webhook:v1.5.0@sha256:ca354b6c2ae6f648bc63798d19a1316fe37cb142246bf08ebb7ce27e6e8f21b2 +image: ghcr.io/cozystack/cozystack/kubeovn-webhook:v1.5.1@sha256:ca354b6c2ae6f648bc63798d19a1316fe37cb142246bf08ebb7ce27e6e8f21b2 diff --git a/packages/system/kubevirt-csi-node/values.yaml b/packages/system/kubevirt-csi-node/values.yaml index 16570bf2ab..b97d74c4ea 100644 --- a/packages/system/kubevirt-csi-node/values.yaml +++ b/packages/system/kubevirt-csi-node/values.yaml @@ -1,3 +1,3 @@ storageClass: replicated csiDriver: - image: ghcr.io/cozystack/cozystack/kubevirt-csi-driver:v1.5.0@sha256:dad088fc04cc54af6d5f487ca9296669331a5345f143ed64cd9027b1bb20b5de + image: ghcr.io/cozystack/cozystack/kubevirt-csi-driver:v1.5.1@sha256:d01f1c0377ce00c8bb46f1576fa9348da3b226679130bf7b46a8bc50d684c38d diff --git a/packages/system/lineage-controller-webhook/values.yaml b/packages/system/lineage-controller-webhook/values.yaml index 4b36c58946..044c1e0b30 100644 --- a/packages/system/lineage-controller-webhook/values.yaml +++ b/packages/system/lineage-controller-webhook/values.yaml @@ -1,5 +1,5 @@ lineageControllerWebhook: - image: ghcr.io/cozystack/cozystack/lineage-controller-webhook:v1.5.0@sha256:5868789439d1384cf78d9fb2f20d5457060b0d5dfcc4f93f96a13ea27ed73b81 + image: ghcr.io/cozystack/cozystack/lineage-controller-webhook:v1.5.1@sha256:5868789439d1384cf78d9fb2f20d5457060b0d5dfcc4f93f96a13ea27ed73b81 debug: false replicas: 2 # DEPRECATED. Injects KUBERNETES_SERVICE_HOST=status.hostIP and diff --git a/packages/system/linstor-gui/values.yaml b/packages/system/linstor-gui/values.yaml index a5bd893108..b92d2c340d 100644 --- a/packages/system/linstor-gui/values.yaml +++ b/packages/system/linstor-gui/values.yaml @@ -3,7 +3,7 @@ ## @param image.tag LINSTOR GUI container image tag (digest recommended) image: repository: ghcr.io/cozystack/cozystack/linstor-gui - tag: v1.5.0@sha256:6e11829de86709f5e21636cdd1a1ada1239a7dcc1741f91e9d95d9f6ade603a2 + tag: v1.5.1@sha256:6e11829de86709f5e21636cdd1a1ada1239a7dcc1741f91e9d95d9f6ade603a2 ## @section Deployment ## @param replicas Number of linstor-gui replicas replicas: 1 diff --git a/packages/system/linstor/values.yaml b/packages/system/linstor/values.yaml index 09cf14cdaf..23c3c81986 100644 --- a/packages/system/linstor/values.yaml +++ b/packages/system/linstor/values.yaml @@ -1,7 +1,7 @@ piraeusServer: image: repository: ghcr.io/cozystack/cozystack/piraeus-server - tag: v1.5.0@sha256:1a00af3642c4814c3552d1964daf2831a430bc451c55e786e5eee278fa50e9e1 + tag: v1.5.1@sha256:1a00af3642c4814c3552d1964daf2831a430bc451c55e786e5eee278fa50e9e1 # Talos-specific workarounds (disable for generic Linux like Ubuntu/Debian) talos: enabled: true @@ -13,4 +13,4 @@ linstor: linstorCSI: image: repository: ghcr.io/cozystack/cozystack/linstor-csi - tag: v1.5.0@sha256:c60519d12aa8d078e4891d6e436c709ff53f2cc96457994390ca9424b6505903 + tag: v1.5.1@sha256:c60519d12aa8d078e4891d6e436c709ff53f2cc96457994390ca9424b6505903 diff --git a/packages/system/metallb/values.yaml b/packages/system/metallb/values.yaml index c269dbe96f..2d2a6ba3d6 100644 --- a/packages/system/metallb/values.yaml +++ b/packages/system/metallb/values.yaml @@ -37,11 +37,11 @@ metallb: controller: image: repository: ghcr.io/cozystack/cozystack/metallb-controller - tag: v1.5.0@sha256:b290031f40bfc072c0b329e07d7c17eca3301e4aad2e1988fc962645d92b5fc0 + tag: v1.5.1@sha256:b290031f40bfc072c0b329e07d7c17eca3301e4aad2e1988fc962645d92b5fc0 speaker: image: repository: ghcr.io/cozystack/cozystack/metallb-speaker - tag: v1.5.0@sha256:3aad3ac50e8da94342332b363099c04316d8eda0dcb8cdc3d89f1bcf5dc04e7d + tag: v1.5.1@sha256:3aad3ac50e8da94342332b363099c04316d8eda0dcb8cdc3d89f1bcf5dc04e7d # The vendored metallb chart's values.yaml leaves `frr-k8s.prometheus:` as # a YAML block containing only commented examples — it parses to null. # Helm-controller's deep-merge then writes that null over the frr-k8s diff --git a/packages/system/monitoring/images/grafana.tag b/packages/system/monitoring/images/grafana.tag index 2eb03cd00a..f92724d612 100644 --- a/packages/system/monitoring/images/grafana.tag +++ b/packages/system/monitoring/images/grafana.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/grafana:v1.5.0@sha256:086f1502edf64d2aabcda6e6505b7ee4673b0d3a14cdb7ca9e90a1d91d6e78f4 +ghcr.io/cozystack/cozystack/grafana:v1.5.1@sha256:086f1502edf64d2aabcda6e6505b7ee4673b0d3a14cdb7ca9e90a1d91d6e78f4 diff --git a/packages/system/multus/templates/multus-daemonset-thick.yml b/packages/system/multus/templates/multus-daemonset-thick.yml index 6664eb2144..98dff22d06 100644 --- a/packages/system/multus/templates/multus-daemonset-thick.yml +++ b/packages/system/multus/templates/multus-daemonset-thick.yml @@ -155,7 +155,7 @@ spec: serviceAccountName: multus containers: - name: kube-multus - image: ghcr.io/cozystack/cozystack/multus-cni:v1.5.0@sha256:f41d0d45bb3a654a6f024a8e416884541550f4c70481f1579647c6dd434f04d8 + image: ghcr.io/cozystack/cozystack/multus-cni:v1.5.1@sha256:f41d0d45bb3a654a6f024a8e416884541550f4c70481f1579647c6dd434f04d8 command: [ "/usr/src/multus-cni/bin/multus-daemon" ] resources: requests: @@ -201,7 +201,7 @@ spec: fieldPath: spec.nodeName initContainers: - name: install-multus-binary - image: ghcr.io/cozystack/cozystack/multus-cni:v1.5.0@sha256:f41d0d45bb3a654a6f024a8e416884541550f4c70481f1579647c6dd434f04d8 + image: ghcr.io/cozystack/cozystack/multus-cni:v1.5.1@sha256:f41d0d45bb3a654a6f024a8e416884541550f4c70481f1579647c6dd434f04d8 command: - "/usr/src/multus-cni/bin/install_multus" - "-d" diff --git a/packages/system/objectstorage-controller/values.yaml b/packages/system/objectstorage-controller/values.yaml index 5fc47baad0..a8bb959d52 100644 --- a/packages/system/objectstorage-controller/values.yaml +++ b/packages/system/objectstorage-controller/values.yaml @@ -1,3 +1,3 @@ objectstorage: controller: - image: "ghcr.io/cozystack/cozystack/objectstorage-controller:v1.5.0@sha256:e27da94691155fc3eb14a22cf23fba78e53c4ba524ca3042709efae3753c565b" + image: "ghcr.io/cozystack/cozystack/objectstorage-controller:v1.5.1@sha256:e27da94691155fc3eb14a22cf23fba78e53c4ba524ca3042709efae3753c565b" diff --git a/packages/system/seaweedfs/values.yaml b/packages/system/seaweedfs/values.yaml index c22027f63d..59295b2a5b 100644 --- a/packages/system/seaweedfs/values.yaml +++ b/packages/system/seaweedfs/values.yaml @@ -197,7 +197,7 @@ seaweedfs: bucketClassName: "seaweedfs" region: "" sidecar: - image: "ghcr.io/cozystack/cozystack/objectstorage-sidecar:v1.5.0@sha256:fdc09634688d5579a329b48aabf6706cc55bcd9503b0331fa47764706f0d6f62" + image: "ghcr.io/cozystack/cozystack/objectstorage-sidecar:v1.5.1@sha256:fdc09634688d5579a329b48aabf6706cc55bcd9503b0331fa47764706f0d6f62" certificates: commonName: "SeaweedFS CA" ipAddresses: [] From 459ae5324bd4982189ca60374f3722440a5f2ee2 Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Tue, 16 Jun 2026 17:36:27 +0500 Subject: [PATCH 03/69] fix(kafka): set entity-operator resources to avoid OOM under tenant LimitRange entityOperator.topicOperator/userOperator carried no resources, so in tenant namespaces (the LimitRange defaults containers to 128Mi) the JVM topic-operator and user-operator are OOMKilled on startup. The entity-operator stays in CrashLoopBackOff, KafkaTopics/KafkaUsers never reconcile, and on teardown topics get stuck on the strimzi.io/topic-operator finalizer. Set explicit requests/limits (256Mi/512Mi) so they no longer inherit the namespace default; add a template test. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Myasnikov Daniil (cherry picked from commit 1ab5f17caca0a950596f6dfd6e64eda318739e1e) --- packages/apps/kafka/templates/kafka.yaml | 21 +++++++++++++++++-- .../tests/entityoperator_template_test.yaml | 12 +++++++++++ 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/packages/apps/kafka/templates/kafka.yaml b/packages/apps/kafka/templates/kafka.yaml index 49876bacf7..b2a65d24f8 100644 --- a/packages/apps/kafka/templates/kafka.yaml +++ b/packages/apps/kafka/templates/kafka.yaml @@ -97,8 +97,25 @@ spec: name: {{ .Release.Name }}-metrics key: kafka-metrics-config.yml entityOperator: - topicOperator: {} - userOperator: {} + # Tenant namespaces ship a LimitRange that defaults containers to 128Mi. + # The topic-operator and user-operator are JVM processes and OOMKill at + # that default, leaving the entity-operator in CrashLoopBackOff so + # KafkaTopics/KafkaUsers never reconcile. Set explicit resources so they + # don't inherit the namespace default. + topicOperator: + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + memory: 512Mi + userOperator: + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + memory: 512Mi template: # `pod` accepts metadata + a flat set of pod-spec fields # (enableServiceLinks, securityContext, affinity, …) directly. diff --git a/packages/apps/kafka/tests/entityoperator_template_test.yaml b/packages/apps/kafka/tests/entityoperator_template_test.yaml index 95b7921387..b179400130 100644 --- a/packages/apps/kafka/tests/entityoperator_template_test.yaml +++ b/packages/apps/kafka/tests/entityoperator_template_test.yaml @@ -20,3 +20,15 @@ tests: # the phantom template.spec key must not reappear - notExists: path: spec.entityOperator.template.spec + + - it: topic-operator and user-operator set explicit resources (avoid 128Mi LimitRange OOM) + release: + name: test-kafka + namespace: tenant-test + asserts: + - equal: + path: spec.entityOperator.topicOperator.resources.limits.memory + value: 512Mi + - equal: + path: spec.entityOperator.userOperator.resources.limits.memory + value: 512Mi From 0240dc128fba5575a81daedad8f2cb05dc8c2152 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Wed, 24 Jun 2026 22:18:05 +0300 Subject: [PATCH 04/69] fix(platform): order backupstrategy-controller after cozystack-basics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit backupstrategy-controller creates the cozy-backups Bucket into the tenant-root namespace, which cozystack-basics provisions. On a fresh install the Bucket patch can race namespace creation and fail with `namespaces "tenant-root" not found`, which then races the install wait deadline and fails the run — on changes that have nothing to do with backup ordering. Add the cozystack-basics dependsOn edge so the target namespace exists first, pinned with a helm-unittest regression guard. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin (cherry picked from commit ee853b00bd046646c7d799bed27fb5cae4eec883) --- .../sources/backupstrategy-controller.yaml | 8 +++++ .../platform/sources/cozystack-basics.yaml | 7 ++-- ...sources_backupstrategy_dependson_test.yaml | 34 +++++++++++++++++++ 3 files changed, 47 insertions(+), 2 deletions(-) create mode 100644 packages/core/platform/tests/sources_backupstrategy_dependson_test.yaml diff --git a/packages/core/platform/sources/backupstrategy-controller.yaml b/packages/core/platform/sources/backupstrategy-controller.yaml index 2306c755f5..d28e599912 100644 --- a/packages/core/platform/sources/backupstrategy-controller.yaml +++ b/packages/core/platform/sources/backupstrategy-controller.yaml @@ -18,8 +18,16 @@ spec: # All three must be installed before the templates render, otherwise # the Helm release fails with "no matches for kind". Order is enforced # via dependsOn so a fresh-cluster bootstrap cannot race the CRDs. + # + # The cozy-backups Bucket is also created INTO the tenant-root namespace, + # which is provisioned by cozystack-basics (the static tenant-root release). + # Without the cozystack-basics edge the Bucket patch can race namespace + # creation on a fresh install and fail with `namespaces "tenant-root" not + # found`, which then races the install script's wait deadline. Depend on + # cozystack-basics so the target namespace exists first. dependsOn: - cozystack.networking + - cozystack.cozystack-basics - cozystack.backup-controller - cozystack.bucket-application - cozystack.objectstorage-controller diff --git a/packages/core/platform/sources/cozystack-basics.yaml b/packages/core/platform/sources/cozystack-basics.yaml index 8a5313ee66..0a71a24615 100644 --- a/packages/core/platform/sources/cozystack-basics.yaml +++ b/packages/core/platform/sources/cozystack-basics.yaml @@ -28,8 +28,11 @@ spec: # Drift detection is off on operator-generated HelmReleases, so a # capability gate in the templates would render the policy out at first # install and never add it back; ordering via dependsOn is the reliable - # fix. cozystack-basics is a dependency-graph leaf (nothing dependsOn it) - # and engine components consume none of its outputs, so this adds no cycle. + # fix. The cozystack-engine and gateway-api-crds packages it depends on + # consume none of cozystack-basics' outputs and do not depend back on it, + # so these edges add no cycle. (Downstream packages such as backupstrategy- + # controller dependsOn cozystack-basics, but those do not feed engine or + # gateway-api-crds, so they cannot close a loop here.) - cozystack.cozystack-engine - cozystack.gateway-api-crds components: diff --git a/packages/core/platform/tests/sources_backupstrategy_dependson_test.yaml b/packages/core/platform/tests/sources_backupstrategy_dependson_test.yaml new file mode 100644 index 0000000000..e1234972bb --- /dev/null +++ b/packages/core/platform/tests/sources_backupstrategy_dependson_test.yaml @@ -0,0 +1,34 @@ +suite: backupstrategy-controller waits for the tenant-root namespace and its CRDs +# Regression guard for a fresh-install ordering race: backupstrategy-controller +# creates the cozy-backups Bucket INTO the tenant-root namespace, which is +# provisioned by cozystack-basics. Without the cozystack-basics edge the Bucket +# patch races namespace creation and fails with `namespaces "tenant-root" not +# found`, which then races the install script's wait deadline. The CRD edges +# (backup-controller / bucket-application / objectstorage-controller / velero) +# guard the `no matches for kind` race. All edges below must stay. +templates: + - templates/sources.yaml +release: + name: cozystack + namespace: cozy-system +tests: + - it: backupstrategy-controller dependsOn cozystack-basics and the CRD providers + documentSelector: + path: metadata.name + value: cozystack.backupstrategy-controller + asserts: + - contains: + path: spec.variants[0].dependsOn + content: cozystack.cozystack-basics + - contains: + path: spec.variants[0].dependsOn + content: cozystack.backup-controller + - contains: + path: spec.variants[0].dependsOn + content: cozystack.bucket-application + - contains: + path: spec.variants[0].dependsOn + content: cozystack.objectstorage-controller + - contains: + path: spec.variants[0].dependsOn + content: cozystack.velero From 71ed3be6e43564c29ac9aea51f11c3275edb79dc Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Sat, 27 Jun 2026 21:59:56 +0300 Subject: [PATCH 05/69] fix(seaweedfs): point S3 consumers at the reachable S3 service The s3-service-name.patch renamed the S3 Service to -s3 (seaweedfs-s3), but several consumers still resolved it through componentName -> -s3. name and fullname only coincide when the release name equals the chart name; the shipped release is seaweedfs-system, so they pointed at seaweedfs-system-s3 while the real service stayed seaweedfs-s3: - the S3 ingress backend and the iceberg ingress backend, which left ingress-nginx returning 503 on the external S3 endpoint. The COSI provisioner in-cluster ENDPOINT was wrong on the same axis and also on the port. Both the s3-gateway branch and the filer-S3 branch rendered a portless URL that defaults to 443, while the s3 service only listens on the S3 port (8333); the filer branch additionally targeted the filer service, which serves no S3. Both in-cluster branches now resolve seaweedfs-s3 on the configured S3 port. The default path (s3.ingress.enabled=true) is repaired by the ingress fix; the COSI in-cluster paths are latent until an operator disables the ingress. Resolve every consumer through the same name helper the Service uses, and pin the contract with unittests rendered under the production release name so the mismatch cannot recur. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin (cherry picked from commit 8a14de9abfee64589c4911a567cd8aff9520487d) --- packages/system/seaweedfs/Makefile | 1 + .../templates/cosi/cosi-deployment.yaml | 4 +- .../templates/s3/s3-iceberg-ingress.yaml | 4 +- .../seaweedfs/templates/s3/s3-ingress.yaml | 2 +- .../patches/s3-service-name-consumers.patch | 47 ++++++++ .../tests/s3_service_name_consumers_test.yaml | 100 ++++++++++++++++++ 6 files changed, 153 insertions(+), 5 deletions(-) create mode 100644 packages/system/seaweedfs/patches/s3-service-name-consumers.patch create mode 100644 packages/system/seaweedfs/tests/s3_service_name_consumers_test.yaml diff --git a/packages/system/seaweedfs/Makefile b/packages/system/seaweedfs/Makefile index cf43956f2d..af7c7b3274 100644 --- a/packages/system/seaweedfs/Makefile +++ b/packages/system/seaweedfs/Makefile @@ -18,5 +18,6 @@ update: patch --no-backup-if-mismatch -p4 < patches/cosi-provisioner-sa-name.patch patch --no-backup-if-mismatch -p4 < patches/cosi-bucket-class-lock-readonly.patch patch --no-backup-if-mismatch -p4 < patches/s3-service-name.patch + patch --no-backup-if-mismatch -p4 < patches/s3-service-name-consumers.patch #patch --no-backup-if-mismatch -p4 < patches/retention-policy-delete.yaml diff --git a/packages/system/seaweedfs/charts/seaweedfs/templates/cosi/cosi-deployment.yaml b/packages/system/seaweedfs/charts/seaweedfs/templates/cosi/cosi-deployment.yaml index d485e06787..d581b085db 100644 --- a/packages/system/seaweedfs/charts/seaweedfs/templates/cosi/cosi-deployment.yaml +++ b/packages/system/seaweedfs/charts/seaweedfs/templates/cosi/cosi-deployment.yaml @@ -79,9 +79,9 @@ spec: {{- else if .Values.s3.ingress.enabled }} value: "{{ printf "https://%s" .Values.s3.ingress.host }}" {{- else if .Values.s3.enabled }} - value: "{{ printf "https://%s.%s.svc" (include "seaweedfs.componentName" (list . "s3")) .Release.Namespace }}" + value: "{{ printf "https://%s-s3.%s.svc:%v" (include "seaweedfs.name" .) .Release.Namespace .Values.s3.port }}" {{- else }} - value: "{{ printf "https://%s.%s.svc" (include "seaweedfs.componentName" (list . "filer")) .Release.Namespace }}" + value: "{{ printf "https://%s-s3.%s.svc:%v" (include "seaweedfs.name" .) .Release.Namespace .Values.filer.s3.port }}" {{- end }} {{- with .Values.cosi.region }} - name: REGION diff --git a/packages/system/seaweedfs/charts/seaweedfs/templates/s3/s3-iceberg-ingress.yaml b/packages/system/seaweedfs/charts/seaweedfs/templates/s3/s3-iceberg-ingress.yaml index 5a760d7e9e..68db538dcf 100644 --- a/packages/system/seaweedfs/charts/seaweedfs/templates/s3/s3-iceberg-ingress.yaml +++ b/packages/system/seaweedfs/charts/seaweedfs/templates/s3/s3-iceberg-ingress.yaml @@ -5,11 +5,11 @@ paths: backend: {{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion }} service: - name: {{ include "seaweedfs.componentName" (list . "s3") }} + name: {{ template "seaweedfs.name" . }}-s3 port: number: {{ .Values.s3.icebergPort }} {{- else }} - serviceName: {{ include "seaweedfs.componentName" (list . "s3") }} + serviceName: {{ template "seaweedfs.name" . }}-s3 servicePort: {{ .Values.s3.icebergPort }} {{- end }} {{- end -}} diff --git a/packages/system/seaweedfs/charts/seaweedfs/templates/s3/s3-ingress.yaml b/packages/system/seaweedfs/charts/seaweedfs/templates/s3/s3-ingress.yaml index bc4eee1642..c92063f8c4 100644 --- a/packages/system/seaweedfs/charts/seaweedfs/templates/s3/s3-ingress.yaml +++ b/packages/system/seaweedfs/charts/seaweedfs/templates/s3/s3-ingress.yaml @@ -2,7 +2,7 @@ {{- $s3Enabled := or .Values.s3.enabled (and .Values.filer.s3.enabled (not .Values.allInOne.enabled)) (and .Values.allInOne.enabled .Values.allInOne.s3.enabled) }} {{- if and $s3Enabled .Values.s3.ingress.enabled }} {{- /* Determine service name based on deployment mode */}} -{{- $serviceName := ternary (include "seaweedfs.componentName" (list . "all-in-one")) (include "seaweedfs.componentName" (list . "s3")) .Values.allInOne.enabled }} +{{- $serviceName := ternary (include "seaweedfs.componentName" (list . "all-in-one")) (printf "%s-s3" (include "seaweedfs.name" .)) .Values.allInOne.enabled }} {{- $s3Port := .Values.allInOne.s3.port | default .Values.s3.port }} {{- /* Build hosts list - support both legacy .host (string) and new .hosts (array) for backwards compatibility */}} {{- $hosts := list }} diff --git a/packages/system/seaweedfs/patches/s3-service-name-consumers.patch b/packages/system/seaweedfs/patches/s3-service-name-consumers.patch new file mode 100644 index 0000000000..73b09dc15f --- /dev/null +++ b/packages/system/seaweedfs/patches/s3-service-name-consumers.patch @@ -0,0 +1,47 @@ +diff --git a/packages/system/seaweedfs/charts/seaweedfs/templates/cosi/cosi-deployment.yaml b/packages/system/seaweedfs/charts/seaweedfs/templates/cosi/cosi-deployment.yaml +index d485e0678..d581b085d 100644 +--- a/packages/system/seaweedfs/charts/seaweedfs/templates/cosi/cosi-deployment.yaml ++++ b/packages/system/seaweedfs/charts/seaweedfs/templates/cosi/cosi-deployment.yaml +@@ -79,9 +79,9 @@ spec: + {{- else if .Values.s3.ingress.enabled }} + value: "{{ printf "https://%s" .Values.s3.ingress.host }}" + {{- else if .Values.s3.enabled }} +- value: "{{ printf "https://%s.%s.svc" (include "seaweedfs.componentName" (list . "s3")) .Release.Namespace }}" ++ value: "{{ printf "https://%s-s3.%s.svc:%v" (include "seaweedfs.name" .) .Release.Namespace .Values.s3.port }}" + {{- else }} +- value: "{{ printf "https://%s.%s.svc" (include "seaweedfs.componentName" (list . "filer")) .Release.Namespace }}" ++ value: "{{ printf "https://%s-s3.%s.svc:%v" (include "seaweedfs.name" .) .Release.Namespace .Values.filer.s3.port }}" + {{- end }} + {{- with .Values.cosi.region }} + - name: REGION +diff --git a/packages/system/seaweedfs/charts/seaweedfs/templates/s3/s3-iceberg-ingress.yaml b/packages/system/seaweedfs/charts/seaweedfs/templates/s3/s3-iceberg-ingress.yaml +index 5a760d7e9..68db538dc 100644 +--- a/packages/system/seaweedfs/charts/seaweedfs/templates/s3/s3-iceberg-ingress.yaml ++++ b/packages/system/seaweedfs/charts/seaweedfs/templates/s3/s3-iceberg-ingress.yaml +@@ -5,11 +5,11 @@ paths: + backend: + {{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion }} + service: +- name: {{ include "seaweedfs.componentName" (list . "s3") }} ++ name: {{ template "seaweedfs.name" . }}-s3 + port: + number: {{ .Values.s3.icebergPort }} + {{- else }} +- serviceName: {{ include "seaweedfs.componentName" (list . "s3") }} ++ serviceName: {{ template "seaweedfs.name" . }}-s3 + servicePort: {{ .Values.s3.icebergPort }} + {{- end }} + {{- end -}} +diff --git a/packages/system/seaweedfs/charts/seaweedfs/templates/s3/s3-ingress.yaml b/packages/system/seaweedfs/charts/seaweedfs/templates/s3/s3-ingress.yaml +index bc4eee164..c92063f8c 100644 +--- a/packages/system/seaweedfs/charts/seaweedfs/templates/s3/s3-ingress.yaml ++++ b/packages/system/seaweedfs/charts/seaweedfs/templates/s3/s3-ingress.yaml +@@ -2,7 +2,7 @@ + {{- $s3Enabled := or .Values.s3.enabled (and .Values.filer.s3.enabled (not .Values.allInOne.enabled)) (and .Values.allInOne.enabled .Values.allInOne.s3.enabled) }} + {{- if and $s3Enabled .Values.s3.ingress.enabled }} + {{- /* Determine service name based on deployment mode */}} +-{{- $serviceName := ternary (include "seaweedfs.componentName" (list . "all-in-one")) (include "seaweedfs.componentName" (list . "s3")) .Values.allInOne.enabled }} ++{{- $serviceName := ternary (include "seaweedfs.componentName" (list . "all-in-one")) (printf "%s-s3" (include "seaweedfs.name" .)) .Values.allInOne.enabled }} + {{- $s3Port := .Values.allInOne.s3.port | default .Values.s3.port }} + {{- /* Build hosts list - support both legacy .host (string) and new .hosts (array) for backwards compatibility */}} + {{- $hosts := list }} diff --git a/packages/system/seaweedfs/tests/s3_service_name_consumers_test.yaml b/packages/system/seaweedfs/tests/s3_service_name_consumers_test.yaml new file mode 100644 index 0000000000..628d2713be --- /dev/null +++ b/packages/system/seaweedfs/tests/s3_service_name_consumers_test.yaml @@ -0,0 +1,100 @@ +suite: seaweedfs s3 consumers resolve the renamed s3 service + +# Pins the regression where consumers of the S3 service 503 / fail to connect +# because they still resolve a service name that no longer exists. +# +# The s3-service-name.patch renamed the S3 Service to `-s3` (seaweedfs-s3), +# but several consumers still resolved it via componentName -> `-s3`: +# - the S3 ingress backend, +# - the iceberg ingress backend, +# - the COSI provisioner ENDPOINT (in-cluster, s3.ingress.enabled=false path). +# name and fullname only coincide when the release name equals the chart name; +# the shipped release is `seaweedfs-system`, so those consumers pointed at +# `seaweedfs-system-s3` while the real service is `seaweedfs-s3`. With no +# matching endpoints, ingress-nginx returns 503 and COSI cannot reach S3. +# +# All tests render under the real release name so name != fullname, the exact +# condition that exposes the bug. The all-in-one case documents the other +# ternary branch, which is unchanged and correct. + +templates: + - charts/seaweedfs/templates/s3/s3-service.yaml + - charts/seaweedfs/templates/s3/s3-ingress.yaml + - charts/seaweedfs/templates/s3/s3-iceberg-ingress.yaml + - charts/seaweedfs/templates/cosi/cosi-deployment.yaml + +release: + name: seaweedfs-system + namespace: cozy-seaweedfs + +tests: + - it: s3 service is named with the name helper, not componentName + template: charts/seaweedfs/templates/s3/s3-service.yaml + set: + seaweedfs.s3.enabled: true + asserts: + - equal: + path: metadata.name + value: seaweedfs-s3 + + - it: s3 ingress backend points at the s3 service, not a fullname-derived name + template: charts/seaweedfs/templates/s3/s3-ingress.yaml + set: + seaweedfs.s3.enabled: true + seaweedfs.s3.ingress.enabled: true + seaweedfs.s3.ingress.host: s3.example.com + asserts: + - equal: + path: spec.rules[0].http.paths[0].backend.service.name + value: seaweedfs-s3 + + - it: all-in-one ingress backend still resolves to the all-in-one service + template: charts/seaweedfs/templates/s3/s3-ingress.yaml + set: + seaweedfs.allInOne.enabled: true + seaweedfs.allInOne.s3.enabled: true + seaweedfs.s3.ingress.enabled: true + seaweedfs.s3.ingress.host: s3.example.com + asserts: + - equal: + path: spec.rules[0].http.paths[0].backend.service.name + value: seaweedfs-system-all-in-one + + - it: iceberg ingress backend points at the s3 service too + template: charts/seaweedfs/templates/s3/s3-iceberg-ingress.yaml + set: + seaweedfs.s3.enabled: true + seaweedfs.s3.icebergPort: 8334 + seaweedfs.s3.icebergIngress.enabled: true + seaweedfs.s3.icebergIngress.host: iceberg.example.com + asserts: + - equal: + path: spec.rules[0].http.paths[0].backend.service.name + value: seaweedfs-s3 + + - it: COSI in-cluster ENDPOINT resolves the s3 service when ingress is disabled + template: charts/seaweedfs/templates/cosi/cosi-deployment.yaml + set: + seaweedfs.cosi.enabled: true + seaweedfs.s3.enabled: true + seaweedfs.s3.ingress.enabled: false + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: ENDPOINT + value: https://seaweedfs-s3.cozy-seaweedfs.svc:8333 + + - it: COSI in-cluster ENDPOINT (filer S3 mode) targets the s3 service on the filer s3 port + template: charts/seaweedfs/templates/cosi/cosi-deployment.yaml + set: + seaweedfs.cosi.enabled: true + seaweedfs.s3.enabled: false + seaweedfs.filer.s3.enabled: true + seaweedfs.s3.ingress.enabled: false + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: ENDPOINT + value: https://seaweedfs-s3.cozy-seaweedfs.svc:8333 From dad745874c8f3532e3eaefb710bafb0480790ef9 Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Tue, 16 Jun 2026 14:49:10 +0500 Subject: [PATCH 06/69] fix(harbor): preserve jobservice and trivy storageClass on upgrade The cozy-harbor wrapper never passed storageClass to the jobservice jobLog PVC, so it was created without a storageClassName and the default-StorageClass admission stamped the cluster default. Because PVC storageClassName is immutable, this caused rendered-vs-live drift on upgrade. Preserve an existing PVC's class via lookup and fall back to the configured storageClass for fresh installs. The trivy StatefulSet's volumeClaimTemplate had the same latent drift -- .Values.storageClass was forwarded blindly into a VCT that cannot be patched at all -- so it gets the same lookup-preserve treatment, keyed off data--trivy-0. A live object pinned to an explicit "" (no dynamic provisioning) now round-trips through upstream's "-" sentinel instead of collapsing to an omitted storageClassName, and the spec lookup is guarded with `| default dict` so a PVC carrying no storageClassName key stays omitted. Adds a helm-unittest regression suite and wires a test target so make unit-tests stops skipping packages/apps/harbor. Fixes #2368 Assisted-By: Claude Signed-off-by: Myasnikov Daniil (cherry picked from commit bdf520bf74a7ed17a62bfeaa669ed2eef7b45509) --- packages/apps/harbor/templates/harbor.yaml | 53 ++++++++++++++- .../tests/jobservice_storageclass_test.yaml | 64 +++++++++++++++++++ 2 files changed, 114 insertions(+), 3 deletions(-) create mode 100644 packages/apps/harbor/tests/jobservice_storageclass_test.yaml diff --git a/packages/apps/harbor/templates/harbor.yaml b/packages/apps/harbor/templates/harbor.yaml index 72bfa61a4c..8cb8b38c08 100644 --- a/packages/apps/harbor/templates/harbor.yaml +++ b/packages/apps/harbor/templates/harbor.yaml @@ -23,6 +23,46 @@ {{- end }} {{- end }} +{{- /* + storageClassName on a PVC is immutable, and a StatefulSet's volumeClaimTemplate + cannot be patched at all. The upstream chart renders both the jobservice PVC and + the trivy VCT without a storageClassName unless one is passed, so on first install + the default-StorageClass admission stamps the cluster default onto the live object. + To avoid an immutable-field rejection on upgrade we keep the rendered value equal + to the live one: preserve an existing object's storageClassName, and only fall back + to the configured storageClass for a fresh install. See issue #2368. + + A live object pinned to an explicit "" (no dynamic provisioning) must round-trip + too: upstream maps the "-" sentinel back to storageClassName: "", so forward "-" + rather than letting "" collapse to an omitted field. An object carrying no + storageClassName key at all keeps the empty fallback so the field stays omitted, + matching the live object. +*/}} +{{- $jobservicePvc := lookup "v1" "PersistentVolumeClaim" .Release.Namespace (printf "%s-jobservice" .Release.Name) }} +{{- $jobserviceStorageClass := .Values.storageClass }} +{{- if $jobservicePvc }} + {{- $jobserviceSpec := $jobservicePvc.spec | default dict }} + {{- if hasKey $jobserviceSpec "storageClassName" }} + {{- $jobserviceStorageClass = $jobserviceSpec.storageClassName | default "-" }} + {{- else }} + {{- $jobserviceStorageClass = "" }} + {{- end }} +{{- end }} + +{{- /* The trivy StatefulSet's VCT is named "data", so its PVCs are + data--trivy-0..N; the class is identical across ordinals, so the + first ordinal is representative. */}} +{{- $trivyPvc := lookup "v1" "PersistentVolumeClaim" .Release.Namespace (printf "data-%s-trivy-0" .Release.Name) }} +{{- $trivyStorageClass := .Values.storageClass }} +{{- if $trivyPvc }} + {{- $trivySpec := $trivyPvc.spec | default dict }} + {{- if hasKey $trivySpec "storageClassName" }} + {{- $trivyStorageClass = $trivySpec.storageClassName | default "-" }} + {{- else }} + {{- $trivyStorageClass = "" }} + {{- end }} +{{- end }} + apiVersion: v1 kind: Secret metadata: @@ -108,13 +148,20 @@ spec: bucket: {{ .Release.Name }}-registry secure: false v4auth: true - {{- if .Values.trivy.enabled }} + {{- if or $jobserviceStorageClass .Values.trivy.enabled }} persistentVolumeClaim: + {{- with $jobserviceStorageClass }} + jobservice: + jobLog: + storageClass: {{ . | quote }} + {{- end }} + {{- if .Values.trivy.enabled }} trivy: size: {{ .Values.trivy.size }} - {{- with .Values.storageClass }} - storageClass: {{ . }} + {{- with $trivyStorageClass }} + storageClass: {{ . | quote }} {{- end }} + {{- end }} {{- end }} portal: resources: {{- include "cozy-lib.resources.defaultingSanitize" (list "t1.nano" (dict) $) | nindent 10 }} diff --git a/packages/apps/harbor/tests/jobservice_storageclass_test.yaml b/packages/apps/harbor/tests/jobservice_storageclass_test.yaml new file mode 100644 index 0000000000..e1aac99339 --- /dev/null +++ b/packages/apps/harbor/tests/jobservice_storageclass_test.yaml @@ -0,0 +1,64 @@ +suite: jobservice storageClass propagation + +release: + name: harbor + namespace: tenant-test + +# Regression test for issue #2368: the jobservice PVC must receive the +# configured storageClass so it is created with an explicit storageClassName +# instead of relying on the default-StorageClass admission. Because PVC +# storageClassName is immutable, the template preserves an existing PVC's +# class when one is found; under `helm template`/unittest the cluster lookup +# returns empty, so these cases exercise the fresh-install fallback path. +# +# The rendered HelmRelease (documentIndex 1) carries the values that the +# cozy-harbor chart forwards to the upstream harbor subchart. +# helm-unittest renders every template in the chart, so the platform-injected +# globals consumed by the ingress/httproute/bucket templates must be supplied +# even though only templates/harbor.yaml is asserted on. +set: + _namespace: + host: example.org + ingress: nginx + gateway: "" + seaweedfs: seaweedfs + _cluster: + solver: http01 + +tests: + - it: forwards storageClass to jobservice jobLog on a fresh install + template: templates/harbor.yaml + documentIndex: 1 + set: + storageClass: local-3rep + asserts: + - equal: + path: spec.values.harbor.persistence.persistentVolumeClaim.jobservice.jobLog.storageClass + value: local-3rep + + - it: still forwards storageClass to trivy alongside jobservice + template: templates/harbor.yaml + documentIndex: 1 + set: + storageClass: local-3rep + asserts: + - equal: + path: spec.values.harbor.persistence.persistentVolumeClaim.trivy.storageClass + value: local-3rep + + - it: omits the jobservice block when storageClass is unset + template: templates/harbor.yaml + documentIndex: 1 + asserts: + - notExists: + path: spec.values.harbor.persistence.persistentVolumeClaim.jobservice + + - it: emits no persistentVolumeClaim block when storageClass is unset and trivy is disabled + template: templates/harbor.yaml + documentIndex: 1 + set: + trivy: + enabled: false + asserts: + - notExists: + path: spec.values.harbor.persistence.persistentVolumeClaim From 46d5d9d1e90cff58cc8d1940f4c93abeb1790668 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Mon, 29 Jun 2026 18:58:36 +0300 Subject: [PATCH 07/69] fix(seaweedfs): enforce read-only bucket access by bumping cosi-driver to v0.3.1 The seaweedfs COSI driver shipped at v0.1.2 hardcodes read-write S3 actions (Read, Write, List, Tagging) for every BucketAccess and never reads the accessPolicy parameter. The chart already defines a -readonly BucketAccessClass carrying parameters.accessPolicy: readonly, but v0.1.2 ignores it, so a credential meant to be read-only can still write (upload and delete) objects. Bump the driver to v0.3.1, which honours accessPolicy: readonly by issuing Read,List-only credentials (read-write and the empty default stay Read,Write,List,Tagging). The override lives in the package values.yaml rather than the vendored chart so it survives make update; the sister extra/seaweedfs chart already runs v0.3.1. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin (cherry picked from commit 0c6426feaa5ded22f838ed48e6e32c5f6b76cb8d) --- .../seaweedfs/tests/cosi_image_test.yaml | 38 +++++++++++++++++++ packages/system/seaweedfs/values.yaml | 5 +++ 2 files changed, 43 insertions(+) create mode 100644 packages/system/seaweedfs/tests/cosi_image_test.yaml diff --git a/packages/system/seaweedfs/tests/cosi_image_test.yaml b/packages/system/seaweedfs/tests/cosi_image_test.yaml new file mode 100644 index 0000000000..7840d200de --- /dev/null +++ b/packages/system/seaweedfs/tests/cosi_image_test.yaml @@ -0,0 +1,38 @@ +suite: seaweedfs COSI driver image pin + +# Pins the COSI driver image to a release that enforces read-only bucket access. +# +# The vendored chart default is seaweedfs-cosi-driver v0.1.2 (see +# charts/seaweedfs/values.yaml), which hardcodes read-write S3 actions for every +# BucketAccess and ignores the accessPolicy parameter. The chart ships a +# -readonly BucketAccessClass with parameters.accessPolicy: readonly, so on +# v0.1.2 a credential meant to be read-only is silently granted read-write. +# The package values.yaml overrides cosi.image to v0.3.1, which honours +# accessPolicy: readonly. +# +# This test does not set the image itself; it relies on the package values.yaml +# override, so it goes red if that override is removed and the image falls back +# to the vulnerable v0.1.2 — turning a silent security regression into a build +# failure. Note: `make update` re-vendors charts/ and resets the vendored +# default to v0.1.2 but does not touch the package values.yaml, so the override +# (and this guard) must be preserved across any re-vendor. + +templates: + - charts/seaweedfs/templates/cosi/cosi-deployment.yaml + +release: + name: seaweedfs + namespace: cozy-seaweedfs + +tests: + - it: pins the cosi-driver container to v0.3.1 (read-only access enforcement) + template: charts/seaweedfs/templates/cosi/cosi-deployment.yaml + set: + seaweedfs.cosi.enabled: true + asserts: + - equal: + path: spec.template.spec.containers[0].name + value: seaweedfs-cosi-driver + - equal: + path: spec.template.spec.containers[0].image + value: ghcr.io/seaweedfs/seaweedfs-cosi-driver:v0.3.1 diff --git a/packages/system/seaweedfs/values.yaml b/packages/system/seaweedfs/values.yaml index 59295b2a5b..eba5bcada5 100644 --- a/packages/system/seaweedfs/values.yaml +++ b/packages/system/seaweedfs/values.yaml @@ -191,6 +191,11 @@ seaweedfs: topologyKey: kubernetes.io/hostname cosi: enabled: true + # Override the vendored chart default (v0.1.2), which hardcodes read-write + # S3 actions for every BucketAccess and ignores the accessPolicy parameter. + # v0.3.x honours accessPolicy: readonly, so a -readonly BucketAccessClass + # actually issues read-only (Read,List) credentials instead of read-write. + image: "ghcr.io/seaweedfs/seaweedfs-cosi-driver:v0.3.1" podLabels: policy.cozystack.io/allow-to-apiserver: "true" driverName: "seaweedfs.objectstorage.k8s.io" From 0dbd0e15973ac398e2d5aa59c789b4f833fb2627 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Sun, 28 Jun 2026 03:09:02 +0300 Subject: [PATCH 08/69] fix(kubernetes): make node group resources optional and prefer them over instanceType MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A node group is sized either by instanceType or by explicit resources, but the generated values schema required resources on every node group. A node group with instanceType and no resources — or an empty resources dropped during typed-CR to HelmRelease translation — failed schema validation with "missing property 'resources'" even though sizing was fully specified. Mark resources optional in the node group definition and regenerate the schema, API types, README, and ApplicationDefinition. The chart template emits domain.cpu/memory only when resources is present, so a node group sized purely by instanceType now passes validation and installs. When a node group sets both instanceType and explicit resources, explicit resources take precedence: the instancetype is omitted from the rendered VM and the per-group instancetype lookup is skipped, so KubeVirt accepts the explicit domain.cpu/memory with no instancetype to conflict with. instanceType stays required and defaulted (no schema change), so the console's required instanceType dropdown keeps working — sending the default instanceType plus explicit resources simply sizes the VM by resources. This replaces an earlier render-time hard failure on the both-set case, which was unreachable to clear from the dashboard (the required dropdown cannot be emptied). Explicit resources are all-or-nothing alongside an instanceType: a partial spec (only cpu, or only memory) on a group that carries a non-empty instanceType would emit one domain override next to the instancetype — which KubeVirt rejects — so it now fails at render time with a clear error. Partial resources on a group without an instanceType (used to tune kubelet reservations) are unaffected. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin (cherry picked from commit a349c753ad16a68e309177562b18f54cf86ce4f9) --- api/apps/v1alpha1/kubernetes/types.go | 4 +- packages/apps/kubernetes/README.md | 46 +++--- .../apps/kubernetes/templates/cluster.yaml | 27 +++- .../apps/kubernetes/tests/cluster_test.yaml | 140 ++++++++++++++++++ packages/apps/kubernetes/values.schema.json | 5 +- packages/apps/kubernetes/values.yaml | 2 +- .../kubernetes-rd/cozyrds/kubernetes.yaml | 2 +- 7 files changed, 193 insertions(+), 33 deletions(-) diff --git a/api/apps/v1alpha1/kubernetes/types.go b/api/apps/v1alpha1/kubernetes/types.go index 57153057db..cdfe8d6b0c 100644 --- a/api/apps/v1alpha1/kubernetes/types.go +++ b/api/apps/v1alpha1/kubernetes/types.go @@ -255,8 +255,8 @@ type NodeGroup struct { // Minimum number of replicas. // +kubebuilder:default:=0 MinReplicas int `json:"minReplicas"` - // CPU and memory resources for each worker node. - Resources Resources `json:"resources"` + // Explicit CPU and memory for each worker node, as an alternative to `instanceType` sizing. Optional: when omitted, the node is sized by `instanceType`. When both `cpu` and `memory` are set, they take precedence and `instanceType` is ignored for that node group (the instancetype is omitted from the VM, since KubeVirt cannot override an instancetype's CPU/memory). Set both `cpu` and `memory` together or neither; setting only one is rejected at render time. + Resources Resources `json:"resources,omitempty"` // List of node roles. Roles []string `json:"roles,omitempty"` // StorageClass for worker node persistent disks. When empty, uses the management cluster default StorageClass (the one annotated storageclass.kubernetes.io/is-default-class: true). NOTE: deliberately not marked immutable — the field is optional and undefaulted, so a strict `self == oldSelf` rule would block any future attempt to set it on an existing node group. diff --git a/packages/apps/kubernetes/README.md b/packages/apps/kubernetes/README.md index 1912419f02..52417f8635 100644 --- a/packages/apps/kubernetes/README.md +++ b/packages/apps/kubernetes/README.md @@ -100,29 +100,29 @@ See the reference for components utilized in this service: ### Application-specific Parameters -| Name | Description | Type | Value | -| ----------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------- | ----------- | -| `nodeGroups` | Worker nodes configuration map. | `map[string]object` | `{...}` | -| `nodeGroups[name].minReplicas` | Minimum number of replicas. | `int` | `0` | -| `nodeGroups[name].maxReplicas` | Maximum number of replicas. | `int` | `10` | -| `nodeGroups[name].instanceType` | Virtual machine instance type. | `string` | `u1.medium` | -| `nodeGroups[name].diskSize` | Persistent disk size for kubelet and containerd data. | `quantity` | `20Gi` | -| `nodeGroups[name].storageClass` | StorageClass for worker node persistent disks. When empty, uses the management cluster default StorageClass (the one annotated storageclass.kubernetes.io/is-default-class: true). NOTE: deliberately not marked immutable — the field is optional and undefaulted, so a strict `self == oldSelf` rule would block any future attempt to set it on an existing node group. | `string` | `""` | -| `nodeGroups[name].roles` | List of node roles. | `[]string` | `[]` | -| `nodeGroups[name].resources` | CPU and memory resources for each worker node. | `object` | `{}` | -| `nodeGroups[name].resources.cpu` | CPU available. | `quantity` | `""` | -| `nodeGroups[name].resources.memory` | Memory (RAM) available. | `quantity` | `""` | -| `nodeGroups[name].gpus` | List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM). | `[]object` | `[]` | -| `nodeGroups[name].gpus[i].name` | Name of GPU, such as "nvidia.com/AD102GL_L40S". | `string` | `""` | -| `nodeGroups[name].kubelet` | Kubelet resource reservations for this node group. | `object` | `{}` | -| `nodeGroups[name].kubelet.systemReservedMemory` | Memory reserved for host OS. Auto-computed from instanceType if empty. | `string` | `""` | -| `nodeGroups[name].kubelet.kubeReservedMemory` | Memory reserved for kubelet and container runtime. Auto-computed from instanceType if empty. | `string` | `""` | -| `nodeGroups[name].kubelet.systemReservedCpu` | CPU reserved for host OS. Auto-computed from instanceType if empty. | `string` | `""` | -| `nodeGroups[name].kubelet.kubeReservedCpu` | CPU reserved for kubelet and container runtime. Auto-computed from instanceType if empty. | `string` | `""` | -| `nodeGroups[name].kubelet.evictionHardMemory` | Hard eviction threshold for memory (absolute like 200Mi or percentage like 7%). | `string` | `7%` | -| `nodeGroups[name].kubelet.evictionSoftMemory` | Soft eviction threshold for memory (absolute like 1Gi or percentage like 10%). | `string` | `10%` | -| `version` | Kubernetes major.minor version to deploy | `string` | `v1.35` | -| `host` | External hostname for Kubernetes cluster. Defaults to `.` if empty. | `string` | `""` | +| Name | Description | Type | Value | +| ----------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------- | ----------- | +| `nodeGroups` | Worker nodes configuration map. | `map[string]object` | `{...}` | +| `nodeGroups[name].minReplicas` | Minimum number of replicas. | `int` | `0` | +| `nodeGroups[name].maxReplicas` | Maximum number of replicas. | `int` | `10` | +| `nodeGroups[name].instanceType` | Virtual machine instance type. | `string` | `u1.medium` | +| `nodeGroups[name].diskSize` | Persistent disk size for kubelet and containerd data. | `quantity` | `20Gi` | +| `nodeGroups[name].storageClass` | StorageClass for worker node persistent disks. When empty, uses the management cluster default StorageClass (the one annotated storageclass.kubernetes.io/is-default-class: true). NOTE: deliberately not marked immutable — the field is optional and undefaulted, so a strict `self == oldSelf` rule would block any future attempt to set it on an existing node group. | `string` | `""` | +| `nodeGroups[name].roles` | List of node roles. | `[]string` | `[]` | +| `nodeGroups[name].resources` | Explicit CPU and memory for each worker node, as an alternative to `instanceType` sizing. Optional: when omitted, the node is sized by `instanceType`. When both `cpu` and `memory` are set, they take precedence and `instanceType` is ignored for that node group (the instancetype is omitted from the VM, since KubeVirt cannot override an instancetype's CPU/memory). Set both `cpu` and `memory` together or neither; setting only one is rejected at render time. | `object` | `{}` | +| `nodeGroups[name].resources.cpu` | CPU available. | `quantity` | `""` | +| `nodeGroups[name].resources.memory` | Memory (RAM) available. | `quantity` | `""` | +| `nodeGroups[name].gpus` | List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM). | `[]object` | `[]` | +| `nodeGroups[name].gpus[i].name` | Name of GPU, such as "nvidia.com/AD102GL_L40S". | `string` | `""` | +| `nodeGroups[name].kubelet` | Kubelet resource reservations for this node group. | `object` | `{}` | +| `nodeGroups[name].kubelet.systemReservedMemory` | Memory reserved for host OS. Auto-computed from instanceType if empty. | `string` | `""` | +| `nodeGroups[name].kubelet.kubeReservedMemory` | Memory reserved for kubelet and container runtime. Auto-computed from instanceType if empty. | `string` | `""` | +| `nodeGroups[name].kubelet.systemReservedCpu` | CPU reserved for host OS. Auto-computed from instanceType if empty. | `string` | `""` | +| `nodeGroups[name].kubelet.kubeReservedCpu` | CPU reserved for kubelet and container runtime. Auto-computed from instanceType if empty. | `string` | `""` | +| `nodeGroups[name].kubelet.evictionHardMemory` | Hard eviction threshold for memory (absolute like 200Mi or percentage like 7%). | `string` | `7%` | +| `nodeGroups[name].kubelet.evictionSoftMemory` | Soft eviction threshold for memory (absolute like 1Gi or percentage like 10%). | `string` | `10%` | +| `version` | Kubernetes major.minor version to deploy | `string` | `v1.35` | +| `host` | External hostname for Kubernetes cluster. Defaults to `.` if empty. | `string` | `""` | ### Cluster Addons diff --git a/packages/apps/kubernetes/templates/cluster.yaml b/packages/apps/kubernetes/templates/cluster.yaml index 0cc8d17554..fbc10a2f08 100644 --- a/packages/apps/kubernetes/templates/cluster.yaml +++ b/packages/apps/kubernetes/templates/cluster.yaml @@ -25,10 +25,16 @@ spec: node-role.kubernetes.io/{{ . }}: "" {{- end }} spec: - {{- with .group.instanceType }} + {{- /* Explicit resources take precedence over instanceType. When a node + group is sized by resources (both cpu and memory set), the + instancetype is omitted from the VM: KubeVirt rejects a + VirtualMachine that references an instancetype and also overrides + either domain.cpu or domain.memory. */}} + {{- $sizedByResources := and .group.resources .group.resources.cpu .group.resources.memory }} + {{- if and .group.instanceType (not $sizedByResources) }} instancetype: kind: VirtualMachineClusterInstancetype - name: {{ . }} + name: {{ .group.instanceType }} {{- end }} runStrategy: Always dataVolumeTemplates: @@ -238,9 +244,24 @@ metadata: name: {{ .Release.Name }} namespace: {{ .Release.Namespace }} {{- range $groupName, $group := .Values.nodeGroups }} +{{- /* instanceType and explicit resources are alternative sizing sources, and + explicit resources win. KubeVirt rejects a VirtualMachine that references + an instancetype while also overriding either domain.cpu or domain.memory, + so when a group is sized by resources (both cpu and memory set) the + instancetype is omitted from the VM (see the VM spec above) and the lookup + below is skipped — a stale instancetype name cannot fail an otherwise + valid, resource-sized group. Explicit resources are all-or-nothing: a + partial spec (only one of cpu/memory) would emit one domain override + alongside the instancetype, which KubeVirt rejects, so it fails here with + a clear render-time error instead of a late admission rejection. */}} +{{- $hasCpu := and $group.resources $group.resources.cpu }} +{{- $hasMem := and $group.resources $group.resources.memory }} +{{- if and $group.instanceType (or (and $hasCpu (not $hasMem)) (and $hasMem (not $hasCpu))) }} +{{- fail (printf "nodeGroup %s: set both resources.cpu and resources.memory, or neither" $groupName) }} +{{- end }} {{/* Resolve instanceType once per group — reused for kubelet reservations and capacity annotations. */}} {{- $instanceType := dict }} -{{- if $group.instanceType }} +{{- if and $group.instanceType (not (and $hasCpu $hasMem)) }} {{- $instanceType = (lookup "instancetype.kubevirt.io/v1beta1" "VirtualMachineClusterInstancetype" "" $group.instanceType) }} {{- if not $instanceType }} {{- fail (printf "nodeGroup %s: specified instanceType %q not found in cluster" $groupName $group.instanceType) }} diff --git a/packages/apps/kubernetes/tests/cluster_test.yaml b/packages/apps/kubernetes/tests/cluster_test.yaml index 73ffa5d480..52a49180f3 100644 --- a/packages/apps/kubernetes/tests/cluster_test.yaml +++ b/packages/apps/kubernetes/tests/cluster_test.yaml @@ -493,3 +493,143 @@ tests: path: spec.template.spec.virtualMachineTemplate.spec.template.metadata.labels["apps.cozystack.io/application.name"] value: foo documentIndex: 5 + + ############################################### + # resources is optional when instanceType set # + ############################################### + + # A node group may be sized either by instanceType or by explicit resources. + # When instanceType is the sizing source, resources may be omitted entirely. + # helm-unittest enforces values.schema.json, so the render below only succeeds + # if the schema does NOT require resources on each node group. + # + # instanceType is set to "" here only to bypass the live-cluster lookup() that + # validates a real VirtualMachineClusterInstancetype; the schema acceptance of + # an omitted resources key is independent of the instanceType value. + # + # The "worker" group sorts after "md0". On this release line each node group + # renders a KubeadmConfigTemplate before its KubevirtMachineTemplate, so the + # per-group document stride is 5 and worker's KubevirtMachineTemplate is + # document index 10 (md0 → 5, worker → 10). + + - it: accepts a node group that omits resources and renders no explicit cpu/memory + release: + name: test-k8s + namespace: tenant-test + set: + nodeGroups: + worker: + minReplicas: 1 + maxReplicas: 3 + instanceType: "" + diskSize: "20Gi" + roles: + - worker + asserts: + - isKind: + of: KubevirtMachineTemplate + documentIndex: 10 + - equal: + path: spec.template.spec.virtualMachineTemplate.spec.template.metadata.labels["cluster.x-k8s.io/deployment-name"] + value: test-k8s-worker + documentIndex: 10 + - notExists: + path: spec.template.spec.virtualMachineTemplate.spec.template.spec.domain.cpu + documentIndex: 10 + - notExists: + path: spec.template.spec.virtualMachineTemplate.spec.template.spec.domain.memory + documentIndex: 10 + + # Complementary case: when explicit resources ARE provided (md0 in common.yaml + # has cpu: 2 / memory: 4Gi), they take effect on the VM domain. Confirms the fix + # only relaxes the schema requirement and does not drop honoured resources. + - it: still renders explicit cpu and memory when resources are provided + release: + name: test-k8s + namespace: tenant-test + asserts: + - isKind: + of: KubevirtMachineTemplate + documentIndex: 5 + - equal: + path: spec.template.spec.virtualMachineTemplate.spec.template.spec.domain.cpu.cores + value: 2 + documentIndex: 5 + - equal: + path: spec.template.spec.virtualMachineTemplate.spec.template.spec.domain.memory.guest + value: 4Gi + documentIndex: 5 + + # instanceType and explicit resources are alternative sizing sources, and + # explicit resources take precedence. When both are set the VM is sized by the + # explicit resources and the instancetype is omitted, so KubeVirt does not + # reject a VM that both references an instancetype and overrides domain.cpu/memory. + # (md0 in common.yaml already carries cpu: 2 / memory: 4Gi; this only adds an + # instanceType on top to exercise the both-set case.) + - it: prefers explicit resources over instanceType when both are set + release: + name: test-k8s + namespace: tenant-test + set: + nodeGroups.md0.instanceType: "u1.medium" + asserts: + - isKind: + of: KubevirtMachineTemplate + documentIndex: 5 + - equal: + path: spec.template.spec.virtualMachineTemplate.spec.template.spec.domain.cpu.cores + value: 2 + documentIndex: 5 + - equal: + path: spec.template.spec.virtualMachineTemplate.spec.template.spec.domain.memory.guest + value: 4Gi + documentIndex: 5 + - notExists: + path: spec.template.spec.virtualMachineTemplate.spec.instancetype + documentIndex: 5 + + # Explicit resources are all-or-nothing when an instanceType is present. A + # partial spec (only cpu, or only memory) on a group that also carries a + # non-empty instanceType would emit one domain override alongside the + # instancetype, which KubeVirt rejects. Fail fast with a clear render-time + # error instead of shipping a VM the admission webhook rejects at apply time. + # (Fresh group names are used so the partial spec is real — a `set` on md0 + # would deep-merge with common.yaml's md0, which already carries both cpu and + # memory.) + - it: fails when only resources.cpu is set without resources.memory + release: + name: test-k8s + namespace: tenant-test + set: + nodeGroups: + mdcpuonly: + minReplicas: 0 + maxReplicas: 10 + instanceType: "u1.medium" + diskSize: 20Gi + roles: + - worker + resources: + cpu: 2 + asserts: + - failedTemplate: + errorMessage: "nodeGroup mdcpuonly: set both resources.cpu and resources.memory, or neither" + + - it: fails when only resources.memory is set without resources.cpu + release: + name: test-k8s + namespace: tenant-test + set: + nodeGroups: + mdmemonly: + minReplicas: 0 + maxReplicas: 10 + instanceType: "u1.medium" + diskSize: 20Gi + roles: + - worker + resources: + memory: 4Gi + asserts: + - failedTemplate: + errorMessage: "nodeGroup mdmemonly: set both resources.cpu and resources.memory, or neither" diff --git a/packages/apps/kubernetes/values.schema.json b/packages/apps/kubernetes/values.schema.json index e1492b787d..db71b285b5 100644 --- a/packages/apps/kubernetes/values.schema.json +++ b/packages/apps/kubernetes/values.schema.json @@ -40,8 +40,7 @@ "diskSize", "instanceType", "maxReplicas", - "minReplicas", - "resources" + "minReplicas" ], "properties": { "diskSize": { @@ -128,7 +127,7 @@ "default": 0 }, "resources": { - "description": "CPU and memory resources for each worker node.", + "description": "Explicit CPU and memory for each worker node, as an alternative to `instanceType` sizing. Optional: when omitted, the node is sized by `instanceType`. When both `cpu` and `memory` are set, they take precedence and `instanceType` is ignored for that node group (the instancetype is omitted from the VM, since KubeVirt cannot override an instancetype's CPU/memory). Set both `cpu` and `memory` together or neither; setting only one is rejected at render time.", "type": "object", "properties": { "cpu": { diff --git a/packages/apps/kubernetes/values.yaml b/packages/apps/kubernetes/values.yaml index 9c1405b550..d8aab8cd1f 100644 --- a/packages/apps/kubernetes/values.yaml +++ b/packages/apps/kubernetes/values.yaml @@ -85,7 +85,7 @@ storageClass: replicated ## @field {string} [storageClass] - StorageClass for worker node persistent disks. When empty, uses the management cluster default StorageClass (the one annotated storageclass.kubernetes.io/is-default-class: true). NOTE: deliberately not marked immutable — the field is optional and undefaulted, so a strict `self == oldSelf` rule would block any future attempt to set it on an existing node group. ## @x-cozystack-options {source: storageclass} ## @field {[]string} roles - List of node roles. -## @field {Resources} resources - CPU and memory resources for each worker node. +## @field {Resources} [resources] - Explicit CPU and memory for each worker node, as an alternative to `instanceType` sizing. Optional: when omitted, the node is sized by `instanceType`. When both `cpu` and `memory` are set, they take precedence and `instanceType` is ignored for that node group (the instancetype is omitted from the VM, since KubeVirt cannot override an instancetype's CPU/memory). Set both `cpu` and `memory` together or neither; setting only one is rejected at render time. ## @field {[]GPU} gpus - List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM). ## @field {Kubelet} [kubelet] - Kubelet resource reservations for this node group. diff --git a/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml b/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml index 9ce1798390..44d325d3dd 100644 --- a/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml +++ b/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml @@ -22,7 +22,7 @@ spec: singular: kubernetes plural: kuberneteses openAPISchema: |- - {"title":"Chart Values","type":"object","properties":{"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":"replicated","x-kubernetes-validations":[{"rule":"self == oldSelf","message":"storageClass is immutable"}],"x-cozystack-options":{"source":"storageclass"}},"nodeGroups":{"description":"Worker nodes configuration map.","type":"object","default":{"md0":{"diskSize":"20Gi","gpus":[],"instanceType":"u1.medium","kubelet":{},"maxReplicas":10,"minReplicas":0,"resources":{},"roles":["ingress-nginx"],"storageClass":""}},"additionalProperties":{"type":"object","required":["diskSize","instanceType","maxReplicas","minReplicas","resources"],"properties":{"diskSize":{"description":"Persistent disk size for kubelet and containerd data.","default":"20Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"gpus":{"description":"List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).","type":"array","items":{"type":"object","required":["name"],"properties":{"name":{"description":"Name of GPU, such as \"nvidia.com/AD102GL_L40S\".","type":"string","x-cozystack-options":{"source":"gpu"}}}}},"instanceType":{"description":"Virtual machine instance type.","type":"string","default":"u1.medium","x-cozystack-options":{"source":"instancetype"}},"kubelet":{"description":"Kubelet resource reservations for this node group.","type":"object","properties":{"evictionHardMemory":{"description":"Hard eviction threshold for memory (absolute like 200Mi or percentage like 7%).","type":"string","default":"7%"},"evictionSoftMemory":{"description":"Soft eviction threshold for memory (absolute like 1Gi or percentage like 10%).","type":"string","default":"10%"},"kubeReservedCpu":{"description":"CPU reserved for kubelet and container runtime. Auto-computed from instanceType if empty.","type":"string"},"kubeReservedMemory":{"description":"Memory reserved for kubelet and container runtime. Auto-computed from instanceType if empty.","type":"string"},"systemReservedCpu":{"description":"CPU reserved for host OS. Auto-computed from instanceType if empty.","type":"string"},"systemReservedMemory":{"description":"Memory reserved for host OS. Auto-computed from instanceType if empty.","type":"string"}}},"maxReplicas":{"description":"Maximum number of replicas.","type":"integer","default":10},"minReplicas":{"description":"Minimum number of replicas.","type":"integer","default":0},"resources":{"description":"CPU and memory resources for each worker node.","type":"object","properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"roles":{"description":"List of node roles.","type":"array","items":{"type":"string"}},"storageClass":{"description":"StorageClass for worker node persistent disks. When empty, uses the management cluster default StorageClass (the one annotated storageclass.kubernetes.io/is-default-class: true). NOTE: deliberately not marked immutable — the field is optional and undefaulted, so a strict `self == oldSelf` rule would block any future attempt to set it on an existing node group.","type":"string","x-cozystack-options":{"source":"storageclass"}}}}},"version":{"description":"Kubernetes major.minor version to deploy","type":"string","default":"v1.35","enum":["v1.35","v1.34","v1.33","v1.32","v1.31","v1.30"]},"host":{"description":"External hostname for Kubernetes cluster. Defaults to `.` if empty.","type":"string","default":""},"addons":{"description":"Cluster addons configuration.","type":"object","default":{},"required":["certManager","cilium","coredns","fluxcd","gatewayAPI","gpuOperator","hami","ingressNginx","monitoringAgents","ouroboros","velero","verticalPodAutoscaler"],"properties":{"certManager":{"description":"Cert-manager addon.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable cert-manager.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"cilium":{"description":"Cilium CNI plugin.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"coredns":{"description":"CoreDNS addon.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"fluxcd":{"description":"FluxCD GitOps operator.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable FluxCD.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"gatewayAPI":{"description":"Gateway API addon.","type":"object","default":{},"required":["enabled"],"properties":{"enabled":{"description":"Enable Gateway API.","type":"boolean","default":false}}},"gpuOperator":{"description":"NVIDIA GPU Operator.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable GPU Operator.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"hami":{"description":"HAMi GPU virtualization middleware.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable HAMi (requires GPU Operator).","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"ingressNginx":{"description":"Ingress-NGINX controller.","type":"object","default":{},"required":["enabled","exposeMethod","valuesOverride"],"properties":{"enabled":{"description":"Enable the controller (requires nodes labeled `ingress-nginx`).","type":"boolean","default":false},"exposeMethod":{"description":"Method to expose the controller. Allowed values: `Proxied`, `LoadBalancer`.","type":"string","default":"Proxied","enum":["Proxied","LoadBalancer"]},"hosts":{"description":"Domains routed to this tenant cluster when `exposeMethod` is `Proxied`.","type":"array","default":[],"items":{"type":"string"}},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"monitoringAgents":{"description":"Monitoring agents.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable monitoring agents.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"ouroboros":{"description":"Hairpin-NAT fix for ingress-nginx with PROXY-protocol.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable ouroboros. Requires addons.ingressNginx.enabled (chart-render fail otherwise). Only useful when PROXY-protocol is wired on the tenant ingress-nginx via valuesOverride.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides. Operator-key wins over cozystack defaults.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"velero":{"description":"Velero backup/restore addon.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable Velero.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"verticalPodAutoscaler":{"description":"Vertical Pod Autoscaler.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}}}},"controlPlane":{"description":"Kubernetes control-plane configuration.","type":"object","default":{},"required":["apiServer","controllerManager","konnectivity","replicas","scheduler"],"properties":{"apiServer":{"description":"API Server configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for API Server.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"c1.medium","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]}}},"controllerManager":{"description":"Controller Manager configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Controller Manager.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"t1.micro","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]}}},"konnectivity":{"description":"Konnectivity configuration.","type":"object","default":{},"required":["server"],"properties":{"server":{"description":"Konnectivity Server configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Konnectivity.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"t1.micro","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]}}}}},"replicas":{"description":"Number of control-plane replicas.","type":"integer","default":2},"scheduler":{"description":"Scheduler configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Scheduler.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"t1.micro","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]}}}}},"images":{"description":"Optional image overrides for air-gapped or rate-limited registries.","type":"object","default":{},"properties":{"waitForKubeconfig":{"description":"Image used by the wait-for-kubeconfig init container. Empty falls back to images/busybox.tag.","type":"string","default":""}}}}} + {"title":"Chart Values","type":"object","properties":{"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":"replicated","x-kubernetes-validations":[{"rule":"self == oldSelf","message":"storageClass is immutable"}],"x-cozystack-options":{"source":"storageclass"}},"nodeGroups":{"description":"Worker nodes configuration map.","type":"object","default":{"md0":{"diskSize":"20Gi","gpus":[],"instanceType":"u1.medium","kubelet":{},"maxReplicas":10,"minReplicas":0,"resources":{},"roles":["ingress-nginx"],"storageClass":""}},"additionalProperties":{"type":"object","required":["diskSize","instanceType","maxReplicas","minReplicas"],"properties":{"diskSize":{"description":"Persistent disk size for kubelet and containerd data.","default":"20Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"gpus":{"description":"List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).","type":"array","items":{"type":"object","required":["name"],"properties":{"name":{"description":"Name of GPU, such as \"nvidia.com/AD102GL_L40S\".","type":"string","x-cozystack-options":{"source":"gpu"}}}}},"instanceType":{"description":"Virtual machine instance type.","type":"string","default":"u1.medium","x-cozystack-options":{"source":"instancetype"}},"kubelet":{"description":"Kubelet resource reservations for this node group.","type":"object","properties":{"evictionHardMemory":{"description":"Hard eviction threshold for memory (absolute like 200Mi or percentage like 7%).","type":"string","default":"7%"},"evictionSoftMemory":{"description":"Soft eviction threshold for memory (absolute like 1Gi or percentage like 10%).","type":"string","default":"10%"},"kubeReservedCpu":{"description":"CPU reserved for kubelet and container runtime. Auto-computed from instanceType if empty.","type":"string"},"kubeReservedMemory":{"description":"Memory reserved for kubelet and container runtime. Auto-computed from instanceType if empty.","type":"string"},"systemReservedCpu":{"description":"CPU reserved for host OS. Auto-computed from instanceType if empty.","type":"string"},"systemReservedMemory":{"description":"Memory reserved for host OS. Auto-computed from instanceType if empty.","type":"string"}}},"maxReplicas":{"description":"Maximum number of replicas.","type":"integer","default":10},"minReplicas":{"description":"Minimum number of replicas.","type":"integer","default":0},"resources":{"description":"Explicit CPU and memory for each worker node, as an alternative to `instanceType` sizing. Optional: when omitted, the node is sized by `instanceType`. When both `cpu` and `memory` are set, they take precedence and `instanceType` is ignored for that node group (the instancetype is omitted from the VM, since KubeVirt cannot override an instancetype's CPU/memory). Set both `cpu` and `memory` together or neither; setting only one is rejected at render time.","type":"object","properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"roles":{"description":"List of node roles.","type":"array","items":{"type":"string"}},"storageClass":{"description":"StorageClass for worker node persistent disks. When empty, uses the management cluster default StorageClass (the one annotated storageclass.kubernetes.io/is-default-class: true). NOTE: deliberately not marked immutable — the field is optional and undefaulted, so a strict `self == oldSelf` rule would block any future attempt to set it on an existing node group.","type":"string","x-cozystack-options":{"source":"storageclass"}}}}},"version":{"description":"Kubernetes major.minor version to deploy","type":"string","default":"v1.35","enum":["v1.35","v1.34","v1.33","v1.32","v1.31","v1.30"]},"host":{"description":"External hostname for Kubernetes cluster. Defaults to `.` if empty.","type":"string","default":""},"addons":{"description":"Cluster addons configuration.","type":"object","default":{},"required":["certManager","cilium","coredns","fluxcd","gatewayAPI","gpuOperator","hami","ingressNginx","monitoringAgents","ouroboros","velero","verticalPodAutoscaler"],"properties":{"certManager":{"description":"Cert-manager addon.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable cert-manager.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"cilium":{"description":"Cilium CNI plugin.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"coredns":{"description":"CoreDNS addon.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"fluxcd":{"description":"FluxCD GitOps operator.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable FluxCD.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"gatewayAPI":{"description":"Gateway API addon.","type":"object","default":{},"required":["enabled"],"properties":{"enabled":{"description":"Enable Gateway API.","type":"boolean","default":false}}},"gpuOperator":{"description":"NVIDIA GPU Operator.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable GPU Operator.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"hami":{"description":"HAMi GPU virtualization middleware.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable HAMi (requires GPU Operator).","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"ingressNginx":{"description":"Ingress-NGINX controller.","type":"object","default":{},"required":["enabled","exposeMethod","valuesOverride"],"properties":{"enabled":{"description":"Enable the controller (requires nodes labeled `ingress-nginx`).","type":"boolean","default":false},"exposeMethod":{"description":"Method to expose the controller. Allowed values: `Proxied`, `LoadBalancer`.","type":"string","default":"Proxied","enum":["Proxied","LoadBalancer"]},"hosts":{"description":"Domains routed to this tenant cluster when `exposeMethod` is `Proxied`.","type":"array","default":[],"items":{"type":"string"}},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"monitoringAgents":{"description":"Monitoring agents.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable monitoring agents.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"ouroboros":{"description":"Hairpin-NAT fix for ingress-nginx with PROXY-protocol.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable ouroboros. Requires addons.ingressNginx.enabled (chart-render fail otherwise). Only useful when PROXY-protocol is wired on the tenant ingress-nginx via valuesOverride.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides. Operator-key wins over cozystack defaults.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"velero":{"description":"Velero backup/restore addon.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable Velero.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"verticalPodAutoscaler":{"description":"Vertical Pod Autoscaler.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}}}},"controlPlane":{"description":"Kubernetes control-plane configuration.","type":"object","default":{},"required":["apiServer","controllerManager","konnectivity","replicas","scheduler"],"properties":{"apiServer":{"description":"API Server configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for API Server.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"c1.medium","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]}}},"controllerManager":{"description":"Controller Manager configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Controller Manager.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"t1.micro","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]}}},"konnectivity":{"description":"Konnectivity configuration.","type":"object","default":{},"required":["server"],"properties":{"server":{"description":"Konnectivity Server configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Konnectivity.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"t1.micro","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]}}}}},"replicas":{"description":"Number of control-plane replicas.","type":"integer","default":2},"scheduler":{"description":"Scheduler configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Scheduler.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"t1.micro","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]}}}}},"images":{"description":"Optional image overrides for air-gapped or rate-limited registries.","type":"object","default":{},"properties":{"waitForKubeconfig":{"description":"Image used by the wait-for-kubeconfig init container. Empty falls back to images/busybox.tag.","type":"string","default":""}}}}} release: prefix: kubernetes- labels: From 6cca847421ec86a7501c1ed2593ca0595d321e98 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Tue, 30 Jun 2026 00:08:48 +0300 Subject: [PATCH 09/69] fix(tenant): gate VM-CR-creating HelmReleases on victoria-metrics-operator The tenant etcd, ingress and monitoring HelmReleases create VM* custom resources (VMPodScrape directly in etcd/ingress; VMCluster, VMAlert, VLCluster, VMAlertmanager and VMServiceScrape via the monitoring-system child of monitoring) guarded by the victoria-metrics-operator validating webhook with failurePolicy: Fail. On a cold install the operator pod can be briefly unavailable on :9443, so these releases reconcile before the webhook backend is serving and are rejected with connection refused. They carry install.remediation.retries: -1 and would eventually recover, but the failed-install -> rollback -> retry churn can outlast a cold-install readiness deadline and surfaces as spurious install failures. Add a Flux spec.dependsOn from each of these releases to the victoria-metrics-operator HelmRelease in cozy-victoria-metrics-operator, so they wait for the operator to be Ready before creating VM* resources. The system monitoring components gate on the operator the same way, via a variant-level dependsOn. The cross-shard reference (tenant releases are sharded, the operator release is not) resolves via helm-controller's uncached dependency check (APIReader, v1.1.0+). Assisted-By: Claude Signed-off-by: Aleksei Sviridkin (cherry picked from commit 6db5ae9c6fd82728f0492475cfbe41be9327356a) --- packages/apps/tenant/templates/etcd.yaml | 9 +++ packages/apps/tenant/templates/ingress.yaml | 9 +++ .../apps/tenant/templates/monitoring.yaml | 11 +++ .../tests/vmo_webhook_dependson_test.yaml | 80 +++++++++++++++++++ 4 files changed, 109 insertions(+) create mode 100644 packages/apps/tenant/tests/vmo_webhook_dependson_test.yaml diff --git a/packages/apps/tenant/templates/etcd.yaml b/packages/apps/tenant/templates/etcd.yaml index 71e0b548d6..38da293aa1 100644 --- a/packages/apps/tenant/templates/etcd.yaml +++ b/packages/apps/tenant/templates/etcd.yaml @@ -13,6 +13,15 @@ metadata: apps.cozystack.io/application.group: apps.cozystack.io apps.cozystack.io/application.name: etcd spec: + # This release creates a VMPodScrape resource guarded by the + # victoria-metrics-operator admission webhook (failurePolicy: Fail). On a cold + # install the operator can be briefly unavailable on :9443, so gate on its + # HelmRelease becoming Ready to avoid an install failure → rollback. The system + # monitoring components gate on the operator the same way, via a variant-level + # dependsOn in packages/core/platform/sources/monitoring.yaml. + dependsOn: + - name: victoria-metrics-operator + namespace: cozy-victoria-metrics-operator chartRef: kind: ExternalArtifact name: cozystack-etcd-application-default-etcd diff --git a/packages/apps/tenant/templates/ingress.yaml b/packages/apps/tenant/templates/ingress.yaml index 229ee7c62d..b5ed2288f2 100644 --- a/packages/apps/tenant/templates/ingress.yaml +++ b/packages/apps/tenant/templates/ingress.yaml @@ -13,6 +13,15 @@ metadata: apps.cozystack.io/application.group: apps.cozystack.io apps.cozystack.io/application.name: ingress spec: + # This release creates VMPodScrape resources guarded by the + # victoria-metrics-operator admission webhook (failurePolicy: Fail). On a cold + # install the operator can be briefly unavailable on :9443, so gate on its + # HelmRelease becoming Ready to avoid an install failure → rollback. The system + # monitoring components gate on the operator the same way, via a variant-level + # dependsOn in packages/core/platform/sources/monitoring.yaml. + dependsOn: + - name: victoria-metrics-operator + namespace: cozy-victoria-metrics-operator chartRef: kind: ExternalArtifact name: cozystack-ingress-application-default-ingress diff --git a/packages/apps/tenant/templates/monitoring.yaml b/packages/apps/tenant/templates/monitoring.yaml index 29dbf95ab8..a66b11d45e 100644 --- a/packages/apps/tenant/templates/monitoring.yaml +++ b/packages/apps/tenant/templates/monitoring.yaml @@ -13,6 +13,17 @@ metadata: apps.cozystack.io/application.group: apps.cozystack.io apps.cozystack.io/application.name: monitoring spec: + # This release (via its monitoring-system child) creates VM* custom resources + # guarded by the victoria-metrics-operator admission webhook (failurePolicy: + # Fail). On a cold install the operator can be briefly unavailable on :9443, so + # gate on its HelmRelease becoming Ready to avoid an install failure → rollback. + # Gating the parent keeps the monitoring-system child (which emits the VM* CRs) + # from being created until the operator is Ready. The system monitoring + # components gate on the operator the same way, via a variant-level dependsOn in + # packages/core/platform/sources/monitoring.yaml. + dependsOn: + - name: victoria-metrics-operator + namespace: cozy-victoria-metrics-operator chartRef: kind: ExternalArtifact name: cozystack-monitoring-application-default-monitoring diff --git a/packages/apps/tenant/tests/vmo_webhook_dependson_test.yaml b/packages/apps/tenant/tests/vmo_webhook_dependson_test.yaml new file mode 100644 index 0000000000..bd2ac1ddec --- /dev/null +++ b/packages/apps/tenant/tests/vmo_webhook_dependson_test.yaml @@ -0,0 +1,80 @@ +suite: tenant HelmReleases gate on the victoria-metrics-operator + +# The victoria-metrics-operator admission webhook is configured with +# failurePolicy: Fail and rejects the VM* custom resources these tenant +# releases create (etcd/ingress: VMPodScrape; monitoring via its +# monitoring-system child: VMCluster/VMAlert/VLCluster/VMAlertmanager/ +# VMServiceScrape) whenever the operator pod is not yet serving on :9443. +# On a cold install the operator can be briefly unavailable, so every tenant +# release that creates VM* resources must depend on the operator HelmRelease +# becoming Ready — otherwise the release reconciles too early, the webhook +# rejects with connection refused, the install fails and rolls back. Pin the +# dependsOn so the gate cannot regress. + +# Restrict rendering to the gated HelmRelease templates so the chart's +# namespace.yaml `lookup` (which errors under helm-unittest) is not evaluated. +templates: + - templates/etcd.yaml + - templates/ingress.yaml + - templates/monitoring.yaml + - templates/seaweedfs.yaml + - templates/gateway.yaml + +release: + name: tenant-test + namespace: tenant-test + +set: + _cluster: + expose-ingress: tenant-root + expose-external-ips: "192.0.2.10" + gateway-enabled: "true" + etcd: true + ingress: true + monitoring: true + seaweedfs: true + gateway: true + host: "" + +tests: + - it: etcd HR depends on the victoria-metrics-operator + asserts: + - template: templates/etcd.yaml + equal: + path: spec.dependsOn + value: + - name: victoria-metrics-operator + namespace: cozy-victoria-metrics-operator + + - it: ingress HR depends on the victoria-metrics-operator + asserts: + - template: templates/ingress.yaml + equal: + path: spec.dependsOn + value: + - name: victoria-metrics-operator + namespace: cozy-victoria-metrics-operator + + - it: monitoring HR depends on the victoria-metrics-operator + asserts: + - template: templates/monitoring.yaml + equal: + path: spec.dependsOn + value: + - name: victoria-metrics-operator + namespace: cozy-victoria-metrics-operator + + # The gate is intentionally narrow: only the releases that create VM* custom + # resources carry it. Tenant releases that create none (seaweedfs, gateway) + # must not be needlessly blocked on the operator. + - it: seaweedfs HR carries no operator gate + asserts: + - template: templates/seaweedfs.yaml + notExists: + path: spec.dependsOn + + - it: gateway HR carries no operator gate + asserts: + - template: templates/gateway.yaml + notExists: + path: spec.dependsOn From b5a09cd14b1d7a5aae0d23332925c715d998a664 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D0=BB=D0=B5=D0=BA=D1=81=D0=B5=D0=B9=20=D0=90=D1=80?= =?UTF-8?q?=D1=82=D0=B0=D0=BC=D0=BE=D0=BD=D0=BE=D0=B2?= Date: Thu, 2 Jul 2026 15:48:29 +0300 Subject: [PATCH 10/69] fix(mariadb): re-guard replication so single-replica passes the operator webhook MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR #2279 made `replication.enabled: true` unconditional so the operator would always create -primary/-secondary services. But the mariadb-operator validating webhook (vmariadb-v1alpha1.kb.io) rejects replication when spec.replicas == 1, so single-replica MariaDB could no longer be created — the HelmRelease never became Ready. Re-guard replication on replicas>1 and teach the service-name consumers about the bare service the operator provisions in both topologies: - mariadb.yaml: gate `replication` on replicas>1; restore single-replica external access via `service.type: LoadBalancer` and keep `primaryService` LoadBalancer for replicas>1 only. - backup-cronjob.yaml: route single-replica dumps to the bare service (no -secondary exists without replication). - dashboard-resourcemap.yaml: always grant RBAC on the bare service, plus -primary/-secondary when replicas>1, matching the ApplicationDefinition. - mariadb-rd: list the bare service in services.include so single-replica installs are visible in the dashboard. Adds a single-replica e2e test that reproduces the webhook rejection and guards against regression. Assisted-By: Claude Signed-off-by: Алексей Артамонов (cherry picked from commit c7e8f4d975f349c2fb0c6ae6dab06c9d6daa1745) --- hack/e2e-apps/mariadb.bats | 59 +++++++++++++++++++ .../mariadb/templates/backup-cronjob.yaml | 4 +- .../templates/dashboard-resourcemap.yaml | 6 ++ packages/apps/mariadb/templates/mariadb.yaml | 7 ++- .../system/mariadb-rd/cozyrds/mariadb.yaml | 1 + 5 files changed, 75 insertions(+), 2 deletions(-) diff --git a/hack/e2e-apps/mariadb.bats b/hack/e2e-apps/mariadb.bats index 0c1e3a5e86..74db318a9d 100644 --- a/hack/e2e-apps/mariadb.bats +++ b/hack/e2e-apps/mariadb.bats @@ -49,3 +49,62 @@ EOF kubectl -n tenant-test wait deployment.apps/mariadb-$name-metrics --timeout=90s --for=jsonpath='{.status.replicas}'=1 kubectl -n tenant-test delete mariadbs.apps.cozystack.io $name } + +@test "Create single-replica MariaDB" { + name='single' + kubectl -n tenant-test delete mariadbs.apps.cozystack.io $name --ignore-not-found --timeout=2m + kubectl apply -f- </dev/null 2>&1; do sleep 2; done" + # A single-replica MariaDB must be accepted by the operator's validating + # webhook: replication is guarded on replicas>1, so replicas=1 renders a CR + # the webhook admits. Dump the HR + rendered CR on failure so a webhook + # rejection (the regression this guards against) is legible in the log. + kubectl -n tenant-test wait hr mariadb-$name --timeout=5m --for=condition=ready \ + || { echo "HR mariadb-$name not ready — dumping state:"; \ + kubectl -n tenant-test describe hr mariadb-$name; \ + kubectl -n tenant-test get mariadbs.k8s.mariadb.com $name -o yaml; false; } + # With replicas=1 the operator provisions the bare service (no + # -primary/-secondary): assert it exists and has an endpoint. + timeout 80 sh -ec "until kubectl -n tenant-test get svc mariadb-$name -o jsonpath='{.spec.ports[0].port}' | grep -q '3306'; do sleep 10; done" + timeout 80 sh -ec "until kubectl -n tenant-test get endpoints mariadb-$name -o jsonpath='{.subsets[*].addresses[*].ip}' | grep -q '[0-9]'; do sleep 10; done" + timeout 60 sh -ec "until kubectl -n tenant-test get statefulset.apps/mariadb-$name >/dev/null 2>&1; do sleep 2; done" + kubectl -n tenant-test wait statefulset.apps/mariadb-$name --timeout=110s --for=jsonpath='{.status.replicas}'=1 + timeout 80 sh -ec "until kubectl -n tenant-test get svc mariadb-$name-metrics -o jsonpath='{.spec.ports[0].port}' | grep -q '9104'; do sleep 10; done" + timeout 40 sh -ec "until kubectl -n tenant-test get endpoints mariadb-$name-metrics -o jsonpath='{.subsets[*].addresses[*].ip}' | grep -q '[0-9]'; do sleep 10; done" + timeout 60 sh -ec "until kubectl -n tenant-test get deployment.apps/mariadb-$name-metrics >/dev/null 2>&1; do sleep 2; done" + kubectl -n tenant-test wait deployment.apps/mariadb-$name-metrics --timeout=90s --for=jsonpath='{.status.replicas}'=1 + kubectl -n tenant-test delete mariadbs.apps.cozystack.io $name --ignore-not-found +} diff --git a/packages/apps/mariadb/templates/backup-cronjob.yaml b/packages/apps/mariadb/templates/backup-cronjob.yaml index ddb237cddd..9316e37a94 100644 --- a/packages/apps/mariadb/templates/backup-cronjob.yaml +++ b/packages/apps/mariadb/templates/backup-cronjob.yaml @@ -41,7 +41,9 @@ spec: name: {{ .Release.Name }} key: root-password - name: MYSQL_HOST - value: "{{ .Release.Name }}-{{ if eq (int .Values.replicas) 1 }}primary{{ else }}secondary{{ end }}" + # replicas>1 enables replication (offload dumps to a secondary); + # replicas=1 has no replication and only the bare service exists. + value: "{{ .Release.Name }}{{- if gt (int .Values.replicas) 1 }}-secondary{{- end }}" - name: AWS_ACCESS_KEY_ID valueFrom: secretKeyRef: diff --git a/packages/apps/mariadb/templates/dashboard-resourcemap.yaml b/packages/apps/mariadb/templates/dashboard-resourcemap.yaml index af89b39c03..a7c5731ea6 100644 --- a/packages/apps/mariadb/templates/dashboard-resourcemap.yaml +++ b/packages/apps/mariadb/templates/dashboard-resourcemap.yaml @@ -8,8 +8,14 @@ rules: resources: - services resourceNames: + # The bare service exists in both topologies; -primary/-secondary + # only when replication is on (replicas>1). Grant exactly what exists so the + # RBAC matches the services the ApplicationDefinition (mariadb-rd) lists. + - {{ .Release.Name }} + {{- if gt (int .Values.replicas) 1 }} - {{ .Release.Name }}-primary - {{ .Release.Name }}-secondary + {{- end }} verbs: ["get", "list", "watch"] - apiGroups: - "" diff --git a/packages/apps/mariadb/templates/mariadb.yaml b/packages/apps/mariadb/templates/mariadb.yaml index d3d5e6367b..8a530134cd 100644 --- a/packages/apps/mariadb/templates/mariadb.yaml +++ b/packages/apps/mariadb/templates/mariadb.yaml @@ -29,11 +29,13 @@ spec: - {{ .Release.Name }} topologyKey: "kubernetes.io/hostname" + {{- if gt (int .Values.replicas) 1 }} replication: enabled: true #primary: # podIndex: 0 # automaticFailover: true + {{- end }} podMetadata: labels: @@ -63,6 +65,9 @@ spec: metadata: labels: app.kubernetes.io/instance: {{ $.Release.Name }} + {{- if and .Values.external (eq (int .Values.replicas) 1) }} + type: LoadBalancer + {{- end }} storage: size: {{ .Values.size }} resizeInUseVolumes: true @@ -71,7 +76,7 @@ spec: storageClassName: {{ . }} {{- end }} - {{- if .Values.external }} + {{- if and .Values.external (gt (int .Values.replicas) 1) }} primaryService: type: LoadBalancer {{- end }} diff --git a/packages/system/mariadb-rd/cozyrds/mariadb.yaml b/packages/system/mariadb-rd/cozyrds/mariadb.yaml index ab1249b2fe..122814052a 100644 --- a/packages/system/mariadb-rd/cozyrds/mariadb.yaml +++ b/packages/system/mariadb-rd/cozyrds/mariadb.yaml @@ -35,5 +35,6 @@ spec: exclude: [] include: - resourceNames: + - mariadb-{{ .name }} - mariadb-{{ .name }}-primary - mariadb-{{ .name }}-secondary From e115e62fb14b2ceb8720e64e980c83517ee86fdd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D0=BB=D0=B5=D0=BA=D1=81=D0=B5=D0=B9=20=D0=90=D1=80?= =?UTF-8?q?=D1=82=D0=B0=D0=BC=D0=BE=D0=BD=D0=BE=D0=B2?= Date: Thu, 2 Jul 2026 17:47:01 +0300 Subject: [PATCH 11/69] test(mariadb): fix operator MariaDB CR name in single-replica failure dump MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The failure-path dump queried mariadbs.k8s.mariadb.com with $name, but the operator CR is named after the Helm release ({{ .Release.Name }} = mariadb-$name), so the dump would hit NotFound instead of showing the rejected CR. Assisted-By: Claude Signed-off-by: Алексей Артамонов (cherry picked from commit f8e4222b7d7d42eac2ae2a39ef33da46c981b4ae) --- hack/e2e-apps/mariadb.bats | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/e2e-apps/mariadb.bats b/hack/e2e-apps/mariadb.bats index 74db318a9d..e099018b10 100644 --- a/hack/e2e-apps/mariadb.bats +++ b/hack/e2e-apps/mariadb.bats @@ -95,7 +95,7 @@ EOF kubectl -n tenant-test wait hr mariadb-$name --timeout=5m --for=condition=ready \ || { echo "HR mariadb-$name not ready — dumping state:"; \ kubectl -n tenant-test describe hr mariadb-$name; \ - kubectl -n tenant-test get mariadbs.k8s.mariadb.com $name -o yaml; false; } + kubectl -n tenant-test get mariadbs.k8s.mariadb.com mariadb-$name -o yaml; false; } # With replicas=1 the operator provisions the bare service (no # -primary/-secondary): assert it exists and has an endpoint. timeout 80 sh -ec "until kubectl -n tenant-test get svc mariadb-$name -o jsonpath='{.spec.ports[0].port}' | grep -q '3306'; do sleep 10; done" From 5378a531574824282b1b9b5661f3e61fa7a0f809 Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Mon, 29 Jun 2026 16:03:15 +0500 Subject: [PATCH 12/69] fix(kubernetes): gate vmop cert-manager-crds dependency on certManager.enabled The victoria-metrics-operator HelmRelease is gated on monitoringAgents.enabled but listed cert-manager-crds in dependsOn unconditionally, while the cert-manager-crds HelmRelease is only created when certManager.enabled is true. The valid combination monitoringAgents.enabled=true + certManager.enabled=false therefore left vmop blocked on a non-existent dependency ("...cert-manager-crds ... not found"), cascading to monitoring-agents and vertical-pod-autoscaler. Gate the dependency on the same condition that creates it. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Myasnikov Daniil (cherry picked from commit 2b1d3e904df40b2c09dbe5cfa217606187dc512c) --- .../victoria-metrics-operator.yaml | 2 + .../tests/victoria-metrics-operator_test.yaml | 77 +++++++++++++++++++ 2 files changed, 79 insertions(+) create mode 100644 packages/apps/kubernetes/tests/victoria-metrics-operator_test.yaml diff --git a/packages/apps/kubernetes/templates/helmreleases/victoria-metrics-operator.yaml b/packages/apps/kubernetes/templates/helmreleases/victoria-metrics-operator.yaml index d6c068ca65..476e470d04 100644 --- a/packages/apps/kubernetes/templates/helmreleases/victoria-metrics-operator.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/victoria-metrics-operator.yaml @@ -31,6 +31,8 @@ spec: dependsOn: - name: {{ .Release.Name }}-cilium namespace: {{ .Release.Namespace }} + {{- if .Values.addons.certManager.enabled }} - name: {{ .Release.Name }}-cert-manager-crds namespace: {{ .Release.Namespace }} + {{- end }} {{- end }} diff --git a/packages/apps/kubernetes/tests/victoria-metrics-operator_test.yaml b/packages/apps/kubernetes/tests/victoria-metrics-operator_test.yaml new file mode 100644 index 0000000000..d62a995efc --- /dev/null +++ b/packages/apps/kubernetes/tests/victoria-metrics-operator_test.yaml @@ -0,0 +1,77 @@ +suite: victoria-metrics-operator.yaml cert-manager-crds dependency gating + +templates: + - templates/helmreleases/victoria-metrics-operator.yaml + +values: + - values/common.yaml + +# The vmop HelmRelease is gated on monitoringAgents.enabled, but the +# cert-manager-crds HelmRelease it can depend on is only created when +# certManager.enabled is true. The dependsOn entry must therefore be gated on +# the same condition, otherwise the valid combination +# monitoringAgents.enabled=true + certManager.enabled=false leaves vmop blocked +# on a non-existent dependency. + +tests: + - it: renders the vmop HelmRelease when monitoringAgents is enabled + release: + name: kubernetes-test + namespace: tenant-test + set: + addons.monitoringAgents.enabled: true + asserts: + - hasDocuments: + count: 1 + - isKind: + of: HelmRelease + - equal: + path: metadata.name + value: kubernetes-test-cozy-victoria-metrics-operator + + - it: always depends on cilium + release: + name: kubernetes-test + namespace: tenant-test + set: + addons.monitoringAgents.enabled: true + asserts: + - contains: + path: spec.dependsOn + content: + name: kubernetes-test-cilium + namespace: tenant-test + + - it: does NOT depend on cert-manager-crds when certManager is disabled + release: + name: kubernetes-test + namespace: tenant-test + set: + addons.monitoringAgents.enabled: true + addons.certManager.enabled: false + asserts: + - notContains: + path: spec.dependsOn + content: + name: kubernetes-test-cert-manager-crds + namespace: tenant-test + - lengthEqual: + path: spec.dependsOn + count: 1 + + - it: depends on cert-manager-crds when certManager is enabled + release: + name: kubernetes-test + namespace: tenant-test + set: + addons.monitoringAgents.enabled: true + addons.certManager.enabled: true + asserts: + - contains: + path: spec.dependsOn + content: + name: kubernetes-test-cert-manager-crds + namespace: tenant-test + - lengthEqual: + path: spec.dependsOn + count: 2 From a114736fd3b7f3d5def0b331c00d568dac9bc3b1 Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Tue, 23 Jun 2026 17:18:30 +0500 Subject: [PATCH 13/69] fix(kamaji): backport datastore unused-deletion deadlock fix (clastix/kamaji#1122) Deleting a Tenant whose etcd DataStore has 0 referencing TenantControlPlanes left the DataStore stuck Terminating (finalizer kamaji.clastix.io/TenantControlPlane), hanging the etcd helm uninstall and wedging the tenant namespace. Backports clastix/kamaji#1122 (fixes clastix/kamaji#1115) as a source patch on the pinned 26.3.6-edge build. Refs #3018. Signed-off-by: Myasnikov Daniil (cherry picked from commit 47638f8569dc231f2b0040e13b0f9419830ff065) --- .../patches/fix-datastore-unused-deletion.diff | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 packages/system/kamaji/images/kamaji/patches/fix-datastore-unused-deletion.diff diff --git a/packages/system/kamaji/images/kamaji/patches/fix-datastore-unused-deletion.diff b/packages/system/kamaji/images/kamaji/patches/fix-datastore-unused-deletion.diff new file mode 100644 index 0000000000..53073699b1 --- /dev/null +++ b/packages/system/kamaji/images/kamaji/patches/fix-datastore-unused-deletion.diff @@ -0,0 +1,12 @@ +diff --git a/controllers/datastore_controller.go b/controllers/datastore_controller.go +--- a/controllers/datastore_controller.go ++++ b/controllers/datastore_controller.go +@@ -151,7 +151,7 @@ func (r *DataStore) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { + return reconcile.Result{}, nil + } + +- if meta.IsStatusConditionFalse(ds.Status.Conditions, kamajiv1alpha1.DataStoreConditionAllowedDeletionType) { ++ if meta.IsStatusConditionFalse(ds.Status.Conditions, kamajiv1alpha1.DataStoreConditionAllowedDeletionType) || len(tcpList.Items) == 0 { + logger.Info("DataStore is not used by any TenantControlPlane object") + + meta.SetStatusCondition(&ds.Status.Conditions, metav1.Condition{ From 7cc5df5139fb1335f11d119876d154c29539936f Mon Sep 17 00:00:00 2001 From: "cozystack-ci[bot]" <274107086+cozystack-ci[bot]@users.noreply.github.com> Date: Fri, 3 Jul 2026 10:12:24 +0000 Subject: [PATCH 14/69] Prepare release v1.5.2 Signed-off-by: cozystack-ci[bot] <274107086+cozystack-ci[bot]@users.noreply.github.com> --- .../apps/clickhouse/images/altinity-clickhouse-backup.tag | 2 +- packages/apps/clickhouse/images/clickhouse-backup.tag | 2 +- packages/apps/http-cache/images/nginx-cache.tag | 2 +- packages/apps/kubernetes/images/cluster-autoscaler.tag | 2 +- packages/apps/kubernetes/images/kubevirt-cloud-provider.tag | 2 +- packages/apps/kubernetes/images/kubevirt-csi-driver.tag | 2 +- .../apps/kubernetes/images/ubuntu-container-disk-v1.30.tag | 2 +- .../apps/kubernetes/images/ubuntu-container-disk-v1.31.tag | 2 +- .../apps/kubernetes/images/ubuntu-container-disk-v1.32.tag | 2 +- .../apps/kubernetes/images/ubuntu-container-disk-v1.33.tag | 2 +- .../apps/kubernetes/images/ubuntu-container-disk-v1.34.tag | 2 +- .../apps/kubernetes/images/ubuntu-container-disk-v1.35.tag | 2 +- packages/apps/mariadb/images/mariadb-backup.tag | 2 +- packages/core/installer/values.yaml | 4 ++-- packages/core/platform/values.yaml | 2 +- packages/core/testing/values.yaml | 2 +- packages/extra/bootbox/images/matchbox.tag | 2 +- packages/extra/seaweedfs/images/objectstorage-sidecar.tag | 2 +- packages/system/backup-controller/values.yaml | 2 +- packages/system/backupstrategy-controller/values.yaml | 2 +- packages/system/bucket/images/s3manager.tag | 2 +- packages/system/cilium/values.yaml | 2 +- packages/system/cozystack-api/values.yaml | 2 +- packages/system/cozystack-controller/values.yaml | 2 +- packages/system/dashboard/values.yaml | 4 ++-- packages/system/flux-shard-operator/values.yaml | 2 +- .../system/grafana-operator/images/grafana-dashboards.tag | 2 +- packages/system/kamaji/values.yaml | 4 ++-- packages/system/kubeovn-plunger/values.yaml | 2 +- packages/system/kubeovn-webhook/values.yaml | 2 +- packages/system/kubevirt-csi-node/values.yaml | 2 +- packages/system/lineage-controller-webhook/values.yaml | 2 +- packages/system/linstor-gui/values.yaml | 2 +- packages/system/linstor/values.yaml | 4 ++-- packages/system/metallb/values.yaml | 4 ++-- packages/system/monitoring/images/grafana.tag | 2 +- packages/system/multus/templates/multus-daemonset-thick.yml | 4 ++-- packages/system/objectstorage-controller/values.yaml | 2 +- packages/system/seaweedfs/values.yaml | 2 +- 39 files changed, 45 insertions(+), 45 deletions(-) diff --git a/packages/apps/clickhouse/images/altinity-clickhouse-backup.tag b/packages/apps/clickhouse/images/altinity-clickhouse-backup.tag index d183e11dd1..15cb7286de 100644 --- a/packages/apps/clickhouse/images/altinity-clickhouse-backup.tag +++ b/packages/apps/clickhouse/images/altinity-clickhouse-backup.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/altinity-clickhouse-backup:v1.5.1@sha256:65d15779d062680ddb69210c1dfaf8920ea4be2beb069d028bcb5934870a4d72 +ghcr.io/cozystack/cozystack/altinity-clickhouse-backup:v1.5.2@sha256:65d15779d062680ddb69210c1dfaf8920ea4be2beb069d028bcb5934870a4d72 diff --git a/packages/apps/clickhouse/images/clickhouse-backup.tag b/packages/apps/clickhouse/images/clickhouse-backup.tag index 4c59cafde7..1190edad5f 100644 --- a/packages/apps/clickhouse/images/clickhouse-backup.tag +++ b/packages/apps/clickhouse/images/clickhouse-backup.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/clickhouse-backup:v1.5.1@sha256:8efef62d59fb82d39544823f81a797c2b9ac130a50241fee48e701161ce2670d +ghcr.io/cozystack/cozystack/clickhouse-backup:v1.5.2@sha256:8efef62d59fb82d39544823f81a797c2b9ac130a50241fee48e701161ce2670d diff --git a/packages/apps/http-cache/images/nginx-cache.tag b/packages/apps/http-cache/images/nginx-cache.tag index ee7c322612..6d34d56a48 100644 --- a/packages/apps/http-cache/images/nginx-cache.tag +++ b/packages/apps/http-cache/images/nginx-cache.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/nginx-cache:v1.5.1@sha256:6468799a55dacc6026198db15fa0c8add516cc481e89c47d107427b0f2ea1626 +ghcr.io/cozystack/cozystack/nginx-cache:v1.5.2@sha256:479e1bb78cb7e0c2b9bf496d6ad1207dfceacbba62c028171c3f2548d1d74c5c diff --git a/packages/apps/kubernetes/images/cluster-autoscaler.tag b/packages/apps/kubernetes/images/cluster-autoscaler.tag index bd6ceb49d7..75c24bf9c9 100644 --- a/packages/apps/kubernetes/images/cluster-autoscaler.tag +++ b/packages/apps/kubernetes/images/cluster-autoscaler.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/cluster-autoscaler:v1.5.1@sha256:1767a51cf78d6700cb0b39ea1562b1443421afa416cb6d85e724225499859616 +ghcr.io/cozystack/cozystack/cluster-autoscaler:v1.5.2@sha256:1767a51cf78d6700cb0b39ea1562b1443421afa416cb6d85e724225499859616 diff --git a/packages/apps/kubernetes/images/kubevirt-cloud-provider.tag b/packages/apps/kubernetes/images/kubevirt-cloud-provider.tag index 1f46fbc300..ff00fb596c 100644 --- a/packages/apps/kubernetes/images/kubevirt-cloud-provider.tag +++ b/packages/apps/kubernetes/images/kubevirt-cloud-provider.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/kubevirt-cloud-provider:v1.5.1@sha256:c7224442a074a6a5e7af7ddec14be6cabf8ae9dd2569bfa38794a0dce9188ef9 +ghcr.io/cozystack/cozystack/kubevirt-cloud-provider:v1.5.2@sha256:c7224442a074a6a5e7af7ddec14be6cabf8ae9dd2569bfa38794a0dce9188ef9 diff --git a/packages/apps/kubernetes/images/kubevirt-csi-driver.tag b/packages/apps/kubernetes/images/kubevirt-csi-driver.tag index 3f3fc4c683..63a9710981 100644 --- a/packages/apps/kubernetes/images/kubevirt-csi-driver.tag +++ b/packages/apps/kubernetes/images/kubevirt-csi-driver.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/kubevirt-csi-driver:v1.5.1@sha256:d01f1c0377ce00c8bb46f1576fa9348da3b226679130bf7b46a8bc50d684c38d +ghcr.io/cozystack/cozystack/kubevirt-csi-driver:v1.5.2@sha256:808553aa64adf49c7c555a05d16f68ba5fb68fbd266e27eeff497e4d97d6dea1 diff --git a/packages/apps/kubernetes/images/ubuntu-container-disk-v1.30.tag b/packages/apps/kubernetes/images/ubuntu-container-disk-v1.30.tag index 1bc5ea9a8d..3e16716187 100644 --- a/packages/apps/kubernetes/images/ubuntu-container-disk-v1.30.tag +++ b/packages/apps/kubernetes/images/ubuntu-container-disk-v1.30.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/ubuntu-container-disk:v1.30-v1.5.1@sha256:98b80b2d18cb1384c96b95e1c8b869618f1a6e21962dcc1ab2dc874e5dff71dd +ghcr.io/cozystack/cozystack/ubuntu-container-disk:v1.30-v1.5.2@sha256:dc2e4794e75b861bf087e93037a2522fec398ffd8ac2b69591c4b0a50260f431 diff --git a/packages/apps/kubernetes/images/ubuntu-container-disk-v1.31.tag b/packages/apps/kubernetes/images/ubuntu-container-disk-v1.31.tag index a44c35d429..31ef9a6ef6 100644 --- a/packages/apps/kubernetes/images/ubuntu-container-disk-v1.31.tag +++ b/packages/apps/kubernetes/images/ubuntu-container-disk-v1.31.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/ubuntu-container-disk:v1.31-v1.5.1@sha256:dd0094f5b43e00538ca236110f89792cb5aafa234626518f3bdae353b0060f70 +ghcr.io/cozystack/cozystack/ubuntu-container-disk:v1.31-v1.5.2@sha256:4e3479b4f469581d87574c72246995d1df6262be42e60022084aca658ead8755 diff --git a/packages/apps/kubernetes/images/ubuntu-container-disk-v1.32.tag b/packages/apps/kubernetes/images/ubuntu-container-disk-v1.32.tag index a028858058..e8ac0ea62d 100644 --- a/packages/apps/kubernetes/images/ubuntu-container-disk-v1.32.tag +++ b/packages/apps/kubernetes/images/ubuntu-container-disk-v1.32.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/ubuntu-container-disk:v1.32-v1.5.1@sha256:99f400892d464afc66289c7ac7f9db3e50ef664376dba262fd181252498622a9 +ghcr.io/cozystack/cozystack/ubuntu-container-disk:v1.32-v1.5.2@sha256:79df2a0da2b1c13eee44f32793e4ca726bf4987935cecc34d204898d58cbbce1 diff --git a/packages/apps/kubernetes/images/ubuntu-container-disk-v1.33.tag b/packages/apps/kubernetes/images/ubuntu-container-disk-v1.33.tag index 10673b8bd1..4fefbb977c 100644 --- a/packages/apps/kubernetes/images/ubuntu-container-disk-v1.33.tag +++ b/packages/apps/kubernetes/images/ubuntu-container-disk-v1.33.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/ubuntu-container-disk:v1.33-v1.5.1@sha256:57ca7239396346ff55293588f8f3b10c11a8e8bd0a9f0198c05343582db6c184 +ghcr.io/cozystack/cozystack/ubuntu-container-disk:v1.33-v1.5.2@sha256:200d41bfb8108d43e1e938df47175dd4dbaae23d48e6d1c20ae6c6432213eeee diff --git a/packages/apps/kubernetes/images/ubuntu-container-disk-v1.34.tag b/packages/apps/kubernetes/images/ubuntu-container-disk-v1.34.tag index 3ded9e92ca..ea724cd7f6 100644 --- a/packages/apps/kubernetes/images/ubuntu-container-disk-v1.34.tag +++ b/packages/apps/kubernetes/images/ubuntu-container-disk-v1.34.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/ubuntu-container-disk:v1.34-v1.5.1@sha256:1f0af945d8c42af3bef9e17149e1530e3afc4d95ff79434dc81ba0fe729d7574 +ghcr.io/cozystack/cozystack/ubuntu-container-disk:v1.34-v1.5.2@sha256:21c0112abe14caba7a001b542d587ec1dab477e5f0fe65dba4383cac331c206c diff --git a/packages/apps/kubernetes/images/ubuntu-container-disk-v1.35.tag b/packages/apps/kubernetes/images/ubuntu-container-disk-v1.35.tag index d9e7c51932..fabef3b996 100644 --- a/packages/apps/kubernetes/images/ubuntu-container-disk-v1.35.tag +++ b/packages/apps/kubernetes/images/ubuntu-container-disk-v1.35.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/ubuntu-container-disk:v1.35-v1.5.1@sha256:aaf3993143e6f1f60a5e758dcb8e134a2c56c0bef3b44763dd097a5134b12cf2 +ghcr.io/cozystack/cozystack/ubuntu-container-disk:v1.35-v1.5.2@sha256:f90a024f2e2b4ef1b3ea653f5f47699218114e41bbd9b7c46e7c0ed00558aebb diff --git a/packages/apps/mariadb/images/mariadb-backup.tag b/packages/apps/mariadb/images/mariadb-backup.tag index 4b8eb06e58..ade215ad6e 100644 --- a/packages/apps/mariadb/images/mariadb-backup.tag +++ b/packages/apps/mariadb/images/mariadb-backup.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/mariadb-backup:v1.5.1@sha256:3cdaed3f19531586756902516feb099a5bece89c2c6685754d945486f0e99999 +ghcr.io/cozystack/cozystack/mariadb-backup:v1.5.2@sha256:3cdaed3f19531586756902516feb099a5bece89c2c6685754d945486f0e99999 diff --git a/packages/core/installer/values.yaml b/packages/core/installer/values.yaml index 0615a6359e..f99b06af7a 100644 --- a/packages/core/installer/values.yaml +++ b/packages/core/installer/values.yaml @@ -7,9 +7,9 @@ bareNamespace: false cozystackOperator: # Deployment variant: talos, generic, hosted variant: talos - image: ghcr.io/cozystack/cozystack/cozystack-operator:v1.5.1@sha256:8f0a898c1e1f75b432b8224bcedf1fdb17c8961e7da64b2919ae24fb4393c883 + image: ghcr.io/cozystack/cozystack/cozystack-operator:v1.5.2@sha256:43fe115a1d55b80d1b56b8e9c589a8dc34a6d454111c91b3b161c7704ea23358 platformSourceUrl: 'oci://ghcr.io/cozystack/cozystack/cozystack-packages' - platformSourceRef: 'digest=sha256:978596eada21f88a78e5654bfa9a6da97937242e69259a64dc0e85f0b766d980' + platformSourceRef: 'digest=sha256:b2d3fc7a706b6e755d98385dcfc514945610166c4f5e61ba8cdc1a1acd8188e9' # When non-empty, overrides the operator's --helmrelease-interval flag # (operator default: 5m). E2E sets this to 30s; production should leave empty. helmReleaseInterval: "" diff --git a/packages/core/platform/values.yaml b/packages/core/platform/values.yaml index e31130b81d..47598fe319 100644 --- a/packages/core/platform/values.yaml +++ b/packages/core/platform/values.yaml @@ -5,7 +5,7 @@ sourceRef: path: / migrations: enabled: false - image: ghcr.io/cozystack/cozystack/platform-migrations:v1.5.1@sha256:5d601b0709c578945f2e04d655b77d29008c94666a4350e67960c9f6bf9d2bde + image: ghcr.io/cozystack/cozystack/platform-migrations:v1.5.2@sha256:5d601b0709c578945f2e04d655b77d29008c94666a4350e67960c9f6bf9d2bde targetVersion: 45 # Bundle deployment configuration bundles: diff --git a/packages/core/testing/values.yaml b/packages/core/testing/values.yaml index 94610c9fc5..dfc3ecf4a6 100644 --- a/packages/core/testing/values.yaml +++ b/packages/core/testing/values.yaml @@ -1,2 +1,2 @@ e2e: - image: ghcr.io/cozystack/cozystack/e2e-sandbox:v1.5.1@sha256:91b9a2985bcadea4e74cf9cd644a6d4c19442a7270fe14ac52e766b4ec0c93d8 + image: ghcr.io/cozystack/cozystack/e2e-sandbox:v1.5.2@sha256:e664363d5ce0b24431f7fd06dd27a4e397a0a33439247f922f5bf0d9474f759f diff --git a/packages/extra/bootbox/images/matchbox.tag b/packages/extra/bootbox/images/matchbox.tag index bff481c4ee..00ba23f5f8 100644 --- a/packages/extra/bootbox/images/matchbox.tag +++ b/packages/extra/bootbox/images/matchbox.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/matchbox:v1.5.1@sha256:2fca5e260f5d441bd045292b7e81ee2a95bcd09904ae429388f51cbd1059c361 +ghcr.io/cozystack/cozystack/matchbox:v1.5.2@sha256:8fd8c75c9bfb70ed1f9263166ddb562a3c4b4f3c954fb8556ac2b096e84d7de3 diff --git a/packages/extra/seaweedfs/images/objectstorage-sidecar.tag b/packages/extra/seaweedfs/images/objectstorage-sidecar.tag index 7619994d92..1bb9c23f5f 100644 --- a/packages/extra/seaweedfs/images/objectstorage-sidecar.tag +++ b/packages/extra/seaweedfs/images/objectstorage-sidecar.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/objectstorage-sidecar:v1.5.1@sha256:fdc09634688d5579a329b48aabf6706cc55bcd9503b0331fa47764706f0d6f62 +ghcr.io/cozystack/cozystack/objectstorage-sidecar:v1.5.2@sha256:fdc09634688d5579a329b48aabf6706cc55bcd9503b0331fa47764706f0d6f62 diff --git a/packages/system/backup-controller/values.yaml b/packages/system/backup-controller/values.yaml index f000bf7c60..b41126fabb 100644 --- a/packages/system/backup-controller/values.yaml +++ b/packages/system/backup-controller/values.yaml @@ -1,5 +1,5 @@ backupController: - image: "ghcr.io/cozystack/cozystack/backup-controller:v1.5.1@sha256:11d55b79cf136f9ec161410e2357dde7bc063246f905d5ac962ad8432af04962" + image: "ghcr.io/cozystack/cozystack/backup-controller:v1.5.2@sha256:11d55b79cf136f9ec161410e2357dde7bc063246f905d5ac962ad8432af04962" replicas: 2 debug: false metrics: diff --git a/packages/system/backupstrategy-controller/values.yaml b/packages/system/backupstrategy-controller/values.yaml index b34e081a67..80223adf32 100644 --- a/packages/system/backupstrategy-controller/values.yaml +++ b/packages/system/backupstrategy-controller/values.yaml @@ -1,5 +1,5 @@ backupStrategyController: - image: "ghcr.io/cozystack/cozystack/backupstrategy-controller:v1.5.1@sha256:d965283f935e4b24b655bc76e37dcb3b731bec832bb5c2e3e407b0284c7526a3" + image: "ghcr.io/cozystack/cozystack/backupstrategy-controller:v1.5.2@sha256:d965283f935e4b24b655bc76e37dcb3b731bec832bb5c2e3e407b0284c7526a3" # chBackupClientImage is the image rendered into the Altinity strategy # Pod: it drives clickhouse-backup's HTTP API via curl + jq. We reuse # platform-migrations (already pre-built and digested by release CI in diff --git a/packages/system/bucket/images/s3manager.tag b/packages/system/bucket/images/s3manager.tag index 655c9e78e6..918a9386e4 100644 --- a/packages/system/bucket/images/s3manager.tag +++ b/packages/system/bucket/images/s3manager.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/s3manager:v1.5.1@sha256:8b36ab67904e34b2fb843b964c20d4822f3a6673f6745afb6a834efd02e017e2 +ghcr.io/cozystack/cozystack/s3manager:v1.5.2@sha256:8b36ab67904e34b2fb843b964c20d4822f3a6673f6745afb6a834efd02e017e2 diff --git a/packages/system/cilium/values.yaml b/packages/system/cilium/values.yaml index fc702c0805..411e2a2e3f 100644 --- a/packages/system/cilium/values.yaml +++ b/packages/system/cilium/values.yaml @@ -15,7 +15,7 @@ cilium: mode: "kubernetes" image: repository: ghcr.io/cozystack/cozystack/cilium - tag: v1.5.1 + tag: v1.5.2 digest: "sha256:94dc74c9301f0da5695c80e77b77ee0486ad0086b5be60a94136a2cafd410b73" envoy: enabled: true diff --git a/packages/system/cozystack-api/values.yaml b/packages/system/cozystack-api/values.yaml index 42d4d9d5f0..e8409ac932 100644 --- a/packages/system/cozystack-api/values.yaml +++ b/packages/system/cozystack-api/values.yaml @@ -1,3 +1,3 @@ cozystackAPI: - image: ghcr.io/cozystack/cozystack/cozystack-api:v1.5.1@sha256:ef50b155e419cf8f0b4d57af9fa385ecbacd5c1248f13591ffcca75c2f9f2a07 + image: ghcr.io/cozystack/cozystack/cozystack-api:v1.5.2@sha256:ef50b155e419cf8f0b4d57af9fa385ecbacd5c1248f13591ffcca75c2f9f2a07 replicas: 2 diff --git a/packages/system/cozystack-controller/values.yaml b/packages/system/cozystack-controller/values.yaml index 355daf70ce..edd418094f 100644 --- a/packages/system/cozystack-controller/values.yaml +++ b/packages/system/cozystack-controller/values.yaml @@ -1,4 +1,4 @@ cozystackController: - image: ghcr.io/cozystack/cozystack/cozystack-controller:v1.5.1@sha256:c27aa360d8081d8bf3fa3d4ec833d38a96b33bda10b3a41183b46389ce091c07 + image: ghcr.io/cozystack/cozystack/cozystack-controller:v1.5.2@sha256:464c9cda4dba5e83e17f26acf386cafaf3688be7c24e267692c97a3acb018a15 debug: false disableTelemetry: false diff --git a/packages/system/dashboard/values.yaml b/packages/system/dashboard/values.yaml index 36aab09226..2feec3b17b 100644 --- a/packages/system/dashboard/values.yaml +++ b/packages/system/dashboard/values.yaml @@ -1,4 +1,4 @@ console: - image: ghcr.io/cozystack/cozystack/cozystack-ui:v1.5.1@sha256:932b11c6f3faf42cb1ec8828412ac417727e2c3baef8af48c667822f692fd9bb + image: ghcr.io/cozystack/cozystack/cozystack-ui:v1.5.2@sha256:c8f86e56952044999b9fa0aa1bbeb2b7e8fa4cfcd2f56ce3cff82e8276a87f68 tokenProxy: - image: ghcr.io/cozystack/cozystack/token-proxy:v1.5.1@sha256:0382e8a9014a842d2eddf1bac5b598f42a4ada7a9be673d9272f1610d47631c4 + image: ghcr.io/cozystack/cozystack/token-proxy:v1.5.2@sha256:0382e8a9014a842d2eddf1bac5b598f42a4ada7a9be673d9272f1610d47631c4 diff --git a/packages/system/flux-shard-operator/values.yaml b/packages/system/flux-shard-operator/values.yaml index 17c6762973..4ab207720b 100644 --- a/packages/system/flux-shard-operator/values.yaml +++ b/packages/system/flux-shard-operator/values.yaml @@ -1,5 +1,5 @@ fluxShardOperator: - image: ghcr.io/cozystack/cozystack/flux-shard-operator:v1.5.1@sha256:56f5ef64a853a9efa5270f73f2794edcd993d3472ba482d7ef9630dce8b8df4a + image: ghcr.io/cozystack/cozystack/flux-shard-operator:v1.5.2@sha256:a75b88c03e5d969b322d418957f30c92941ae2ec8e1b1852719fa7d9eb9aa2a5 debug: false replicas: 2 ## Number of helm-controller shards to provision and distribute tenants diff --git a/packages/system/grafana-operator/images/grafana-dashboards.tag b/packages/system/grafana-operator/images/grafana-dashboards.tag index ae75a6387c..78a5db48d2 100644 --- a/packages/system/grafana-operator/images/grafana-dashboards.tag +++ b/packages/system/grafana-operator/images/grafana-dashboards.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/grafana-dashboards:v1.5.1@sha256:ec97b9ee9aa591b5737da595ed7772b22714b9517f9a365d67e7977dd45dd77c +ghcr.io/cozystack/cozystack/grafana-dashboards:v1.5.2@sha256:ec97b9ee9aa591b5737da595ed7772b22714b9517f9a365d67e7977dd45dd77c diff --git a/packages/system/kamaji/values.yaml b/packages/system/kamaji/values.yaml index 7e99299964..97c536b381 100644 --- a/packages/system/kamaji/values.yaml +++ b/packages/system/kamaji/values.yaml @@ -3,7 +3,7 @@ kamaji: deploy: false image: pullPolicy: IfNotPresent - tag: v1.5.1@sha256:130d5013a9b35be0013b96efefc656210cc30c6fe92a8c39c9d2ab8d47f791e0 + tag: v1.5.2@sha256:fcf5f4451dc935d141d12ef40a966cb6f7bdb46461431a64803e1f018b0c89de repository: ghcr.io/cozystack/cozystack/kamaji resources: limits: @@ -36,4 +36,4 @@ kamaji: periodSeconds: 10 timeoutSeconds: 1 extraArgs: - - --migrate-image=ghcr.io/cozystack/cozystack/kamaji:v1.5.1@sha256:130d5013a9b35be0013b96efefc656210cc30c6fe92a8c39c9d2ab8d47f791e0 + - --migrate-image=ghcr.io/cozystack/cozystack/kamaji:v1.5.2@sha256:fcf5f4451dc935d141d12ef40a966cb6f7bdb46461431a64803e1f018b0c89de diff --git a/packages/system/kubeovn-plunger/values.yaml b/packages/system/kubeovn-plunger/values.yaml index 88f56ea1b6..dbdf45cbab 100644 --- a/packages/system/kubeovn-plunger/values.yaml +++ b/packages/system/kubeovn-plunger/values.yaml @@ -1,4 +1,4 @@ portSecurity: true routes: "" -image: ghcr.io/cozystack/cozystack/kubeovn-plunger:v1.5.1@sha256:908b5903c014f6fa4348f57b1fced929a2ba823aadfbcda7ad9b76134bcf3fce +image: ghcr.io/cozystack/cozystack/kubeovn-plunger:v1.5.2@sha256:908b5903c014f6fa4348f57b1fced929a2ba823aadfbcda7ad9b76134bcf3fce ovnCentralName: ovn-central diff --git a/packages/system/kubeovn-webhook/values.yaml b/packages/system/kubeovn-webhook/values.yaml index 5ed0d24baf..d55271c83b 100644 --- a/packages/system/kubeovn-webhook/values.yaml +++ b/packages/system/kubeovn-webhook/values.yaml @@ -1,3 +1,3 @@ portSecurity: true routes: "" -image: ghcr.io/cozystack/cozystack/kubeovn-webhook:v1.5.1@sha256:ca354b6c2ae6f648bc63798d19a1316fe37cb142246bf08ebb7ce27e6e8f21b2 +image: ghcr.io/cozystack/cozystack/kubeovn-webhook:v1.5.2@sha256:ca354b6c2ae6f648bc63798d19a1316fe37cb142246bf08ebb7ce27e6e8f21b2 diff --git a/packages/system/kubevirt-csi-node/values.yaml b/packages/system/kubevirt-csi-node/values.yaml index b97d74c4ea..b65b551268 100644 --- a/packages/system/kubevirt-csi-node/values.yaml +++ b/packages/system/kubevirt-csi-node/values.yaml @@ -1,3 +1,3 @@ storageClass: replicated csiDriver: - image: ghcr.io/cozystack/cozystack/kubevirt-csi-driver:v1.5.1@sha256:d01f1c0377ce00c8bb46f1576fa9348da3b226679130bf7b46a8bc50d684c38d + image: ghcr.io/cozystack/cozystack/kubevirt-csi-driver:v1.5.2@sha256:808553aa64adf49c7c555a05d16f68ba5fb68fbd266e27eeff497e4d97d6dea1 diff --git a/packages/system/lineage-controller-webhook/values.yaml b/packages/system/lineage-controller-webhook/values.yaml index 044c1e0b30..6b1dadd08b 100644 --- a/packages/system/lineage-controller-webhook/values.yaml +++ b/packages/system/lineage-controller-webhook/values.yaml @@ -1,5 +1,5 @@ lineageControllerWebhook: - image: ghcr.io/cozystack/cozystack/lineage-controller-webhook:v1.5.1@sha256:5868789439d1384cf78d9fb2f20d5457060b0d5dfcc4f93f96a13ea27ed73b81 + image: ghcr.io/cozystack/cozystack/lineage-controller-webhook:v1.5.2@sha256:5868789439d1384cf78d9fb2f20d5457060b0d5dfcc4f93f96a13ea27ed73b81 debug: false replicas: 2 # DEPRECATED. Injects KUBERNETES_SERVICE_HOST=status.hostIP and diff --git a/packages/system/linstor-gui/values.yaml b/packages/system/linstor-gui/values.yaml index b92d2c340d..fbcd8c806e 100644 --- a/packages/system/linstor-gui/values.yaml +++ b/packages/system/linstor-gui/values.yaml @@ -3,7 +3,7 @@ ## @param image.tag LINSTOR GUI container image tag (digest recommended) image: repository: ghcr.io/cozystack/cozystack/linstor-gui - tag: v1.5.1@sha256:6e11829de86709f5e21636cdd1a1ada1239a7dcc1741f91e9d95d9f6ade603a2 + tag: v1.5.2@sha256:6e11829de86709f5e21636cdd1a1ada1239a7dcc1741f91e9d95d9f6ade603a2 ## @section Deployment ## @param replicas Number of linstor-gui replicas replicas: 1 diff --git a/packages/system/linstor/values.yaml b/packages/system/linstor/values.yaml index 23c3c81986..105fb101c3 100644 --- a/packages/system/linstor/values.yaml +++ b/packages/system/linstor/values.yaml @@ -1,7 +1,7 @@ piraeusServer: image: repository: ghcr.io/cozystack/cozystack/piraeus-server - tag: v1.5.1@sha256:1a00af3642c4814c3552d1964daf2831a430bc451c55e786e5eee278fa50e9e1 + tag: v1.5.2@sha256:8a4038a01032a4bbf65ca8da767c52b95ca21e6cdef73460b4860953cc0eeb4f # Talos-specific workarounds (disable for generic Linux like Ubuntu/Debian) talos: enabled: true @@ -13,4 +13,4 @@ linstor: linstorCSI: image: repository: ghcr.io/cozystack/cozystack/linstor-csi - tag: v1.5.1@sha256:c60519d12aa8d078e4891d6e436c709ff53f2cc96457994390ca9424b6505903 + tag: v1.5.2@sha256:c7ff3f3a19f1079bd0fe67b26319fbc07ee318a826f207ee7ec92a6c7b8a4761 diff --git a/packages/system/metallb/values.yaml b/packages/system/metallb/values.yaml index 2d2a6ba3d6..a44f269de3 100644 --- a/packages/system/metallb/values.yaml +++ b/packages/system/metallb/values.yaml @@ -37,11 +37,11 @@ metallb: controller: image: repository: ghcr.io/cozystack/cozystack/metallb-controller - tag: v1.5.1@sha256:b290031f40bfc072c0b329e07d7c17eca3301e4aad2e1988fc962645d92b5fc0 + tag: v1.5.2@sha256:b290031f40bfc072c0b329e07d7c17eca3301e4aad2e1988fc962645d92b5fc0 speaker: image: repository: ghcr.io/cozystack/cozystack/metallb-speaker - tag: v1.5.1@sha256:3aad3ac50e8da94342332b363099c04316d8eda0dcb8cdc3d89f1bcf5dc04e7d + tag: v1.5.2@sha256:3aad3ac50e8da94342332b363099c04316d8eda0dcb8cdc3d89f1bcf5dc04e7d # The vendored metallb chart's values.yaml leaves `frr-k8s.prometheus:` as # a YAML block containing only commented examples — it parses to null. # Helm-controller's deep-merge then writes that null over the frr-k8s diff --git a/packages/system/monitoring/images/grafana.tag b/packages/system/monitoring/images/grafana.tag index f92724d612..48c341e4bf 100644 --- a/packages/system/monitoring/images/grafana.tag +++ b/packages/system/monitoring/images/grafana.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/grafana:v1.5.1@sha256:086f1502edf64d2aabcda6e6505b7ee4673b0d3a14cdb7ca9e90a1d91d6e78f4 +ghcr.io/cozystack/cozystack/grafana:v1.5.2@sha256:086f1502edf64d2aabcda6e6505b7ee4673b0d3a14cdb7ca9e90a1d91d6e78f4 diff --git a/packages/system/multus/templates/multus-daemonset-thick.yml b/packages/system/multus/templates/multus-daemonset-thick.yml index 98dff22d06..e2bf51f364 100644 --- a/packages/system/multus/templates/multus-daemonset-thick.yml +++ b/packages/system/multus/templates/multus-daemonset-thick.yml @@ -155,7 +155,7 @@ spec: serviceAccountName: multus containers: - name: kube-multus - image: ghcr.io/cozystack/cozystack/multus-cni:v1.5.1@sha256:f41d0d45bb3a654a6f024a8e416884541550f4c70481f1579647c6dd434f04d8 + image: ghcr.io/cozystack/cozystack/multus-cni:v1.5.2@sha256:679ca9637b685c886d6760cfcd881b2f199ed068abee88daceded142edebd0c1 command: [ "/usr/src/multus-cni/bin/multus-daemon" ] resources: requests: @@ -201,7 +201,7 @@ spec: fieldPath: spec.nodeName initContainers: - name: install-multus-binary - image: ghcr.io/cozystack/cozystack/multus-cni:v1.5.1@sha256:f41d0d45bb3a654a6f024a8e416884541550f4c70481f1579647c6dd434f04d8 + image: ghcr.io/cozystack/cozystack/multus-cni:v1.5.2@sha256:679ca9637b685c886d6760cfcd881b2f199ed068abee88daceded142edebd0c1 command: - "/usr/src/multus-cni/bin/install_multus" - "-d" diff --git a/packages/system/objectstorage-controller/values.yaml b/packages/system/objectstorage-controller/values.yaml index a8bb959d52..b3a238cc32 100644 --- a/packages/system/objectstorage-controller/values.yaml +++ b/packages/system/objectstorage-controller/values.yaml @@ -1,3 +1,3 @@ objectstorage: controller: - image: "ghcr.io/cozystack/cozystack/objectstorage-controller:v1.5.1@sha256:e27da94691155fc3eb14a22cf23fba78e53c4ba524ca3042709efae3753c565b" + image: "ghcr.io/cozystack/cozystack/objectstorage-controller:v1.5.2@sha256:e27da94691155fc3eb14a22cf23fba78e53c4ba524ca3042709efae3753c565b" diff --git a/packages/system/seaweedfs/values.yaml b/packages/system/seaweedfs/values.yaml index eba5bcada5..c54c797c07 100644 --- a/packages/system/seaweedfs/values.yaml +++ b/packages/system/seaweedfs/values.yaml @@ -202,7 +202,7 @@ seaweedfs: bucketClassName: "seaweedfs" region: "" sidecar: - image: "ghcr.io/cozystack/cozystack/objectstorage-sidecar:v1.5.1@sha256:fdc09634688d5579a329b48aabf6706cc55bcd9503b0331fa47764706f0d6f62" + image: "ghcr.io/cozystack/cozystack/objectstorage-sidecar:v1.5.2@sha256:fdc09634688d5579a329b48aabf6706cc55bcd9503b0331fa47764706f0d6f62" certificates: commonName: "SeaweedFS CA" ipAddresses: [] From 20b053cbbe64db5c6c1295a0f081b2ebe638e28e Mon Sep 17 00:00:00 2001 From: mattia-eleuteri Date: Fri, 12 Jun 2026 10:29:08 +0200 Subject: [PATCH 15/69] fix(seaweedfs): configure postgres2 connection pool for the filer The vendored seaweedfs chart ships connection pool settings only for the mysql store, which cozystack does not use. The postgres2 store, which is enabled, has no pool settings, so the Go sql pool keeps zero idle connections and every filer metadata lookup opens a fresh PostgreSQL connection (TCP + TLS + SCRAM, ~300ms each). With ~5 path lookups per S3 operation this adds roughly 2 seconds of latency to every S3 request, regardless of cluster load. Measured on a production cluster (12 nodes, 2 filers): PUT of a 4KiB object went from 10s (p50) to 0.28s, HEAD from 2.6s to 0.16s after setting these values; pg_stat_activity confirmed connections were previously recycled every ~200ms and are now reused. max_open is capped at 40 so that two filer replicas stay below the CNPG default max_connections of 100. Signed-off-by: mattia-eleuteri (cherry picked from commit 49cbea8c7376259e2899b93fe42edd10145f4d73) --- packages/system/seaweedfs/values.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/packages/system/seaweedfs/values.yaml b/packages/system/seaweedfs/values.yaml index c54c797c07..70af9489e8 100644 --- a/packages/system/seaweedfs/values.yaml +++ b/packages/system/seaweedfs/values.yaml @@ -57,6 +57,14 @@ seaweedfs: extraEnvironmentVars: WEED_LEVELDB2_ENABLED: "false" WEED_POSTGRES2_ENABLED: "true" + # Without explicit pool settings the Go sql pool keeps zero idle + # connections, so every filer metadata lookup opens a fresh PostgreSQL + # connection (TCP + TLS + SCRAM, ~300ms each), adding seconds of latency + # to every S3 operation. Keep max_open * filer.replicas below the + # database max_connections (CNPG default: 100). + WEED_POSTGRES2_CONNECTION_MAX_IDLE: "20" + WEED_POSTGRES2_CONNECTION_MAX_OPEN: "40" + WEED_POSTGRES2_CONNECTION_MAX_LIFETIME_SECONDS: "600" WEED_POSTGRES2_CREATETABLE: | CREATE TABLE IF NOT EXISTS "%s" ( dirhash BIGINT, From f52af1922cc2e5836a29f5db78f429d572cdc466 Mon Sep 17 00:00:00 2001 From: Andrei Kvapil Date: Tue, 30 Jun 2026 15:01:10 +0200 Subject: [PATCH 16/69] fix(kubevirt-instancetypes): restore persistent EFI/TPM state MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reverts #3006. Dropping persistent EFI/TPM was motivated by a live-migration concern that rested on an outdated assumption — that an RWO Filesystem backend-storage PVC pins the VM to its node. KubeVirt has migrated RWO-Filesystem backend storage since v1.4 (kubevirt/kubevirt#12629): on migration it creates a fresh target state PVC and copies the small state blob, so persistent-EFI/TPM VMs on the default replicated storage live-migrate fine (verified: the VM reports LiveMigratable=True and the copy-on-target PVC is created on migration). Restore upstream's persistent firmware state for the windows.* preferences. Assisted-By: Claude Signed-off-by: Andrei Kvapil (cherry picked from commit 59f035ff5345206b6cf26b71ea6360c1b82d7f13) --- .../system/kubevirt-instancetypes/Makefile | 6 ----- .../templates/preferences.yaml | 24 ++++++++++++++----- 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/packages/system/kubevirt-instancetypes/Makefile b/packages/system/kubevirt-instancetypes/Makefile index 15ac774d50..99771e976c 100644 --- a/packages/system/kubevirt-instancetypes/Makefile +++ b/packages/system/kubevirt-instancetypes/Makefile @@ -16,12 +16,6 @@ update: mkdir templates kustomize build https://github.com/kubevirt/common-instancetypes/preferences?ref=$(COMMON_INSTANCETYPES_REF) > templates/preferences.yaml yq -i 'select(.kind != "VirtualMachinePreference")' templates/preferences.yaml - # TEMPORARY (see #3005): drop persistent EFI/TPM state. With persistence on, - # KubeVirt creates a backend-storage PVC that is RWO on the default StorageClass, - # which pins the VM to its node and blocks live-migration / drain. Use `del` (not - # `sed -d`) so an emptied preferredTPM stays `{}` rather than null, which the - # KubeVirt v1.8 CRD structural schema rejects. Revert once vm-state can request RWX. - yq -i 'del(.spec.firmware.preferredEfi.persistent) | del(.spec.devices.preferredTPM.persistent)' templates/preferences.yaml kustomize build https://github.com/kubevirt/common-instancetypes/instancetypes?ref=$(COMMON_INSTANCETYPES_REF) > templates/instancetypes.yaml yq -i 'select(.kind != "VirtualMachineInstancetype")' templates/instancetypes.yaml # Re-append Cozystack-local instancetypes the pinned catalog no longer ships diff --git a/packages/system/kubevirt-instancetypes/templates/preferences.yaml b/packages/system/kubevirt-instancetypes/templates/preferences.yaml index 6a42e6304f..458d8bd7b3 100644 --- a/packages/system/kubevirt-instancetypes/templates/preferences.yaml +++ b/packages/system/kubevirt-instancetypes/templates/preferences.yaml @@ -1352,7 +1352,8 @@ spec: preferredAutoattachInputDevice: true preferredDiskBus: sata preferredInterfaceModel: e1000e - preferredTPM: {} + preferredTPM: + persistent: true features: preferredAcpi: {} preferredApic: {} @@ -1374,6 +1375,7 @@ spec: preferredSmm: {} firmware: preferredEfi: + persistent: true secureBoot: true preferredTerminationGracePeriodSeconds: 3600 requirements: @@ -1422,7 +1424,8 @@ spec: preferredInputBus: virtio preferredInputType: tablet preferredInterfaceModel: virtio - preferredTPM: {} + preferredTPM: + persistent: true features: preferredAcpi: {} preferredApic: {} @@ -1444,6 +1447,7 @@ spec: preferredSmm: {} firmware: preferredEfi: + persistent: true secureBoot: true preferredTerminationGracePeriodSeconds: 3600 requirements: @@ -1890,7 +1894,8 @@ spec: preferredAutoattachInputDevice: true preferredDiskBus: sata preferredInterfaceModel: e1000e - preferredTPM: {} + preferredTPM: + persistent: true features: preferredAcpi: {} preferredApic: {} @@ -1912,6 +1917,7 @@ spec: preferredSmm: {} firmware: preferredEfi: + persistent: true secureBoot: true preferredTerminationGracePeriodSeconds: 3600 requirements: @@ -1960,7 +1966,8 @@ spec: preferredInputBus: virtio preferredInputType: tablet preferredInterfaceModel: virtio - preferredTPM: {} + preferredTPM: + persistent: true features: preferredAcpi: {} preferredApic: {} @@ -1982,6 +1989,7 @@ spec: preferredSmm: {} firmware: preferredEfi: + persistent: true secureBoot: true preferredTerminationGracePeriodSeconds: 3600 requirements: @@ -2028,7 +2036,8 @@ spec: preferredAutoattachInputDevice: true preferredDiskBus: sata preferredInterfaceModel: e1000e - preferredTPM: {} + preferredTPM: + persistent: true features: preferredAcpi: {} preferredApic: {} @@ -2050,6 +2059,7 @@ spec: preferredSmm: {} firmware: preferredEfi: + persistent: true secureBoot: true preferredTerminationGracePeriodSeconds: 3600 requirements: @@ -2098,7 +2108,8 @@ spec: preferredInputBus: virtio preferredInputType: tablet preferredInterfaceModel: virtio - preferredTPM: {} + preferredTPM: + persistent: true features: preferredAcpi: {} preferredApic: {} @@ -2120,6 +2131,7 @@ spec: preferredSmm: {} firmware: preferredEfi: + persistent: true secureBoot: true preferredTerminationGracePeriodSeconds: 3600 requirements: From d1fc12609b2e0d44d288b3add4c4375943e95fa7 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Wed, 17 Jun 2026 12:47:02 +0300 Subject: [PATCH 17/69] chore(kubevirt-operator): update to v1.8.4 Refresh the vendored virt-operator manifest from v1.8.2 to v1.8.4. The release backports validation of the VMI checksum status fields as a uint32 range, required on Kubernetes 1.36 where strict CRD numeric format validation otherwise leaves VMIs stuck in Scheduled. Remaining changes are bug fixes and CVE remediations; the embedded CRDs and RBAC are unchanged, so only the operator image tag moves. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin (cherry picked from commit 192246e8b64ce40fbdebd25340e2b7537355e83b) --- packages/system/kubevirt-operator/Makefile | 4 ++-- .../kubevirt-operator/templates/kubevirt-operator.yaml | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/system/kubevirt-operator/Makefile b/packages/system/kubevirt-operator/Makefile index 9aeb6de1a1..3f3a22649a 100644 --- a/packages/system/kubevirt-operator/Makefile +++ b/packages/system/kubevirt-operator/Makefile @@ -6,8 +6,8 @@ include ../../../hack/package.mk update: rm -rf templates mkdir templates - # v1.7.0 blocked by https://github.com/kubevirt/kubevirt/issues/16386 - export RELEASE=v1.8.2 && \ + # Skipping v1.7.x: live-migration regression, https://github.com/kubevirt/kubevirt/issues/16386 + export RELEASE=v1.8.4 && \ wget https://github.com/kubevirt/kubevirt/releases/download/$${RELEASE}/kubevirt-operator.yaml -O templates/kubevirt-operator.yaml && \ sed -i 's/namespace: kubevirt/namespace: $(NAMESPACE)/g' templates/kubevirt-operator.yaml awk -i inplace -v RS="---" '!/kind: Namespace/{printf "%s", $$0 RS}' templates/kubevirt-operator.yaml diff --git a/packages/system/kubevirt-operator/templates/kubevirt-operator.yaml b/packages/system/kubevirt-operator/templates/kubevirt-operator.yaml index 8d73f98226..f6acbc37d5 100644 --- a/packages/system/kubevirt-operator/templates/kubevirt-operator.yaml +++ b/packages/system/kubevirt-operator/templates/kubevirt-operator.yaml @@ -8536,14 +8536,14 @@ spec: - virt-operator env: - name: VIRT_OPERATOR_IMAGE - value: quay.io/kubevirt/virt-operator:v1.8.2 + value: quay.io/kubevirt/virt-operator:v1.8.4 - name: WATCH_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.annotations['olm.targetNamespaces'] - name: KUBEVIRT_VERSION - value: v1.8.2 - image: quay.io/kubevirt/virt-operator:v1.8.2 + value: v1.8.4 + image: quay.io/kubevirt/virt-operator:v1.8.4 imagePullPolicy: IfNotPresent livenessProbe: httpGet: From 1b5ccddb322432f64439b9c718492ffda9e835d7 Mon Sep 17 00:00:00 2001 From: Ivan Okhotnikov Date: Mon, 13 Jul 2026 23:32:24 +0300 Subject: [PATCH 18/69] fix(apps/vpn): remove invalid foo field from urls Secret A leftover debug field "foo" in the -urls Secret template violates the core/v1 Secret schema. It went unnoticed under client-side apply but breaks Helm upgrades under server-side apply with ".foo: field not declared in schema", leaving the release in UpgradeFailed state. Drop the debug block. Signed-off-by: Ivan Okhotnikov (cherry picked from commit 70a2284551c76d2ae932dbba9b3c980360592eed) --- packages/apps/vpn/templates/secret.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/apps/vpn/templates/secret.yaml b/packages/apps/vpn/templates/secret.yaml index 3ef4ac4b32..595bfd32ae 100644 --- a/packages/apps/vpn/templates/secret.yaml +++ b/packages/apps/vpn/templates/secret.yaml @@ -59,8 +59,6 @@ kind: Secret metadata: name: {{ .Release.Name }}-urls type: Opaque -foo: | - {{ toJson $passwords }} stringData: {{- range $user, $u := .Values.users }} "{{ $user }}": "ss://{{ regexReplaceAll "=" (replace "/" "_" (replace "+" "-" (printf "chacha20-ietf-poly1305:%s" (index $passwords $user) | b64enc))) "" }}@{{ $.Values.host | default (printf "%s.%s" $.Release.Name $host) }}:40000/?outline=1#{{ $.Release.Name }}" From 78709bba883c4a5389272827275d928f64d38f90 Mon Sep 17 00:00:00 2001 From: Ivan Okhotnikov Date: Tue, 2 Jun 2026 21:05:15 +0300 Subject: [PATCH 19/69] fix(dashboard): set webOrigins to "+" and add --api-route for k8s API paths Two related issues caused the dashboard SPA to break with CORS errors after the kc-access cookie expired: 1. KeycloakClient for dashboard had no webOrigins set, so the operator never touched the field in Keycloak after initial creation. After any rootUrl change the existing webOrigins value became stale and blocked CORS from the SPA. Setting "+" tells Keycloak to derive allowed origins from redirectUris. 2. oauth2-proxy returned 302 -> Keycloak /auth for unauthenticated XHR/fetch on /api and /apis. The login page does not emit Access-Control-Allow-Origin, so the SPA's fetch failed with CORS. --api-route makes oauth2-proxy answer 401 without redirect, so the SPA can handle the re-auth itself via top-level navigation. Signed-off-by: Ivan Okhotnikov (cherry picked from commit be8f6b473914e4a59709942c12cd475fb807f9ee) --- packages/system/dashboard/templates/gatekeeper.yaml | 2 ++ packages/system/dashboard/templates/keycloakclient.yaml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/packages/system/dashboard/templates/gatekeeper.yaml b/packages/system/dashboard/templates/gatekeeper.yaml index 8d4b13f74b..3aa6c5bef8 100644 --- a/packages/system/dashboard/templates/gatekeeper.yaml +++ b/packages/system/dashboard/templates/gatekeeper.yaml @@ -68,6 +68,8 @@ spec: {{- end }} - --whitelist-domain=keycloak.{{ $host }} - --email-domain=* + - --api-route=^/api(/|$) + - --api-route=^/apis(/|$) - --pass-access-token=true - --pass-authorization-header=true - --cookie-refresh=3m diff --git a/packages/system/dashboard/templates/keycloakclient.yaml b/packages/system/dashboard/templates/keycloakclient.yaml index fa33fea4a8..cd455af7ff 100644 --- a/packages/system/dashboard/templates/keycloakclient.yaml +++ b/packages/system/dashboard/templates/keycloakclient.yaml @@ -64,6 +64,8 @@ spec: directAccess: true public: false webUrl: "https://dashboard.{{ $host }}" + webOrigins: + - "+" defaultClientScopes: - groups - kubernetes-client From aaf54886fe5cb5ea2e1cd4f92aa91980f5e3bcb1 Mon Sep 17 00:00:00 2001 From: Ivan Okhotnikov Date: Tue, 14 Jul 2026 12:02:23 +0300 Subject: [PATCH 20/69] fix(fluxcd): omit empty distribution.artifact in FluxInstance The flux-instance template rendered spec.distribution.artifact unconditionally, unlike the sibling optional fields (artifactPullSecret, imagePullSecret, variant). The umbrella default is empty to use the operator's embedded manifests (the air-gapped default from #964), but an empty value renders as YAML null. In the flux-operator v0.50.0 CRD the field is optional yet typed 'string' with pattern ^oci://.*$ and is not nullable, so the rendered null (like an empty string) fails schema validation and the API rejects the FluxInstance. Guest Kubernetes clusters with the fluxcd addon enabled could therefore never install it: the sub-release looped install/uninstall, the parent kubernetes release stayed InProgress, and the cluster never reached Ready. Guard the field so an empty value is omitted (the operator falls back to its embedded manifests, preserving the air-gapped default) while a non-empty artifact still renders. The template lives in the vendored subchart, so the change is carried as a patch applied by 'make update'. Signed-off-by: Ivan Okhotnikov (cherry picked from commit da47b70e2cc2fd2124b554c8151bdb45beee4a52) --- packages/system/fluxcd/Makefile | 3 ++- .../flux-instance/templates/instance.yaml | 2 ++ .../patches/guard-distribution-artifact.diff | 22 +++++++++++++++++++ 3 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 packages/system/fluxcd/patches/guard-distribution-artifact.diff diff --git a/packages/system/fluxcd/Makefile b/packages/system/fluxcd/Makefile index 9aa32fe9b1..c398a1b7e6 100644 --- a/packages/system/fluxcd/Makefile +++ b/packages/system/fluxcd/Makefile @@ -8,4 +8,5 @@ apply-locally: update: rm -rf charts - helm pull oci://ghcr.io/controlplaneio-fluxcd/charts/flux-instance --untar --untardir charts + helm pull oci://ghcr.io/controlplaneio-fluxcd/charts/flux-instance --version 0.50.0 --untar --untardir charts + patch --no-backup-if-mismatch -p1 < patches/guard-distribution-artifact.diff diff --git a/packages/system/fluxcd/charts/flux-instance/templates/instance.yaml b/packages/system/fluxcd/charts/flux-instance/templates/instance.yaml index f1f58309e3..5217c24d9c 100644 --- a/packages/system/fluxcd/charts/flux-instance/templates/instance.yaml +++ b/packages/system/fluxcd/charts/flux-instance/templates/instance.yaml @@ -16,7 +16,9 @@ spec: distribution: version: {{ .Values.instance.distribution.version | quote }} registry: {{ .Values.instance.distribution.registry }} + {{- if .Values.instance.distribution.artifact }} artifact: {{ .Values.instance.distribution.artifact }} + {{- end }} {{- if .Values.instance.distribution.artifactPullSecret }} artifactPullSecret: {{ .Values.instance.distribution.artifactPullSecret }} {{- end }} diff --git a/packages/system/fluxcd/patches/guard-distribution-artifact.diff b/packages/system/fluxcd/patches/guard-distribution-artifact.diff new file mode 100644 index 0000000000..57348afe5f --- /dev/null +++ b/packages/system/fluxcd/patches/guard-distribution-artifact.diff @@ -0,0 +1,22 @@ +# Upstream bug: flux-instance's instance.yaml renders spec.distribution.artifact +# unconditionally, unlike the sibling optional fields (artifactPullSecret, +# imagePullSecret, variant). The umbrella's empty default then renders as +# `artifact: null`, which the FluxInstance CRD rejects (type: string, +# pattern ^oci://.*$, and the field is optional). Guard it like its siblings. +# +# Chart source: https://github.com/controlplaneio-fluxcd/charts (flux-instance) +# Not yet reported upstream; drop this hunk once upstream guards the field. +diff --git a/charts/flux-instance/templates/instance.yaml b/charts/flux-instance/templates/instance.yaml +index f1f58309e..5217c24d9 100644 +--- a/charts/flux-instance/templates/instance.yaml ++++ b/charts/flux-instance/templates/instance.yaml +@@ -16,7 +16,9 @@ spec: + distribution: + version: {{ .Values.instance.distribution.version | quote }} + registry: {{ .Values.instance.distribution.registry }} ++ {{- if .Values.instance.distribution.artifact }} + artifact: {{ .Values.instance.distribution.artifact }} ++ {{- end }} + {{- if .Values.instance.distribution.artifactPullSecret }} + artifactPullSecret: {{ .Values.instance.distribution.artifactPullSecret }} + {{- end }} From 9e11998ae9caef35f1a29e6fe481c5946d5cd3ed Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Mon, 27 Jul 2026 20:46:38 +0500 Subject: [PATCH 21/69] fix(ci, kubernetes): build container disks concurrently to unblock Build Every release-1.5 PR Build job dies at the 30-minute timeout, and all of it is spent in one target. `image-ubuntu-container-disk` builds a disk per Kubernetes minor -- six of them -- and each spends ~4-5min inside a libguestfs appliance installing that minor's kubelet/kubeadm into its own copy of the cloud image. Serially that is ~28min of the 30min budget: v1.35 6:00, v1.34 4:21, v1.33 4:27, v1.32 4:39, v1.31 4:42, v1.30 cancelled None of it is cached, and no warmer will fix that on its own. Only build-main.yaml sets WRITE_CACHE=1, and main dropped this image from `make image` in 39cdc8866 six days before release-1.5 was cut, so the :*-buildcache refs it reads have never been written -- the CI log shows all six importers failing with "not found". release-1.5 is now the only branch that builds this image at all. The six builds share no per-version work, so run them concurrently instead. A sub-make carries its own -j, keeping the root `make build` serial, and buildkit dedups the shared guestfish and cloud-image stages across the concurrent solves. --output-sync=target groups each version's log under its target name; six interleaved buildx progress streams have no per-solve identifier and are unreadable. Verified the per-version buildx/echo/rm command set is byte-identical to the loop it replaces across PUBLISH_VERSIONED, PUBLISH_FLOATING and WRITE_CACHE combinations. Moving off the 4cpu/16gb shape because six concurrent libguestfs appliances want ~9gb of RAM and a core apiece. The comment justifying the small shape assumed a warm cache, which no longer holds here. This also stops swallowing failures. The loop separated versions with `;`, so make saw only the last version's exit status: a failed build for any other version left its committed .tag file stale and the job green. Under the sub-make a failure in any version fails the target. Assisted-By: Claude Signed-off-by: Myasnikov Daniil --- .github/workflows/pull-requests.yaml | 17 +++++++++----- packages/apps/kubernetes/Makefile | 35 ++++++++++++++++++---------- 2 files changed, 34 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pull-requests.yaml b/.github/workflows/pull-requests.yaml index e9664b015a..e92272a472 100644 --- a/.github/workflows/pull-requests.yaml +++ b/.github/workflows/pull-requests.yaml @@ -33,12 +33,17 @@ jobs: # buildkit; concurrent build jobs serialize on buildkit's single-writer bbolt # cache lock + exporter mutex and stall to the 30-min timeout (proven via a # live SIGUSR1 goroutine dump). One VM per job => one buildkit per job => that - # contention is severed by construction. `make build` is serial (one image at - # a time), so a small shape suffices -- 4cpu/16gb, not the 24cpu the e2e job - # uses -- and it leans on the warm mode=max cache (#2938) to stay under the - # 30-min wall. The `debug` label still routes to self-hosted for the - # breakpoint path. Phase 0.5 of #2937. - runs-on: ${{ contains(github.event.pull_request.labels.*.name, 'debug') && 'self-hosted' || 'oracle-vm-4cpu-16gb-x86-64' }} + # contention is severed by construction. Phase 0.5 of #2937. + # + # 24cpu/96gb rather than the 4cpu/16gb this used to run on: `make build` is + # still serial across images, but this branch is the only one that still + # builds ubuntu-container-disk, and it does so once per Kubernetes minor with + # no shared build cache to warm it (main dropped the image in 39cdc8866, six + # days before release-1.5 was cut, so nothing writes its :*-buildcache refs). + # Those six builds now run concurrently, and each drives its own libguestfs + # appliance -- ~9gb of RAM and a core apiece, which the small shape cannot + # give. The `debug` label still routes to self-hosted for the breakpoint path. + runs-on: ${{ contains(github.event.pull_request.labels.*.name, 'debug') && 'self-hosted' || 'oracle-vm-24cpu-96gb-x86-64' }} timeout-minutes: 30 permissions: contents: read diff --git a/packages/apps/kubernetes/Makefile b/packages/apps/kubernetes/Makefile index c2f44309b5..186e564296 100644 --- a/packages/apps/kubernetes/Makefile +++ b/packages/apps/kubernetes/Makefile @@ -20,19 +20,30 @@ update: image: image-ubuntu-container-disk image-kubevirt-cloud-provider image-kubevirt-csi-driver image-cluster-autoscaler +# One container disk per Kubernetes minor. Each build spends ~4-5min inside a +# libguestfs appliance installing that minor's kubelet/kubeadm into its own copy +# of the cloud image, and the versions share no per-version work, so build them +# concurrently: the serial loop this replaced spent ~28min of the Build job's +# 30min budget on this target alone. The sub-make carries its own -j so the root +# `make build` stays serial, and buildkit dedups the shared guestfish and +# cloud-image stages across the concurrent solves. --output-sync=target groups +# each version's log under its target name -- six interleaved buildx progress +# streams are unreadable otherwise. image-ubuntu-container-disk: - $(foreach ver,$(KUBERNETES_VERSIONS), \ - docker buildx build images/ubuntu-container-disk \ - --build-arg KUBERNETES_VERSION=$(ver) \ - --tag $(REGISTRY)/ubuntu-container-disk:$(ver)-$(IMAGE_TAG) \ - $(if $(filter 1,$(PUBLISH_VERSIONED)),--tag $(REGISTRY)/ubuntu-container-disk:$(ver)) \ - $(call cache-args,ubuntu-container-disk,$(ver)-buildcache) \ - --metadata-file images/ubuntu-container-disk-$(ver).json \ - $(BUILDX_ARGS) && \ - echo "$(REGISTRY)/ubuntu-container-disk:$(ver)-$(IMAGE_TAG)@$$(yq e '."containerimage.digest"' images/ubuntu-container-disk-$(ver).json -o json -r)" \ - > images/ubuntu-container-disk-$(ver).tag && \ - rm -f images/ubuntu-container-disk-$(ver).json; \ - ) + $(MAKE) -j$(words $(KUBERNETES_VERSIONS)) --output-sync=target \ + $(addprefix image-ubuntu-container-disk-,$(KUBERNETES_VERSIONS)) + +image-ubuntu-container-disk-%: + docker buildx build images/ubuntu-container-disk \ + --build-arg KUBERNETES_VERSION=$* \ + --tag $(REGISTRY)/ubuntu-container-disk:$*-$(IMAGE_TAG) \ + $(if $(filter 1,$(PUBLISH_VERSIONED)),--tag $(REGISTRY)/ubuntu-container-disk:$*) \ + $(call cache-args,ubuntu-container-disk,$*-buildcache) \ + --metadata-file images/ubuntu-container-disk-$*.json \ + $(BUILDX_ARGS) + echo "$(REGISTRY)/ubuntu-container-disk:$*-$(IMAGE_TAG)@$$(yq e '."containerimage.digest"' images/ubuntu-container-disk-$*.json -o json -r)" \ + > images/ubuntu-container-disk-$*.tag + rm -f images/ubuntu-container-disk-$*.json image-kubevirt-cloud-provider: docker buildx build images/kubevirt-cloud-provider \ From 428fe40e93d933a8705c88e2eec02b5465ef5a21 Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Tue, 28 Jul 2026 12:30:56 +0500 Subject: [PATCH 22/69] ci: warm the build cache on release-1.5 pushes Nothing has ever warmed this branch's build cache. This workflow is a copy of main's warmer that still triggers on `push: branches: [main]`, so on release-1.5 it has never fired once -- and it targets [self-hosted], the decommissioned cozy-runner-1 (#2937), so it could not have run even if it did fire. The consequence is a PR Build that runs cold end to end and exceeds its 30-minute timeout. Two separate reasons, both traceable to 39cdc8866 ("bootstrap tenant workers with Talos via CABPT") landing on main six days before this branch was cut: * that commit dropped ubuntu-container-disk from `make image`, so main stopped writing its six :-buildcache refs. release-1.5 is the only branch that still builds the image, and the CI log shows all six importers failing with "not found" -- ~28min serial, cut to ~10min by parallelising the target in #3457. * every other ref carries main's sources, which have diverged from this branch, so most of the remaining 28 packages miss too -- ~38min across the full build, with no single hot spot left to fix. Point it at this branch, move it to the ephemeral shape main's warmer already uses, and write to a branch-scoped CACHE_REGISTRY namespace. The namespace matters: writing the shared refs from here would replace main's layers with release-1.5's, so main's next PR build would miss on everything and blow its own 30-min timeout until the next main push re-warmed it. pull-requests.yaml has to read the same namespace, which the following commit does. The filename stays build-main.yaml even though it no longer warms main. Renaming it would make any later backport touching main's copy arrive as a whole new file rather than a modification, which is the failure mode the backport bot already hits on this branch. Assisted-By: Claude Signed-off-by: Myasnikov Daniil --- .github/workflows/build-main.yaml | 43 ++++++++++++++++++++++++------- 1 file changed, 33 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build-main.yaml b/.github/workflows/build-main.yaml index 927794676d..55e61fa7d2 100644 --- a/.github/workflows/build-main.yaml +++ b/.github/workflows/build-main.yaml @@ -1,9 +1,23 @@ -name: Build cache (main) +name: Build cache (release-1.5) -# Warms the shared mode=max build cache so PR builds start hot. +# Warms the mode=max build cache for THIS branch so its PR builds start hot. +# +# On main this workflow fires on main pushes; on a release branch it has to fire +# on that branch's own pushes, because the cache main writes is main's content. +# Nothing had been warming release-1.5: main dropped ubuntu-container-disk from +# `make image` in 39cdc8866, six days before this branch was cut, so its six +# :*-buildcache refs were never written at all, and every other ref carries +# main's sources rather than this branch's. The result was a PR Build that ran +# cold end to end and timed out -- ~28min of it in ubuntu-container-disk alone +# before #3457 parallelised that target, and ~38min across all 28 packages after. +# +# The cache is written to its OWN namespace (CACHE_REGISTRY below), not the +# shared one. Writing the shared refs from here would replace main's layers with +# this branch's, so main's next PR build would miss on everything and blow its +# own 30-min timeout until the next main push re-warmed it. # # PR builds (pull-requests.yaml) read the cache but never write it -# (WRITE_CACHE unset -> 0). Only this serialized main build writes +# (WRITE_CACHE unset -> 0). Only this serialized branch build writes # CACHE_REGISTRY/:buildcache, so concurrent PR builds never race on the # cache manifest -- the 409 collisions PR #2711 fixed for image tags would # otherwise reappear on the cache ref. Cache is mode=max so all multistage @@ -19,23 +33,32 @@ name: Build cache (main) env: # Same per-CI registry as pull-requests.yaml; the cache is co-located here. REGISTRY: iad.ocir.io/idyksih5sir9/cozystack + # Branch-scoped cache namespace, kept in sync with the Build job in + # pull-requests.yaml. Both must point at the same place or PR builds read a + # cache nobody writes. + CACHE_REGISTRY: iad.ocir.io/idyksih5sir9/cozystack-cache-release-1.5 on: push: - branches: [main] + branches: [release-1.5] paths-ignore: - 'docs/**' -# Only the newest main commit's cache matters; cancel older in-flight warmers. +# Only the newest commit's cache matters; cancel older in-flight warmers. concurrency: - group: build-main-cache + group: build-release-1.5-cache cancel-in-progress: true jobs: warm-cache: name: Warm build cache - runs-on: [self-hosted] - timeout-minutes: 90 + # Was [self-hosted], which is the decommissioned cozy-runner-1 (#2937) -- this + # workflow could not have run on this branch even once. Same ephemeral shape + # the main warmer moved to; a cold serial `make build` here is CPU-bound. + runs-on: oracle-vm-24cpu-96gb-x86-64 + # The first run is fully cold and builds six ubuntu-container-disk images on + # top of the other 28 packages; later runs mostly re-export a warm cache. + timeout-minutes: 120 permissions: contents: read packages: write @@ -92,7 +115,7 @@ jobs: BUILDER: ${{ steps.buildx.outputs.name }} # WRITE_CACHE=1 turns on --cache-to (mode=max) for every image. WRITE_CACHE: '1' - # Publish a floating :main handle; do not move :latest (releases do). - IMAGE_TAG: main + # Floating handle for this branch; do not move :latest (releases do). + IMAGE_TAG: release-1.5 PUBLISH_VERSIONED: '0' PUBLISH_FLOATING: '0' From 55d25aa5516f115ce3ef6b1818de69d681e93658 Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Tue, 28 Jul 2026 12:32:43 +0500 Subject: [PATCH 23/69] ci: read the branch-scoped build cache in PR builds Point the Build job at the namespace build-main.yaml now writes for this branch. Without this the warmer would run and nothing would read it: the cache-from refs would still resolve to the shared namespace main writes, whose layers carry main's sources and miss on most of this branch's 28 packages. Builds only read the cache -- WRITE_CACHE defaults to 0, so concurrent PR builds still never race on a cache manifest, and a miss degrades to a cold build rather than an error. Raise the timeout to 75 minutes at the same time. A warm build is ~15min, but the namespace starts empty, so this PR and the next one run cold at ~45min: six ubuntu-container-disk images plus 28 packages from scratch. Anything that later invalidates the cache broadly -- a Dockerfile edit, a go.mod bump -- costs the same again. 30 minutes only ever fit a warm build, and when it did not fit, the job reported a timeout rather than whatever actually went wrong. Assisted-By: Claude Signed-off-by: Myasnikov Daniil --- .github/workflows/pull-requests.yaml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pull-requests.yaml b/.github/workflows/pull-requests.yaml index e92272a472..20c4236bf7 100644 --- a/.github/workflows/pull-requests.yaml +++ b/.github/workflows/pull-requests.yaml @@ -3,6 +3,11 @@ name: Pull Request env: # TODO: unhardcode this REGISTRY: iad.ocir.io/idyksih5sir9/cozystack + # Read the cache build-main.yaml writes for THIS branch, not the shared refs + # main writes -- those carry main's sources and mostly miss here. Must stay in + # sync with CACHE_REGISTRY in build-main.yaml. Builds only read it + # (WRITE_CACHE defaults to 0), so a miss is a cold build, never an error. + CACHE_REGISTRY: iad.ocir.io/idyksih5sir9/cozystack-cache-release-1.5 on: pull_request: types: [opened, synchronize, reopened] @@ -44,7 +49,13 @@ jobs: # appliance -- ~9gb of RAM and a core apiece, which the small shape cannot # give. The `debug` label still routes to self-hosted for the breakpoint path. runs-on: ${{ contains(github.event.pull_request.labels.*.name, 'debug') && 'self-hosted' || 'oracle-vm-24cpu-96gb-x86-64' }} - timeout-minutes: 30 + # A warm cache puts this job around 15min. 30 was not enough for a cold one: + # the branch-scoped cache starts empty, and anything that invalidates it + # (a Dockerfile edit, a go.mod bump) drops a later build back to ~45min -- + # six container disks plus 28 packages built from scratch. Sized for that + # case rather than for the warm one, so a cold build reports its real + # failure instead of a timeout that says nothing about why. + timeout-minutes: 75 permissions: contents: read packages: write From 922a6c1c0388ce14b2c1c250e17504d60182d6ca Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Wed, 29 Jul 2026 14:53:44 +0500 Subject: [PATCH 24/69] ci: install flux in the release-1.5 cache warmer `make build` finishes at packages/core/installer, whose image-packages target shells out to `flux push artifact`. The warmer used to target the self-hosted runner, which had flux baked in; #3469 moved it to the ephemeral shape without carrying over the toolchain step that main's warmer and pull-requests.yaml both have for exactly this reason. Confirmed on the first run after #3469 merged (run 30430753152): it failed at 49 minutes with `/bin/sh: 1: flux: not found` on Makefile:35, the last of 28 packages. Everything before it had already exported its cache -- 87 export operations, 70 cache manifests, 34 repos created under CACHE_REGISTRY, including all six ubuntu-container-disk refs -- so the cache came out warm and only the cozystack-packages artifact push was lost. A red warmer is still a broken signal, and nothing else would have reported it. Assisted-By: Claude Signed-off-by: Myasnikov Daniil --- .github/workflows/build-main.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/build-main.yaml b/.github/workflows/build-main.yaml index 55e61fa7d2..cfef7f3038 100644 --- a/.github/workflows/build-main.yaml +++ b/.github/workflows/build-main.yaml @@ -74,6 +74,16 @@ jobs: # .git/config where a later build step could read it. persist-credentials: false + # `make build` ends at packages/core/installer, whose image-packages target + # shells out to `flux push artifact`. The self-hosted runner this workflow + # used to target had flux baked in; the ephemeral shape does not, so without + # this the warmer builds all 28 packages and then fails on the last one. + # pull-requests.yaml installs it for the same reason. Idempotent. + - name: Set up build toolchain + run: | + command -v flux >/dev/null \ + || curl -fsSL https://fluxcd.io/install.sh | sudo bash + - name: Set up Docker config run: | if [ -d ~/.docker ]; then From fe035d9874feb87be65980f3aba8984b5ef33d8b Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Wed, 15 Jul 2026 12:53:34 +0300 Subject: [PATCH 25/69] fix(seaweedfs): decouple workload names from the Helm release name Upstream chart 4.31 switched resource names from the chart name to the release name. The data-plane release is -system, so every StatefulSet was renamed to -system-*. StatefulSet names are immutable, so the upgrade could not rename in place: Helm stood up a second, empty set beside the running one while the data stayed on the original data1-seaweedfs-volume-* PVCs. Where the node count let both sets run, they shared one S3 Service and one filer metadata database, so the empty set answered reads and mixed volume IDs from an independent sequence into shared metadata. Pin fullnameOverride so the rendered names no longer follow the release name, matching the convention already used by other system packages (cozy-proxy, flux-operator, linstor-scheduler, victoria-metrics-operator). The s3-consumer suite pinned the name/fullname divergence this override removes by default, so it now re-creates that divergence explicitly and keeps guarding it regardless of the shipped default. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin (cherry picked from commit 7bcd8e07bb77a4b8bc941862a012ef1b188f2895) (cherry picked from commit 1da45ead8262334636919c097019afdf71da412e) --- .../tests/fullname_override_test.yaml | 62 +++++++++++++++++++ .../tests/s3_service_name_consumers_test.yaml | 15 ++++- packages/system/seaweedfs/values.yaml | 11 ++++ 3 files changed, 85 insertions(+), 3 deletions(-) create mode 100644 packages/system/seaweedfs/tests/fullname_override_test.yaml diff --git a/packages/system/seaweedfs/tests/fullname_override_test.yaml b/packages/system/seaweedfs/tests/fullname_override_test.yaml new file mode 100644 index 0000000000..545e949b55 --- /dev/null +++ b/packages/system/seaweedfs/tests/fullname_override_test.yaml @@ -0,0 +1,62 @@ +suite: seaweedfs workload naming is pinned to the chart name + +# Upstream chart 4.31 names workloads after the Helm release (`seaweedfs.fullname`) +# instead of after the chart (`seaweedfs.name`, pre-4.31). The shipped release is +# `seaweedfs-system`, so the bump silently renamed every StatefulSet/Deployment to +# `seaweedfs-system-*`. Those names are immutable, so an upgrade cannot rename in +# place — Helm brings up a second, duplicate set beside the running one while the +# data stays on the original `data1-seaweedfs-volume-*` PVCs. +# +# values.yaml pins `fullnameOverride: seaweedfs` so the rendered names stay +# chart-based regardless of the release name, matching the convention already used +# by other system packages (cozy-proxy, flux-operator, linstor-scheduler, ...). +# These tests pin that contract: names must NOT follow the release name. + +release: + name: seaweedfs-system + namespace: tenant-root + +tests: + - it: names the master StatefulSet after the chart, not the release + template: charts/seaweedfs/templates/master/master-statefulset.yaml + asserts: + - equal: + path: metadata.name + value: seaweedfs-master + - equal: + path: spec.serviceName + value: seaweedfs-master + + - it: names the volume StatefulSet after the chart, so its data PVCs keep the legacy names + template: charts/seaweedfs/templates/volume/volume-statefulset.yaml + asserts: + - equal: + path: metadata.name + value: seaweedfs-volume + # Together with the StatefulSet name this yields data1-seaweedfs-volume-N — + # the PVC names the running clusters already hold their data on. + - equal: + path: spec.volumeClaimTemplates[0].metadata.name + value: data1 + + - it: names the filer StatefulSet after the chart + template: charts/seaweedfs/templates/filer/filer-statefulset.yaml + asserts: + - equal: + path: metadata.name + value: seaweedfs-filer + + - it: keeps the selector labels release-scoped so adopted pods keep matching + template: charts/seaweedfs/templates/master/master-statefulset.yaml + asserts: + # A StatefulSet selector is immutable: in-place adoption only works because + # 4.31 left these labels identical to 4.05. + - equal: + path: spec.selector.matchLabels["app.kubernetes.io/name"] + value: seaweedfs + - equal: + path: spec.selector.matchLabels["app.kubernetes.io/instance"] + value: seaweedfs-system + - equal: + path: spec.selector.matchLabels["app.kubernetes.io/component"] + value: master diff --git a/packages/system/seaweedfs/tests/s3_service_name_consumers_test.yaml b/packages/system/seaweedfs/tests/s3_service_name_consumers_test.yaml index 628d2713be..b536b7c57d 100644 --- a/packages/system/seaweedfs/tests/s3_service_name_consumers_test.yaml +++ b/packages/system/seaweedfs/tests/s3_service_name_consumers_test.yaml @@ -13,9 +13,12 @@ suite: seaweedfs s3 consumers resolve the renamed s3 service # `seaweedfs-system-s3` while the real service is `seaweedfs-s3`. With no # matching endpoints, ingress-nginx returns 503 and COSI cannot reach S3. # -# All tests render under the real release name so name != fullname, the exact -# condition that exposes the bug. The all-in-one case documents the other -# ternary branch, which is unchanged and correct. +# values.yaml now pins fullnameOverride: seaweedfs, which makes name == fullname +# and would hide this bug by accident. So each test re-creates the original +# condition explicitly (fullnameOverride: seaweedfs-system => name != fullname) +# and keeps guarding the divergence itself, independent of what the shipped +# default happens to be. The all-in-one case documents the other ternary branch, +# which is unchanged and correct. templates: - charts/seaweedfs/templates/s3/s3-service.yaml @@ -31,6 +34,7 @@ tests: - it: s3 service is named with the name helper, not componentName template: charts/seaweedfs/templates/s3/s3-service.yaml set: + seaweedfs.fullnameOverride: seaweedfs-system seaweedfs.s3.enabled: true asserts: - equal: @@ -40,6 +44,7 @@ tests: - it: s3 ingress backend points at the s3 service, not a fullname-derived name template: charts/seaweedfs/templates/s3/s3-ingress.yaml set: + seaweedfs.fullnameOverride: seaweedfs-system seaweedfs.s3.enabled: true seaweedfs.s3.ingress.enabled: true seaweedfs.s3.ingress.host: s3.example.com @@ -51,6 +56,7 @@ tests: - it: all-in-one ingress backend still resolves to the all-in-one service template: charts/seaweedfs/templates/s3/s3-ingress.yaml set: + seaweedfs.fullnameOverride: seaweedfs-system seaweedfs.allInOne.enabled: true seaweedfs.allInOne.s3.enabled: true seaweedfs.s3.ingress.enabled: true @@ -63,6 +69,7 @@ tests: - it: iceberg ingress backend points at the s3 service too template: charts/seaweedfs/templates/s3/s3-iceberg-ingress.yaml set: + seaweedfs.fullnameOverride: seaweedfs-system seaweedfs.s3.enabled: true seaweedfs.s3.icebergPort: 8334 seaweedfs.s3.icebergIngress.enabled: true @@ -75,6 +82,7 @@ tests: - it: COSI in-cluster ENDPOINT resolves the s3 service when ingress is disabled template: charts/seaweedfs/templates/cosi/cosi-deployment.yaml set: + seaweedfs.fullnameOverride: seaweedfs-system seaweedfs.cosi.enabled: true seaweedfs.s3.enabled: true seaweedfs.s3.ingress.enabled: false @@ -88,6 +96,7 @@ tests: - it: COSI in-cluster ENDPOINT (filer S3 mode) targets the s3 service on the filer s3 port template: charts/seaweedfs/templates/cosi/cosi-deployment.yaml set: + seaweedfs.fullnameOverride: seaweedfs-system seaweedfs.cosi.enabled: true seaweedfs.s3.enabled: false seaweedfs.filer.s3.enabled: true diff --git a/packages/system/seaweedfs/values.yaml b/packages/system/seaweedfs/values.yaml index 70af9489e8..d360d82ac6 100644 --- a/packages/system/seaweedfs/values.yaml +++ b/packages/system/seaweedfs/values.yaml @@ -11,6 +11,17 @@ global: monitoring: enabled: true seaweedfs: + # Decouple the workload names from the Helm release name. Upstream 4.31 started + # naming resources after the release, and the data-plane release is + # `-system`, so every StatefulSet was renamed to `-system-*`. Those + # names are immutable, so Helm could not rename in place — it stood up a second, + # empty set beside the running one while the data stayed on `data1-seaweedfs-volume-*`. + # Pinning the chart name restores the pre-4.31 names, so the running workloads + # and their volumes are adopted across the bump. extra/seaweedfs does NOT override + # this per release — it relies on this pin — and guards the one case the pin + # cannot serve (a tenant installed fresh on 1.5.x, whose data is on the renamed + # volumes) by refusing to render. + fullnameOverride: seaweedfs master: # 30 GB per volume × 10 GiB PVC = 0 → rounded to 1 maxVolume per # volume server. With defaultReplication=001 (one replica on a From f73796804f4afb1b527e4c9510f16d8e1194de06 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Wed, 15 Jul 2026 12:53:34 +0300 Subject: [PATCH 26/69] fix(seaweedfs): refuse to rename a running SeaweedFS away from its data With the chart-based names pinned back, an upgrade adopts the running workloads and their volumes in place. Before 4.31 the chart ignored the release name, so this holds for an instance running under any name: its data sits on the chart-named data1-seaweedfs-volume-* PVCs either way. Two states cannot be adopted and stop the render instead, each pointing the operator at the matching recovery step: - Fresh on 1.5.x: only the renamed volumes exist, so adopting the chart-based name would rename the workloads away from that data and bring up an empty cluster. Helm cannot move data between PVCs, so the operator is sent to the PV re-bind procedure. - D-split: both sets exist AND the renamed volume servers are live, so they may hold objects written through the split endpoint. Adopting would strand them, so the operator is sent to reconcile the split first. A duplicate that never served (zero ready replicas) is safe to adopt and renders through. The guard mirrors the chart's fullname helper to recognise what 4.31 named this instance (the helper appends the chart name when the release name does not contain it, then truncates), and matches the renamed volumes by shape plus the chart labels on the StatefulSet rather than reconstructing the exact name. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin (cherry picked from commit 052ce2e4a3b7d2d9c0e548c44b995d59f4c9e02f) (cherry picked from commit 5a724e0bc0d43c8afb621a6da4aab5b13a86e774) --- .../extra/seaweedfs/templates/seaweedfs.yaml | 64 ++++ .../tests/fullname_override_named_test.yaml | 167 +++++++++ .../tests/fullname_override_test.yaml | 322 ++++++++++++++++++ 3 files changed, 553 insertions(+) create mode 100644 packages/extra/seaweedfs/tests/fullname_override_named_test.yaml create mode 100644 packages/extra/seaweedfs/tests/fullname_override_test.yaml diff --git a/packages/extra/seaweedfs/templates/seaweedfs.yaml b/packages/extra/seaweedfs/templates/seaweedfs.yaml index 0ae2139770..3eb28515f5 100644 --- a/packages/extra/seaweedfs/templates/seaweedfs.yaml +++ b/packages/extra/seaweedfs/templates/seaweedfs.yaml @@ -98,6 +98,70 @@ {{- $solver := (index .Values._cluster "solver") | default "http01" }} {{- $clusterIssuer := (index .Values._cluster "issuer-name") | default "letsencrypt-prod" }} {{- $wildcardSecret := (index .Values._cluster "wildcard-secret-name") | default "" }} +{{- /* Refuse to rename a running SeaweedFS away from its data. + + Before 4.31 the chart named workloads after the CHART, ignoring the release + name, so every instance ran as `seaweedfs-*` with its data on + `data1-seaweedfs-volume-*`. 4.31 names them after the release instead, so the + upgrade could not rename in place (StatefulSet names are immutable) and Helm + stood up a second, empty set while the data stayed put. system/seaweedfs pins + the chart-based name back, so those workloads and volumes are adopted in place. + + Two states cannot be adopted and stop the render instead: + + - FRESH on 1.5.x: only the renamed volumes exist, so the data was written under + them. Adopting the chart-based name would rename the workloads AWAY from that + data and bring up an empty cluster beside it. Helm cannot move data between + PVCs, so the operator is sent to the PV re-bind runbook (Step 2). + - D-split: both sets exist AND the renamed volume servers are live, so they may + hold objects written through the split endpoint. Adopting the chart-based set + would strand those. The operator is sent to reconcile the split first (Step 3). + A duplicate that never served (D-wedged: zero ready replicas) is safe to adopt + and renders through. + + The renamed volumes are matched by SHAPE rather than by reconstructing the + name: a volume PVC is always `data1--volume[-]-N`, and + `` is `seaweedfs` only for the adopted naming. Reconstructing it + instead would have to replay the chart's fullname helper AND its 56-character + component truncation, and would silently miss a tenant whose name is long or + does not contain the chart name. + + PVCs hold the data and outlive any workload, but carry no chart labels, so they + are matched on the name shape and must mention the chart to avoid tripping over + an unrelated `data1-*-volume-*` claim. StatefulSets do carry the labels, so they + are matched on those, which also covers a tenant whose PVCs are not provisioned + yet and one whose name the chart truncated past the chart name; the renamed + volume StatefulSet is also where the D-split liveness signal is read. */}} +{{- $legacyData := false }} +{{- $systemData := false }} +{{- $systemRunning := false }} +{{- range (lookup "v1" "PersistentVolumeClaim" .Release.Namespace "").items | default list }} +{{- if and (hasPrefix "data1-" .metadata.name) (contains "-volume" .metadata.name) (contains "seaweedfs" .metadata.name) }} +{{- if hasPrefix "data1-seaweedfs-volume" .metadata.name }} +{{- $legacyData = true }} +{{- else }} +{{- $systemData = true }} +{{- end }} +{{- end }} +{{- end }} +{{- range (lookup "apps/v1" "StatefulSet" .Release.Namespace "").items | default list }} +{{- if and (eq (dig "app.kubernetes.io/name" "" (.metadata.labels | default dict)) "seaweedfs") (contains "volume" .metadata.name) }} +{{- if hasPrefix "seaweedfs-volume" .metadata.name }} +{{- $legacyData = true }} +{{- else }} +{{- $systemData = true }} +{{- if gt (int (dig "status" "readyReplicas" 0 .)) 0 }} +{{- $systemRunning = true }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} +{{- if and $legacyData $systemData $systemRunning }} +{{- fail (printf "SeaweedFS %s in namespace %s is running two live SeaweedFS sets (D-split): the renamed set has ready volume servers alongside the adopted set, and both write to the shared seaweedfs-db metadata. Rendering would adopt the chart-named set and strand the objects the renamed set wrote. Stop the split and reconcile the two sets before upgrading — see docs/operations/seaweedfs-431-rename-recovery.md (Step 3)." .Release.Name .Release.Namespace) }} +{{- end }} +{{- if and $systemData (not $legacyData) }} +{{- fail (printf "SeaweedFS %s in namespace %s keeps its data on volumes named after the Helm release (this tenant was installed on Cozystack 1.5.x). Workloads are named seaweedfs-*, so rendering would rename them away from that data and bring up an empty cluster. Re-bind the existing PVs onto the data1-seaweedfs-volume-* PVC names before upgrading — see docs/operations/seaweedfs-431-rename-recovery.md" .Release.Name .Release.Namespace) }} +{{- end }} apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: diff --git a/packages/extra/seaweedfs/tests/fullname_override_named_test.yaml b/packages/extra/seaweedfs/tests/fullname_override_named_test.yaml new file mode 100644 index 0000000000..ff627c9bbb --- /dev/null +++ b/packages/extra/seaweedfs/tests/fullname_override_named_test.yaml @@ -0,0 +1,167 @@ +suite: seaweedfs naming guard for a non-default instance name + +# SeaweedFS is a user-creatable kind, so an instance can run under a name other +# than `seaweedfs`. Two things follow, and the guard has to get both right: +# - Before 4.31 the chart ignored the release name, so such an instance ALSO +# stored its data on the chart-named data1-seaweedfs-volume-* PVCs. +# - On 4.31 the fullname helper appends the chart name when the release name +# does not contain it, so a fresh 1.5.x `archive` wrote its data to +# data1-archive-system-seaweedfs-volume-*, NOT data1-archive-system-volume-*. + +templates: + - templates/seaweedfs.yaml + +release: + name: archive + namespace: tenant-root + +set: &values + topology: Simple + replicationFactor: 2 + _namespace: + host: example.org + ingress: tenant-root + _cluster: + issuer-name: letsencrypt-prod + solver: http01 + +tests: + - it: adopts a pre-4.31 instance whose data is on the chart-named PVCs + set: *values + kubernetesProvider: + scheme: + "v1/PersistentVolumeClaim": + gvr: + group: "" + version: "v1" + resource: "persistentvolumeclaims" + namespaced: true + "apps/v1/StatefulSet": + gvr: + group: "apps" + version: "v1" + resource: "statefulsets" + namespaced: true + objects: + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: data1-seaweedfs-volume-0 + namespace: tenant-root + - kind: StatefulSet + apiVersion: apps/v1 + metadata: + name: unrelated-app + namespace: tenant-root + documentSelector: + path: kind + value: HelmRelease + asserts: + - equal: + path: metadata.name + value: archive-system + + - it: refuses to render when this release's data is on its renamed volumes + set: *values + kubernetesProvider: + scheme: + "v1/PersistentVolumeClaim": + gvr: + group: "" + version: "v1" + resource: "persistentvolumeclaims" + namespaced: true + "apps/v1/StatefulSet": + gvr: + group: "apps" + version: "v1" + resource: "statefulsets" + namespaced: true + objects: + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: data1-archive-system-seaweedfs-volume-0 + namespace: tenant-root + - kind: StatefulSet + apiVersion: apps/v1 + metadata: + name: unrelated-app + namespace: tenant-root + asserts: + - failedTemplate: + errorPattern: "SeaweedFS archive in namespace tenant-root keeps its data on volumes named after the Helm release" + + - it: catches a long instance name whose renamed workloads the chart truncated + set: *values + kubernetesProvider: + scheme: + "v1/PersistentVolumeClaim": + gvr: + group: "" + version: "v1" + resource: "persistentvolumeclaims" + namespaced: true + "apps/v1/StatefulSet": + gvr: + group: "apps" + version: "v1" + resource: "statefulsets" + namespaced: true + objects: + # 4.31 truncates to 56 chars before appending -volume, so a long + # name loses the chart name from its tail and the PVC name alone no longer + # identifies it. The StatefulSet still carries the chart labels, so the + # guard sees it there. + - kind: StatefulSet + apiVersion: apps/v1 + metadata: + name: archive-system-a-very-long-instance-name-that-gets-trunca-volume + namespace: tenant-root + labels: + app.kubernetes.io/name: seaweedfs + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: unrelated-app-data-0 + namespace: tenant-root + asserts: + - failedTemplate: + errorPattern: "keeps its data on volumes named after the Helm release" + + - it: ignores an unrelated data1-*-volume-* PVC belonging to another app + set: *values + kubernetesProvider: + scheme: + "v1/PersistentVolumeClaim": + gvr: + group: "" + version: "v1" + resource: "persistentvolumeclaims" + namespaced: true + "apps/v1/StatefulSet": + gvr: + group: "apps" + version: "v1" + resource: "statefulsets" + namespaced: true + objects: + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: data1-clickhouse-volume-0 + namespace: tenant-root + - kind: StatefulSet + apiVersion: apps/v1 + metadata: + name: clickhouse-volume + namespace: tenant-root + labels: + app.kubernetes.io/name: clickhouse + documentSelector: + path: kind + value: HelmRelease + asserts: + - equal: + path: metadata.name + value: archive-system diff --git a/packages/extra/seaweedfs/tests/fullname_override_test.yaml b/packages/extra/seaweedfs/tests/fullname_override_test.yaml new file mode 100644 index 0000000000..cb18fccbe3 --- /dev/null +++ b/packages/extra/seaweedfs/tests/fullname_override_test.yaml @@ -0,0 +1,322 @@ +suite: seaweedfs legacy-naming guard + +# system/seaweedfs pins the chart-based name, so workloads keep their pre-4.31 +# names (`seaweedfs-master`, `data1-seaweedfs-volume-N`) and an upgrade past the +# 4.31 rename adopts the running set in place. +# +# Two states stop the render instead of adopting: a tenant installed FRESH on +# 1.5.x (only the renamed volumes exist → PV re-bind, Step 2), and a D-split +# tenant (both sets present AND the renamed volume servers are live → reconcile +# the split, Step 3). A duplicate that never served (D-wedged, zero ready +# replicas) renders through and is adopted. +# +# Two helm-unittest quirks the mocks work around: +# - The template LISTs both PVCs and StatefulSets, so every mocked provider +# must register both list kinds — a fake client errors on an unregistered +# LIST rather than returning empty. +# - status.readyReplicas is quoted ("1"/"0"): the fake client cannot deep-copy +# a bare Go int in a mocked object's status, and the template's `int` coerces +# the string back either way. + +templates: + - templates/seaweedfs.yaml + +release: + name: seaweedfs + namespace: tenant-root + +set: &values + topology: Simple + replicationFactor: 2 + _namespace: + host: example.org + ingress: tenant-root + _cluster: + issuer-name: letsencrypt-prod + solver: http01 + +tests: + - it: renders for a net-new install (no SeaweedFS data in the namespace) + set: *values + kubernetesProvider: + scheme: + "v1/PersistentVolumeClaim": + gvr: + group: "" + version: "v1" + resource: "persistentvolumeclaims" + namespaced: true + "apps/v1/StatefulSet": + gvr: + group: "apps" + version: "v1" + resource: "statefulsets" + namespaced: true + objects: + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: unrelated-app-data-0 + namespace: tenant-root + - kind: StatefulSet + apiVersion: apps/v1 + metadata: + name: unrelated-app + namespace: tenant-root + documentSelector: + path: kind + value: HelmRelease + asserts: + - equal: + path: metadata.name + value: seaweedfs-system + # The chart-based name is pinned in system/seaweedfs values, not re-pinned here. + - notExists: + path: spec.values.seaweedfs.fullnameOverride + + - it: renders for a cluster whose data is on the legacy seaweedfs-* PVCs + set: *values + kubernetesProvider: + scheme: + "v1/PersistentVolumeClaim": + gvr: + group: "" + version: "v1" + resource: "persistentvolumeclaims" + namespaced: true + "apps/v1/StatefulSet": + gvr: + group: "apps" + version: "v1" + resource: "statefulsets" + namespaced: true + objects: + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: data1-seaweedfs-volume-0 + namespace: tenant-root + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: unrelated-app-data-0 + namespace: tenant-root + - kind: StatefulSet + apiVersion: apps/v1 + metadata: + name: unrelated-app + namespace: tenant-root + documentSelector: + path: kind + value: HelmRelease + asserts: + - equal: + path: metadata.name + value: seaweedfs-system + + - it: renders on a D-wedged cluster (duplicate present but never served) — adopts the legacy set + set: *values + kubernetesProvider: + scheme: + "v1/PersistentVolumeClaim": + gvr: + group: "" + version: "v1" + resource: "persistentvolumeclaims" + namespaced: true + "apps/v1/StatefulSet": + gvr: + group: "apps" + version: "v1" + resource: "statefulsets" + namespaced: true + objects: + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: data1-seaweedfs-volume-0 + namespace: tenant-root + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: data1-seaweedfs-system-volume-0 + namespace: tenant-root + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: unrelated-app-data-0 + namespace: tenant-root + # The duplicate volume set exists but never scheduled (anti-affinity): + # zero ready replicas, so its PVCs are empty and adoption is safe. + - kind: StatefulSet + apiVersion: apps/v1 + metadata: + name: seaweedfs-system-volume + namespace: tenant-root + labels: + app.kubernetes.io/name: seaweedfs + status: + readyReplicas: "0" + documentSelector: + path: kind + value: HelmRelease + asserts: + - equal: + path: metadata.name + value: seaweedfs-system + + - it: refuses to render on a D-split cluster (duplicate volume servers are live) + set: *values + kubernetesProvider: + scheme: + "v1/PersistentVolumeClaim": + gvr: + group: "" + version: "v1" + resource: "persistentvolumeclaims" + namespaced: true + "apps/v1/StatefulSet": + gvr: + group: "apps" + version: "v1" + resource: "statefulsets" + namespaced: true + objects: + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: data1-seaweedfs-volume-0 + namespace: tenant-root + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: data1-seaweedfs-system-volume-0 + namespace: tenant-root + # The duplicate volume set is live: it may hold objects written through + # the split endpoint, so adoption would strand them. + - kind: StatefulSet + apiVersion: apps/v1 + metadata: + name: seaweedfs-system-volume + namespace: tenant-root + labels: + app.kubernetes.io/name: seaweedfs + status: + readyReplicas: "1" + asserts: + - failedTemplate: + errorPattern: "is running two live SeaweedFS sets.*Stop the split" + + - it: renders for a zoned legacy cluster (volume StatefulSets are per zone) + set: *values + kubernetesProvider: + scheme: + "v1/PersistentVolumeClaim": + gvr: + group: "" + version: "v1" + resource: "persistentvolumeclaims" + namespaced: true + "apps/v1/StatefulSet": + gvr: + group: "apps" + version: "v1" + resource: "statefulsets" + namespaced: true + objects: + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: data1-seaweedfs-volume-zone-a-0 + namespace: tenant-root + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: unrelated-app-data-0 + namespace: tenant-root + - kind: StatefulSet + apiVersion: apps/v1 + metadata: + name: unrelated-app + namespace: tenant-root + documentSelector: + path: kind + value: HelmRelease + asserts: + - equal: + path: metadata.name + value: seaweedfs-system + + - it: refuses to render when the data lives only on seaweedfs-system-* PVCs + set: *values + kubernetesProvider: + scheme: + "v1/PersistentVolumeClaim": + gvr: + group: "" + version: "v1" + resource: "persistentvolumeclaims" + namespaced: true + "apps/v1/StatefulSet": + gvr: + group: "apps" + version: "v1" + resource: "statefulsets" + namespaced: true + objects: + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: data1-seaweedfs-system-volume-0 + namespace: tenant-root + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: unrelated-app-data-0 + namespace: tenant-root + - kind: StatefulSet + apiVersion: apps/v1 + metadata: + name: unrelated-app + namespace: tenant-root + asserts: + - failedTemplate: + errorPattern: "keeps its data on volumes named after the Helm release.*Re-bind" + + - it: refuses to render on a seaweedfs-system-* cluster detected by its StatefulSet + set: *values + kubernetesProvider: + scheme: + "v1/PersistentVolumeClaim": + gvr: + group: "" + version: "v1" + resource: "persistentvolumeclaims" + namespaced: true + "apps/v1/StatefulSet": + gvr: + group: "apps" + version: "v1" + resource: "statefulsets" + namespaced: true + objects: + - kind: StatefulSet + apiVersion: apps/v1 + metadata: + name: seaweedfs-system-volume + namespace: tenant-root + labels: + app.kubernetes.io/name: seaweedfs + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: unrelated-app-data-0 + namespace: tenant-root + - kind: StatefulSet + apiVersion: apps/v1 + metadata: + name: unrelated-app + namespace: tenant-root + asserts: + - failedTemplate: + errorPattern: "keeps its data on volumes named after the Helm release.*Re-bind" From 35ebc1188db4900baa0ec8e45d822b6d3003370f Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Wed, 15 Jul 2026 12:53:34 +0300 Subject: [PATCH 27/69] docs(seaweedfs): add 4.31 rename audit & recovery runbook Classify each tenant from its volume PVCs, re-bind the volumes of a tenant installed fresh on 1.5.x so the adopted names pick them up, stop a split cluster before upgrading it, and clear the leftovers the upgrade cannot remove itself. The recovery steps select the renamed set by a precise filter and derive secret names from the fullname helper, so they work for an instance running under a non-default name and for pooled or zoned volumes. Co-authored-by: Myasnikov Daniil Assisted-By: Claude Signed-off-by: Aleksei Sviridkin (cherry picked from commit 088669fb5ae5106aa1d4d22e3226488f557e4b02) --- .../seaweedfs-431-rename-recovery.md | 197 ++++++++++++++++++ 1 file changed, 197 insertions(+) create mode 100644 docs/operations/seaweedfs-431-rename-recovery.md diff --git a/docs/operations/seaweedfs-431-rename-recovery.md b/docs/operations/seaweedfs-431-rename-recovery.md new file mode 100644 index 0000000000..adbedc4485 --- /dev/null +++ b/docs/operations/seaweedfs-431-rename-recovery.md @@ -0,0 +1,197 @@ +# SeaweedFS 4.31 rename — audit & recovery + +This runbook covers clusters affected by the SeaweedFS chart-rename regression introduced when the vendored chart was bumped from `4.0.405` to `4.31.0` (Cozystack v1.5.0). Use it to classify each tenant, and to recover the ones that need an operator before they can be upgraded. + +## Background + +Before 4.31 the chart named workloads after the chart (`seaweedfs-*`), ignoring the release name. 4.31 names them after the release, and the data-plane HelmRelease is `-system`, so every StatefulSet wanted to become `seaweedfs-system-*`. StatefulSet names are immutable, so the upgrade could not rename in place — Helm stood up a second, duplicate set beside the running one. Depending on cluster size the duplicate either deadlocks or splits: + +- **D-wedged** — with as many nodes as master replicas, the new masters cannot schedule (hard pod anti-affinity against the old masters). The new set stays `Pending`/`CrashLoopBackOff`, the old set keeps serving. No data at risk. +- **D-split** — with more nodes than masters, the new (empty) set comes up. Both sets carry identical pod labels, so the `seaweedfs-s3` Service load-balances across them, and both filers write to the **same `seaweedfs-db` Postgres** metadata store while pointing at different volume servers. This is a data-integrity incident, not just a duplicate: reads of existing objects through the new endpoint miss, new writes land on empty volumes, and the two master sets hand out volume IDs from independent sequences into one shared metadata table. + +The fix pins `fullnameOverride: seaweedfs` in `system/seaweedfs` values, so workloads are always named after the chart, exactly as they were before 4.31. Upgrading past the bump therefore **adopts the running set and its volumes in place**. + +One class cannot be adopted that way: a tenant installed **fresh on 1.5.x**, whose data was written under the release-based names and lives on `data1-seaweedfs-system-volume-*` PVCs. Pinning the chart name there would rename the workloads *away* from that data. Helm cannot move data between PVCs, so `extra/seaweedfs` refuses to render for such a tenant and points here. Re-bind its volumes (below) before upgrading. + +## Step 1 — Audit the fleet (read-only) + +Run per cluster (`KUBECONFIG` pointed at each). It mutates nothing. Classification is driven by the **PVCs**, because they hold the data and outlive any workload. + +```sh +#!/usr/bin/env bash +set -euo pipefail +printf '%-28s %-10s %s\n' NAMESPACE CLASS NOTE +printf '%-28s %-10s %s\n' --------- ----- ---- +# A volume PVC is data1--volume[-]-N. Pre-4.31 was +# always the chart name, so legacy data is data1-seaweedfs-volume-*. On 4.31 it is +# the release name, plus the chart name when the release does not contain it: +# seaweedfs-system-* for the default instance, -system-seaweedfs-* otherwise. +nss=$(kubectl get pvc -A -o jsonpath='{range .items[*]}{.metadata.namespace}{" "}{.metadata.name}{"\n"}{end}' \ + | awk '$2 ~ /^data1-.*volume/ && $2 ~ /seaweedfs/ {print $1}' | sort -u) +[ -z "${nss}" ] && { echo "(no SeaweedFS volume PVCs found)"; exit 0; } +for ns in ${nss}; do + pvcs=$(kubectl get pvc -n "$ns" -o name | sed 's|persistentvolumeclaim/||') + legacy=$(echo "$pvcs" | grep -cE '^data1-seaweedfs-volume' || true) + system=$(echo "$pvcs" | grep -E '^data1-.*volume' | grep -E 'seaweedfs' | grep -vE '^data1-seaweedfs-volume' | grep -c . || true) + running_new=$(kubectl get pods -n "$ns" \ + -l app.kubernetes.io/name=seaweedfs,app.kubernetes.io/component=volume \ + --field-selector=status.phase=Running -o name 2>/dev/null \ + | grep -vc '/seaweedfs-volume-' || true) + if [ "$legacy" -gt 0 ] && [ "$system" -gt 0 ]; then + if [ "${running_new:-0}" -gt 0 ]; then + printf '%-28s %-10s %s\n' "$ns" "D-split" "DANGER: duplicate volume servers Running — audit before upgrading" + else + printf '%-28s %-10s %s\n' "$ns" "D-wedged" "duplicate set never served; upgrade adopts the legacy set" + fi + elif [ "$legacy" -gt 0 ]; then + printf '%-28s %-10s %s\n' "$ns" "L" "pre-1.5 naming; upgrade adopts in place, nothing to do" + else + printf '%-28s %-10s %s\n' "$ns" "S" "fresh 1.5.x install; re-bind volumes (Step 2) BEFORE upgrading" + fi +done +``` + +Act on the classes in this order: resolve every **D-split** and every **S** tenant first (both need an operator), then upgrade. **L** and **D-wedged** need nothing. + +## Step 2 — `S` tenants: re-bind the volumes before upgrading + +The data is on `data1-seaweedfs-system-volume-N`; the fixed chart expects `data1-seaweedfs-volume-N`. Rather than copying objects, re-point the same PersistentVolume at a PVC with the new name. Nothing is written or moved; only the claim is renamed. Expect downtime for this tenant's S3. + +```sh +ns= + +# 1. Stop the operator from fighting the change. +app= # the SeaweedFS resource name; `seaweedfs` unless you renamed it +kubectl -n "$ns" patch helmrelease "${app}-system" --type merge -p '{"spec":{"suspend":true}}' +# Select the renamed workloads precisely. An S tenant has only the renamed set, +# but exclude the pre-4.31 chart-named seaweedfs-master/-filer/-volume[-] +# anyway so the same selector is reused in Step 3, where both sets coexist. +# seaweedfs-system-* and -system-seaweedfs-* are kept. +renamed_sts() { + kubectl -n "$ns" get sts -l app.kubernetes.io/name=seaweedfs -o name \ + | sed 's|statefulset.apps/||' | grep -vE '^seaweedfs-(master|filer|volume)($|-)' +} +for sts in $(renamed_sts); do + kubectl -n "$ns" scale sts "$sts" --replicas=0 +done +kubectl -n "$ns" scale deploy -l app.kubernetes.io/name=seaweedfs,app.kubernetes.io/component=s3 --replicas=0 + +# 2. For every volume PVC: protect its PV, then re-bind it under the new name. +for pvc in $(kubectl -n "$ns" get pvc -o name | sed 's|persistentvolumeclaim/||' \ + | grep -E '^data1-.*volume' | grep seaweedfs | grep -vE '^data1-seaweedfs-volume'); do + # data1--volume[-]-N -> data1-seaweedfs-volume[-]-N + new_pvc=$(echo "$pvc" | sed -E 's/^data1-.*-volume-/data1-seaweedfs-volume-/') + pv=$(kubectl -n "$ns" get pvc "$pvc" -o jsonpath='{.spec.volumeName}') + sc=$(kubectl -n "$ns" get pvc "$pvc" -o jsonpath='{.spec.storageClassName}') + size=$(kubectl -n "$ns" get pvc "$pvc" -o jsonpath='{.spec.resources.requests.storage}') + # Remember the PV's own reclaim policy: it is restored verbatim below, so a volume + # the cluster deliberately set to Retain does not silently come back as Delete. + reclaim=$(kubectl get pv "$pv" -o jsonpath='{.spec.persistentVolumeReclaimPolicy}') + + # Keep the PV (and the data) when the claim goes away. + kubectl patch pv "$pv" -p '{"spec":{"persistentVolumeReclaimPolicy":"Retain"}}' + kubectl -n "$ns" delete pvc "$pvc" + # A Released PV cannot be re-bound until its old claimRef is cleared. + kubectl patch pv "$pv" --type json -p '[{"op":"remove","path":"/spec/claimRef"}]' + + kubectl -n "$ns" apply -f - <-system-seaweedfs-volume-*` for an instance running under another name, because 4.31's name helper appends the chart name when the release name does not contain it. + +## Step 3 — `D-split` tenants: stop the split before upgrading + +Do **not** upgrade first and do not delete the duplicate PVCs — they may hold objects written through the split endpoint. + +1. Take the live duplicate out of the `seaweedfs-s3` Service rotation so nothing else is written to it. Select the renamed set precisely — the pre-4.31 chart-named `seaweedfs-master/-filer/-volume[-]` is the authoritative set and must keep serving: + + ```sh + ns= + kubectl -n "$ns" get sts -l app.kubernetes.io/name=seaweedfs -o name | sed 's|statefulset.apps/||' \ + | grep -vE '^seaweedfs-(master|filer|volume)($|-)' \ + | xargs -r -I{} kubectl -n "$ns" scale sts {} --replicas=0 + # The renamed s3 Deployment is the one whose pod template is NOT the chart-named set; + # scale every seaweedfs s3 Deployment except the adopted `seaweedfs-s3`. + kubectl -n "$ns" get deploy -l app.kubernetes.io/name=seaweedfs,app.kubernetes.io/component=s3 -o name \ + | sed 's|deployment.apps/||' | grep -v '^seaweedfs-s3$' \ + | xargs -r -I{} kubectl -n "$ns" scale deploy {} --replicas=0 + ``` + +2. Confirm the legacy set (`seaweedfs-*`) is the authoritative one and is serving. +3. Audit what the split wrote: enumerate filer entries whose fids resolve only on the duplicate volume servers (objects PUT while both sets were live). Export anything that must survive, then re-upload it through the authoritative endpoint. +4. Only then upgrade, and clean up the duplicate as in Step 4. + +## Step 4 — Upgrade, then clear the leftovers + +Upgrade the cluster to a Cozystack version carrying the fix. On reconcile the `seaweedfs-system` release renders the chart-based names, adopts the running workloads and their volumes, and drops the duplicate `seaweedfs-system-*` StatefulSets and Deployments (they are no longer part of the release). + +Helm does not delete PVCs it did not template, and cert-manager Secrets have no owner reference, so the duplicate's volumes and certificates survive the upgrade as inert leftovers. Remove them by hand once the tenant is verified healthy: + +```sh +ns= +# Duplicate volume PVCs, BY NAME. Never by label: the live data PVCs carry the +# same app.kubernetes.io/instance=seaweedfs-system label and would match too. +kubectl -n "$ns" get pvc -o name | sed 's|persistentvolumeclaim/||' | grep -E '^data1-.*volume' \ + | grep seaweedfs | grep -vE '^data1-seaweedfs-volume' | xargs -r -I{} kubectl -n "$ns" delete pvc {} +# Orphaned certificate/db secrets of the duplicate set. cert-manager names them +# --cert, so the renamed set's are everything matching the cert/db +# suffix EXCEPT the adopted chart-named seaweedfs--cert / seaweedfs-db-secret. +# This covers the default (seaweedfs-system-*) and any non-default (-system- +# seaweedfs-*) instance without hardcoding the name. +kubectl -n "$ns" get secret -o name | sed 's|secret/||' \ + | grep -E '(-cert|-db-secret)$' | grep seaweedfs \ + | grep -vE '^seaweedfs-(admin|ca|client|filer|master|volume|worker)-cert$|^seaweedfs-db-secret$' \ + | xargs -r -I{} kubectl -n "$ns" delete secret {} --ignore-not-found +``` + +Never delete `data1-seaweedfs-volume-*` — those are the live data PVCs of the adopted set. + +If a tenant's `seaweedfs-system` HelmRelease was suspended during triage, resume it so the fix can reconcile: + +```sh +kubectl -n patch helmrelease seaweedfs-system --type merge -p '{"spec":{"suspend":false}}' +kubectl -n annotate helmrelease seaweedfs-system reconcile.fluxcd.io/requestedAt="$(date +%s)" --overwrite +``` + +## Step 5 — Verify + +```sh +ns= +# Exactly one set, named after the chart, healthy. +kubectl -n "$ns" get sts -l app.kubernetes.io/name=seaweedfs +# All three HelmReleases Ready. +kubectl -n "$ns" get helmrelease seaweedfs seaweedfs-system seaweedfs-db +# S3 endpoints resolve only to the adopted set's ready pods. +kubectl -n "$ns" get endpoints seaweedfs-s3 +# Objects are readable through S3 (bucket list via any tenant bucket). +``` + +A recovered tenant has a single `seaweedfs-*` set, all three HelmReleases `Ready`, no `seaweedfs-system-*` StatefulSets, and no `data1-seaweedfs-system-volume-*` PVCs left behind. From 33611dfe7e88a7ac510e5d7a70181ca10cd29b16 Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Mon, 27 Jul 2026 19:40:50 +0500 Subject: [PATCH 28/69] fix(seaweedfs): close the 4.31 rename fallout on the 1.5.x upgrade path Backport of #3339 onto release-1.5, on top of the #3282 adopt-in-place fix (#3326). Three differences from main, all forced by what release-1.5 does not carry: * The repair migration is renumbered 53 -> 45 and stamps 46, since release-1.5 tops out at migration 44 with targetVersion 45. platform values.yaml goes to targetVersion 46. * Migrations 43 and 45 keep release-1.5's inline labelled stamp instead of lib/cozystack-version.sh. That helper is the shared stamp refactor (#2980), whose backport (#3065) was closed, so it does not exist on this branch. Only lib/seaweedfs-db-adopt.sh -- the part that actually carries the fix -- is sourced. * The extra/seaweedfs post-delete cleanup hook and its three tests are dropped. The hook arrived with #3092, a breaking change that was never backported, so there is nothing here for #3339's changes to it to apply to. Comments in lib/seaweedfs-db-adopt.sh and the recovery runbook that promised the hook would reclaim an orphaned volume are corrected to say it does not on 1.5.x. The migration renumbering is carried through the shared helper, the bats suite (STAMP 54 -> 46) and the runbook prose. Verified locally: helm unittest green for both seaweedfs charts (51 + 16 assertions) including both blind-upgrade guard canaries, and 34 bats assertions across migration-seaweedfs-db-adopt, seaweedfs-naming-audit and seaweedfs-guard-parity. Assisted-By: Claude Signed-off-by: Myasnikov Daniil (cherry picked from commit 88e9fdcbbbff8aeab317b08557ca6bd6c8b5706d) --- .../seaweedfs-431-rename-recovery.md | 287 +++++++-- hack/migration-seaweedfs-db-adopt.bats | 333 ++++++++++ hack/seaweedfs-guard-parity.bats | 119 ++++ hack/seaweedfs-naming-audit.bats | 140 +++++ hack/seaweedfs-naming-audit.sh | 237 ++++++++ hack/testdata/migration-seaweedfs-db/kubectl | 101 ++++ .../platform/images/migrations/migrations/43 | 42 +- .../platform/images/migrations/migrations/45 | 65 ++ .../migrations/lib/seaweedfs-db-adopt.sh | 179 ++++++ packages/core/platform/values.yaml | 10 +- packages/extra/seaweedfs/Makefile | 16 + .../extra/seaweedfs/templates/_naming.tpl | 48 ++ .../extra/seaweedfs/templates/seaweedfs.yaml | 83 ++- .../tests/fullname_override_named_test.yaml | 115 ++-- .../tests/fullname_override_test.yaml | 215 +++---- .../tests/guard_fail_closed_test.yaml | 42 ++ packages/system/seaweedfs/Makefile | 15 + .../templates/cosi/cosi-cluster-role.yaml | 6 +- .../templates/shared/cluster-role.yaml | 6 +- .../cluster-scoped-names-per-namespace.patch | 60 ++ .../system/seaweedfs/templates/_naming.tpl | 48 ++ .../templates/cluster-scoped-rbac-guard.yaml | 69 +++ .../seaweedfs/templates/naming-guard.yaml | 154 +++++ .../tests/cluster_scoped_names_test.yaml | 138 +++++ .../tests/cluster_scoped_rbac_guard_test.yaml | 177 ++++++ .../seaweedfs/tests/naming_guard_test.yaml | 568 ++++++++++++++++++ packages/system/seaweedfs/values.yaml | 9 +- 27 files changed, 3024 insertions(+), 258 deletions(-) create mode 100644 hack/migration-seaweedfs-db-adopt.bats create mode 100644 hack/seaweedfs-guard-parity.bats create mode 100644 hack/seaweedfs-naming-audit.bats create mode 100755 hack/seaweedfs-naming-audit.sh create mode 100755 hack/testdata/migration-seaweedfs-db/kubectl create mode 100755 packages/core/platform/images/migrations/migrations/45 create mode 100644 packages/core/platform/images/migrations/migrations/lib/seaweedfs-db-adopt.sh create mode 100644 packages/extra/seaweedfs/templates/_naming.tpl create mode 100644 packages/extra/seaweedfs/tests/guard_fail_closed_test.yaml create mode 100644 packages/system/seaweedfs/patches/cluster-scoped-names-per-namespace.patch create mode 100644 packages/system/seaweedfs/templates/_naming.tpl create mode 100644 packages/system/seaweedfs/templates/cluster-scoped-rbac-guard.yaml create mode 100644 packages/system/seaweedfs/templates/naming-guard.yaml create mode 100644 packages/system/seaweedfs/tests/cluster_scoped_names_test.yaml create mode 100644 packages/system/seaweedfs/tests/cluster_scoped_rbac_guard_test.yaml create mode 100644 packages/system/seaweedfs/tests/naming_guard_test.yaml diff --git a/docs/operations/seaweedfs-431-rename-recovery.md b/docs/operations/seaweedfs-431-rename-recovery.md index adbedc4485..ed4b092441 100644 --- a/docs/operations/seaweedfs-431-rename-recovery.md +++ b/docs/operations/seaweedfs-431-rename-recovery.md @@ -2,56 +2,115 @@ This runbook covers clusters affected by the SeaweedFS chart-rename regression introduced when the vendored chart was bumped from `4.0.405` to `4.31.0` (Cozystack v1.5.0). Use it to classify each tenant, and to recover the ones that need an operator before they can be upgraded. +**Scope — the default instance name is assumed throughout.** The supported way to run SeaweedFS is the tenant module: the tenant chart creates the instance under the fixed name `seaweedfs` (`packages/apps/tenant/templates/seaweedfs.yaml` hardcodes it; a tenant only enables or disables the module). Every shell selector below assumes that name. The API does not yet enforce it, so an instance created directly against `seaweedfses.apps.cozystack.io` under another name can exist — the audit script still classifies it (its release is `-system`), but **do not run the shell loops here against it: escalate instead**, because the name-based `grep seaweedfs` filters cannot see claims whose names the chart truncated (instance names of roughly 30+ characters), and the chart guard's reconstruction does not cover the zone/pool volume components of long-named instances. One accepted limit applies even to default-named instances: a zone or pool **key** of roughly 40+ characters pushes `seaweedfs-system-volume-` past the chart's truncation limit and similarly out of the guard's reconstruction — do not use keys that long. + ## Background Before 4.31 the chart named workloads after the chart (`seaweedfs-*`), ignoring the release name. 4.31 names them after the release, and the data-plane HelmRelease is `-system`, so every StatefulSet wanted to become `seaweedfs-system-*`. StatefulSet names are immutable, so the upgrade could not rename in place — Helm stood up a second, duplicate set beside the running one. Depending on cluster size the duplicate either deadlocks or splits: -- **D-wedged** — with as many nodes as master replicas, the new masters cannot schedule (hard pod anti-affinity against the old masters). The new set stays `Pending`/`CrashLoopBackOff`, the old set keeps serving. No data at risk. +- **D-wedged** — with as many nodes as master replicas, the new masters cannot schedule (hard pod anti-affinity against the old masters). The new set stays `Pending`/`CrashLoopBackOff`, the old set keeps serving. Usually no data at risk — but "usually" is not something a Helm render can verify (a duplicate that served writes and later crashed or was scaled down is indistinguishable from one that never started), so the chart refuses these too rather than adopt on an assumption. - **D-split** — with more nodes than masters, the new (empty) set comes up. Both sets carry identical pod labels, so the `seaweedfs-s3` Service load-balances across them, and both filers write to the **same `seaweedfs-db` Postgres** metadata store while pointing at different volume servers. This is a data-integrity incident, not just a duplicate: reads of existing objects through the new endpoint miss, new writes land on empty volumes, and the two master sets hand out volume IDs from independent sequences into one shared metadata table. The fix pins `fullnameOverride: seaweedfs` in `system/seaweedfs` values, so workloads are always named after the chart, exactly as they were before 4.31. Upgrading past the bump therefore **adopts the running set and its volumes in place**. -One class cannot be adopted that way: a tenant installed **fresh on 1.5.x**, whose data was written under the release-based names and lives on `data1-seaweedfs-system-volume-*` PVCs. Pinning the chart name there would rename the workloads *away* from that data. Helm cannot move data between PVCs, so `extra/seaweedfs` refuses to render for such a tenant and points here. Re-bind its volumes (below) before upgrading. +Two states cannot be adopted that way, and the charts refuse to render for both rather than guess: + +- A tenant installed **fresh on 1.5.x**, whose data was written under the release-based names and lives on `data1-seaweedfs-system-volume-*` PVCs. Pinning the chart name there would rename the workloads *away* from that data, and Helm cannot move data between PVCs. Re-bind its volumes (Step 2) before upgrading. +- A tenant where **both** naming generations exist. One of them is an empty duplicate and one holds the data — but nothing durable in the object graph says which. Claim timestamps are not evidence: Step 2's own re-bind deletes and recreates claims, so a tenant interrupted mid-recovery has a brand-new claim holding real data. StatefulSets are recreated by the adoption hook. `readyReplicas: 0` does not prove a duplicate never served. Rendering would adopt the chart-named set, so a wrong guess strands or destroys data. Step 1 classifies these with signals a template does not have; once the empty generation is deleted, exactly one remains and the render proceeds on its own. + +The **enforcing** guard lives in `system/seaweedfs` (`templates/naming-guard.yaml`) — the `-system` HelmRelease pulls that chart straight from a platform-managed ExternalArtifact, so a platform upgrade re-renders it directly and nothing else stands between the upgrade and the tenant's workloads; `extra/seaweedfs` carries a sibling copy so the refusal is also visible on the SeaweedFS application itself. + +A tenant upgrading **1.4.x straight to 1.6 never renames**, so it only ever has one generation and is unaffected by any of this. Duplicates exist only on tenants that passed through 1.5.x. + +## Step 0 — `seaweedfs-db` ownership check (read-only, do this FIRST) + +Unrelated to the rename, but it lands on the same upgrade and it destroys data rather than duplicating it, so clear it before anything else. + +The v1.5.0 db split moved the CNPG `Cluster/seaweedfs-db` — the filer metadata store, i.e. the index for every object in the tenant's S3 — out of the `-system` release into its own `-db` release. Migration 43 performs the hand-over: it re-owns the Cluster to `-db` and stamps `helm.sh/resource-policy: keep` so the `-system` upgrade, whose chart no longer renders the Cluster, does not delete it as a removed resource. **Migration 43 shipped comparing the owning release name against the literal `seaweedfs-system`**, so it only ever fired for an instance named `seaweedfs`. `SeaweedFS` is a user-creatable kind: an instance named `foo` is owned by `foo-system`, was skipped, and had its Cluster pruned — CNPG takes the PVC with it. + +The prune is not a one-shot. Helm computes deletions by diffing the **last deployed** revision against the new manifest, so a tenant whose `-system` last succeeded on a pre-split revision recomputes the same deletion on *every* upgrade attempt — including attempts that fail for unrelated reasons and never become the new deployed revision. Such a tenant re-deletes the Cluster each time `-db` recreates it. + +Migration 43 is fixed to match the `-system` suffix, and migration 45 re-runs the hand-over for clusters that already ran the hardcoded version. Both are pre-upgrade hooks, so they land before `-system` re-renders. Audit anyway — a Cluster already deleted cannot be recovered by either: + +```sh +kubectl get cluster.postgresql.cnpg.io -A \ + -o custom-columns='NS:.metadata.namespace,NAME:.metadata.name,OWNER:.metadata.annotations.meta\.helm\.sh/release-name,KEEP:.metadata.annotations.helm\.sh/resource-policy' +``` + +Read the rows for `NAME=seaweedfs-db`: + +| OWNER | KEEP | Meaning | +|---|---|---| +| `-db` | `keep` | Handed over. Nothing to do. | +| `-db` | *(none)* | Installed fresh on ≥1.5.0. Safe: `-system` never rendered the Cluster, so it is not in that release's prune baseline. | +| `-system` | *(none)* | **At risk.** Migration 45 hands it over on the next upgrade. Do not reconcile `-system` before the migration runs. | +| *(no row at all)* | | **Already lost** — see below. | + +A tenant with a SeaweedFS instance but **no `seaweedfs-db` row** has already had its metadata deleted. Its S3 returns 500 and its objects are unreachable even though the volume PVCs still hold the bytes. Nothing in this runbook or in any migration can rebuild that index: the Cluster and its PVC are gone. Restore the `seaweedfs-db` Postgres from a backup, or treat the tenant's object storage as lost. Note that `-db` may report **Ready** while this is true — it rendered its Cluster successfully and a later `-system` prune removed it; Flux has not re-checked. Trust the `kubectl get cluster` output, not the HelmRelease status. ## Step 1 — Audit the fleet (read-only) -Run per cluster (`KUBECONFIG` pointed at each). It mutates nothing. Classification is driven by the **PVCs**, because they hold the data and outlive any workload. +```sh +hack/seaweedfs-naming-audit.sh # whole cluster +hack/seaweedfs-naming-audit.sh tenant-foo # or named namespaces +``` + +It mutates nothing. Earlier revisions of this runbook inlined the classification as a shell snippet here; it is a tested script now (`hack/seaweedfs-naming-audit.bats`), because it is what the chart's refusal hands you to and acting on it deletes PVCs. Two inline versions shipped wrong — one whose selector matched both generations at once and so inverted its own primary rule, one that could not see a long instance name at all — so it is not a snippet any more. + +It reports one class per SeaweedFS instance, matching exactly what the chart's guard decides: + +| CLASS | Meaning | Action | +|---|---|---| +| `L` | Only the chart-named generation. | None. The upgrade adopts it in place. | +| `S` | Only the release-named generation — installed fresh on 1.5.x, or a long instance name. | Step 2, before upgrading. | +| `MIXED` | Both generations. The chart **refuses**. | Below. | + +For `MIXED` it also names which generation is **original**, from two independent durable signals: the naming scheme revision 1 of the `-system` release was installed with, and each generation's **PersistentVolume** creation timestamps. It reads PV timestamps, never claim timestamps — Step 2 deletes each release-named claim and recreates it under the chart name against the same PV, so claim age is not durable and inverts for a tenant interrupted mid-re-bind. The direction rule is **relative, never a clock**: a generation is the candidate duplicate only when *every* one of its bound PVs is strictly newer than every bound PV of the other generation — the same precondition Step 2a enforces. A tenant interrupted mid-re-bind has both generations on original-vintage PVs, so their ranges **overlap** and the audit names no candidate: finish Step 2, do not run Step 2a. + +**Read the audit's own warning.** "Original" is not "the other one is empty", and the gap is exactly where it matters. A duplicate that never scheduled (safe to delete) and one that served writes and later crashed or was scaled down (holds unique objects, deleting destroys them) are **identical on every durable signal** — same revision-1 scheme, same `first_deployed` deltas. The audit narrows the question to one generation; it does not answer it. Before deleting anything, establish that the candidate is empty: + +Empty means: no volume files (`.dat`/`.idx`/`.vif`) in any data directory. Do **not** `kubectl exec` into the candidate's own pods to check — a wedged duplicate's pods never start, so a check that needs the pod running is unexecutable exactly for the class where it matters most. Mount the claims instead: ```sh -#!/usr/bin/env bash -set -euo pipefail -printf '%-28s %-10s %s\n' NAMESPACE CLASS NOTE -printf '%-28s %-10s %s\n' --------- ----- ---- -# A volume PVC is data1--volume[-]-N. Pre-4.31 was -# always the chart name, so legacy data is data1-seaweedfs-volume-*. On 4.31 it is -# the release name, plus the chart name when the release does not contain it: -# seaweedfs-system-* for the default instance, -system-seaweedfs-* otherwise. -nss=$(kubectl get pvc -A -o jsonpath='{range .items[*]}{.metadata.namespace}{" "}{.metadata.name}{"\n"}{end}' \ - | awk '$2 ~ /^data1-.*volume/ && $2 ~ /seaweedfs/ {print $1}' | sort -u) -[ -z "${nss}" ] && { echo "(no SeaweedFS volume PVCs found)"; exit 0; } -for ns in ${nss}; do - pvcs=$(kubectl get pvc -n "$ns" -o name | sed 's|persistentvolumeclaim/||') - legacy=$(echo "$pvcs" | grep -cE '^data1-seaweedfs-volume' || true) - system=$(echo "$pvcs" | grep -E '^data1-.*volume' | grep -E 'seaweedfs' | grep -vE '^data1-seaweedfs-volume' | grep -c . || true) - running_new=$(kubectl get pods -n "$ns" \ - -l app.kubernetes.io/name=seaweedfs,app.kubernetes.io/component=volume \ - --field-selector=status.phase=Running -o name 2>/dev/null \ - | grep -vc '/seaweedfs-volume-' || true) - if [ "$legacy" -gt 0 ] && [ "$system" -gt 0 ]; then - if [ "${running_new:-0}" -gt 0 ]; then - printf '%-28s %-10s %s\n' "$ns" "D-split" "DANGER: duplicate volume servers Running — audit before upgrading" - else - printf '%-28s %-10s %s\n' "$ns" "D-wedged" "duplicate set never served; upgrade adopts the legacy set" - fi - elif [ "$legacy" -gt 0 ]; then - printf '%-28s %-10s %s\n' "$ns" "L" "pre-1.5 naming; upgrade adopts in place, nothing to do" - else - printf '%-28s %-10s %s\n' "$ns" "S" "fresh 1.5.x install; re-bind volumes (Step 2) BEFORE upgrading" - fi -done +ns= +# 1. Stop the candidate's workloads so its RWO claims can be mounted elsewhere. +# Reversible, and it is Step 3's first action anyway (Step 2a's for S-damaged). +# 2. A candidate claim still Pending has no PV and therefore no data — empty by +# construction; skip it. +# 3. Mount each BOUND candidate claim read-only in a scratch pod: +pvc= # e.g. data1-seaweedfs-system-volume-0; repeat per claim +kubectl -n "$ns" apply -f - < -- weed shell -c "volume.list" ``` -Act on the classes in this order: resolve every **D-split** and every **S** tenant first (both need an operator), then upgrade. **L** and **D-wedged** need nothing. +If a candidate claim cannot be inspected, or the two views disagree, **stop and escalate**. Both generations holding real data is recoverable; deleting the wrong one is not. + +**D-wedged** — a duplicate that never scheduled — is `MIXED` too, and the chart refuses it like any other duplicate. On main it rendered through, because a duplicate reading `readyReplicas: 0` was taken as proof it never served. That is not proof: a duplicate that served writes and later crashed or was scaled down reads identically. Confirm emptiness as above, then remove it via Step 3 **before** upgrading. + +On a cluster with no spare nodes (nodes ≤ replicas) a wedged duplicate also blocks the adoption rollout even once the render passes: its pods are still *scheduled*, carry the same labels as the adopted set (including `app.kubernetes.io/instance`), and their hard pod anti-affinity keeps the adopted set's rolled pods from landing anywhere (observed as `seaweedfs-filer-1`/`seaweedfs-master-2` stuck Pending on `didn't match pod anti-affinity rules`, wedging the `-system` HelmRelease in upgrade/rollback loops). Step 3 removes it, which resolves that too. ## Step 2 — `S` tenants: re-bind the volumes before upgrading @@ -84,22 +143,36 @@ for pvc in $(kubectl -n "$ns" get pvc -o name | sed 's|persistentvolumeclaim/||' pv=$(kubectl -n "$ns" get pvc "$pvc" -o jsonpath='{.spec.volumeName}') sc=$(kubectl -n "$ns" get pvc "$pvc" -o jsonpath='{.spec.storageClassName}') size=$(kubectl -n "$ns" get pvc "$pvc" -o jsonpath='{.spec.resources.requests.storage}') - # Remember the PV's own reclaim policy: it is restored verbatim below, so a volume - # the cluster deliberately set to Retain does not silently come back as Delete. + # Stash the PV's own reclaim policy ON THE PV before changing it, so a volume the + # cluster deliberately set to Retain does not silently come back as Delete -- and + # so the record survives this loop being interrupted, which a shell variable would + # not. Step 5 restores it from the annotation once the tenant is verified healthy. reclaim=$(kubectl get pv "$pv" -o jsonpath='{.spec.persistentVolumeReclaimPolicy}') + if [ -z "$(kubectl get pv "$pv" -o jsonpath='{.metadata.annotations.cozystack\.io/original-reclaim-policy}')" ]; then + kubectl annotate pv "$pv" "cozystack.io/original-reclaim-policy=${reclaim}" || { echo "FAILED to record the original reclaim policy on $pv; aborting before any PVC is deleted" >&2; break; } + fi # Keep the PV (and the data) when the claim goes away. - kubectl patch pv "$pv" -p '{"spec":{"persistentVolumeReclaimPolicy":"Retain"}}' + kubectl patch pv "$pv" -p '{"spec":{"persistentVolumeReclaimPolicy":"Retain"}}' || { echo "FAILED to set Retain on $pv; aborting before deleting its PVC" >&2; break; } kubectl -n "$ns" delete pvc "$pvc" # A Released PV cannot be re-bound until its old claimRef is cleared. kubectl patch pv "$pv" --type json -p '[{"op":"remove","path":"/spec/claimRef"}]' + # The labels matter: they are what the StatefulSet's volumeClaimTemplate renders, + # and reconciliation does NOT retrofit claim-template labels onto an existing PVC. + # A claim recreated without them stays unlabelled for good, so any tooling that + # selects volume PVCs by app.kubernetes.io/name + instance -- including the + # post-delete volume reclaim 1.6 adds -- cannot see a recovered tenant's volumes, + # and they leak on a later app deletion. kubectl -n "$ns" apply -f - <-system-seaweedfs-volume-*` for an instance running under another name, because 4.31's name helper appends the chart name when the release name does not contain it. -## Step 3 — `D-split` tenants: stop the split before upgrading +## Step 2a — `S-damaged` tenants: remove the empty chart-named set first + +An `S` tenant that an unguarded 1.6.0 upgrade already reached has an extra problem: the upgrade **created** chart-named workloads (`seaweedfs-master/-filer/-volume`, an s3 Deployment) and **empty** `data1-seaweedfs-volume-*` PVCs beside the live renamed set, and deleted the renamed `-s3` Service (only `seaweedfs-s3` remains — its endpoints may still resolve to the renamed set's pods because both sets carry identical labels, which is luck, not design: if the chart-named pods ever become Ready, the Service splits reads across a live set and an empty one). -Do **not** upgrade first and do not delete the duplicate PVCs — they may hold objects written through the split endpoint. +> **STOP if you are resuming an interrupted Step 2.** This step deletes every `data1-seaweedfs-volume-*` claim. If Step 2 already re-bound some of them, those claims hold your data and are bound to the original PVs — deleting them is the data-loss path this step used to be reachable through by misclassification. Finish Step 2 instead. The check below refuses in that case, but read the Step 1 classification first and be sure. -1. Take the live duplicate out of the `seaweedfs-s3` Service rotation so nothing else is written to it. Select the renamed set precisely — the pre-4.31 chart-named `seaweedfs-master/-filer/-volume[-]` is the authoritative set and must keep serving: +Verify the direction before touching anything. The chart-named generation must be the **newer** one, judged on the **PV** ages (claims are recreated by Step 2's re-bind, so their timestamps prove nothing). The precondition below refuses unless every chart-named claim is bound to a PV strictly newer than every release-named PV: + +```sh +#!/usr/bin/env bash +set -euo pipefail +ns= +app= + +# PRECONDITION. Every chart-named claim must be bound to a PV strictly newer than +# every release-named PV. A chart-named claim sitting on an OLD PV means Step 2 +# already re-bound it: it holds data, and this step would delete it. +pv_age() { kubectl get pv "$(kubectl -n "$ns" get pvc "$1" -o jsonpath='{.spec.volumeName}')" \ + -o jsonpath='{.metadata.creationTimestamp}'; } +newest_release_pv="" +for pvc in $(kubectl -n "$ns" get pvc -o name | sed 's|persistentvolumeclaim/||' \ + | grep -E '^data1-.*volume' | grep seaweedfs | grep -vE '^data1-seaweedfs-volume'); do + a=$(pv_age "$pvc"); [ "$a" \> "$newest_release_pv" ] && newest_release_pv="$a" +done +[ -n "$newest_release_pv" ] || { echo "REFUSING: no release-named volumes found; this is not an S-damaged tenant"; exit 1; } +for pvc in $(kubectl -n "$ns" get pvc -o name | sed 's|persistentvolumeclaim/||' | grep -E '^data1-seaweedfs-volume'); do + a=$(pv_age "$pvc") + if [ ! "$a" \> "$newest_release_pv" ]; then + echo "REFUSING: $pvc is bound to a PV created $a, NOT newer than the newest release-named PV ($newest_release_pv)." + echo "That claim is not an empty duplicate — Step 2 has most likely already re-bound it. Finish Step 2; do not run this step." + exit 1 + fi +done +echo "ok: every chart-named claim is bound to a strictly newer PV — safe to clear the duplicate" + +kubectl -n "$ns" patch helmrelease "${app}-system" --type merge -p '{"spec":{"suspend":true}}' +# The chart-named workloads were created by the aborted upgrade and never held +# data. Delete them so the re-bind can take over their names. +kubectl -n "$ns" delete sts seaweedfs-master seaweedfs-filer --ignore-not-found +kubectl -n "$ns" get sts -o name | sed 's|statefulset.apps/||' | grep -E '^seaweedfs-volume($|-)' \ + | xargs -r -I{} kubectl -n "$ns" delete sts {} +# The EMPTY chart-named claims block Step 2's re-bind (same names). +kubectl -n "$ns" get pvc -o name | sed 's|persistentvolumeclaim/||' | grep -E '^data1-seaweedfs-volume' \ + | xargs -r -I{} kubectl -n "$ns" delete pvc {} +``` + +Leave the HelmRelease suspended and continue with Step 2 (skip its suspend line; the renamed StatefulSets it scales down are the ones holding the data, exactly as in a plain `S` tenant). Do NOT scale down or delete anything named `*-system-*` before its volumes are re-bound. + +## Step 3 — `MIXED` tenants: remove the duplicate before upgrading + +The chart refuses while both generations exist, so the duplicate must be **gone before** the upgrade, not cleaned up after it. Do not skip to Step 4: it cannot run until this is done. + +Do **not** start by deleting PVCs. If the duplicate ever served, they may hold objects written through the split endpoint. + +1. **Take the duplicate out of service** so nothing else is written to it. Select it precisely — the generation Step 1 named as ORIGINAL is authoritative and must keep serving. For the common case (chart-named original, release-named duplicate): ```sh ns= kubectl -n "$ns" get sts -l app.kubernetes.io/name=seaweedfs -o name | sed 's|statefulset.apps/||' \ | grep -vE '^seaweedfs-(master|filer|volume)($|-)' \ | xargs -r -I{} kubectl -n "$ns" scale sts {} --replicas=0 - # The renamed s3 Deployment is the one whose pod template is NOT the chart-named set; - # scale every seaweedfs s3 Deployment except the adopted `seaweedfs-s3`. + # The renamed s3 Deployment is the one that is NOT the chart-named `seaweedfs-s3`. kubectl -n "$ns" get deploy -l app.kubernetes.io/name=seaweedfs,app.kubernetes.io/component=s3 -o name \ | sed 's|deployment.apps/||' | grep -v '^seaweedfs-s3$' \ | xargs -r -I{} kubectl -n "$ns" scale deploy {} --replicas=0 ``` -2. Confirm the legacy set (`seaweedfs-*`) is the authoritative one and is serving. -3. Audit what the split wrote: enumerate filer entries whose fids resolve only on the duplicate volume servers (objects PUT while both sets were live). Export anything that must survive, then re-upload it through the authoritative endpoint. -4. Only then upgrade, and clean up the duplicate as in Step 4. + If Step 1 named the **release-named** generation as original, this tenant is `S-damaged`: the duplicate is the chart-named set. Use **Step 2a** instead, then Step 2 — the selectors are inverted there. + +2. **Confirm the authoritative set is serving**, and that the duplicate is out of the `seaweedfs-s3` endpoints: + + ```sh + kubectl -n "$ns" get endpoints seaweedfs-s3 -o yaml + ``` + +3. **If the duplicate ever served, escalate.** Step 1's emptiness check is what decides this. Recovering objects that exist only on a duplicate is not a procedure this runbook can give you: both master sets allocate volume IDs from independent sequences into one shared `seaweedfs-db`, so a fid written through the split endpoint can collide with a fid on the authoritative set, and there is no supported tool that reconciles two volume-ID spaces against one metadata store. Do not improvise it. Involve someone who can plan a per-object export, and treat the tenant as an incident. + +4. **Delete the duplicate's StatefulSets, Deployments and PVCs.** Only once (3) is settled, and only for a duplicate confirmed empty (or whose contents have been exported): + + ```sh + ns= + # StatefulSets and Deployments of the duplicate generation. + kubectl -n "$ns" get sts -l app.kubernetes.io/name=seaweedfs -o name | sed 's|statefulset.apps/||' \ + | grep -vE '^seaweedfs-(master|filer|volume)($|-)' \ + | xargs -r -I{} kubectl -n "$ns" delete sts {} + kubectl -n "$ns" get deploy -l app.kubernetes.io/name=seaweedfs -o name | sed 's|deployment.apps/||' \ + | grep -vE '^seaweedfs-(s3|objectstorage-provisioner)$' \ + | xargs -r -I{} kubectl -n "$ns" delete deploy {} + # Duplicate volume PVCs, BY NAME. Never by label: the live data PVCs carry the + # same app.kubernetes.io/instance=-system label and would match too. + kubectl -n "$ns" get pvc -o name | sed 's|persistentvolumeclaim/||' | grep -E '^data1-.*volume' \ + | grep seaweedfs | grep -vE '^data1-seaweedfs-volume' | xargs -r -I{} kubectl -n "$ns" delete pvc {} + ``` + + Never delete `data1-seaweedfs-volume-*` here — those are the live data PVCs of the authoritative set. (For an `S-damaged` tenant it is the other way round; that is Step 2a's job, and it has its own precondition check.) + +5. **Re-run the audit.** The tenant must now read `L` (or `S`, if you are on the Step 2 path). Only then upgrade. + + ```sh + hack/seaweedfs-naming-audit.sh "$ns" + ``` ## Step 4 — Upgrade, then clear the leftovers -Upgrade the cluster to a Cozystack version carrying the fix. On reconcile the `seaweedfs-system` release renders the chart-based names, adopts the running workloads and their volumes, and drops the duplicate `seaweedfs-system-*` StatefulSets and Deployments (they are no longer part of the release). +Every tenant must read `L` or `S` in the audit before you start: the chart refuses to render while both generations exist, so a `MIXED` tenant does not upgrade at all — Steps 2/2a/3 come first, not after. Once the fleet is clean, upgrade to a Cozystack version carrying the fix. On reconcile the `-system` release renders the chart-based names and adopts the running workloads and their volumes in place. -Helm does not delete PVCs it did not template, and cert-manager Secrets have no owner reference, so the duplicate's volumes and certificates survive the upgrade as inert leftovers. Remove them by hand once the tenant is verified healthy: +A duplicate's StatefulSets and PVCs are removed in Step 3, before the upgrade. What can still be left behind afterwards are objects no release templated and nothing owns: cert-manager Secrets have no owner reference, and PVCs Helm never templated are never GC'd. Remove those once the tenant is verified healthy: ```sh ns= @@ -174,6 +329,26 @@ kubectl -n "$ns" get secret -o name | sed 's|secret/||' \ Never delete `data1-seaweedfs-volume-*` — those are the live data PVCs of the adopted set. +The same 4.31 bump also renamed the tenant's four **cluster-scoped** RBAC objects, and those leftovers are cluster-wide rather than namespaced. Pre-4.31 they were named after the per-namespace service account (`-seaweedfs-*`); 4.31 named them after the Helm release, which is identical in every tenant, so the fleet collided on one object. The fix puts them back on the service account name, which is where a 1.4.x tenant's objects already are — those are adopted in place and need no cleanup. A tenant that passed **through** 1.5.x, however, leaves behind whatever name that release used, and because more than one tenant claimed it, it may not be pruned by any release's manifest: + +```sh +# Stale shared/renamed cluster-scoped RBAC. The go-forward names all start with a +# tenant namespace; anything on the release-based names is a leftover. The fourth +# pre-4.31 object is matched separately: 4.31 renamed the master-rw BINDING from +# upstream's username-shaped system:serviceaccount::default to -rw-crb, so +# the old one carries neither suffix and no tenant prefix. +kubectl get clusterrole,clusterrolebinding \ + -o custom-columns='KIND:.kind,NAME:.metadata.name,OWNER:.metadata.annotations.meta\.helm\.sh/release-name,NS:.metadata.annotations.meta\.helm\.sh/release-namespace' \ + | grep -E 'objectstorage-provisioner|-rw-cr|^\S+\s+system:serviceaccount:.*:default' \ + | grep -vE '^\S+\s+tenant-' +``` + +A `system:serviceaccount:-seaweedfs:default` row is the pre-4.31 master-rw binding. It is superseded by `-seaweedfs-rw-crb` and is pruned automatically by the tenant's own upgrade (it is in that release's manifest), so it should not survive — if it does, the tenant has not upgraded yet. + +Each surviving row is inert once every tenant is upgraded and verified — no release renders those names any more. Confirm the tenant listed in `OWNER`/`NS` is healthy on the go-forward names first, then delete by name. Do not delete anything named `-seaweedfs-*`: those are live. + +**During** a rolling fleet upgrade there is a window, and it is worth expecting rather than debugging: Helm prunes by name without checking ownership, so the first tenant to reconcile onto the fixed chart deletes the shared `seaweedfs-objectstorage-provisioner` / `seaweedfs-rw-cr` objects that a not-yet-upgraded tenant is still bound through. Those tenants' COSI provisioners get 403s on bucket operations until they reconcile onto their own per-namespace RBAC. Existing buckets keep serving — S3 traffic does not go through the provisioner — so this is a provisioning outage, not a data-plane one, and it closes on its own as the remaining tenants reconcile. + If a tenant's `seaweedfs-system` HelmRelease was suspended during triage, resume it so the fix can reconcile: ```sh @@ -195,3 +370,15 @@ kubectl -n "$ns" get endpoints seaweedfs-s3 ``` A recovered tenant has a single `seaweedfs-*` set, all three HelmReleases `Ready`, no `seaweedfs-system-*` StatefulSets, and no `data1-seaweedfs-system-volume-*` PVCs left behind. + +Only once that is true, restore the reclaim policy Step 2 stashed on each PV. Until this runs the volumes are `Retain`, which is deliberate: it is what makes an accidental claim deletion during recovery survivable. + +```sh +ns= +for pvc in $(kubectl -n "$ns" get pvc -o name | sed 's|persistentvolumeclaim/||' | grep -E '^data1-seaweedfs-volume'); do + pv=$(kubectl -n "$ns" get pvc "$pvc" -o jsonpath='{.spec.volumeName}') + orig=$(kubectl get pv "$pv" -o jsonpath='{.metadata.annotations.cozystack\.io/original-reclaim-policy}') + [ -n "$orig" ] || continue + kubectl patch pv "$pv" -p "{\"spec\":{\"persistentVolumeReclaimPolicy\":\"${orig}\"}}" && kubectl annotate pv "$pv" cozystack.io/original-reclaim-policy- +done +``` diff --git a/hack/migration-seaweedfs-db-adopt.bats b/hack/migration-seaweedfs-db-adopt.bats new file mode 100644 index 0000000000..47af1d2d27 --- /dev/null +++ b/hack/migration-seaweedfs-db-adopt.bats @@ -0,0 +1,333 @@ +#!/usr/bin/env bats +# ----------------------------------------------------------------------------- +# Unit tests for the SeaweedFS db-split hand-over shared by platform migrations +# 43 (original) and 45 (repair) — lib/seaweedfs-db-adopt.sh. +# +# The 1.5.0 split (PR #2601) moved Cluster/seaweedfs-db out of the -system +# release into a new -db release. The hand-over must, before -system +# next renders, re-own the Cluster to -db AND stamp +# helm.sh/resource-policy: keep — otherwise the -system upgrade prunes the +# Cluster as a removed resource, CNPG takes its PVC with it, and the tenant's +# filer metadata (all of its S3) is gone. +# +# Three properties are pinned here, each of which shipped broken: +# +# 1. INSTANCE NAME. Migration 43 compared the owner against the literal +# "seaweedfs-system". `SeaweedFS` is user-creatable, so an instance named +# `foo` is owned by `foo-system` and was silently skipped. Observed live: +# four default-named tenants carry release-name=seaweedfs-db + keep and their +# Clusters survived; the one tenant running `foo` has no Cluster at all. +# +# 2. OWNERSHIP IS NOT SAFETY. A Cluster already owned by -db can still +# need keep: where the hand-over was skipped, -system prunes it and +# -db RECREATES it under its own ownership with no keep, while +# -system's prune baseline still lists it. Live proof that the shape is +# real: tenant-l and tenant-root are -db-owned and their -system +# deployed revision still contains the Cluster — only keep saves them. +# +# 3. FAIL CLOSED. Migrations never re-run, so a swallowed error permanently +# leaves at-risk tenants exposed. `for ns in $(kubectl ...)` does not trip +# errexit: on failure the loop runs zero times and the script stamps the +# version anyway. Only "the resource type is not served" (no CNPG at all) and +# "gone between scan and read" may be treated as empty. +# +# These drive the real migration scripts end-to-end against a fake kubectl +# (hack/testdata/migration-seaweedfs-db/), mocking only the cluster boundary. +# +# SHELL. Production runs these under /bin/sh = busybox ash: the migrations image +# is FROM alpine, and run-migrations.sh execs `/migrations/` BY PATH, so the +# kernel honours the `#!/bin/sh` shebang. `set -euo pipefail` and +# errexit-in-function semantics differ from bash, and both are load-bearing here: +# the fail-closed returns from adopt_seaweedfs_db_clusters have to abort the +# script before it reaches its stamp. Asserting fail-closed in a shell that never +# runs it would be asserting nothing. +# +# So these tests do not invoke the migrations through the runner's shell at all: +# run_migration() executes them by path inside the image's own pinned base, read +# from the migrations Dockerfile. Same base image, same interpreter, same +# invocation form — production, rather than an approximation of it. This needs a +# working docker; there is deliberately no host-shell fallback, because every +# fallback available is a shell production never uses. +# +# What that replaced was `sh "$MIG_DIR/"`, which is neither production nor +# bash: on the GitHub ubuntu runner /bin/sh is dash, which has no `set -o +# pipefail` and aborts on line 1 with "set: Illegal option -o pipefail" before +# the script does any work, while on a developer box /bin/sh is often bash, which +# runs green and proves nothing about ash. +# +# cozytest.sh's awk parser recognizes only @test blocks and a bare `}` on its +# own line; there is no bats `run`/`$status`/`setup`. Assertions are direct +# shell tests that exit non-zero on failure. +# +# Run with: hack/cozytest.sh hack/migration-seaweedfs-db-adopt.bats +# ----------------------------------------------------------------------------- + +FAKEBIN="$PWD/hack/testdata/migration-seaweedfs-db" +MIG_DIR="$PWD/packages/core/platform/images/migrations/migrations" + +# The production base image, read out of the migrations Dockerfile rather than +# repeated here, so the interpreter under test cannot drift from the one the +# migrations actually ship on when that pin is bumped. +ALPINE=$(sed -n 's/^FROM \(alpine:[^ ]*\).*$/\1/p' \ + "$PWD/packages/core/platform/images/migrations/Dockerfile" | head -1) + +# run_migration -- run migrations/ the way run-migrations.sh does. +# +# By path, not `sh `: that is what makes the shebang, and therefore the +# interpreter, part of what is under test. The fake kubectl goes on PATH inside +# the container and $WORK is bind-mounted, so $FAKE_CMDLOG is the same file the +# assertions read back on the host. --network none because nothing here may +# reach a real cluster; --user keeps $WORK removable by the test afterwards. +# +# The explicit `return` is load-bearing: cozytest.sh's awk generator rewrites +# every bare `}` in column 0 into `return 0` + `}`, so a helper that falls off +# its own end returns 0 no matter what it ran, and every fail-closed assertion +# below would pass vacuously. Capture the status and return it by hand. +run_migration() { + _run_migration_rc=0 + docker run --rm --network none \ + --user "$(id -u):$(id -g)" \ + -v "$MIG_DIR:/migrations:ro" \ + -v "$FAKEBIN:/fakebin:ro" \ + -v "$WORK:/work" \ + -e PATH=/fakebin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ + -e FAKE_CMDLOG=/work/cmdlog \ + -e NAMESPACE="${NAMESPACE-}" \ + -e FAKE_CLUSTERS="${FAKE_CLUSTERS-}" \ + -e FAKE_LIST_FAIL="${FAKE_LIST_FAIL-}" \ + -e FAKE_GET_FAIL="${FAKE_GET_FAIL-}" \ + -e FAKE_ANNOTATE_FAIL="${FAKE_ANNOTATE_FAIL-}" \ + "$ALPINE" "/migrations/$1" || _run_migration_rc=$? + return "$_run_migration_rc" +} + +# prep resets env to a clean scenario. Tests set FAKE_* afterwards. +prep() { + # Fail here rather than at the first docker run, so the reason is legible. + docker info >/dev/null 2>&1 || { + echo "docker is required: these tests run the migrations inside $ALPINE," >&2 + echo "the base image of the migrations image, so that they exercise busybox" >&2 + echo "ash — the interpreter run-migrations.sh actually gives them." >&2 + return 1 + } + chmod +x "$FAKEBIN/kubectl" + WORK=$(mktemp -d) + export FAKE_CMDLOG="$WORK/cmdlog" + : > "$FAKE_CMDLOG" + export NAMESPACE=cozy-system + export FAKE_CLUSTERS="" + unset FAKE_LIST_FAIL FAKE_GET_FAIL FAKE_ANNOTATE_FAIL || true +} + +# --- 1. instance name ------------------------------------------------------- + +@test "hands over a default-named instance (seaweedfs-system -> seaweedfs-db)" { + prep + export FAKE_CLUSTERS="tenant-root seaweedfs-system -" + rc=0 + run_migration 43 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -eq 0 ] + grep -qF -- "ANNOTATE tenant-root release-name=seaweedfs-db resource-policy=keep" "$FAKE_CMDLOG" + # Migration 43 stamps 44 — asserting the number, not a bare "STAMP": a wrong + # version would loop run-migrations.sh forever. + grep -qF -- "STAMP 44" "$FAKE_CMDLOG" + rm -rf "$WORK" +} + +# THE original regression. Unfixed (owner compared against the literal +# "seaweedfs-system") this namespace is skipped entirely: no ANNOTATE line, and +# the Cluster is left with no keep for the foo-system upgrade to prune. +@test "hands over a NON-default instance name (foo-system -> foo-db)" { + prep + export FAKE_CLUSTERS="tenant-named foo-system -" + rc=0 + run_migration 43 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -eq 0 ] + grep -qF -- "ANNOTATE tenant-named release-name=foo-db resource-policy=keep" "$FAKE_CMDLOG" + rm -rf "$WORK" +} + +@test "migration 45 repairs a non-default instance the hardcoded 43 skipped, and stamps 46" { + prep + export FAKE_CLUSTERS="tenant-named foo-system -" + rc=0 + run_migration 45 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -eq 0 ] + grep -qF -- "ANNOTATE tenant-named release-name=foo-db resource-policy=keep" "$FAKE_CMDLOG" + grep -qF -- "STAMP 46" "$FAKE_CMDLOG" + rm -rf "$WORK" +} + +@test "handles a mixed fleet: every -system owner is handed over, in its own namespace" { + prep + # The shape of the upgrade stand: default-named tenants plus one `foo`. + export FAKE_CLUSTERS="tenant-root seaweedfs-system - +tenant-dsplit seaweedfs-system - +tenant-l seaweedfs-system - +tenant-named foo-system -" + rc=0 + run_migration 45 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -eq 0 ] + grep -qF -- "ANNOTATE tenant-root release-name=seaweedfs-db resource-policy=keep" "$FAKE_CMDLOG" + grep -qF -- "ANNOTATE tenant-dsplit release-name=seaweedfs-db resource-policy=keep" "$FAKE_CMDLOG" + grep -qF -- "ANNOTATE tenant-l release-name=seaweedfs-db resource-policy=keep" "$FAKE_CMDLOG" + grep -qF -- "ANNOTATE tenant-named release-name=foo-db resource-policy=keep" "$FAKE_CMDLOG" + [ "$(grep -c 'ANNOTATE' "$FAKE_CMDLOG")" -eq 4 ] + rm -rf "$WORK" +} + +# --- 2. ownership is not safety -------------------------------------------- + +# A -db-owned Cluster WITHOUT keep is exposed, not done: -system's +# prune baseline may still list the Cluster (live on the stand for tenant-l and +# tenant-root), in which case its next reconcile deletes it. Skipping on +# ownership alone — the shape the previous revision of this helper shipped — +# leaves the database to be pruned. +@test "protects a -db-owned Cluster that is still missing keep" { + prep + export FAKE_CLUSTERS="tenant-named foo-db -" + rc=0 + run_migration 45 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -eq 0 ] + grep -qF -- "ANNOTATE tenant-named release-name= resource-policy=keep" "$FAKE_CMDLOG" + rm -rf "$WORK" +} + +@test "protects a default-named -db-owned Cluster that is still missing keep" { + prep + export FAKE_CLUSTERS="tenant-fresh seaweedfs-db -" + rc=0 + run_migration 45 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -eq 0 ] + grep -qF -- "ANNOTATE tenant-fresh release-name= resource-policy=keep" "$FAKE_CMDLOG" + rm -rf "$WORK" +} + +@test "idempotent: a Cluster already owned by -db AND carrying keep is left alone" { + prep + export FAKE_CLUSTERS="tenant-root seaweedfs-db keep +tenant-named foo-db keep" + rc=0 + run_migration 45 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -eq 0 ] + ! grep -q 'ANNOTATE' "$FAKE_CMDLOG" + grep -qF -- "STAMP 46" "$FAKE_CMDLOG" + rm -rf "$WORK" +} + +@test "leaves a Cluster with no Helm owner annotation alone, but says so" { + prep + # Not Helm-managed. Guessing an owner would be worse than doing nothing, but + # an unowned SeaweedFS database must not pass silently. + export FAKE_CLUSTERS="tenant-manual - -" + rc=0 + run_migration 45 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -eq 0 ] + ! grep -q 'ANNOTATE' "$FAKE_CMDLOG" + grep -qF -- "carries no meta.helm.sh/release-name" "$WORK/out" + rm -rf "$WORK" +} + +@test "leaves a Cluster owned by an unrelated release alone" { + prep + export FAKE_CLUSTERS="tenant-x some-other-release -" + rc=0 + run_migration 45 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -eq 0 ] + ! grep -q 'ANNOTATE' "$FAKE_CMDLOG" + grep -qF -- "owned by unrelated release" "$WORK/out" + rm -rf "$WORK" +} + +@test "refuses a release literally named -system rather than annotating owner -db" { + prep + # "${current%-system}" would be empty, yielding release-name=-db, which no + # release will ever claim: the Cluster would be orphaned by the very step + # meant to protect it. + export FAKE_CLUSTERS="tenant-weird -system -" + rc=0 + run_migration 45 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -eq 0 ] + ! grep -q 'ANNOTATE' "$FAKE_CMDLOG" + grep -qF -- "no instance name" "$WORK/out" + rm -rf "$WORK" +} + +# --- 3. fail closed --------------------------------------------------------- + +@test "a failing fleet scan aborts the migration instead of stamping past it" { + prep + export FAKE_CLUSTERS="tenant-named foo-system -" + export FAKE_LIST_FAIL="Error from server (Timeout): the server was unable to return a response in the time allotted" + rc=0 + run_migration 45 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + # Must propagate: the Job retries rather than advancing the version. + [ "$rc" -ne 0 ] + grep -qF -- "refusing to stamp past an unverified fleet" "$WORK/out" + ! grep -q 'STAMP' "$FAKE_CMDLOG" + rm -rf "$WORK" +} + +@test "an unreadable owner annotation aborts rather than being read as not-Helm-managed" { + prep + export FAKE_CLUSTERS="tenant-named foo-system -" + export FAKE_GET_FAIL="Error from server (Forbidden): clusters.postgresql.cnpg.io \"seaweedfs-db\" is forbidden" + rc=0 + run_migration 45 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -ne 0 ] + grep -qF -- "cannot read the Helm owner" "$WORK/out" + ! grep -q 'STAMP' "$FAKE_CMDLOG" + rm -rf "$WORK" +} + +@test "a failed hand-over aborts rather than stamping a half-migrated fleet" { + prep + export FAKE_CLUSTERS="tenant-named foo-system -" + export FAKE_ANNOTATE_FAIL="Error from server (Conflict): the object has been modified" + rc=0 + run_migration 45 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -ne 0 ] + ! grep -q 'STAMP' "$FAKE_CMDLOG" + rm -rf "$WORK" +} + +# The fail-open that IS load-bearing: a cluster with no CNPG at all must not be +# blocked from upgrading. "The server doesn't have a resource type" is the only +# list failure allowed to mean "nothing to do". +@test "a cluster with no CNPG resource type stamps cleanly without annotating" { + prep + export FAKE_LIST_FAIL="error: the server doesn't have a resource type \"cluster\" in group \"postgresql.cnpg.io\"" + rc=0 + run_migration 45 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -eq 0 ] + grep -qF -- "resource type is not served" "$WORK/out" + ! grep -q 'ANNOTATE' "$FAKE_CMDLOG" + grep -qF -- "STAMP 46" "$FAKE_CMDLOG" + rm -rf "$WORK" +} + +@test "an empty fleet stamps without annotating" { + prep + export FAKE_CLUSTERS="" + rc=0 + run_migration 45 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -eq 0 ] + ! grep -q 'ANNOTATE' "$FAKE_CMDLOG" + grep -qF -- "STAMP 46" "$FAKE_CMDLOG" + rm -rf "$WORK" +} diff --git a/hack/seaweedfs-guard-parity.bats b/hack/seaweedfs-guard-parity.bats new file mode 100644 index 0000000000..cdca9311a4 --- /dev/null +++ b/hack/seaweedfs-guard-parity.bats @@ -0,0 +1,119 @@ +#!/usr/bin/env bats +# ----------------------------------------------------------------------------- +# The SeaweedFS naming guard exists in two copies: +# +# packages/system/seaweedfs/templates/naming-guard.yaml (ENFORCING — the +# -system HelmRelease pulls this chart from a platform-managed +# ExternalArtifact, so a platform upgrade re-renders it directly and nothing +# else stands between the upgrade and the tenant's workloads) +# packages/extra/seaweedfs/templates/seaweedfs.yaml (sibling — warns on +# the SeaweedFS application itself, where the operator looks first) +# +# They cannot be shared: the two packages are separate charts and neither depends +# on cozy-lib, so there is no library either could include the classifier from. +# They were kept in sync by a comment saying "keep the two classifications in +# sync" — and this whole branch exists because the guard lived in the wrong +# chart. Silent drift between the copies is the same failure class, so pin it. +# +# Only the DETECTION block is compared: the two `fail` messages differ by design +# ("SeaweedFS release " vs "SeaweedFS ", matching each chart's voice) +# and the branch structure is asserted separately. It is the detection — which +# objects establish which generation — that must never diverge. +# +# The block starts at the reconstruction, not at the flag declarations: the line +# above it is deliberately different per chart (system/ IS the -system +# release; extra/ is and derives the child), and that is the ONLY sanctioned +# difference. Both are asserted separately below. +# +# Run with: hack/cozytest.sh hack/seaweedfs-guard-parity.bats +# ----------------------------------------------------------------------------- + +SYS="$PWD/packages/system/seaweedfs/templates/naming-guard.yaml" +EXTRA="$PWD/packages/extra/seaweedfs/templates/seaweedfs.yaml" + +# detection_block -- the generation-detection lines, from the first flag +# declaration through the two derived generation booleans. +detection_block() { + sed -n '/\$renamedVol := include/,/\$systemGen := or/p' "$1" +} + +@test "both charts detect naming generations with byte-identical logic" { + a=$(mktemp); b=$(mktemp) + detection_block "$SYS" > "$a" + detection_block "$EXTRA" > "$b" + # Non-empty: a sed range that matched nothing would make this test vacuous. + [ -s "$a" ] + [ -s "$b" ] + diff -u "$a" "$b" + rm -f "$a" "$b" +} + +@test "each chart feeds the reconstruction the -system release name" { + # system/seaweedfs IS that release; extra/seaweedfs is and must derive it. + # Getting this wrong silently reconstructs the wrong prefix, so neither + # generation matches and the guard renders through. + grep -qF -- '$sysRelease := .Release.Name' "$SYS" + grep -qF -- '$sysRelease := printf "%s-system" .Release.Name' "$EXTRA" +} + +@test "both charts reconstruct the renamed prefix rather than prefix-matching alone" { + # An instance legitimately named `seaweedfs-volume` renders release-named objects + # (seaweedfs-volume-system-volume, data1-seaweedfs-volume-system-volume-0) that + # ALSO satisfy the chart-named prefixes. Release-named must be tested FIRST, + # against a reconstructed prefix, or live storage reads as legacy. + for f in "$SYS" "$EXTRA"; do + grep -qF -- '$renamedVol := include "seaweedfs.renamedVolumePrefix" $sysRelease' "$f" + # release-named branch precedes the chart-named fallback in both scans + pv=$(grep -n 'hasPrefix (printf "data1-%s" $renamedVol)' "$f" | cut -d: -f1) + lv=$(grep -n 'hasPrefix "data1-seaweedfs-volume"' "$f" | cut -d: -f1) + [ -n "$pv" ] && [ -n "$lv" ] && [ "$pv" -lt "$lv" ] + ps=$(grep -n 'hasPrefix $renamedVol .metadata.name' "$f" | cut -d: -f1) + ls=$(grep -n 'hasPrefix "seaweedfs-volume" .metadata.name' "$f" | cut -d: -f1) + [ -n "$ps" ] && [ -n "$ls" ] && [ "$ps" -lt "$ls" ] + done +} + +@test "both charts derive the generation flags from PVC and StatefulSet evidence" { + # The OR is load-bearing: a tenant whose PVCs are not provisioned yet is only + # visible through its label-matched StatefulSet. + for f in "$SYS" "$EXTRA"; do + grep -qF -- '$legacyGen := or $legacyPVC $legacySTS' "$f" + grep -qF -- '$systemGen := or $systemPVC $systemSTS' "$f" + done +} + +@test "both charts refuse when both generations are present" { + for f in "$SYS" "$EXTRA"; do + grep -qF -- 'if and $legacyGen $systemGen' "$f" + grep -qF -- 'has BOTH naming generations present' "$f" + done +} + +@test "both charts refuse a release-named-only tenant (class S)" { + for f in "$SYS" "$EXTRA"; do + grep -qF -- 'else if $systemGen' "$f" + grep -qF -- 'keeps its data on volumes named after the Helm release' "$f" + done +} + +@test "neither chart classifies on mutable claim timestamps or liveness" { + # The premise "PVCs are never recreated in place" is false: the runbook's own + # Step 2 re-bind deletes and recreates each claim, so a tenant interrupted + # part-way through reads as the exact inverse of the truth — and the step the + # old classification pointed at deletes the claim Step 2 just re-bound. + # readyReplicas is likewise only a snapshot, not proof a duplicate never + # served. Neither may come back as a discriminator. + for f in "$SYS" "$EXTRA"; do + ! grep -qF -- 'creationTimestamp' "$f" + ! grep -qF -- 'readyReplicas' "$f" + ! grep -qF -- '$systemOldest' "$f" + ! grep -qF -- '$legacyOldest' "$f" + done +} + +@test "both charts gate the guard behind the same cluster-view canary" { + for f in "$SYS" "$EXTRA"; do + grep -qF -- '$canary := lookup "v1" "Namespace" "" .Release.Namespace' "$f" + grep -qF -- 'refusing to upgrade blind' "$f" + done +} diff --git a/hack/seaweedfs-naming-audit.bats b/hack/seaweedfs-naming-audit.bats new file mode 100644 index 0000000000..eb6356e025 --- /dev/null +++ b/hack/seaweedfs-naming-audit.bats @@ -0,0 +1,140 @@ +#!/usr/bin/env bats +# ----------------------------------------------------------------------------- +# Unit tests for hack/seaweedfs-naming-audit.sh. +# +# The chart's naming guard refuses whenever both naming generations exist, so the +# audit IS the classifier — it is what an operator acts on, and acting on it +# deletes PVCs. Two earlier revisions of this classification shipped as an +# untested shell snippet inside the runbook, and both were wrong in ways that +# routed a live tenant into the step that strands its data: +# +# - a selector `^data1-(.*seaweedfs.*)-volume` also matched the CHART-named +# claims, so the release-named age range spanned both generations, every +# tenant read as "ranges overlap / mid-rebind", and a genuine S-damaged tenant +# was routed AWAY from Step 2a (which would have been correct) into Step 2, +# which deletes its data claim; +# - matching claims by name with `grep seaweedfs` cannot see a long instance +# name, whose claims the chart truncates past `seaweedfs`, so such a tenant +# read as "L — nothing to do" while the chart refused it. +# +# Moving the classifier into a file with tests is the point. These drive it +# against a fake kubectl, mocking only the cluster boundary. +# +# cozytest.sh's awk parser recognizes only @test blocks and a bare `}` on its own +# line; there is no bats `run`/`$status`/`setup`. +# +# Run with: hack/cozytest.sh hack/seaweedfs-naming-audit.bats +# ----------------------------------------------------------------------------- + +SEAWEEDFS_AUDIT_LIB=1 +export SEAWEEDFS_AUDIT_LIB +# shellcheck source=seaweedfs-naming-audit.sh +. "$PWD/hack/seaweedfs-naming-audit.sh" + +@test "reconstructs the renamed volume prefix for a default instance" { + [ "$(renamed_volume_prefix seaweedfs-system)" = "seaweedfs-system-volume" ] +} + +@test "reconstructs the renamed volume prefix for a non-default instance" { + # The release name does not contain the chart name, so 4.31 appends it. + [ "$(renamed_volume_prefix foo-system)" = "foo-system-seaweedfs-volume" ] +} + +@test "reconstructs the truncated prefix for a long instance name" { + # componentName cuts the fullname to 62-len("volume")=56 before appending, so + # `seaweedfs` falls off the tail — the case a `grep seaweedfs` name match cannot + # see, and the reason this is reconstructed rather than pattern-matched. + got=$(renamed_volume_prefix archive-of-quarterly-financial-statements-x1-system) + [ "$got" = "archive-of-quarterly-financial-statements-x1-system-seaw-volume" ] + # 56 chars of fullname + "-volume" + [ "${#got}" -eq 63 ] +} + +@test "reconstructs a distinct prefix for an instance named seaweedfs-volume" { + # The pathological case: this instance's RELEASE-named objects + # (seaweedfs-volume-system-volume, data1-seaweedfs-volume-system-volume-0) also + # satisfy the CHART-named prefixes. The reconstruction must return the + # release-named prefix so the release-named branch can be tested first. + got=$(renamed_volume_prefix seaweedfs-volume-system) + [ "$got" = "seaweedfs-volume-system-volume" ] + # It must NOT collide with the chart-named prefix. + [ "$got" != "seaweedfs-volume" ] +} + +@test "the reconstructed prefix never equals the chart-named prefix" { + # If it did, both generations would match one branch and the guard/audit could + # not separate them at all. + for r in seaweedfs-system foo-system seaweedfs-volume-system a-system; do + [ "$(renamed_volume_prefix "$r")" != "seaweedfs-volume" ] + done +} + +@test "clean duplicate: release-named PVs all strictly newer => legacy is original" { + # A tenant that passed through the 4.31 rename: legacy PVs at install time, + # duplicate PVs provisioned by the bad upgrade much later. + [ "$(classify_mixed_direction 1000 1010 5000 5020)" = "legacy-original" ] +} + +@test "S-damaged: chart-named PVs all strictly newer => release-named is original" { + # Installed fresh on 1.5.x (release-named PVs first); an unguarded 1.6 upgrade + # then created empty chart-named claims beside them. + [ "$(classify_mixed_direction 5000 5020 1000 1010)" = "renamed-original" ] +} + +@test "interrupted Step 2 re-bind: overlapping vintages => no candidate" { + # Step 2 re-binds release-named claims onto their ORIGINAL PVs under chart + # names, one claim at a time. Interrupted part-way, BOTH generations sit on + # original-vintage PVs, so the ranges interleave. The old absolute-window + # classifier fell through to a coin flip here and its advice deleted the + # un-re-bound claims; the relative rule must refuse instead. + [ "$(classify_mixed_direction 1000 1010 1005 1015)" = "overlap" ] +} + +@test "a tie is overlap, not a candidate" { + # Second-resolution timestamps: a duplicate provisioned within the same second + # as the newest original PV is not STRICTLY newer. Refusing is recoverable; + # naming the wrong candidate is not. + [ "$(classify_mixed_direction 1000 1010 1010 1020)" = "overlap" ] +} + +@test "direction needs no clock: vintages far from any anchor still classify" { + # The shipped StorageClasses are WaitForFirstConsumer, so PVs appear at + # pod-SCHEDULE time — on a cold cluster minutes after first_deployed. The rule + # must not care: only the two generations' ranges relative to EACH OTHER count. + # (The previous classifier anchored a 120s window on first_deployed and + # misclassified exactly this case.) + [ "$(classify_mixed_direction 100000 100600 200000 200600)" = "legacy-original" ] +} + +@test "the audit's reconstruction agrees with the chart helper it mirrors" { + # hack/seaweedfs-naming-audit.sh and + # packages/system/seaweedfs/templates/_naming.tpl reimplement the same two + # upstream helpers in two languages. If they drift, the audit classifies a + # tenant differently from the render that refuses it, and the operator is + # working from a different picture than the chart. Render the chart helper + # through helm and compare it to this script's output, release by release. + chart=$(mktemp -d) + printf 'apiVersion: v2\nname: probe\nversion: 0.0.0\n' > "$chart/Chart.yaml" + mkdir -p "$chart/templates" + cp packages/system/seaweedfs/templates/_naming.tpl "$chart/templates/" + cat > "$chart/templates/out.yaml" <<'EOF' +apiVersion: v1 +kind: ConfigMap +metadata: + name: probe +data: +{{- range $r := list "seaweedfs-system" "foo-system" "seaweedfs-volume-system" "archive-of-quarterly-financial-statements-x1-system" "a-system" }} + {{ $r }}: {{ include "seaweedfs.renamedVolumePrefix" $r }} +{{- end }} +EOF + helm template probe "$chart" 2>/dev/null | sed -n 's/^ \([a-z0-9-]*-system\): \(.*\)$/\1=\2/p' > "$chart/rendered" + [ -s "$chart/rendered" ] + [ "$(wc -l < "$chart/rendered")" -eq 5 ] + while IFS='=' read -r rel expected; do + [ -n "$rel" ] || continue + got=$(renamed_volume_prefix "$rel") + echo "chart: $rel -> $expected ; audit: $got" + [ "$got" = "$expected" ] + done < "$chart/rendered" + rm -rf "$chart" +} diff --git a/hack/seaweedfs-naming-audit.sh b/hack/seaweedfs-naming-audit.sh new file mode 100755 index 0000000000..a1e28c4c41 --- /dev/null +++ b/hack/seaweedfs-naming-audit.sh @@ -0,0 +1,237 @@ +#!/bin/sh +# SeaweedFS 4.31 rename — fleet audit (READ-ONLY). +# +# Classifies every SeaweedFS tenant into the three states the chart's naming guard +# (packages/system/seaweedfs/templates/naming-guard.yaml) decides between: +# +# L exactly the chart-named generation -> the upgrade adopts it. Nothing to do. +# S exactly the release-named generation -> re-bind first (runbook Step 2). +# MIXED both generations -> the chart REFUSES; an operator must remove the empty one. +# +# For MIXED it also reports which generation is ORIGINAL, from durable evidence: +# +# sh.helm.release.v1.-system.v1 — the manifest of revision 1 names either +# seaweedfs-master (born pre-4.31) or -master (born 4.31). Exact, but +# revision 1 may have been pruned by Helm's history limit. +# PersistentVolume creation timestamps — the original generation's PVs predate the +# duplicate's, which were provisioned later by the bad upgrade. Compared only +# against each other (see the relative rule below); info.first_deployed is +# printed as context but decides nothing. +# +# PV timestamps, not PVC ones: runbook Step 2 deletes each release-named claim and +# recreates it under the chart name against the SAME PV, so claim age is not durable +# and inverts for a tenant interrupted mid-re-bind. The PV survives and keeps its +# creationTimestamp. +# +# Direction is decided by a RELATIVE rule, never a clock window: a generation is +# the candidate duplicate only when EVERY one of its bound PVs is strictly newer +# than every bound PV of the other generation — the same precondition runbook +# Step 2a enforces before deleting anything. Overlapping vintages mean an +# interrupted Step 2 re-bind (both generations on original PVs) or interleaved +# provisioning, and the audit refuses to name a candidate. An absolute window +# measured from first_deployed was tried first and is unsound: the shipped +# StorageClasses are WaitForFirstConsumer, so PVs appear at pod-SCHEDULE time, +# and ordinary cold-cluster latency puts even the original generation's PVs +# minutes past first_deployed — which pushed a mid-rebind tenant into the branch +# whose advice deletes the un-re-bound claims. +# +# IMPORTANT — what this does NOT tell you. "Which generation is original" is not +# "the other one is empty". A duplicate that never scheduled (safe to delete) and one +# that served writes and later crashed (holds unique objects) are identical on every +# durable signal — same revision-1 scheme, same first_deployed deltas. Establishing +# emptiness needs the duplicate's volume files and the filer's volume list, which this +# script does not inspect. It narrows the question; it does not answer it. +# +# Usage: hack/seaweedfs-naming-audit.sh [namespace...] +# KUBECONFIG= hack/seaweedfs-naming-audit.sh +# Exit: 0 always (audit only). Nothing is mutated. +# +# POSIX sh, deliberately. hack/seaweedfs-naming-audit.bats sources this file, and +# cozytest.sh sources the converted test into its own /bin/sh — which is dash on +# the CI runner. A bash shebang would not save it: `.` runs the file's contents in +# the caller's shell and the shebang is just a comment. Keeping the script POSIX +# is what makes the tested shell and the executed shell the same one; anything +# bash-only here is untested in CI and unavailable at run time. `pipefail` in +# particular is not POSIX and dash 0.5.12 (Ubuntu) rejects it outright. +set -u + +# renamed_volume_prefix -- reconstruct the name 4.31 gives the volume +# component of . Mirrors seaweedfs.fullname + seaweedfs.componentName +# (see packages/system/seaweedfs/templates/_naming.tpl, which the chart's guard +# uses for exactly the same purpose): the fullname gains -seaweedfs when the +# release name does not already contain it, is capped at 63, and componentName +# then cuts it to (62 - len("volume")) = 56 before appending -volume. +renamed_volume_prefix() { + release="$1" + case "$release" in + *seaweedfs*) full="$release" ;; + *) full="${release}-seaweedfs" ;; + esac + full=$(printf '%s' "$full" | cut -c1-63 | sed 's/-$//') + printf '%s-volume' "$(printf '%s' "$full" | cut -c1-56 | sed 's/-$//')" +} + +# release_json [rev] -- decode a Helm release secret. +# Helm stores base64(gzip(json)) in Secret.data.release, and Kubernetes base64s +# the data value again, hence the two decodes. +release_json() { + kubectl get secret -n "$1" "sh.helm.release.v1.$2.v${3:-1}" -o jsonpath='{.data.release}' 2>/dev/null \ + | base64 -d 2>/dev/null | base64 -d 2>/dev/null | gunzip 2>/dev/null +} + +# revisions -- retained revision numbers, oldest first. +revisions() { + kubectl get secret -n "$1" -l "name=$2,owner=helm" \ + -o jsonpath='{range .items[*]}{.metadata.labels.version}{"\n"}{end}' 2>/dev/null | sort -n +} + +# system_releases -- the SeaweedFS -system Helm releases in a namespace. +# Filtering on the `-system` suffix alone is not enough: every Cozystack app has a +# -system release (ingress-nginx-system, bucket-*-system, ...), and since the +# generation scan below matches PVCs by NAME across the whole namespace, an +# unrelated release in a namespace that happens to run SeaweedFS would be reported +# as a SeaweedFS tenant. Confirm the chart. +system_releases() { + for rel in $(kubectl get secret -n "$1" \ + -o jsonpath='{range .items[?(@.type=="helm.sh/release.v1")]}{.metadata.labels.name}{"\n"}{end}' 2>/dev/null \ + | grep -E -- '-system$' | sort -u); do + for rev in $(revisions "$1" "$rel"); do + chart=$(release_json "$1" "$rel" "$rev" | sed -n 's/.*"name":"\(cozy-seaweedfs\)".*/\1/p' | head -1) + if [ -n "$chart" ]; then printf '%s\n' "$rel"; fi + break + done + done +} + +# first_deployed -- epoch seconds of the release's first install, +# from any retained revision (the field is identical on all of them). +first_deployed() { + for rev in $(revisions "$1" "$2"); do + ts=$(release_json "$1" "$2" "$rev" | sed -n 's/.*"first_deployed":"\([^"]*\)".*/\1/p' | head -1) + if [ -n "$ts" ]; then date -u -d "$(printf '%s' "$ts" | cut -c1-19)" +%s 2>/dev/null; return; fi + done +} + +# rev1_scheme -- "legacy" | "renamed" | "" (revision 1 pruned). +rev1_scheme() { + m=$(release_json "$1" "$2" 1) + [ -n "$m" ] || return 0 + if printf '%s' "$m" | grep -q 'name: seaweedfs-master'; then printf 'legacy' + elif printf '%s' "$m" | grep -qE "name: $2(-seaweedfs)?-master"; then printf 'renamed' + fi +} + +# pv_epoch -- creation epoch of the PV the claim is BOUND to. +pv_epoch() { + pv=$(kubectl get pvc -n "$1" "$2" -o jsonpath='{.spec.volumeName}' 2>/dev/null) + [ -n "$pv" ] || return 0 + t=$(kubectl get pv "$pv" -o jsonpath='{.metadata.creationTimestamp}' 2>/dev/null) + [ -n "$t" ] || return 0 + date -u -d "$(printf '%s' "$t" | cut -c1-19)" +%s 2>/dev/null +} + +# classify_mixed_direction -- direction of a MIXED +# tenant, from the bound-PV creation-epoch ranges of the chart-named (l*) and +# release-named (r*) generations. Prints one of: +# +# legacy-original every release-named PV strictly newer -> it is the candidate +# duplicate (runbook Step 3) +# renamed-original every chart-named PV strictly newer -> S-damaged, the +# chart-named set is the candidate duplicate (Step 2a, then 2) +# overlap vintages interleave or touch -> no candidate. An interrupted +# Step 2 re-bind puts BOTH generations on original-vintage PVs. +# +# Purely relative — no clock, no window, no first_deployed. Ties (second-resolution +# timestamps) count as overlap: refusing a candidate is recoverable, naming the +# wrong one is not. +classify_mixed_direction() { + if [ "$3" -gt "$2" ]; then printf 'legacy-original' + elif [ "$1" -gt "$4" ]; then printf 'renamed-original' + else printf 'overlap' + fi +} + +audit_ns() { + ns="$1" + for rel in $(system_releases "$ns"); do + prefix=$(renamed_volume_prefix "$rel") + legacy_pvcs=""; renamed_pvcs="" + for pvc in $(kubectl get pvc -n "$ns" -o name 2>/dev/null | sed 's|persistentvolumeclaim/||'); do + case "$pvc" in + "data1-${prefix}"*) renamed_pvcs="$renamed_pvcs $pvc" ;; + data1-seaweedfs-volume*) legacy_pvcs="$legacy_pvcs $pvc" ;; + esac + done + legacy_sts=""; renamed_sts="" + for sts in $(kubectl get sts -n "$ns" -l app.kubernetes.io/name=seaweedfs -o name 2>/dev/null | sed 's|statefulset.apps/||'); do + case "$sts" in + "${prefix}"*) renamed_sts="$renamed_sts $sts" ;; + seaweedfs-volume*) legacy_sts="$legacy_sts $sts" ;; + esac + done + has_legacy=0; has_renamed=0 + [ -n "$legacy_pvcs$legacy_sts" ] && has_legacy=1 + [ -n "$renamed_pvcs$renamed_sts" ] && has_renamed=1 + [ "$has_legacy" = 0 ] && [ "$has_renamed" = 0 ] && continue + + if [ "$has_legacy" = 1 ] && [ "$has_renamed" = 0 ]; then + printf '%-24s %-14s %-8s %s\n' "$ns" "$rel" "L" "chart-named only; the upgrade adopts it, nothing to do" + continue + fi + if [ "$has_renamed" = 1 ] && [ "$has_legacy" = 0 ]; then + printf '%-24s %-14s %-8s %s\n' "$ns" "$rel" "S" "release-named only; re-bind the volumes (Step 2) BEFORE upgrading" + continue + fi + + # Both generations. Report which is ORIGINAL from durable evidence. + fd=$(first_deployed "$ns" "$rel") + scheme=$(rev1_scheme "$ns" "$rel") + printf '%-24s %-14s %-8s %s\n' "$ns" "$rel" "MIXED" "both generations; the chart REFUSES until one is removed" + if [ -n "$scheme" ]; then + printf '%-24s %-14s %-8s revision 1 was installed with the %s names => the %s generation is ORIGINAL\n' \ + "" "" "" "$scheme" "$scheme" + fi + lmin=""; lmax=""; rmin=""; rmax="" + for p in $legacy_pvcs; do + e=$(pv_epoch "$ns" "$p"); [ -n "$e" ] || continue + { [ -z "$lmin" ] || [ "$e" -lt "$lmin" ]; } && lmin=$e + { [ -z "$lmax" ] || [ "$e" -gt "$lmax" ]; } && lmax=$e + done + for p in $renamed_pvcs; do + e=$(pv_epoch "$ns" "$p"); [ -n "$e" ] || continue + { [ -z "$rmin" ] || [ "$e" -lt "$rmin" ]; } && rmin=$e + { [ -z "$rmax" ] || [ "$e" -gt "$rmax" ]; } && rmax=$e + done + if [ -n "$fd" ] && [ -n "$lmin" ] && [ -n "$rmin" ]; then + printf '%-24s %-14s %-8s oldest PV vs first_deployed: chart-named +%ss, release-named +%ss (context only, not the rule)\n' \ + "" "" "" "$((lmin - fd))" "$((rmin - fd))" + fi + if [ -n "$lmin" ] && [ -n "$rmin" ]; then + case $(classify_mixed_direction "$lmin" "$lmax" "$rmin" "$rmax") in + legacy-original) + printf '%-24s %-14s %-8s every release-named PV is strictly newer => chart-named is ORIGINAL, the release-named set is the candidate duplicate (Step 3)\n' "" "" "" ;; + renamed-original) + printf '%-24s %-14s %-8s every chart-named PV is strictly newer => release-named is ORIGINAL, the chart-named set is the candidate duplicate (Step 2a, then Step 2)\n' "" "" "" ;; + overlap) + printf '%-24s %-14s %-8s PV vintages OVERLAP => no candidate. An interrupted Step 2 re-bind looks exactly like this (both generations on original PVs). Finish Step 2 if one is in progress; otherwise escalate. Do NOT run Step 2a.\n' "" "" "" ;; + esac + else + printf '%-24s %-14s %-8s a generation has no bound PVs (Pending claims, or StatefulSets only) => direction cannot be established from PV ages. Resolve the Pending claims or classify by hand.\n' "" "" "" + fi + printf '%-24s %-14s %-8s CANDIDATE ONLY: "original" does not mean the other set is EMPTY. A duplicate that\n' "" "" "" + printf '%-24s %-14s %-8s served writes and later crashed looks identical here. Verify emptiness before deleting.\n' "" "" "" + done +} + +main() { + printf '%-24s %-14s %-8s %s\n' NAMESPACE RELEASE CLASS NOTE + printf '%-24s %-14s %-8s %s\n' --------- ------- ----- ---- + if [ "$#" -gt 0 ]; then + for ns in "$@"; do audit_ns "$ns"; done + else + for ns in $(kubectl get ns -o name 2>/dev/null | sed 's|namespace/||'); do audit_ns "$ns"; done + fi +} + +# Sourced by hack/seaweedfs-naming-audit.bats to exercise the classifier directly. +[ "${SEAWEEDFS_AUDIT_LIB:-0}" = "1" ] || main "$@" diff --git a/hack/testdata/migration-seaweedfs-db/kubectl b/hack/testdata/migration-seaweedfs-db/kubectl new file mode 100755 index 0000000000..dba02f54bf --- /dev/null +++ b/hack/testdata/migration-seaweedfs-db/kubectl @@ -0,0 +1,101 @@ +#!/bin/sh +# Fake kubectl for hack/migration-seaweedfs-db-adopt.bats. It serves only the +# calls lib/seaweedfs-db-adopt.sh makes; behaviour is driven by FAKE_* env and +# every invocation is appended to $FAKE_CMDLOG so the test can assert on which +# namespaces were acted on and with which annotations. +# +# FAKE_CMDLOG file to append a one-line record of each call to +# FAKE_CLUSTERS newline list of " " — one +# per namespace holding a Cluster/seaweedfs-db. Use "-" for +# either annotation to model it being ABSENT (jsonpath prints +# nothing and exits 0). +# FAKE_LIST_FAIL non-empty => the -A fleet scan exits 1 with this as stderr +# FAKE_GET_FAIL non-empty => the per-namespace get exits 1 with this stderr +# FAKE_ANNOTATE_FAIL non-empty => `annotate` exits 1 (a failed hand-over) +# +# The failure knobs exist because the point of the helper is that a kubectl error +# must ABORT the migration rather than let it stamp the version and never run +# again. A fake that can only succeed cannot test that. +set -u + +log() { [ -n "${FAKE_CMDLOG:-}" ] && printf '%s\n' "$*" >> "$FAKE_CMDLOG"; } +log "KUBECTL $*" + +args="$*" + +# arg_after KEY -- echoes the token following KEY in the argument vector. +arg_after() { k="$1"; shift; p=""; for a in "$@"; do [ "$p" = "$k" ] && { printf '%s' "$a"; return; }; p="$a"; done; } + +# field -- echo that annotation for the FAKE_CLUSTERS row of ns. +field() { + printf '%s\n' "${FAKE_CLUSTERS:-}" | while read -r n rel keep; do + if [ "$n" = "$1" ]; then + case "$2" in + rel) [ "${rel:--}" = "-" ] || printf '%s' "$rel" ;; + keep) [ "${keep:--}" = "-" ] || printf '%s' "$keep" ;; + esac + break + fi + done +} + +case "$args" in + # Version stamp: kubectl apply --filename - + *"apply --filename -"*) + manifest=$(cat) + # Record the version actually stamped. A migration that stamped the wrong + # number would loop the runner forever, and asserting a bare "STAMP" would + # not notice. + v=$(printf '%s\n' "$manifest" | sed -n 's/^ version: "\{0,1\}\([0-9]\{1,\}\)"\{0,1\}$/\1/p' | head -1) + log "STAMP ${v:-unknown}" + exit 0 + ;; + + # Fleet scan: list namespaces holding a Cluster named seaweedfs-db. + *"get cluster.postgresql.cnpg.io -A"*) + if [ -n "${FAKE_LIST_FAIL:-}" ]; then + printf '%s\n' "$FAKE_LIST_FAIL" >&2 + exit 1 + fi + printf '%s\n' "${FAKE_CLUSTERS:-}" | while read -r ns rel keep; do + [ -n "$ns" ] && printf '%s\n' "$ns" + done + exit 0 + ;; + + # Per-namespace read of an annotation. + *"get cluster.postgresql.cnpg.io seaweedfs-db"*) + if [ -n "${FAKE_GET_FAIL:-}" ]; then + printf '%s\n' "$FAKE_GET_FAIL" >&2 + exit 1 + fi + ns=$(arg_after -n "$@") + case "$args" in + *"resource-policy"*) field "$ns" keep ;; + *) field "$ns" rel ;; + esac + exit 0 + ;; + + # The hand-over itself. + *"annotate cluster.postgresql.cnpg.io seaweedfs-db"*) + if [ -n "${FAKE_ANNOTATE_FAIL:-}" ]; then + printf '%s\n' "$FAKE_ANNOTATE_FAIL" >&2 + exit 1 + fi + ns=$(arg_after -n "$@") + rel="" + keep="" + for a in "$@"; do + case "$a" in + meta.helm.sh/release-name=*) rel="${a#meta.helm.sh/release-name=}" ;; + helm.sh/resource-policy=*) keep="${a#helm.sh/resource-policy=}" ;; + esac + done + log "ANNOTATE $ns release-name=${rel:-} resource-policy=${keep:-}" + exit 0 + ;; +esac + +log "UNHANDLED $args" +exit 0 diff --git a/packages/core/platform/images/migrations/migrations/43 b/packages/core/platform/images/migrations/migrations/43 index 0c421d1fce..a8d7f74a5f 100755 --- a/packages/core/platform/images/migrations/migrations/43 +++ b/packages/core/platform/images/migrations/migrations/43 @@ -1,31 +1,31 @@ #!/bin/sh # Migration 43 --> 44 -# Adopt existing Cluster/seaweedfs-db resources into the new seaweedfs-db +# Adopt existing Cluster/seaweedfs-db resources into the new -db # HelmRelease introduced in this release. # -# Pre-split, the CNPG Cluster lived inside the seaweedfs-system Helm release. -# Splitting moves it to a new release named seaweedfs-db. We rewrite the -# helm ownership annotations so Helm adopts the existing Cluster instead of -# erroring on the next reconcile, and stamp helm.sh/resource-policy: keep -# so the seaweedfs-system upgrade (which no longer renders the Cluster) does -# not delete it during the transition. +# Pre-split, the CNPG Cluster lived inside the -system Helm release. +# Splitting moves it to a new release named -db. We rewrite the helm +# ownership annotations so Helm adopts the existing Cluster instead of erroring +# on the next reconcile, and stamp helm.sh/resource-policy: keep so the +# -system upgrade (which no longer renders the Cluster) does not delete it +# during the transition. +# +# The hand-over itself lives in lib/seaweedfs-db-adopt.sh, shared with migration +# 45. This migration originally compared the owning release name against the +# literal "seaweedfs-system", which silently skipped every instance NOT named +# `seaweedfs`: SeaweedFS is a user-creatable kind, so an instance `foo` is owned +# by `foo-system`. Those tenants got no `keep`, and the -system upgrade +# pruned their Cluster — taking the filer metadata, and with it all of the +# tenant's S3. The shared helper matches the `-system` SUFFIX instead, covering +# every instance name; migration 45 repairs clusters that already ran the +# hardcoded version. set -euo pipefail -# Iterate every namespace that has a Cluster named "seaweedfs-db". -for ns in $(kubectl get cluster.postgresql.cnpg.io -A \ - -o jsonpath='{range .items[?(@.metadata.name=="seaweedfs-db")]}{.metadata.namespace}{"\n"}{end}'); do - current=$(kubectl get cluster.postgresql.cnpg.io seaweedfs-db -n "$ns" \ - -o jsonpath='{.metadata.annotations.meta\.helm\.sh/release-name}' 2>/dev/null || true) - if [ "$current" = "seaweedfs-system" ]; then - echo "Re-annotating Cluster/seaweedfs-db in $ns: seaweedfs-system -> seaweedfs-db" - kubectl annotate cluster.postgresql.cnpg.io seaweedfs-db -n "$ns" \ - meta.helm.sh/release-name=seaweedfs-db \ - helm.sh/resource-policy=keep \ - --overwrite - # release-namespace stays the same (tenant namespace, e.g. tenant-root) - fi -done +# shellcheck source=lib/seaweedfs-db-adopt.sh +. "$(dirname "$0")/lib/seaweedfs-db-adopt.sh" + +adopt_seaweedfs_db_clusters # Stamp version. # Mirror migration 42's labeled manifest. A label-less apply by the same field diff --git a/packages/core/platform/images/migrations/migrations/45 b/packages/core/platform/images/migrations/migrations/45 new file mode 100755 index 0000000000..8243aea577 --- /dev/null +++ b/packages/core/platform/images/migrations/migrations/45 @@ -0,0 +1,65 @@ +#!/bin/sh +# Migration 45 --> 46 +# Repair the 1.5.0 db-split hand-over for SeaweedFS instances whose name is not +# the default `seaweedfs`. +# +# Migration 43 performs the hand-over of Cluster/seaweedfs-db from the +# -system release to the -db release introduced by the split (PR +# #2601, v1.5.0). It matched the owning release name against the literal +# "seaweedfs-system", so it only ever fired for an instance named `seaweedfs`. +# `SeaweedFS` is a user-creatable kind: an instance named `foo` is owned by +# release `foo-system` and was skipped. Such a tenant kept +# meta.helm.sh/release-name: -system and never received +# helm.sh/resource-policy: keep, so the -system upgrade — whose post-split +# chart no longer renders the Cluster — deleted it as a removed resource. CNPG +# takes the PVC with the Cluster, so the filer metadata, and with it every object +# in that tenant's S3, is unreachable. +# +# Migration 43 is fixed in place, which covers clusters upgrading from before 43. +# Migrations never re-run, so any cluster already at version >= 44 ran the +# hardcoded version and is still exposed: this migration re-checks the fleet and +# performs the hand-over for any Cluster/seaweedfs-db still owned by a +# -system release. Running as a pre-upgrade hook, it lands before the +# platform applies the new artifacts and therefore before -system +# re-renders, closing the window on THIS upgrade. +# +# The prune is not a one-shot: Helm diffs the LAST DEPLOYED revision against the +# new manifest, so a tenant whose -system last succeeded on a pre-split +# revision recomputes the same deletion on EVERY upgrade attempt, including ones +# that fail for unrelated reasons. Such a tenant re-deletes the Cluster each time +# -db recreates it. Stamping `keep` breaks that loop even when the +# -system upgrade itself stays wedged. +# +# Idempotent (a Cluster already owned by -db is left alone), so it is a +# no-op on a correctly migrated cluster and on fresh installs. +# +# LIMIT — this migration cannot recover a Cluster that is ALREADY deleted. There +# is nothing left to re-annotate: the PVC went with it. Such a tenant needs its +# filer metadata restored from a backup; see +# docs/operations/seaweedfs-431-rename-recovery.md. + +set -euo pipefail + +# shellcheck source=lib/seaweedfs-db-adopt.sh +. "$(dirname "$0")/lib/seaweedfs-db-adopt.sh" + +adopt_seaweedfs_db_clusters + +# Stamp version. +# Mirror migration 42's labeled manifest. A label-less apply by the same field +# manager (kubectl create | kubectl apply) would strip the +# platform.cozystack.io/no-delete label migration 42 added, dropping +# cozystack-version out of the cozystack-no-delete-guardrail +# ValidatingAdmissionPolicy. templates/cozystack-version.yaml only re-renders the +# labeled ConfigMap on first install, so the label must be carried inline here. +kubectl apply --filename - <-db release. +# +# The 1.5.0 db split (PR #2601) moved the CNPG Cluster carrying SeaweedFS filer +# metadata out of the -system Helm release into a new -db release. +# The Cluster object itself did not move — only its ownership had to. Two things +# must be true BEFORE -system next renders, and they are independent: +# +# meta.helm.sh/release-name=-db so -db ADOPTS the existing +# Cluster instead of failing its install +# on Helm's ownership check; +# helm.sh/resource-policy=keep so the -system upgrade, whose new +# chart no longer renders the Cluster, +# does not GARBAGE-COLLECT it as a +# removed resource. +# +# Without `keep` the Cluster is deleted, CNPG takes its PVC with it, and the +# tenant's filer metadata — hence all of its S3 — is gone. Data loss, not outage. +# +# The window is not a one-shot. Helm prunes by diffing the LAST DEPLOYED revision +# against the new manifest, so as long as -system's last successful revision +# is a pre-split one that still contains the Cluster, EVERY subsequent upgrade +# attempt — including ones that fail for unrelated reasons and never become the +# new "deployed" revision — re-computes the same deletion. A tenant whose +# -system is wedged therefore re-deletes the Cluster on every retry, racing +# -db, which keeps recreating it. `keep` is what breaks that loop, and it +# must stay for as long as that pre-split revision remains the prune baseline +# (i.e. until -system upgrades successfully at least once). Clearing it +# again is deferred to the 1.7 batch migrations. +# +# Ownership is therefore NOT a proxy for safety. A Cluster can already be owned by +# -db and still need `keep`: where the hand-over was skipped, -system +# prunes the Cluster and -db simply RECREATES it under its own ownership +# with no keep — while -system's prune baseline still lists it, so the next +# reconcile deletes it again. Both shapes are live on the upgrade stand: +# tenant-l and tenant-root are -db-owned WITH keep and their -system +# deployed revision (rev 1) still contains the Cluster, so only keep saves them; +# tenant-fresh is -db-owned WITHOUT keep because it was installed after the +# split and its -system never rendered a Cluster. Telling those two apart +# needs the release's deployed manifest, which this script cannot read cheaply or +# reliably. The costs are asymmetric: stamping keep where it was not needed leaves +# an orphan on app delete (which the 1.6 extra/seaweedfs cleanup hook reclaims; +# on 1.5.x nothing does, so it has to be removed by hand); +# missing one loses the database. So keep is stamped on every Cluster owned by +# either side of the split. +# +# Migration 43 shipped this logic with the owning release name hardcoded to +# "seaweedfs-system", which is only correct for an instance named `seaweedfs`. +# `SeaweedFS` is a user-creatable kind, so an instance named e.g. `foo` is owned +# by `foo-system` and was silently skipped: no re-own, no keep, Cluster pruned. +# Matching on the `-system` SUFFIX instead covers every instance name, so this +# helper is sourced by both migration 43 (the original hand-over, for clusters +# that have not run it yet) and migration 45 (repair, for clusters that already +# ran the hardcoded version). +# +# Idempotent: a Cluster already owned by -db AND carrying keep is left +# alone, so re-running is a no-op and both migrations can safely fire on the same +# cluster. +# +# FAILS CLOSED. Migrations never re-run, so a transient error swallowed here would +# permanently leave at-risk tenants exposed with no later migration to catch them. +# Every kubectl failure is fatal EXCEPT the two that genuinely mean "nothing to +# do": the CNPG resource type not being served at all (a cluster without CNPG), +# and a Cluster disappearing between the scan and the read. A non-zero return +# aborts the migration before it stamps the version, so the Job retries. +# +# Sourced, not executed: +# . "$(dirname "$0")/lib/seaweedfs-db-adopt.sh" + +# _sdb_is_absent_err +# True when a kubectl error means the thing simply is not there, as opposed to the +# API being unreachable, forbidden, throttled, or not yet established. Kept +# deliberately narrow: anything unrecognised is treated as fatal. +_sdb_is_absent_err() { + grep -qiE "server doesn't have a resource type|server could not find the requested resource|could not find the requested resource|not found" "$1" +} + +# adopt_seaweedfs_db_clusters +# Re-own and/or protect every Cluster/seaweedfs-db the split left exposed. +# Returns non-zero (aborting the migration) on any error that is not "absent". +adopt_seaweedfs_db_clusters() { + _sdb_err=$(mktemp) + + # Fleet scan. Assigning inside `if !` keeps errexit from firing so the exit + # status can be inspected — `for ns in $(kubectl ...)` would silently iterate + # zero times on failure and let the caller stamp the version regardless, which + # is the whole bug this guard exists for. + if ! _sdb_namespaces=$(kubectl get cluster.postgresql.cnpg.io -A \ + -o jsonpath='{range .items[?(@.metadata.name=="seaweedfs-db")]}{.metadata.namespace}{"\n"}{end}' \ + 2>"$_sdb_err"); then + if _sdb_is_absent_err "$_sdb_err"; then + echo "CNPG Cluster resource type is not served on this cluster — no SeaweedFS databases to hand over" + rm -f "$_sdb_err" + return 0 + fi + echo "FATAL: cannot list Cluster/seaweedfs-db across namespaces; refusing to stamp past an unverified fleet:" >&2 + cat "$_sdb_err" >&2 + rm -f "$_sdb_err" + return 1 + fi + + for ns in $_sdb_namespaces; do + [ -n "$ns" ] || continue + + if ! _sdb_current=$(kubectl get cluster.postgresql.cnpg.io seaweedfs-db -n "$ns" \ + -o jsonpath='{.metadata.annotations.meta\.helm\.sh/release-name}' 2>"$_sdb_err"); then + if _sdb_is_absent_err "$_sdb_err"; then + echo "Cluster/seaweedfs-db in $ns disappeared between scan and read — skipping" + continue + fi + echo "FATAL: cannot read the Helm owner of Cluster/seaweedfs-db in $ns:" >&2 + cat "$_sdb_err" >&2 + rm -f "$_sdb_err" + return 1 + fi + + if ! _sdb_keep=$(kubectl get cluster.postgresql.cnpg.io seaweedfs-db -n "$ns" \ + -o jsonpath='{.metadata.annotations.helm\.sh/resource-policy}' 2>"$_sdb_err"); then + if _sdb_is_absent_err "$_sdb_err"; then + echo "Cluster/seaweedfs-db in $ns disappeared between scan and read — skipping" + continue + fi + echo "FATAL: cannot read the resource-policy of Cluster/seaweedfs-db in $ns:" >&2 + cat "$_sdb_err" >&2 + rm -f "$_sdb_err" + return 1 + fi + + case "$_sdb_current" in + # No Helm ownership annotation at all — distinct from "unreadable", which is + # fatal above. Guessing an owner would be worse than doing nothing, but a + # SeaweedFS database that nobody owns is worth saying out loud. + "") + echo "WARNING: Cluster/seaweedfs-db in $ns carries no meta.helm.sh/release-name — not Helm-managed, leaving it alone. If this tenant runs SeaweedFS, verify by hand that its database is not about to be pruned." >&2 + ;; + + # Owned by the data-plane release: this is the hand-over. The instance name + # is whatever precedes -system, so `foo-system` -> `foo-db` exactly as + # `seaweedfs-system` -> `seaweedfs-db`. + *-system) + _sdb_instance="${_sdb_current%-system}" + # A release literally named "-system" has no instance name; refusing is + # safer than annotating an owner of "-db" that no release will claim. + if [ -z "$_sdb_instance" ]; then + echo "WARNING: Cluster/seaweedfs-db in $ns is owned by a release named '$_sdb_current' with no instance name — skipping" >&2 + continue + fi + echo "Re-annotating Cluster/seaweedfs-db in $ns: $_sdb_current -> ${_sdb_instance}-db (+keep)" + kubectl annotate cluster.postgresql.cnpg.io seaweedfs-db -n "$ns" \ + meta.helm.sh/release-name="${_sdb_instance}-db" \ + helm.sh/resource-policy=keep \ + --overwrite + # release-namespace stays the same (the tenant namespace); both releases + # live there, so the ownership check only needed release-name rewritten. + ;; + + # Already owned by the db release. Ownership is correct, but that does NOT + # imply it is protected — see the note above. Stamp keep if it is missing. + *-db) + if [ "$_sdb_keep" = "keep" ]; then + echo "Cluster/seaweedfs-db in $ns already handed over to $_sdb_current and protected — nothing to do" + else + echo "Protecting Cluster/seaweedfs-db in $ns: owned by $_sdb_current but missing helm.sh/resource-policy=keep" + kubectl annotate cluster.postgresql.cnpg.io seaweedfs-db -n "$ns" \ + helm.sh/resource-policy=keep \ + --overwrite + fi + ;; + + # Owned by some unrelated release. Not ours to touch. + *) + echo "WARNING: Cluster/seaweedfs-db in $ns is owned by unrelated release '$_sdb_current' — leaving it alone" >&2 + ;; + esac + done + + rm -f "$_sdb_err" + return 0 +} diff --git a/packages/core/platform/values.yaml b/packages/core/platform/values.yaml index 47598fe319..f4da8db906 100644 --- a/packages/core/platform/values.yaml +++ b/packages/core/platform/values.yaml @@ -5,8 +5,16 @@ sourceRef: path: / migrations: enabled: false + # The image digest is NOT hand-edited alongside targetVersion. The release + # build rebuilds the migrations image from this same source tree (which carries + # migrations/) and restamps this pin in the same commit via + # `make image-migrations` (yq --inplace) — so image and targetVersion are only + # consistent at a release tag, not mid-PR. The digest below therefore lags + # targetVersion in-tree by design; run-migrations.sh refuses to advance past a + # migration file missing from the image (exit 1), so a stale pin fails loudly + # rather than silently skipping the SeaweedFS database hand-over. image: ghcr.io/cozystack/cozystack/platform-migrations:v1.5.2@sha256:5d601b0709c578945f2e04d655b77d29008c94666a4350e67960c9f6bf9d2bde - targetVersion: 45 + targetVersion: 46 # Bundle deployment configuration bundles: system: diff --git a/packages/extra/seaweedfs/Makefile b/packages/extra/seaweedfs/Makefile index fd0ad40fdd..799919b592 100644 --- a/packages/extra/seaweedfs/Makefile +++ b/packages/extra/seaweedfs/Makefile @@ -4,6 +4,22 @@ include ../../../hack/package.mk test: helm unittest . + $(MAKE) test-guard-fail-closed + +# Sibling of the same target in packages/system/seaweedfs: this chart's copy of +# the naming guard must also refuse an UPGRADE it cannot see the cluster for. +# helm-unittest 1.0.3 ignores release.isUpgrade, so tests/guard_fail_closed_test.yaml +# can only cover the render-through side and passes with or without the canary — +# the refusal needs a real renderer. +.PHONY: test-guard-fail-closed +test-guard-fail-closed: + @out=$$(helm template seaweedfs . -n tenant-root --is-upgrade \ + --set topology=Simple --set replicationFactor=2 \ + --set _namespace.host=example.org --set _namespace.ingress=tenant-root \ + --set _cluster.issuer-name=letsencrypt-prod --set _cluster.solver=http01 2>&1); \ + echo "$$out" | grep -q 'refusing to upgrade blind' \ + || { echo "FAIL: a blind upgrade rendered instead of refusing (naming-guard canary is not firing)"; echo "$$out" | head -5; exit 1; } + @echo "ok: naming guard refuses a blind upgrade" generate: cozyvalues-gen -v values.yaml -s values.schema.json -r README.md diff --git a/packages/extra/seaweedfs/templates/_naming.tpl b/packages/extra/seaweedfs/templates/_naming.tpl new file mode 100644 index 0000000000..4f8935db89 --- /dev/null +++ b/packages/extra/seaweedfs/templates/_naming.tpl @@ -0,0 +1,48 @@ +{{- /* seaweedfs.renamedVolumePrefix — reconstruct the name 4.31 gives the volume + component of the `-system` release, i.e. the RELEASE-NAMED generation the + fullnameOverride pin exists to avoid. + + Input: the `-system` release name, as a bare string. + {{ include "seaweedfs.renamedVolumePrefix" "foo-system" }} -> foo-system-seaweedfs-volume + + This replays two upstream helpers, so it must track them if the vendored chart + changes (hack/seaweedfs-guard-parity.bats pins the two copies of the guard + against each other; charts/seaweedfs/templates/shared/_helpers.tpl is the + source of truth for the rules below): + + seaweedfs.fullname — with no fullnameOverride, the release name, plus + `-` when the release name does not + already contain it; truncated to 63. + seaweedfs.componentName — truncates the fullname to (62 - len(suffix)) before + appending `-`, so for `volume` the fullname + is cut to 56. An instance name >= ~40 chars + therefore loses `seaweedfs` from the tail, which is + why a `contains "seaweedfs"` name filter cannot see + its claims and this reconstruction can. + + The guard deliberately does NOT call seaweedfs.fullname directly: that helper + reads .Values.fullnameOverride, which this chart PINS to `seaweedfs`, so it + returns the chart-named generation — the opposite of what is wanted here. + + ACCEPTED LIMIT — zone/pool volume components. MultiZone (and Simple-with- + pools) tenants get per-group components with suffix `volume-`, cut at + (62 - len(suffix)), i.e. SHORTER than the 56 this helper reconstructs. The + prefixes only diverge when the fullname exceeds that shorter cut: for the + supported instance (the tenant module hardcodes the name `seaweedfs`, + fullname 16 chars) that takes a zone/pool key of ~40+ characters, and for + the default `volume` group it can never happen (its suffix IS the 56 cut). + A guard prefix that diverges means a release-named zone component the guard + cannot see. Decided 2026-07-17 (1.6 triage): accepted and documented in + docs/operations/seaweedfs-431-rename-recovery.md (Scope) rather than + reconstructed per-key — instance names are fixed by the tenant module and + absurd keys are the only reachable trigger. Revisit if instance naming is + ever opened up or a key-length clamp lands in extra/seaweedfs. */}} +{{- define "seaweedfs.renamedVolumePrefix" -}} +{{- $release := . -}} +{{- $full := $release -}} +{{- if not (contains "seaweedfs" $release) -}} +{{- $full = printf "%s-seaweedfs" $release -}} +{{- end -}} +{{- $full = $full | trunc 63 | trimSuffix "-" -}} +{{- printf "%s-volume" ($full | trunc 56 | trimSuffix "-") -}} +{{- end -}} diff --git a/packages/extra/seaweedfs/templates/seaweedfs.yaml b/packages/extra/seaweedfs/templates/seaweedfs.yaml index 3eb28515f5..3933234296 100644 --- a/packages/extra/seaweedfs/templates/seaweedfs.yaml +++ b/packages/extra/seaweedfs/templates/seaweedfs.yaml @@ -107,7 +107,7 @@ stood up a second, empty set while the data stayed put. system/seaweedfs pins the chart-based name back, so those workloads and volumes are adopted in place. - Two states cannot be adopted and stop the render instead: + These states cannot be adopted and stop the render instead: - FRESH on 1.5.x: only the renamed volumes exist, so the data was written under them. Adopting the chart-based name would rename the workloads AWAY from that @@ -116,8 +116,9 @@ - D-split: both sets exist AND the renamed volume servers are live, so they may hold objects written through the split endpoint. Adopting the chart-based set would strand those. The operator is sent to reconcile the split first (Step 3). - A duplicate that never served (D-wedged: zero ready replicas) is safe to adopt - and renders through. + A duplicate that never served (D-wedged: zero ready replicas) is refused the + same way: readyReplicas is a snapshot, not proof it never served, so it cannot + be told apart from D-split and is never adopted blind (see the note below). The renamed volumes are matched by SHAPE rather than by reconstructing the name: a volume PVC is always `data1--volume[-]-N`, and @@ -129,38 +130,66 @@ PVCs hold the data and outlive any workload, but carry no chart labels, so they are matched on the name shape and must mention the chart to avoid tripping over an unrelated `data1-*-volume-*` claim. StatefulSets do carry the labels, so they - are matched on those, which also covers a tenant whose PVCs are not provisioned - yet and one whose name the chart truncated past the chart name; the renamed - volume StatefulSet is also where the D-split liveness signal is read. */}} -{{- $legacyData := false }} -{{- $systemData := false }} -{{- $systemRunning := false }} + are matched on those, which is also the ONLY signal for a tenant whose PVCs are + not provisioned yet and for one whose instance name is long enough that the + chart truncated `seaweedfs` off the PVC names. Either establishes a generation's + presence, so the two are OR-ed. + + Two generations present is UNDECIDABLE from inside a render and this guard does + not guess — see the long note in system/seaweedfs/templates/naming-guard.yaml. + In short: the release history DOES record birth order durably, but birth order + is the wrong question. It says which generation is original; adoption needs to + know whether the other one is EMPTY, and D-wedged and D-split are identical on + every durable signal. + + This copy of the guard warns on the SeaweedFS application itself, where the + operator looks first. The ENFORCING copy lives in system/seaweedfs + (templates/naming-guard.yaml): the -system HelmRelease pulls its chart + from a platform-managed ExternalArtifact, so a platform upgrade re-renders + THAT chart directly without ever re-rendering this one. Keep the two + classifications in sync — they are deliberately identical, and + hack/seaweedfs-guard-parity.bats fails the build if they drift. + + Fail-closed: lookup errors abort the render, but a client-side render returns + nothing — the release namespace is used as a canary, and an upgrade that + cannot see the cluster refuses instead of misclassifying. */}} +{{- $canary := lookup "v1" "Namespace" "" .Release.Namespace }} +{{- if and (not $canary) .Release.IsUpgrade }} +{{- fail (printf "SeaweedFS naming-migration guard for %s in namespace %s: cannot see the cluster (lookup returned nothing for the release namespace itself). This render decides whether workloads are renamed away from live data, so refusing to upgrade blind. Renders applied by helm-controller always see the cluster; if templating by hand, render server-side." .Release.Name .Release.Namespace) }} +{{- end }} +{{- if $canary }} +{{- /* extra/seaweedfs is the `` release; the renamed generation is named + after its CHILD `-system` release, so derive it. This one line is the only + difference from the copy in system/seaweedfs. */}} +{{- $sysRelease := printf "%s-system" .Release.Name }} +{{- $renamedVol := include "seaweedfs.renamedVolumePrefix" $sysRelease }} +{{- $legacyPVC := false }} +{{- $systemPVC := false }} +{{- $legacySTS := false }} +{{- $systemSTS := false }} {{- range (lookup "v1" "PersistentVolumeClaim" .Release.Namespace "").items | default list }} -{{- if and (hasPrefix "data1-" .metadata.name) (contains "-volume" .metadata.name) (contains "seaweedfs" .metadata.name) }} -{{- if hasPrefix "data1-seaweedfs-volume" .metadata.name }} -{{- $legacyData = true }} -{{- else }} -{{- $systemData = true }} -{{- end }} +{{- if hasPrefix (printf "data1-%s" $renamedVol) .metadata.name }} +{{- $systemPVC = true }} +{{- else if hasPrefix "data1-seaweedfs-volume" .metadata.name }} +{{- $legacyPVC = true }} {{- end }} {{- end }} {{- range (lookup "apps/v1" "StatefulSet" .Release.Namespace "").items | default list }} -{{- if and (eq (dig "app.kubernetes.io/name" "" (.metadata.labels | default dict)) "seaweedfs") (contains "volume" .metadata.name) }} -{{- if hasPrefix "seaweedfs-volume" .metadata.name }} -{{- $legacyData = true }} -{{- else }} -{{- $systemData = true }} -{{- if gt (int (dig "status" "readyReplicas" 0 .)) 0 }} -{{- $systemRunning = true }} -{{- end }} +{{- if eq (dig "app.kubernetes.io/name" "" (.metadata.labels | default dict)) "seaweedfs" }} +{{- if hasPrefix $renamedVol .metadata.name }} +{{- $systemSTS = true }} +{{- else if hasPrefix "seaweedfs-volume" .metadata.name }} +{{- $legacySTS = true }} {{- end }} {{- end }} {{- end }} -{{- if and $legacyData $systemData $systemRunning }} -{{- fail (printf "SeaweedFS %s in namespace %s is running two live SeaweedFS sets (D-split): the renamed set has ready volume servers alongside the adopted set, and both write to the shared seaweedfs-db metadata. Rendering would adopt the chart-named set and strand the objects the renamed set wrote. Stop the split and reconcile the two sets before upgrading — see docs/operations/seaweedfs-431-rename-recovery.md (Step 3)." .Release.Name .Release.Namespace) }} +{{- $legacyGen := or $legacyPVC $legacySTS }} +{{- $systemGen := or $systemPVC $systemSTS }} +{{- if and $legacyGen $systemGen }} +{{- fail (printf "SeaweedFS %s in namespace %s has BOTH naming generations present: the chart-named set (seaweedfs-volume / data1-seaweedfs-volume-*) AND the release-named set this chart no longer uses. One of them is an empty duplicate and one holds the data, and which is which cannot be established from inside a Helm render — claim timestamps are mutable (the Step 2 re-bind recreates claims), StatefulSets are recreated by the adoption hook, and a duplicate with zero ready replicas may still have served writes. Rendering would adopt the chart-named set, so guessing wrong strands or destroys the data. Refusing instead. Classify the tenant and delete the EMPTY generation so exactly one remains — this render then adopts the survivor with no further action. See docs/operations/seaweedfs-431-rename-recovery.md (Step 1 classifies; Step 2/2a/3 recover). If you are part-way through Step 2's re-bind, finish it." .Release.Name .Release.Namespace) }} +{{- else if $systemGen }} +{{- fail (printf "SeaweedFS %s in namespace %s keeps its data on volumes named after the Helm release — a tenant installed fresh on Cozystack 1.5.x (or one whose instance name is long enough that the chart truncated the volume PVC names, leaving only its StatefulSets to match on). Workloads are named seaweedfs-*, so rendering would rename them away from that data and bring up an empty cluster. Helm cannot move data between PVCs. Re-bind the existing PVs onto the data1-seaweedfs-volume-* PVC names before upgrading — see docs/operations/seaweedfs-431-rename-recovery.md (Step 2)." .Release.Name .Release.Namespace) }} {{- end }} -{{- if and $systemData (not $legacyData) }} -{{- fail (printf "SeaweedFS %s in namespace %s keeps its data on volumes named after the Helm release (this tenant was installed on Cozystack 1.5.x). Workloads are named seaweedfs-*, so rendering would rename them away from that data and bring up an empty cluster. Re-bind the existing PVs onto the data1-seaweedfs-volume-* PVC names before upgrading — see docs/operations/seaweedfs-431-rename-recovery.md" .Release.Name .Release.Namespace) }} {{- end }} apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease diff --git a/packages/extra/seaweedfs/tests/fullname_override_named_test.yaml b/packages/extra/seaweedfs/tests/fullname_override_named_test.yaml index ff627c9bbb..4a6465c3ab 100644 --- a/packages/extra/seaweedfs/tests/fullname_override_named_test.yaml +++ b/packages/extra/seaweedfs/tests/fullname_override_named_test.yaml @@ -7,6 +7,9 @@ suite: seaweedfs naming guard for a non-default instance name # - On 4.31 the fullname helper appends the chart name when the release name # does not contain it, so a fresh 1.5.x `archive` wrote its data to # data1-archive-system-seaweedfs-volume-*, NOT data1-archive-system-volume-*. +# +# The guard only runs when it can see the cluster (the release namespace is its +# canary), so every provider registers v1/Namespace and includes the namespace. templates: - templates/seaweedfs.yaml @@ -29,7 +32,13 @@ tests: - it: adopts a pre-4.31 instance whose data is on the chart-named PVCs set: *values kubernetesProvider: - scheme: + scheme: &scheme + "v1/Namespace": + gvr: + group: "" + version: "v1" + resource: "namespaces" + namespaced: false "v1/PersistentVolumeClaim": gvr: group: "" @@ -43,11 +52,17 @@ tests: resource: "statefulsets" namespaced: true objects: + - &ns + kind: Namespace + apiVersion: v1 + metadata: + name: tenant-root - kind: PersistentVolumeClaim apiVersion: v1 metadata: name: data1-seaweedfs-volume-0 namespace: tenant-root + creationTimestamp: "2025-11-02T10:00:00Z" - kind: StatefulSet apiVersion: apps/v1 metadata: @@ -64,25 +79,15 @@ tests: - it: refuses to render when this release's data is on its renamed volumes set: *values kubernetesProvider: - scheme: - "v1/PersistentVolumeClaim": - gvr: - group: "" - version: "v1" - resource: "persistentvolumeclaims" - namespaced: true - "apps/v1/StatefulSet": - gvr: - group: "apps" - version: "v1" - resource: "statefulsets" - namespaced: true + scheme: *scheme objects: + - *ns - kind: PersistentVolumeClaim apiVersion: v1 metadata: name: data1-archive-system-seaweedfs-volume-0 namespace: tenant-root + creationTimestamp: "2026-06-20T07:01:59Z" - kind: StatefulSet apiVersion: apps/v1 metadata: @@ -92,38 +97,67 @@ tests: - failedTemplate: errorPattern: "SeaweedFS archive in namespace tenant-root keeps its data on volumes named after the Helm release" + - it: refuses when both generations exist (was S-damaged) + set: *values + kubernetesProvider: + scheme: *scheme + objects: + - *ns + # A fresh-1.5.x `archive` that an unguarded upgrade already damaged: the + # renamed volumes are older (the data was born there) and the chart-named + # set is the newer, empty one. Must be S (Step 2a), never D-split. + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: data1-archive-system-seaweedfs-volume-0 + namespace: tenant-root + creationTimestamp: "2026-06-20T07:01:59Z" + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: data1-seaweedfs-volume-0 + namespace: tenant-root + creationTimestamp: "2026-06-20T08:28:58Z" + - kind: StatefulSet + apiVersion: apps/v1 + metadata: + name: archive-system-seaweedfs-volume + namespace: tenant-root + labels: + app.kubernetes.io/name: seaweedfs + status: + readyReplicas: "1" + asserts: + - failedTemplate: + errorPattern: "has BOTH naming generations present" + - it: catches a long instance name whose renamed workloads the chart truncated + # Instance `archive-of-quarterly-financial-statements-x1` -> child release + # archive-of-quarterly-financial-statements-x1-system -> the fullname gains + # `-seaweedfs` (the release name does not contain it) and is 61 chars, so + # componentName cuts it to 56 before appending `-volume`: the chart name is gone + # from the tail and both the workload and its claims read + # ...-x1-system-seaw-volume. A `contains "seaweedfs"` name filter cannot see + # either of them; the reconstructed prefix matches both exactly. + release: + name: archive-of-quarterly-financial-statements-x1 + namespace: tenant-root set: *values kubernetesProvider: - scheme: - "v1/PersistentVolumeClaim": - gvr: - group: "" - version: "v1" - resource: "persistentvolumeclaims" - namespaced: true - "apps/v1/StatefulSet": - gvr: - group: "apps" - version: "v1" - resource: "statefulsets" - namespaced: true + scheme: *scheme objects: - # 4.31 truncates to 56 chars before appending -volume, so a long - # name loses the chart name from its tail and the PVC name alone no longer - # identifies it. The StatefulSet still carries the chart labels, so the - # guard sees it there. + - *ns - kind: StatefulSet apiVersion: apps/v1 metadata: - name: archive-system-a-very-long-instance-name-that-gets-trunca-volume + name: archive-of-quarterly-financial-statements-x1-system-seaw-volume namespace: tenant-root labels: app.kubernetes.io/name: seaweedfs - kind: PersistentVolumeClaim apiVersion: v1 metadata: - name: unrelated-app-data-0 + name: data1-archive-of-quarterly-financial-statements-x1-system-seaw-volume-0 namespace: tenant-root asserts: - failedTemplate: @@ -132,20 +166,9 @@ tests: - it: ignores an unrelated data1-*-volume-* PVC belonging to another app set: *values kubernetesProvider: - scheme: - "v1/PersistentVolumeClaim": - gvr: - group: "" - version: "v1" - resource: "persistentvolumeclaims" - namespaced: true - "apps/v1/StatefulSet": - gvr: - group: "apps" - version: "v1" - resource: "statefulsets" - namespaced: true + scheme: *scheme objects: + - *ns - kind: PersistentVolumeClaim apiVersion: v1 metadata: diff --git a/packages/extra/seaweedfs/tests/fullname_override_test.yaml b/packages/extra/seaweedfs/tests/fullname_override_test.yaml index cb18fccbe3..590a9cf6f2 100644 --- a/packages/extra/seaweedfs/tests/fullname_override_test.yaml +++ b/packages/extra/seaweedfs/tests/fullname_override_test.yaml @@ -4,16 +4,28 @@ suite: seaweedfs legacy-naming guard # names (`seaweedfs-master`, `data1-seaweedfs-volume-N`) and an upgrade past the # 4.31 rename adopts the running set in place. # -# Two states stop the render instead of adopting: a tenant installed FRESH on -# 1.5.x (only the renamed volumes exist → PV re-bind, Step 2), and a D-split -# tenant (both sets present AND the renamed volume servers are live → reconcile -# the split, Step 3). A duplicate that never served (D-wedged, zero ready -# replicas) renders through and is adopted. +# Any tenant with BOTH naming generations present stops the render instead of +# adopting — which set is the empty duplicate cannot be told from inside a render, +# so the guard refuses and sends the operator to classify. The recovery path +# differs by case: a tenant an unguarded upgrade already damaged (the renamed +# volumes are OLDER — the data was born there and the chart-named set is the empty +# one → Step 2a + Step 2), and a D-split tenant (the legacy volumes are older AND +# the renamed volume servers are live → reconcile the split, Step 3). A duplicate +# that never served (D-wedged, zero ready replicas) is refused the same way: +# readyReplicas is a snapshot, not proof it never served, so it is indistinguishable +# from D-split and cannot be adopted blind. A tenant installed FRESH on 1.5.x (only +# the renamed volumes exist, no second generation) is refused separately → PV +# re-bind, Step 2. +# +# The guard only runs when it can see the cluster: the release namespace is its +# canary. Every test that exercises classification therefore registers +# v1/Namespace and provides the release namespace object; the fail-closed +# behaviour of the canary itself is covered in guard_fail_closed_test.yaml. # # Two helm-unittest quirks the mocks work around: -# - The template LISTs both PVCs and StatefulSets, so every mocked provider -# must register both list kinds — a fake client errors on an unregistered -# LIST rather than returning empty. +# - The template GETs the namespace and LISTs both PVCs and StatefulSets, so +# every mocked provider must register all three kinds — a fake client errors +# on an unregistered kind rather than returning empty. # - status.readyReplicas is quoted ("1"/"0"): the fake client cannot deep-copy # a bare Go int in a mocked object's status, and the template's `int` coerces # the string back either way. @@ -39,7 +51,13 @@ tests: - it: renders for a net-new install (no SeaweedFS data in the namespace) set: *values kubernetesProvider: - scheme: + scheme: &scheme + "v1/Namespace": + gvr: + group: "" + version: "v1" + resource: "namespaces" + namespaced: false "v1/PersistentVolumeClaim": gvr: group: "" @@ -53,6 +71,11 @@ tests: resource: "statefulsets" namespaced: true objects: + - &ns + kind: Namespace + apiVersion: v1 + metadata: + name: tenant-root - kind: PersistentVolumeClaim apiVersion: v1 metadata: @@ -77,25 +100,15 @@ tests: - it: renders for a cluster whose data is on the legacy seaweedfs-* PVCs set: *values kubernetesProvider: - scheme: - "v1/PersistentVolumeClaim": - gvr: - group: "" - version: "v1" - resource: "persistentvolumeclaims" - namespaced: true - "apps/v1/StatefulSet": - gvr: - group: "apps" - version: "v1" - resource: "statefulsets" - namespaced: true + scheme: *scheme objects: + - *ns - kind: PersistentVolumeClaim apiVersion: v1 metadata: name: data1-seaweedfs-volume-0 namespace: tenant-root + creationTimestamp: "2025-11-02T10:00:00Z" - kind: PersistentVolumeClaim apiVersion: v1 metadata: @@ -114,40 +127,35 @@ tests: path: metadata.name value: seaweedfs-system - - it: renders on a D-wedged cluster (duplicate present but never served) — adopts the legacy set + # Was: "renders on a D-wedged cluster — adopts the legacy set". A duplicate + # reading zero ready replicas used to be treated as proof it never served, so + # this rendered through. readyReplicas is only a snapshot: a duplicate that + # crashed, was scaled down, or lost readiness AFTER serving writes looks + # identical, and adopting the legacy set would strand whatever it wrote. Both + # generations present is now refused regardless of liveness. + - it: refuses when both generations exist and the duplicate reads zero ready (was D-wedged, adopted) set: *values kubernetesProvider: - scheme: - "v1/PersistentVolumeClaim": - gvr: - group: "" - version: "v1" - resource: "persistentvolumeclaims" - namespaced: true - "apps/v1/StatefulSet": - gvr: - group: "apps" - version: "v1" - resource: "statefulsets" - namespaced: true + scheme: *scheme objects: + - *ns - kind: PersistentVolumeClaim apiVersion: v1 metadata: name: data1-seaweedfs-volume-0 namespace: tenant-root + creationTimestamp: "2025-11-02T10:00:00Z" - kind: PersistentVolumeClaim apiVersion: v1 metadata: name: data1-seaweedfs-system-volume-0 namespace: tenant-root + creationTimestamp: "2026-06-20T12:00:00Z" - kind: PersistentVolumeClaim apiVersion: v1 metadata: name: unrelated-app-data-0 namespace: tenant-root - # The duplicate volume set exists but never scheduled (anti-affinity): - # zero ready replicas, so its PVCs are empty and adoption is safe. - kind: StatefulSet apiVersion: apps/v1 metadata: @@ -157,41 +165,28 @@ tests: app.kubernetes.io/name: seaweedfs status: readyReplicas: "0" - documentSelector: - path: kind - value: HelmRelease asserts: - - equal: - path: metadata.name - value: seaweedfs-system + - failedTemplate: + errorPattern: "has BOTH naming generations present" - - it: refuses to render on a D-split cluster (duplicate volume servers are live) + - it: refuses when both generations exist and the duplicate is live (was D-split) set: *values kubernetesProvider: - scheme: - "v1/PersistentVolumeClaim": - gvr: - group: "" - version: "v1" - resource: "persistentvolumeclaims" - namespaced: true - "apps/v1/StatefulSet": - gvr: - group: "apps" - version: "v1" - resource: "statefulsets" - namespaced: true + scheme: *scheme objects: + - *ns - kind: PersistentVolumeClaim apiVersion: v1 metadata: name: data1-seaweedfs-volume-0 namespace: tenant-root + creationTimestamp: "2025-11-02T10:00:00Z" - kind: PersistentVolumeClaim apiVersion: v1 metadata: name: data1-seaweedfs-system-volume-0 namespace: tenant-root + creationTimestamp: "2026-06-20T12:00:00Z" # The duplicate volume set is live: it may hold objects written through # the split endpoint, so adoption would strand them. - kind: StatefulSet @@ -205,30 +200,65 @@ tests: readyReplicas: "1" asserts: - failedTemplate: - errorPattern: "is running two live SeaweedFS sets.*Stop the split" + errorPattern: "has BOTH naming generations present" + + - it: refuses when both generations exist and the renamed volumes are older (was S-damaged) + set: *values + kubernetesProvider: + scheme: *scheme + objects: + - *ns + # Mirrors a fresh-1.5.x tenant that an unguarded 1.6 upgrade already hit: + # the data was born on the renamed volumes (older), and the upgrade + # created the chart-named set (newer, EMPTY). The renamed set is live — + # which a presence-only guard misreads as D-split, sending the operator + # to quiesce the set that holds all the data. + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: data1-seaweedfs-system-volume-0 + namespace: tenant-root + creationTimestamp: "2026-06-20T07:01:59Z" + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: data1-seaweedfs-volume-0 + namespace: tenant-root + creationTimestamp: "2026-06-20T08:28:58Z" + - kind: StatefulSet + apiVersion: apps/v1 + metadata: + name: seaweedfs-system-volume + namespace: tenant-root + labels: + app.kubernetes.io/name: seaweedfs + status: + readyReplicas: "1" + - kind: StatefulSet + apiVersion: apps/v1 + metadata: + name: seaweedfs-volume + namespace: tenant-root + labels: + app.kubernetes.io/name: seaweedfs + status: + readyReplicas: "0" + asserts: + - failedTemplate: + errorPattern: "has BOTH naming generations present" - it: renders for a zoned legacy cluster (volume StatefulSets are per zone) set: *values kubernetesProvider: - scheme: - "v1/PersistentVolumeClaim": - gvr: - group: "" - version: "v1" - resource: "persistentvolumeclaims" - namespaced: true - "apps/v1/StatefulSet": - gvr: - group: "apps" - version: "v1" - resource: "statefulsets" - namespaced: true + scheme: *scheme objects: + - *ns - kind: PersistentVolumeClaim apiVersion: v1 metadata: name: data1-seaweedfs-volume-zone-a-0 namespace: tenant-root + creationTimestamp: "2025-11-02T10:00:00Z" - kind: PersistentVolumeClaim apiVersion: v1 metadata: @@ -250,30 +280,15 @@ tests: - it: refuses to render when the data lives only on seaweedfs-system-* PVCs set: *values kubernetesProvider: - scheme: - "v1/PersistentVolumeClaim": - gvr: - group: "" - version: "v1" - resource: "persistentvolumeclaims" - namespaced: true - "apps/v1/StatefulSet": - gvr: - group: "apps" - version: "v1" - resource: "statefulsets" - namespaced: true + scheme: *scheme objects: + - *ns - kind: PersistentVolumeClaim apiVersion: v1 metadata: name: data1-seaweedfs-system-volume-0 namespace: tenant-root - - kind: PersistentVolumeClaim - apiVersion: v1 - metadata: - name: unrelated-app-data-0 - namespace: tenant-root + creationTimestamp: "2026-06-20T07:01:59Z" - kind: StatefulSet apiVersion: apps/v1 metadata: @@ -286,20 +301,12 @@ tests: - it: refuses to render on a seaweedfs-system-* cluster detected by its StatefulSet set: *values kubernetesProvider: - scheme: - "v1/PersistentVolumeClaim": - gvr: - group: "" - version: "v1" - resource: "persistentvolumeclaims" - namespaced: true - "apps/v1/StatefulSet": - gvr: - group: "apps" - version: "v1" - resource: "statefulsets" - namespaced: true + scheme: *scheme objects: + - *ns + # No volume PVCs are visible at all (e.g. not provisioned yet, or a long + # name truncated the chart name out of them); the renamed StatefulSet + # alone is enough to refuse. - kind: StatefulSet apiVersion: apps/v1 metadata: @@ -319,4 +326,4 @@ tests: namespace: tenant-root asserts: - failedTemplate: - errorPattern: "keeps its data on volumes named after the Helm release.*Re-bind" + errorPattern: "keeps its data on volumes named after the Helm release" diff --git a/packages/extra/seaweedfs/tests/guard_fail_closed_test.yaml b/packages/extra/seaweedfs/tests/guard_fail_closed_test.yaml new file mode 100644 index 0000000000..3234d3e95e --- /dev/null +++ b/packages/extra/seaweedfs/tests/guard_fail_closed_test.yaml @@ -0,0 +1,42 @@ +suite: seaweedfs naming guard skips cleanly without a cluster view + +# The guard classifies a tenant by looking up its PVCs and StatefulSets. A +# failed API call already aborts the render (Helm propagates lookup errors), +# but a CLIENT-SIDE render (helm template, --dry-run=client) silently returns +# nothing — which, unguarded, would render the chart-based names over a class-S +# tenant. The release namespace is the canary: it always exists for a real +# install or upgrade, so when it cannot be seen, an UPGRADE refuses rather than +# classify blind, while an install without cluster access (CI lint, unittest +# with no provider — this test) skips the guard: it never touches live data. +# +# The refusal branch cannot be modelled here: helm-unittest 1.0.3 ignores +# release.isUpgrade (.Release.IsUpgrade renders false regardless). Verify it +# with a real renderer instead: +# helm template packages/extra/seaweedfs --is-upgrade ... 2>&1 \ +# | grep 'refusing to upgrade blind' + +templates: + - templates/seaweedfs.yaml + +release: + name: seaweedfs + namespace: tenant-root + +tests: + - it: still renders a client-side install (no cluster view, no data at risk) + set: + topology: Simple + replicationFactor: 2 + _namespace: + host: example.org + ingress: tenant-root + _cluster: + issuer-name: letsencrypt-prod + solver: http01 + documentSelector: + path: kind + value: HelmRelease + asserts: + - equal: + path: metadata.name + value: seaweedfs-system diff --git a/packages/system/seaweedfs/Makefile b/packages/system/seaweedfs/Makefile index af7c7b3274..ea2c973a18 100644 --- a/packages/system/seaweedfs/Makefile +++ b/packages/system/seaweedfs/Makefile @@ -5,6 +5,20 @@ include ../../../hack/package.mk test: helm unittest . + $(MAKE) test-guard-fail-closed + +# The naming guard refuses to render an UPGRADE that cannot see the cluster +# (templates/naming-guard.yaml): a client-side lookup silently returns nothing, +# which would classify a class-S tenant as net-new and rename its workloads away +# from its data. helm-unittest 1.0.3 ignores release.isUpgrade, so .Release.IsUpgrade +# always renders false there and the refusal branch gets NO unit coverage — the +# suite passes with or without the canary. Verify with a real renderer instead. +.PHONY: test-guard-fail-closed +test-guard-fail-closed: + @out=$$(helm template seaweedfs-system . -n tenant-root --is-upgrade 2>&1); \ + echo "$$out" | grep -q 'refusing to upgrade blind' \ + || { echo "FAIL: a blind upgrade rendered instead of refusing (naming-guard canary is not firing)"; echo "$$out" | head -5; exit 1; } + @echo "ok: naming guard refuses a blind upgrade" update: rm -rf charts @@ -19,5 +33,6 @@ update: patch --no-backup-if-mismatch -p4 < patches/cosi-bucket-class-lock-readonly.patch patch --no-backup-if-mismatch -p4 < patches/s3-service-name.patch patch --no-backup-if-mismatch -p4 < patches/s3-service-name-consumers.patch + patch --no-backup-if-mismatch -p4 < patches/cluster-scoped-names-per-namespace.patch #patch --no-backup-if-mismatch -p4 < patches/retention-policy-delete.yaml diff --git a/packages/system/seaweedfs/charts/seaweedfs/templates/cosi/cosi-cluster-role.yaml b/packages/system/seaweedfs/charts/seaweedfs/templates/cosi/cosi-cluster-role.yaml index b5c1035757..a6e5b761ee 100644 --- a/packages/system/seaweedfs/charts/seaweedfs/templates/cosi/cosi-cluster-role.yaml +++ b/packages/system/seaweedfs/charts/seaweedfs/templates/cosi/cosi-cluster-role.yaml @@ -4,7 +4,7 @@ kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: - name: {{ include "seaweedfs.fullname" . }}-objectstorage-provisioner + name: {{ include "seaweedfs.serviceAccountName" . }}-objectstorage-provisioner labels: app.kubernetes.io/name: {{ template "seaweedfs.name" . }} helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} @@ -53,7 +53,7 @@ rules: kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: - name: {{ include "seaweedfs.fullname" . }}-objectstorage-provisioner + name: {{ include "seaweedfs.serviceAccountName" . }}-objectstorage-provisioner labels: app.kubernetes.io/name: {{ template "seaweedfs.name" . }} helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} @@ -65,6 +65,6 @@ subjects: namespace: {{ .Release.Namespace }} roleRef: kind: ClusterRole - name: {{ include "seaweedfs.fullname" . }}-objectstorage-provisioner + name: {{ include "seaweedfs.serviceAccountName" . }}-objectstorage-provisioner apiGroup: rbac.authorization.k8s.io {{- end }} diff --git a/packages/system/seaweedfs/charts/seaweedfs/templates/shared/cluster-role.yaml b/packages/system/seaweedfs/charts/seaweedfs/templates/shared/cluster-role.yaml index 14d09f9993..e31c2d5f2f 100644 --- a/packages/system/seaweedfs/charts/seaweedfs/templates/shared/cluster-role.yaml +++ b/packages/system/seaweedfs/charts/seaweedfs/templates/shared/cluster-role.yaml @@ -5,7 +5,7 @@ kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: - name: {{ include "seaweedfs.fullname" . }}-rw-cr + name: {{ include "seaweedfs.serviceAccountName" . }}-rw-cr labels: app.kubernetes.io/name: {{ template "seaweedfs.name" . }} helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} @@ -19,7 +19,7 @@ rules: kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: - name: {{ include "seaweedfs.fullname" . }}-rw-crb + name: {{ include "seaweedfs.serviceAccountName" . }}-rw-crb labels: app.kubernetes.io/name: {{ template "seaweedfs.name" . }} helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} @@ -32,5 +32,5 @@ subjects: roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: {{ include "seaweedfs.fullname" . }}-rw-cr + name: {{ include "seaweedfs.serviceAccountName" . }}-rw-cr {{- end }} \ No newline at end of file diff --git a/packages/system/seaweedfs/patches/cluster-scoped-names-per-namespace.patch b/packages/system/seaweedfs/patches/cluster-scoped-names-per-namespace.patch new file mode 100644 index 0000000000..2091c6d5cf --- /dev/null +++ b/packages/system/seaweedfs/patches/cluster-scoped-names-per-namespace.patch @@ -0,0 +1,60 @@ +diff --git a/packages/system/seaweedfs/charts/seaweedfs/templates/cosi/cosi-cluster-role.yaml b/packages/system/seaweedfs/charts/seaweedfs/templates/cosi/cosi-cluster-role.yaml +index b5c103575..a6e5b761e 100644 +--- a/packages/system/seaweedfs/charts/seaweedfs/templates/cosi/cosi-cluster-role.yaml ++++ b/packages/system/seaweedfs/charts/seaweedfs/templates/cosi/cosi-cluster-role.yaml +@@ -4,7 +4,7 @@ + kind: ClusterRole + apiVersion: rbac.authorization.k8s.io/v1 + metadata: +- name: {{ include "seaweedfs.fullname" . }}-objectstorage-provisioner ++ name: {{ include "seaweedfs.serviceAccountName" . }}-objectstorage-provisioner + labels: + app.kubernetes.io/name: {{ template "seaweedfs.name" . }} + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} +@@ -53,7 +53,7 @@ rules: + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: +- name: {{ include "seaweedfs.fullname" . }}-objectstorage-provisioner ++ name: {{ include "seaweedfs.serviceAccountName" . }}-objectstorage-provisioner + labels: + app.kubernetes.io/name: {{ template "seaweedfs.name" . }} + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} +@@ -65,6 +65,6 @@ subjects: + namespace: {{ .Release.Namespace }} + roleRef: + kind: ClusterRole +- name: {{ include "seaweedfs.fullname" . }}-objectstorage-provisioner ++ name: {{ include "seaweedfs.serviceAccountName" . }}-objectstorage-provisioner + apiGroup: rbac.authorization.k8s.io + {{- end }} +diff --git a/packages/system/seaweedfs/charts/seaweedfs/templates/shared/cluster-role.yaml b/packages/system/seaweedfs/charts/seaweedfs/templates/shared/cluster-role.yaml +index 14d09f999..e31c2d5f2 100644 +--- a/packages/system/seaweedfs/charts/seaweedfs/templates/shared/cluster-role.yaml ++++ b/packages/system/seaweedfs/charts/seaweedfs/templates/shared/cluster-role.yaml +@@ -5,7 +5,7 @@ + kind: ClusterRole + apiVersion: rbac.authorization.k8s.io/v1 + metadata: +- name: {{ include "seaweedfs.fullname" . }}-rw-cr ++ name: {{ include "seaweedfs.serviceAccountName" . }}-rw-cr + labels: + app.kubernetes.io/name: {{ template "seaweedfs.name" . }} + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} +@@ -19,7 +19,7 @@ rules: + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: +- name: {{ include "seaweedfs.fullname" . }}-rw-crb ++ name: {{ include "seaweedfs.serviceAccountName" . }}-rw-crb + labels: + app.kubernetes.io/name: {{ template "seaweedfs.name" . }} + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} +@@ -32,5 +32,5 @@ subjects: + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole +- name: {{ include "seaweedfs.fullname" . }}-rw-cr ++ name: {{ include "seaweedfs.serviceAccountName" . }}-rw-cr + {{- end }} +\ No newline at end of file diff --git a/packages/system/seaweedfs/templates/_naming.tpl b/packages/system/seaweedfs/templates/_naming.tpl new file mode 100644 index 0000000000..4f8935db89 --- /dev/null +++ b/packages/system/seaweedfs/templates/_naming.tpl @@ -0,0 +1,48 @@ +{{- /* seaweedfs.renamedVolumePrefix — reconstruct the name 4.31 gives the volume + component of the `-system` release, i.e. the RELEASE-NAMED generation the + fullnameOverride pin exists to avoid. + + Input: the `-system` release name, as a bare string. + {{ include "seaweedfs.renamedVolumePrefix" "foo-system" }} -> foo-system-seaweedfs-volume + + This replays two upstream helpers, so it must track them if the vendored chart + changes (hack/seaweedfs-guard-parity.bats pins the two copies of the guard + against each other; charts/seaweedfs/templates/shared/_helpers.tpl is the + source of truth for the rules below): + + seaweedfs.fullname — with no fullnameOverride, the release name, plus + `-` when the release name does not + already contain it; truncated to 63. + seaweedfs.componentName — truncates the fullname to (62 - len(suffix)) before + appending `-`, so for `volume` the fullname + is cut to 56. An instance name >= ~40 chars + therefore loses `seaweedfs` from the tail, which is + why a `contains "seaweedfs"` name filter cannot see + its claims and this reconstruction can. + + The guard deliberately does NOT call seaweedfs.fullname directly: that helper + reads .Values.fullnameOverride, which this chart PINS to `seaweedfs`, so it + returns the chart-named generation — the opposite of what is wanted here. + + ACCEPTED LIMIT — zone/pool volume components. MultiZone (and Simple-with- + pools) tenants get per-group components with suffix `volume-`, cut at + (62 - len(suffix)), i.e. SHORTER than the 56 this helper reconstructs. The + prefixes only diverge when the fullname exceeds that shorter cut: for the + supported instance (the tenant module hardcodes the name `seaweedfs`, + fullname 16 chars) that takes a zone/pool key of ~40+ characters, and for + the default `volume` group it can never happen (its suffix IS the 56 cut). + A guard prefix that diverges means a release-named zone component the guard + cannot see. Decided 2026-07-17 (1.6 triage): accepted and documented in + docs/operations/seaweedfs-431-rename-recovery.md (Scope) rather than + reconstructed per-key — instance names are fixed by the tenant module and + absurd keys are the only reachable trigger. Revisit if instance naming is + ever opened up or a key-length clamp lands in extra/seaweedfs. */}} +{{- define "seaweedfs.renamedVolumePrefix" -}} +{{- $release := . -}} +{{- $full := $release -}} +{{- if not (contains "seaweedfs" $release) -}} +{{- $full = printf "%s-seaweedfs" $release -}} +{{- end -}} +{{- $full = $full | trunc 63 | trimSuffix "-" -}} +{{- printf "%s-volume" ($full | trunc 56 | trimSuffix "-") -}} +{{- end -}} diff --git a/packages/system/seaweedfs/templates/cluster-scoped-rbac-guard.yaml b/packages/system/seaweedfs/templates/cluster-scoped-rbac-guard.yaml new file mode 100644 index 0000000000..d14d8d289a --- /dev/null +++ b/packages/system/seaweedfs/templates/cluster-scoped-rbac-guard.yaml @@ -0,0 +1,69 @@ +{{- /* Cluster-scoped RBAC uniqueness guard. + + ClusterRoles and ClusterRoleBindings are CLUSTER-scoped: exactly one object can + carry a given name, cluster-wide. This chart is installed once per tenant, so + every cluster-scoped name it renders must be unique per install. + + That is in direct tension with the adoption pin. `seaweedfs.fullnameOverride` + is pinned to `seaweedfs` (see values.yaml and templates/naming-guard.yaml) so + the NAMESPACED workloads keep their pre-4.31 chart-based names and are adopted + in place across the bump — which means `seaweedfs.fullname` is, by + construction, the SAME string in every tenant. Naming a cluster-scoped object + after it makes every tenant fight over one object. + + Upstream 4.31 did exactly that: it switched the four cluster-scoped names from + global.serviceAccountName to seaweedfs.fullname. Pre-4.31 they were + `-{objectstorage-provisioner,rw-cr}`, and Cozystack + sets that value to `-seaweedfs` (extra/seaweedfs), so they were + unique per tenant. patches/cluster-scoped-names-per-namespace.patch puts them + back on the service account name — restoring the pre-4.31 names byte-for-byte + and, with them, per-tenant uniqueness. + + Uniqueness therefore rests on a VALUE, so assert it instead of trusting it: + the name must carry the release namespace. Without this, a tenant that stops + setting the value per namespace silently re-collides — no error, no ownership + conflict, just one ClusterRoleBinding whose subject flips to whichever tenant + reconciled last while every other tenant's COSI provisioner loses its RBAC. + That failure is invisible until a bucket operation fails, which is why the + collision survived two releases. + + Checked only when the render can see the cluster (same namespace canary as + naming-guard.yaml): system/seaweedfs's own values.yaml carries a static + PLACEHOLDER serviceAccountName that only extra/seaweedfs overrides per + release, so a client-side render (CI lint, helm unittest) legitimately sees + the placeholder and touches no live RBAC. + + The compat shim must run first. extra/seaweedfs sets the OLD flat key + global.serviceAccountName, and seaweedfs.compat is what folds it into the + canonical global.seaweedfs.serviceAccountName (the flat key wins when + present). Every subchart template includes the shim before reading, and this + guard has to read the same value the subchart will, or it compares against + this chart's placeholder default and refuses every real upgrade. Including + the shim rather than re-deriving the precedence here is deliberate: a second + copy of that rule would be free to drift from the one that actually names the + objects. It is idempotent and output-free, so calling it here is safe. */}} +{{- include "seaweedfs.compat" . -}} + +{{- $canary := lookup "v1" "Namespace" "" .Release.Namespace }} +{{- if $canary }} +{{- $sa := include "seaweedfs.serviceAccountName" . }} +{{- /* Equality, not a prefix or substring test. extra/seaweedfs sets exactly + `-seaweedfs`, so that is what to assert. Anything looser accepts a + value belonging to a DIFFERENT namespace: `contains` lets namespace `tenant-a` + through on `tenant-ab-seaweedfs`, and `hasPrefix "tenant-a-"` still lets it + through on `tenant-a-b-seaweedfs` — which is also the natural value for + namespace `tenant-a-b`, i.e. two namespaces would accept one string and + collide on the very object this guard exists to keep unique. + + This asserts the VALUE the names are built from, not the rendered names — it + cannot catch the chart being reverted to name them after the pinned fullname. + tests/cluster_scoped_names_test.yaml covers that axis, by asserting that two + tenants render two different names. + + (The comment marker must be exactly `{{- /*`: Go's lexer looks for `/*` at a + fixed offset past the trim marker, so `{{- /*` lexes as a command and fails + with `unexpected "/" in command`.) */}} +{{- if ne $sa (printf "%s-seaweedfs" .Release.Namespace) }} +{{- fail (printf "SeaweedFS release %s in namespace %s would render cluster-scoped RBAC named %q, which is not this namespace's own -seaweedfs and so is not unique per tenant. Every SeaweedFS instance pins the same fullname, so ClusterRole/ClusterRoleBinding names are taken from global.seaweedfs.serviceAccountName, which extra/seaweedfs must set per namespace. With a shared name only one tenant can own the object, and the ClusterRoleBinding's subject flips to whichever tenant reconciled last — silently revoking every other tenant's COSI provisioner." .Release.Name .Release.Namespace $sa) }} +{{- end }} +{{- end }} diff --git a/packages/system/seaweedfs/templates/naming-guard.yaml b/packages/system/seaweedfs/templates/naming-guard.yaml new file mode 100644 index 0000000000..a60541cb47 --- /dev/null +++ b/packages/system/seaweedfs/templates/naming-guard.yaml @@ -0,0 +1,154 @@ +{{- /* Naming-migration guard — the ENFORCING copy. + + Before 4.31 the vendored chart named workloads after the CHART, ignoring the + release name, so every instance ran as `seaweedfs-*` with its data on + `data1-seaweedfs-volume-*`. 4.31 names them after the release; values.yaml pins + `fullnameOverride: seaweedfs` so an upgrade past the bump adopts the running + set and its volumes in place. This template stops the render instead of letting + Helm rename workloads away from live data, or adopt the wrong one of two sets: + + - exactly the legacy generation (or nothing) — decidable, and adoption is + correct. Renders. + - exactly the release-named generation — a tenant installed fresh on 1.5.x, + whose data is on `data1-[-seaweedfs]-volume-*`. Rendering the pinned + names would rename the workloads AWAY from that data. Refuses → runbook Step 2. + - BOTH generations — undecidable from inside a render, so it refuses rather + than guess. See the classification note below. → runbook Step 1, then 2/2a/3. + + extra/seaweedfs carries the same classification for its own render, but that + render is NOT in the path of a platform upgrade: this chart reaches + helm-controller through an ExternalArtifact, so a platform bump upgrades the + `-system` release directly from the new artifact without ever + re-rendering extra/seaweedfs. The only render guaranteed to sit between a + platform upgrade and the tenant's workloads is THIS one. (v1.6 regression: + the guard lived only in extra/seaweedfs, so upgrading a fresh-1.5.x tenant + created an empty chart-named set beside its live data.) + + Fail-closed: `lookup` errors abort the render (Helm propagates API errors), + but a client-side render (`helm template`, --dry-run=client) silently returns + nothing. The release namespace itself is used as a canary: it always exists + for any real install/upgrade, so an empty canary means the render cannot see + the cluster — refuse on upgrade rather than misclassify, and skip the guard + for client-side installs (CI lint/unittest), which never touch live data. */}} + +{{- /* The classification below and the whole adoption scheme assume chart-based + workload names. Refuse to render at all if the pin is ever lifted. */}} +{{- if ne (dig "fullnameOverride" "" (.Values.seaweedfs | default dict)) "seaweedfs" }} +{{- fail (printf "system/seaweedfs must pin seaweedfs.fullnameOverride=seaweedfs (got %q): workload adoption across the 4.31 rename and the naming-migration guard are both built on chart-based names. Do not lift the pin without a data-migration plan." (dig "fullnameOverride" "" (.Values.seaweedfs | default dict))) }} +{{- end }} + +{{- $canary := lookup "v1" "Namespace" "" .Release.Namespace }} +{{- if and (not $canary) .Release.IsUpgrade }} +{{- fail (printf "SeaweedFS naming-migration guard for release %s in namespace %s: cannot see the cluster (lookup returned nothing for the release namespace itself). This render decides whether workloads are renamed away from live data, so refusing to upgrade blind. Renders applied by helm-controller always see the cluster; if templating by hand, render server-side." .Release.Name .Release.Namespace) }} +{{- end }} +{{- if $canary }} + +{{- /* Two naming GENERATIONS can exist in a namespace: + legacy — the pre-4.31 chart-based names (`seaweedfs-volume[-]`, + `data1-seaweedfs-volume[-]-N`), which this chart pins and renders; + system — the 4.31 release-based names (`[-seaweedfs]-volume`, + `data1-[-seaweedfs]-volume-N`). + + The release-named generation is matched by RECONSTRUCTING the name 4.31 would + give this release's volume component, and the chart-named one by the pinned + name — release-named checked FIRST, because the two prefixes can nest. + Overlapping prefixes alone are not safe: for an instance legitimately named + `seaweedfs-volume`, the release-named StatefulSet `seaweedfs-volume-system-volume` + and claim `data1-seaweedfs-volume-system-volume-0` BOTH satisfy the chart-named + prefixes, so a prefix-only guard reads live release-named storage as legacy and + renders onto the empty chart-named claims, stranding the real generation. + + Reconstructing also removes the blind spot a name match alone has: 4.31's + componentName truncates the fullname to 56 chars before appending `-volume`, so + an instance name >= ~40 chars loses `seaweedfs` from its claim names and a + `contains "seaweedfs"` filter cannot see them. The reconstructed prefix carries + the same truncation, so it matches those claims exactly. + + PVCs hold the data and outlive any workload but carry no chart labels, so they + are matched on name alone. StatefulSets DO carry the labels, and are the only + signal for a tenant whose PVCs are not provisioned yet. Either establishes a + generation's presence, so the two are OR-ed. */}} +{{- /* This chart IS the `-system` release, so its own release name is what + 4.31 named the renamed generation after. extra/seaweedfs is the `` release + and derives the child instead; that one line is the only difference between the + two copies of the block below. */}} +{{- $sysRelease := .Release.Name }} +{{- $renamedVol := include "seaweedfs.renamedVolumePrefix" $sysRelease }} +{{- $legacyPVC := false }} +{{- $systemPVC := false }} +{{- $legacySTS := false }} +{{- $systemSTS := false }} +{{- range (lookup "v1" "PersistentVolumeClaim" .Release.Namespace "").items | default list }} +{{- if hasPrefix (printf "data1-%s" $renamedVol) .metadata.name }} +{{- $systemPVC = true }} +{{- else if hasPrefix "data1-seaweedfs-volume" .metadata.name }} +{{- $legacyPVC = true }} +{{- end }} +{{- end }} +{{- range (lookup "apps/v1" "StatefulSet" .Release.Namespace "").items | default list }} +{{- if eq (dig "app.kubernetes.io/name" "" (.metadata.labels | default dict)) "seaweedfs" }} +{{- if hasPrefix $renamedVol .metadata.name }} +{{- $systemSTS = true }} +{{- else if hasPrefix "seaweedfs-volume" .metadata.name }} +{{- $legacySTS = true }} +{{- end }} +{{- end }} +{{- end }} +{{- $legacyGen := or $legacyPVC $legacySTS }} +{{- $systemGen := or $systemPVC $systemSTS }} + +{{- /* Exactly one generation is decidable; two is not. + + This guard used to try to tell the cases apart by comparing PVC creation + timestamps — older generation wins, because "PVCs are never recreated in + place". That premise is false, and the runbook itself is the counter-example: + Step 2's re-bind DELETES each release-named claim and re-creates it under the + chart name against the same PV. A tenant interrupted part-way through that + loop has a chart-named claim created SECONDS ago holding real data, beside a + release-named claim with the original timestamp also holding real data — the + exact inversion of the rule. The guard then reported S-damaged and sent the + operator to a step that deletes the claim Step 2 had just re-bound. + + A durable signal for BIRTH ORDER does exist, and it is worth naming so nobody + re-derives the broken one: the release history. `sh.helm.release.v1.- + system.v1` records the naming scheme the tenant was installed under (its + manifest names either `seaweedfs-master` or `-master`), and + `info.first_deployed` — present on every retained revision, so it survives + history pruning — anchors the PV creationTimestamps that Step 2's re-bind + preserves. On the upgrade stand that classifies every tenant correctly, + including reporting "no duplicate, mid-rebind" for the interrupted-Step-2 case, + because there BOTH generations sit at first_deployed. + + This render still cannot read it — the release is base64(gzip(json)) in a + Secret and Go templates have no gunzip — but that is NOT the reason the guard + refuses. Birth order is the wrong question. It says which generation is + ORIGINAL; adoption needs to know whether the OTHER one is EMPTY, and those come + apart exactly where it matters: + + D-wedged — duplicate never scheduled, holds nothing -> adopting is safe + D-split — duplicate served writes, holds unique objects -> adopting strands them + + Both are born pre-4.31, so the history signal is IDENTICAL for both (verified: + tenant-root and tenant-dsplit have the same rev-1 scheme and the same + first_deployed deltas — ~10s legacy vs ~35min release-named — yet one duplicate + served and the other did not). The only thing that separated them was + readyReplicas, which is a snapshot and not evidence: a duplicate that crashed, + was scaled down, or lost readiness after serving reads exactly like one that + never started. + + So this render does not guess. Two generations => refuse, and let an operator + establish emptiness with context the template does not have (the duplicate's + volume files, the filer's volume list). Once the empty generation is gone + exactly one remains, which IS decidable, and the render proceeds on its own. + + The cost is bounded: a duplicate only exists on a tenant that passed through + the broken 4.31 rename in 1.5.x, i.e. one that is already damaged and already + needs an operator. A 1.4.x tenant upgrading straight to 1.6 never renames, so + it only ever has the legacy generation and renders untouched. */}} +{{- if and $legacyGen $systemGen }} +{{- fail (printf "SeaweedFS release %s in namespace %s has BOTH naming generations present: the chart-named set (seaweedfs-volume / data1-seaweedfs-volume-*) AND the release-named set this chart no longer uses. One of them is an empty duplicate and one holds the data, and which is which cannot be established from inside a Helm render — claim timestamps are mutable (the Step 2 re-bind recreates claims), StatefulSets are recreated by the adoption hook, and a duplicate with zero ready replicas may still have served writes. Rendering would adopt the chart-named set, so guessing wrong strands or destroys the data. Refusing instead. Classify the tenant and delete the EMPTY generation so exactly one remains — this render then adopts the survivor with no further action. See docs/operations/seaweedfs-431-rename-recovery.md (Step 1 classifies; Step 2/2a/3 recover). If you are part-way through Step 2's re-bind, finish it." .Release.Name .Release.Namespace) }} +{{- else if $systemGen }} +{{- fail (printf "SeaweedFS release %s in namespace %s keeps its data on volumes named after the Helm release — a tenant installed fresh on Cozystack 1.5.x (or one whose instance name is long enough that the chart truncated the volume PVC names, leaving only its StatefulSets to match on). This chart pins the pre-4.31 chart-based names, so rendering would rename the workloads away from that data and stand up an empty cluster beside it. Helm cannot move data between PVCs. Re-bind the existing PVs onto the data1-seaweedfs-volume-* PVC names first — docs/operations/seaweedfs-431-rename-recovery.md (Step 2)." .Release.Name .Release.Namespace) }} +{{- end }} + +{{- end }} diff --git a/packages/system/seaweedfs/tests/cluster_scoped_names_test.yaml b/packages/system/seaweedfs/tests/cluster_scoped_names_test.yaml new file mode 100644 index 0000000000..8574ebc427 --- /dev/null +++ b/packages/system/seaweedfs/tests/cluster_scoped_names_test.yaml @@ -0,0 +1,138 @@ +suite: seaweedfs cluster-scoped RBAC names stay unique per tenant + +# ClusterRoles and ClusterRoleBindings are CLUSTER-scoped: exactly one object can +# carry a given name, cluster-wide. Every SeaweedFS instance pins the same +# `fullnameOverride: seaweedfs` (#3282 — that pin is what adopts the running +# NAMESPACED workloads across the 4.31 rename), so `seaweedfs.fullname` is +# identical in every tenant and must never name a cluster-scoped object. +# +# Upstream 4.31 switched these four names from global.serviceAccountName (which +# Cozystack sets per namespace) to seaweedfs.fullname, which is how five tenants +# ended up fighting over ONE ClusterRole. patches/cluster-scoped-names-per- +# namespace.patch puts them back on the service account name, restoring the +# pre-4.31 names byte-for-byte. +# +# These assertions fail on unpatched code: it renders `seaweedfs-*` for every +# namespace, so the two namespaces below produce the SAME name instead of two. + +templates: + - charts/seaweedfs/templates/cosi/cosi-cluster-role.yaml + - charts/seaweedfs/templates/shared/cluster-role.yaml + +release: + name: seaweedfs-system + namespace: tenant-root + +tests: + - it: names the COSI ClusterRole/Binding after the per-namespace service account + template: charts/seaweedfs/templates/cosi/cosi-cluster-role.yaml + set: + global.seaweedfs.serviceAccountName: tenant-root-seaweedfs + asserts: + # ClusterRole — pre-4.31 name, restored. + - equal: + path: metadata.name + value: tenant-root-seaweedfs-objectstorage-provisioner + documentIndex: 0 + # ClusterRoleBinding — same name, and its roleRef must follow the rename or + # the binding would point at another tenant's ClusterRole. + - equal: + path: metadata.name + value: tenant-root-seaweedfs-objectstorage-provisioner + documentIndex: 1 + - equal: + path: roleRef.name + value: tenant-root-seaweedfs-objectstorage-provisioner + documentIndex: 1 + # The subject was ALREADY per-namespace (cosi-provisioner-sa-name.patch); + # it is what the cluster-scoped names are now aligned with. + - equal: + path: subjects[0].name + value: tenant-root-seaweedfs-objectstorage-provisioner + documentIndex: 1 + - equal: + path: subjects[0].namespace + value: tenant-root + documentIndex: 1 + + - it: names the master-rw ClusterRole/Binding after the per-namespace service account + template: charts/seaweedfs/templates/shared/cluster-role.yaml + set: + global.seaweedfs.serviceAccountName: tenant-root-seaweedfs + asserts: + - equal: + path: metadata.name + value: tenant-root-seaweedfs-rw-cr + documentIndex: 0 + - equal: + path: metadata.name + value: tenant-root-seaweedfs-rw-crb + documentIndex: 1 + - equal: + path: roleRef.name + value: tenant-root-seaweedfs-rw-cr + documentIndex: 1 + - equal: + path: subjects[0].name + value: tenant-root-seaweedfs + documentIndex: 1 + + # THE regression test. A second tenant, running an instance under a DIFFERENT + # name (`foo` -> release foo-system), must not collide with tenant-root above. + # Unpatched, both render `seaweedfs-objectstorage-provisioner` / `seaweedfs-rw-cr` + # — observed live as one ClusterRole annotated to a single owning tenant while + # four others silently lost their COSI RBAC. + - it: renders a different cluster-scoped name for a different tenant (collision regression) + template: charts/seaweedfs/templates/cosi/cosi-cluster-role.yaml + release: + name: foo-system + namespace: tenant-named + set: + global.seaweedfs.serviceAccountName: tenant-named-seaweedfs + asserts: + - equal: + path: metadata.name + value: tenant-named-seaweedfs-objectstorage-provisioner + documentIndex: 0 + - equal: + path: metadata.name + value: tenant-named-seaweedfs-objectstorage-provisioner + documentIndex: 1 + - equal: + path: roleRef.name + value: tenant-named-seaweedfs-objectstorage-provisioner + documentIndex: 1 + + - it: renders a different master-rw name for a different tenant (collision regression) + template: charts/seaweedfs/templates/shared/cluster-role.yaml + release: + name: foo-system + namespace: tenant-named + set: + global.seaweedfs.serviceAccountName: tenant-named-seaweedfs + asserts: + - equal: + path: metadata.name + value: tenant-named-seaweedfs-rw-cr + documentIndex: 0 + - equal: + path: metadata.name + value: tenant-named-seaweedfs-rw-crb + documentIndex: 1 + + # The pin must NOT leak into cluster-scoped names, and must NOT be lifted from + # namespaced ones. Guards the two halves against each other. + - it: keeps the cluster-scoped name independent of the pinned fullname + template: charts/seaweedfs/templates/cosi/cosi-cluster-role.yaml + set: + global.seaweedfs.serviceAccountName: tenant-root-seaweedfs + seaweedfs.fullnameOverride: seaweedfs + asserts: + - notMatchRegex: + path: metadata.name + pattern: "^seaweedfs-" + documentIndex: 0 + - notMatchRegex: + path: metadata.name + pattern: "^seaweedfs-system-" + documentIndex: 0 diff --git a/packages/system/seaweedfs/tests/cluster_scoped_rbac_guard_test.yaml b/packages/system/seaweedfs/tests/cluster_scoped_rbac_guard_test.yaml new file mode 100644 index 0000000000..dd8aeb29a7 --- /dev/null +++ b/packages/system/seaweedfs/tests/cluster_scoped_rbac_guard_test.yaml @@ -0,0 +1,177 @@ +suite: seaweedfs cluster-scoped RBAC uniqueness guard + +# Companion to cluster_scoped_names_test.yaml: that suite pins the NAMES the +# chart renders, this one pins the guard that refuses when those names would not +# be unique per tenant. +# +# The guard reads global.seaweedfs.serviceAccountName and requires it to carry +# the release namespace, because the pinned fullname is identical in every tenant +# and so cannot be what distinguishes a cluster-scoped object. It runs only when +# the render can see the cluster (namespace canary, same as naming-guard.yaml) — +# system/seaweedfs's values.yaml default is a placeholder that only +# extra/seaweedfs overrides per release, so a client-side render must not trip. + +templates: + - templates/cluster-scoped-rbac-guard.yaml + +release: + name: seaweedfs-system + namespace: tenant-fresh + +tests: + - it: renders for a client-side install with no cluster view (CI lint, unittest) + asserts: + - hasDocuments: + count: 0 + + - it: renders when the cluster-scoped RBAC name carries the namespace + set: + global: + seaweedfs: + serviceAccountName: tenant-fresh-seaweedfs + kubernetesProvider: + scheme: &scheme + "v1/Namespace": + gvr: + group: "" + version: "v1" + resource: "namespaces" + namespaced: false + objects: + - &ns + kind: Namespace + apiVersion: v1 + metadata: + name: tenant-fresh + asserts: + - hasDocuments: + count: 0 + + # M2 REGRESSION — the compat shim. Production does NOT set the canonical + # global.seaweedfs.serviceAccountName that the other cases here use: + # extra/seaweedfs sets the OLD FLAT key global.serviceAccountName, which only + # `include "seaweedfs.compat"` folds into the canonical path (and the flat key + # wins when present). Without that include the guard reads this chart's + # placeholder default and refuses EVERY real upgrade on EVERY cluster — a bug + # that only `helm upgrade --dry-run=server` caught, because every mock set the + # canonical key. This case sets the key production actually sets. + - it: accepts the flat global.serviceAccountName that extra/seaweedfs actually sets + set: + global: + serviceAccountName: tenant-fresh-seaweedfs + kubernetesProvider: + scheme: *scheme + objects: + - *ns + asserts: + - hasDocuments: + count: 0 + + # The shim must not become a way to bypass the guard: a flat key that is not + # per-namespace is still refused. + - it: refuses a flat global.serviceAccountName that is not per-namespace + set: + global: + serviceAccountName: seaweedfs + kubernetesProvider: + scheme: *scheme + objects: + - *ns + asserts: + - failedTemplate: + errorPattern: "not unique per tenant" + + # A substring match accepts `tenant-ab-seaweedfs` for namespace `tenant-a`; a + # prefix match still accepts `tenant-a-b-seaweedfs`, which is ALSO the natural + # value for namespace `tenant-a-b` — so two namespaces would accept one string + # and collide on the object this guard exists to keep unique. Only equality with + # `-seaweedfs` — the value extra/seaweedfs actually sets — is safe. + - it: refuses a service account from a namespace that only shares a prefix + release: + name: seaweedfs-system + namespace: tenant-a + set: + global: + seaweedfs: + serviceAccountName: tenant-ab-seaweedfs + kubernetesProvider: + scheme: *scheme + objects: + - kind: Namespace + apiVersion: v1 + metadata: + name: tenant-a + asserts: + - failedTemplate: + errorPattern: "not unique per tenant" + + # The prefix variant: `tenant-a-b-seaweedfs` starts with `tenant-a-` but belongs + # to namespace tenant-a-b. hasPrefix passes this; equality does not. + - it: refuses a service account whose namespace merely extends this one + release: + name: seaweedfs-system + namespace: tenant-a + set: + global: + seaweedfs: + serviceAccountName: tenant-a-b-seaweedfs + kubernetesProvider: + scheme: *scheme + objects: + - kind: Namespace + apiVersion: v1 + metadata: + name: tenant-a + asserts: + - failedTemplate: + errorPattern: "not unique per tenant" + + # The regression this guard exists for: the bare chart default (or any value + # that is the same string in every tenant) collides cluster-wide. + - it: refuses when the RBAC name would be identical in every tenant + set: + global: + seaweedfs: + serviceAccountName: seaweedfs + kubernetesProvider: + scheme: *scheme + objects: + - *ns + asserts: + - failedTemplate: + errorPattern: "not unique per tenant" + + # The placeholder shipped in system/seaweedfs/values.yaml is namespace-shaped + # but belongs to no real namespace — it must not be mistaken for a valid value + # once the render can see the cluster. + - it: refuses when the value is the shipped placeholder, not this namespace + set: + global: + seaweedfs: + serviceAccountName: tenant-foo-seaweedfs + kubernetesProvider: + scheme: *scheme + objects: + - *ns + asserts: + - failedTemplate: + errorPattern: "not unique per tenant" + + - it: refuses for a non-default instance name whose SA is not per-namespace + release: + name: foo-system + namespace: tenant-named + set: + global: + seaweedfs: + serviceAccountName: seaweedfs + kubernetesProvider: + scheme: *scheme + objects: + - kind: Namespace + apiVersion: v1 + metadata: + name: tenant-named + asserts: + - failedTemplate: + errorPattern: "not unique per tenant" diff --git a/packages/system/seaweedfs/tests/naming_guard_test.yaml b/packages/system/seaweedfs/tests/naming_guard_test.yaml new file mode 100644 index 0000000000..293028aca5 --- /dev/null +++ b/packages/system/seaweedfs/tests/naming_guard_test.yaml @@ -0,0 +1,568 @@ +suite: seaweedfs naming-migration guard (enforcing copy) + +# This chart is what a platform upgrade actually re-renders: the -system +# HelmRelease pulls it from a platform-managed ExternalArtifact, so bumping the +# platform upgrades the release directly — extra/seaweedfs (which carries a +# sibling copy of this guard for operator visibility) is NOT in that path. +# +# The guard decides between exactly three outcomes: +# legacy generation only (or nothing) -> render (adoption is correct) +# release-named generation only -> refuse, class S (runbook Step 2) +# BOTH generations -> refuse, undecidable (runbook Step 1) +# +# It used to tell the both-generations cases apart by comparing PVC +# creationTimestamps ("PVCs are never recreated in place, so the older generation +# is where the data was born"). That premise is false — the runbook's own Step 2 +# re-bind DELETES each release-named claim and recreates it under the chart name +# against the same PV — and acting on the wrong answer deletes live data. The +# guard no longer guesses; the tests below pin that. +# +# The guard reads the cluster through lookup with the release namespace as its +# canary, so classification tests register v1/Namespace + v1/PersistentVolumeClaim +# + apps/v1/StatefulSet and include the namespace object. readyReplicas is quoted +# ("1"/"0") because the fake client cannot deep-copy a bare Go int. + +templates: + - templates/naming-guard.yaml + +release: + name: seaweedfs-system + namespace: tenant-fresh + +tests: + - it: renders for a client-side install with no cluster view (CI lint, unittest) + asserts: + - hasDocuments: + count: 0 + + # NOTE: the fail-closed refusal (client-side UPGRADE with no cluster view) + # cannot be modelled here — helm-unittest 1.0.3 ignores release.isUpgrade, so + # .Release.IsUpgrade always renders false. The chart's `make test` target + # covers it with a real renderer (helm template --is-upgrade). + + - it: refuses to render if the fullnameOverride pin is ever lifted + set: + seaweedfs: + fullnameOverride: something-else + asserts: + - failedTemplate: + errorPattern: "must pin seaweedfs.fullnameOverride=seaweedfs" + + - it: renders for a net-new tenant (namespace visible, no SeaweedFS data) + kubernetesProvider: + scheme: &scheme + "v1/Namespace": + gvr: + group: "" + version: "v1" + resource: "namespaces" + namespaced: false + "v1/PersistentVolumeClaim": + gvr: + group: "" + version: "v1" + resource: "persistentvolumeclaims" + namespaced: true + "apps/v1/StatefulSet": + gvr: + group: "apps" + version: "v1" + resource: "statefulsets" + namespaced: true + objects: + - &ns + kind: Namespace + apiVersion: v1 + metadata: + name: tenant-fresh + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: unrelated-app-data-0 + namespace: tenant-fresh + - kind: StatefulSet + apiVersion: apps/v1 + metadata: + name: unrelated-app + namespace: tenant-fresh + asserts: + - hasDocuments: + count: 0 + + - it: renders for a legacy-only tenant (data on the chart-named PVCs, adopt in place) + kubernetesProvider: + scheme: *scheme + objects: + - *ns + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: data1-seaweedfs-volume-0 + namespace: tenant-fresh + - kind: StatefulSet + apiVersion: apps/v1 + metadata: + name: seaweedfs-volume + namespace: tenant-fresh + labels: + app.kubernetes.io/name: seaweedfs + asserts: + - hasDocuments: + count: 0 + + - it: refuses class S — data only on release-named volumes (fresh 1.5.x install) + kubernetesProvider: + scheme: *scheme + objects: + - *ns + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: data1-seaweedfs-system-volume-0 + namespace: tenant-fresh + - kind: StatefulSet + apiVersion: apps/v1 + metadata: + name: seaweedfs-system-volume + namespace: tenant-fresh + labels: + app.kubernetes.io/name: seaweedfs + asserts: + - failedTemplate: + errorPattern: "keeps its data on volumes named after the Helm release.*Step 2" + + - it: refuses class S for a non-default instance name (foo-system release) + release: + name: foo-system + namespace: tenant-fresh + kubernetesProvider: + scheme: *scheme + objects: + - *ns + # 4.31 appends the chart name when the release name does not contain it: + # a fresh 1.5.x `foo` wrote to data1-foo-system-seaweedfs-volume-*. + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: data1-foo-system-seaweedfs-volume-0 + namespace: tenant-fresh + # The fake client builds its list kinds from the objects present, so a + # suite that LISTs StatefulSets must include at least one or the lookup + # errors instead of returning empty. + - &unrelated_sts + kind: StatefulSet + apiVersion: apps/v1 + metadata: + name: unrelated-app + namespace: tenant-fresh + asserts: + - failedTemplate: + errorPattern: "keeps its data on volumes named after the Helm release" + + # --------------------------------------------------------------------------- + # Both generations present => refuse. Four shapes the old timestamp classifier + # routed FOUR different ways; all four get the same answer now. + # --------------------------------------------------------------------------- + + # Old code: legacy older + duplicate live -> "D-split" (Step 3). Now: refuse. + - it: refuses when both generations exist and the duplicate is live (was D-split) + kubernetesProvider: + scheme: *scheme + objects: + - *ns + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: data1-seaweedfs-volume-0 + namespace: tenant-fresh + creationTimestamp: "2025-11-02T10:00:00Z" + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: data1-seaweedfs-system-volume-0 + namespace: tenant-fresh + creationTimestamp: "2026-06-20T12:00:00Z" + - kind: StatefulSet + apiVersion: apps/v1 + metadata: + name: seaweedfs-system-volume + namespace: tenant-fresh + labels: + app.kubernetes.io/name: seaweedfs + status: + readyReplicas: "1" + asserts: + - failedTemplate: + errorPattern: "has BOTH naming generations present" + + # B5 REGRESSION. Old code: legacy older + zero ready replicas -> "D-wedged", + # which rendered THROUGH and adopted. readyReplicas is only a snapshot: a + # duplicate that crashed, was scaled down, or lost readiness AFTER serving + # writes reads identically to one that never served. Adoption would strand + # whatever it wrote. + - it: refuses when both generations exist and the duplicate reads zero ready (was D-wedged, adopted) + kubernetesProvider: + scheme: *scheme + objects: + - *ns + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: data1-seaweedfs-volume-0 + namespace: tenant-fresh + creationTimestamp: "2025-11-02T10:00:00Z" + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: data1-seaweedfs-system-volume-0 + namespace: tenant-fresh + creationTimestamp: "2026-06-20T12:00:00Z" + - kind: StatefulSet + apiVersion: apps/v1 + metadata: + name: seaweedfs-system-volume + namespace: tenant-fresh + labels: + app.kubernetes.io/name: seaweedfs + status: + readyReplicas: "0" + - kind: StatefulSet + apiVersion: apps/v1 + metadata: + name: seaweedfs-volume + namespace: tenant-fresh + labels: + app.kubernetes.io/name: seaweedfs + status: + readyReplicas: "1" + asserts: + - failedTemplate: + errorPattern: "has BOTH naming generations present" + + # Old code: release-named older -> "S-damaged" (Step 2a). Now: refuse. + - it: refuses when both generations exist and the release-named claims are older (was S-damaged) + kubernetesProvider: + scheme: *scheme + objects: + - *ns + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: data1-seaweedfs-system-volume-0 + namespace: tenant-fresh + creationTimestamp: "2026-06-20T07:01:59Z" + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: data1-seaweedfs-volume-0 + namespace: tenant-fresh + creationTimestamp: "2026-06-20T08:28:58Z" + - *unrelated_sts + asserts: + - failedTemplate: + errorPattern: "has BOTH naming generations present" + + # B1 REGRESSION — the data-loss path. A tenant interrupted part-way through the + # runbook's Step 2 re-bind: data1-seaweedfs-volume-0 was DELETED and RECREATED + # against its original PV seconds ago, so it is the NEWEST claim while holding + # real data; data1-seaweedfs-system-volume-1 has not been re-bound yet, so it + # keeps the ORIGINAL timestamp and also holds real data. The old timestamp rule + # read "release-named older" and reported S-damaged, sending the operator to + # Step 2a — which deletes data1-seaweedfs-volume-*, i.e. the claim Step 2 had + # just re-bound, with the PV's reclaim policy already restored to Delete. + - it: refuses a tenant interrupted mid Step-2 re-bind rather than calling it S-damaged + kubernetesProvider: + scheme: *scheme + objects: + - *ns + # Not yet re-bound: original claim, original timestamp, real data. + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: data1-seaweedfs-system-volume-1 + namespace: tenant-fresh + creationTimestamp: "2026-06-20T07:01:59Z" + # Already re-bound by Step 2: brand-new claim, ORIGINAL PV, real data. + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: data1-seaweedfs-volume-0 + namespace: tenant-fresh + creationTimestamp: "2026-07-17T09:30:00Z" + - *unrelated_sts + asserts: + - failedTemplate: + errorPattern: "has BOTH naming generations present" + + # --------------------------------------------------------------------------- + # B2 REGRESSION — long instance name. seaweedfs.componentName truncates the + # fullname to 56 chars before appending -volume, so an instance name >= ~40 + # chars drops `seaweedfs` off the PVC name and the name match cannot see it. + # The StatefulSet is label-matched and still can, which is why the generation + # flags OR the two signals instead of keying off the PVC evidence alone. + # --------------------------------------------------------------------------- + + - it: refuses both-generations for a long instance name whose PVC names were truncated + release: + name: archive-of-quarterly-financial-statements-x1-system + namespace: tenant-fresh + kubernetesProvider: + scheme: *scheme + objects: + - *ns + # The renamed volume PVC, truncated past the chart name: unmatchable. + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: data1-archive-of-quarterly-financial-statements-x1-system-seaw-volume-0 + namespace: tenant-fresh + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: data1-seaweedfs-volume-0 + namespace: tenant-fresh + # Only the label-matched StatefulSet reveals the renamed generation. Old + # code fell through to the D-split branch here, sending the operator to + # quiesce the set holding the data — the exact misclassification the + # parent commit exists to remove. + - kind: StatefulSet + apiVersion: apps/v1 + metadata: + name: archive-of-quarterly-financial-statements-x1-system-seaw-volume + namespace: tenant-fresh + labels: + app.kubernetes.io/name: seaweedfs + status: + readyReplicas: "1" + asserts: + - failedTemplate: + errorPattern: "has BOTH naming generations present" + + - it: refuses a truncated long instance name whose duplicate reads zero ready (rendered unguarded before) + release: + name: archive-of-quarterly-financial-statements-x1-system + namespace: tenant-fresh + kubernetesProvider: + scheme: *scheme + objects: + - *ns + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: data1-archive-of-quarterly-financial-statements-x1-system-seaw-volume-0 + namespace: tenant-fresh + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: data1-seaweedfs-volume-0 + namespace: tenant-fresh + # Old code: no branch fired at all here — it rendered through and adopted + # the empty chart-named set. Blocker fully open. + - kind: StatefulSet + apiVersion: apps/v1 + metadata: + name: archive-of-quarterly-financial-statements-x1-system-seaw-volume + namespace: tenant-fresh + labels: + app.kubernetes.io/name: seaweedfs + status: + readyReplicas: "0" + asserts: + - failedTemplate: + errorPattern: "has BOTH naming generations present" + + - it: refuses class S for a truncated long instance name with no legacy generation + release: + name: archive-of-quarterly-financial-statements-x1-system + namespace: tenant-fresh + kubernetesProvider: + scheme: *scheme + objects: + - *ns + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: data1-archive-of-quarterly-financial-statements-x1-system-seaw-volume-0 + namespace: tenant-fresh + - kind: StatefulSet + apiVersion: apps/v1 + metadata: + name: archive-of-quarterly-financial-statements-x1-system-seaw-volume + namespace: tenant-fresh + labels: + app.kubernetes.io/name: seaweedfs + asserts: + - failedTemplate: + errorPattern: "keeps its data on volumes named after the Helm release" + + # --------------------------------------------------------------------------- + # An instance legitimately named `seaweedfs-volume`. Its release is + # seaweedfs-volume-system, which CONTAINS "seaweedfs", so 4.31 names its volume + # workloads seaweedfs-volume-system-volume and its claims + # data1-seaweedfs-volume-system-volume-N. Both satisfy the chart-named prefixes + # ("seaweedfs-volume" / "data1-seaweedfs-volume"), so a prefix-only guard reads + # this tenant's LIVE release-named storage as legacy, sees no release-named + # generation, renders, and stands an empty cluster on data1-seaweedfs-volume-* + # while the real data is stranded. Reconstructing the renamed prefix and testing + # it FIRST is what separates them. + # --------------------------------------------------------------------------- + + - it: refuses class S for an instance named seaweedfs-volume (release-named storage is not legacy) + release: + name: seaweedfs-volume-system + namespace: tenant-fresh + kubernetesProvider: + scheme: *scheme + objects: + - *ns + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: data1-seaweedfs-volume-system-volume-0 + namespace: tenant-fresh + - kind: StatefulSet + apiVersion: apps/v1 + metadata: + name: seaweedfs-volume-system-volume + namespace: tenant-fresh + labels: + app.kubernetes.io/name: seaweedfs + asserts: + - failedTemplate: + errorPattern: "keeps its data on volumes named after the Helm release" + + - it: refuses both-generations for an instance named seaweedfs-volume + release: + name: seaweedfs-volume-system + namespace: tenant-fresh + kubernetesProvider: + scheme: *scheme + objects: + - *ns + # Live release-named storage... + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: data1-seaweedfs-volume-system-volume-0 + namespace: tenant-fresh + # ...beside a genuine chart-named duplicate. + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: data1-seaweedfs-volume-0 + namespace: tenant-fresh + - kind: StatefulSet + apiVersion: apps/v1 + metadata: + name: seaweedfs-volume-system-volume + namespace: tenant-fresh + labels: + app.kubernetes.io/name: seaweedfs + asserts: + - failedTemplate: + errorPattern: "has BOTH naming generations present" + + # --------------------------------------------------------------------------- + # MultiZone (and Simple-with-pools) tenants have NO plain `volume` component — + # extra/seaweedfs sets volume.enabled=false and renders one component per zone + # key, suffix `volume-`. For the supported instance (the tenant module + # hardcodes the name `seaweedfs`, fullname 16 chars) nothing is truncated, so + # the reconstructed prefix `seaweedfs-system-volume` must prefix-match every + # zone component. These pin that. The reconstruction does NOT cover zone + # components of long-named instances or ~40+ char zone keys — an accepted, + # documented limit (_naming.tpl, runbook Scope), deliberately not tested. + # --------------------------------------------------------------------------- + + - it: renders for a legacy-only MultiZone tenant (zone components adopt in place) + kubernetesProvider: + scheme: *scheme + objects: + - *ns + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: data1-seaweedfs-volume-md-0 + namespace: tenant-fresh + - kind: StatefulSet + apiVersion: apps/v1 + metadata: + name: seaweedfs-volume-md + namespace: tenant-fresh + labels: + app.kubernetes.io/name: seaweedfs + asserts: + - hasDocuments: + count: 0 + + - it: refuses class S for a MultiZone tenant with only zone components (fresh 1.5.x) + kubernetesProvider: + scheme: *scheme + objects: + - *ns + # No plain `volume` component exists on a MultiZone tenant — the zone + # objects are the ONLY release-named evidence, so if the prefix match + # missed them this tenant would render and be renamed away from its data. + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: data1-seaweedfs-system-volume-md-0 + namespace: tenant-fresh + - kind: StatefulSet + apiVersion: apps/v1 + metadata: + name: seaweedfs-system-volume-md + namespace: tenant-fresh + labels: + app.kubernetes.io/name: seaweedfs + asserts: + - failedTemplate: + errorPattern: "keeps its data on volumes named after the Helm release" + + - it: refuses both-generations for a MultiZone tenant + kubernetesProvider: + scheme: *scheme + objects: + - *ns + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: data1-seaweedfs-volume-md-0 + namespace: tenant-fresh + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: data1-seaweedfs-system-volume-md-0 + namespace: tenant-fresh + - kind: StatefulSet + apiVersion: apps/v1 + metadata: + name: seaweedfs-system-volume-md + namespace: tenant-fresh + labels: + app.kubernetes.io/name: seaweedfs + status: + readyReplicas: "0" + asserts: + - failedTemplate: + errorPattern: "has BOTH naming generations present" + + - it: ignores unrelated data1-*-volume-* claims of other apps + kubernetesProvider: + scheme: *scheme + objects: + - *ns + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: data1-clickhouse-volume-0 + namespace: tenant-fresh + - kind: StatefulSet + apiVersion: apps/v1 + metadata: + name: clickhouse-volume + namespace: tenant-fresh + labels: + app.kubernetes.io/name: clickhouse + asserts: + - hasDocuments: + count: 0 diff --git a/packages/system/seaweedfs/values.yaml b/packages/system/seaweedfs/values.yaml index d360d82ac6..2da808d626 100644 --- a/packages/system/seaweedfs/values.yaml +++ b/packages/system/seaweedfs/values.yaml @@ -18,9 +18,12 @@ seaweedfs: # empty set beside the running one while the data stayed on `data1-seaweedfs-volume-*`. # Pinning the chart name restores the pre-4.31 names, so the running workloads # and their volumes are adopted across the bump. extra/seaweedfs does NOT override - # this per release — it relies on this pin — and guards the one case the pin - # cannot serve (a tenant installed fresh on 1.5.x, whose data is on the renamed - # volumes) by refusing to render. + # this per release — it relies on this pin. The cases the pin cannot serve (a + # tenant installed fresh on 1.5.x, whose data is on the renamed volumes, and a + # live D-split duplicate) refuse to render in templates/naming-guard.yaml — the + # guard MUST live in this chart, because a platform upgrade re-renders this + # chart directly from its ExternalArtifact without re-rendering extra/seaweedfs. + # The guard also refuses to render at all if this pin is ever lifted. fullnameOverride: seaweedfs master: # 30 GB per volume × 10 GiB PVC = 0 → rounded to 1 maxVolume per From c9b27f15df94456ff4591468d8911621d8b4ca86 Mon Sep 17 00:00:00 2001 From: Ivan Okhotnikov Date: Wed, 15 Jul 2026 16:59:01 +0300 Subject: [PATCH 29/69] fix(backups/postgres): purge stale recovery cluster on repeat in-place restore A repeat in-place PostgreSQL RestoreJob into an already-restored target silently no-opped: the destructive purge was skipped because the live Cluster still carried spec.bootstrap.recovery left over from the previous restore, and the re-purge guard mistook that for a freshly-recovered Cluster produced by this restore's own purge. The job then reported Succeeded while the PVC, disk, and data were never touched. Tell the two apart by the Cluster's creationTimestamp relative to the RestoreJob's StartedAt: only a recovery Cluster created at/after StartedAt is treated as this restore's own and skipped; one that predates StartedAt is leftover from an earlier completed restore and is purged so the restore re-bootstraps from the backup. Signed-off-by: Ivan Okhotnikov (cherry picked from commit cf1b6ba28e1d6b464dd3b8e887b712e7072f145f) (cherry picked from commit c85c9f38a42765d7da052e8c4b5ed7ab2aef5c7b) (cherry picked from commit 507a48762292665343195da99913d64d4b256906) --- .../cnpgstrategy_controller.go | 76 +++++++-- .../cnpgstrategy_controller_test.go | 149 ++++++++++++++---- 2 files changed, 179 insertions(+), 46 deletions(-) diff --git a/internal/backupcontroller/cnpgstrategy_controller.go b/internal/backupcontroller/cnpgstrategy_controller.go index f4e3236fe1..51d20c0691 100644 --- a/internal/backupcontroller/cnpgstrategy_controller.go +++ b/internal/backupcontroller/cnpgstrategy_controller.go @@ -314,16 +314,46 @@ func cnpgBackupDeadlineExceeded(startedAt *metav1.Time) bool { // destructive step: // 1. The RestoreJob already records that we've purged // (restoreCondTargetPurged=True). Normal idempotent path. -// 2. The live Cluster already has spec.bootstrap.recovery populated. This -// only happens when an earlier reconcile purged successfully but the +// 2. The live Cluster is a freshly-recovered one that THIS restore's own +// purge + chart re-render produced (see cnpgClusterFreshlyRecovered). +// This only happens when an earlier reconcile purged successfully but the // status-condition write failed; the chart has since re-rendered the // Cluster with our restore-shaped values. Re-purging here would delete // the Cluster CNPG is actively bootstrapping from S3. -func cnpgPurgeNeeded(purgedCondition, liveClusterHasRecovery bool) bool { +// +// A live Cluster that carries spec.bootstrap.recovery but predates this +// RestoreJob is NOT a skip signal: it is leftover from an earlier, already- +// completed restore and still holds the old data. Skipping the purge on it +// (the previous behaviour, which keyed only on "has recovery bootstrap") made +// a repeat in-place restore a silent no-op - the job reported Succeeded while +// the PVC, disk, and data were never touched. +func cnpgPurgeNeeded(purgedCondition, liveClusterFreshlyRecovered bool) bool { if purgedCondition { return false } - return !liveClusterHasRecovery + return !liveClusterFreshlyRecovered +} + +// cnpgClusterFreshlyRecovered reports whether a live cnpg.io Cluster that +// carries spec.bootstrap.recovery was produced by THIS RestoreJob's own purge +// + chart re-render (its creationTimestamp is at or after the job's StartedAt) +// rather than being left over from an earlier, already-completed restore. +// +// Only the former is safe to skip re-purging. A leftover recovery Cluster from +// a prior restore predates StartedAt and still holds the prior data, so it must +// be purged for the new restore to re-bootstrap from the backup. The fresh +// Cluster in the status-write-race case is always created after StartedAt (the +// job sets StartedAt on its first reconcile, long before it purges and the +// chart re-renders), so the timestamp comparison cleanly separates the two. +// +// Returns false when freshness cannot be determined (no recovery bootstrap, or +// a missing timestamp): the caller then proceeds to purge, which is the safe +// default for any pre-existing, non-fresh Cluster. +func cnpgClusterFreshlyRecovered(hasRecovery bool, clusterCreatedAt, restoreStartedAt *metav1.Time) bool { + if !hasRecovery || clusterCreatedAt == nil || restoreStartedAt == nil { + return false + } + return !clusterCreatedAt.Time.Before(restoreStartedAt.Time) } // applyClusterBarmanObjectStore SSA-patches the live CNPG Cluster's @@ -614,12 +644,18 @@ func (r *RestoreJobReconciler) reconcileCNPGRestore(ctx context.Context, restore // check the live Cluster for bootstrap.recovery: if present, the chart // has already re-rendered after a previous purge, and we must NOT delete // it again. - hasRecovery, err := r.clusterHasRecoveryBootstrap(ctx, target.Namespace, clusterName) + hasRecovery, clusterCreatedAt, err := r.recoveryBootstrapClusterState(ctx, target.Namespace, clusterName) if err != nil { return ctrl.Result{}, err } purgedCondition := apimeta.IsStatusConditionTrue(restoreJob.Status.Conditions, restoreCondTargetPurged) - if cnpgPurgeNeeded(purgedCondition, hasRecovery) { + // Only skip the destructive purge for a recovery Cluster THIS restore just + // produced (created at/after StartedAt). A recovery Cluster left over from + // an earlier completed restore predates StartedAt and still holds the old + // data, so it must be purged - otherwise a repeat in-place restore silently + // no-ops. + freshlyRecovered := cnpgClusterFreshlyRecovered(hasRecovery, clusterCreatedAt, restoreJob.Status.StartedAt) + if cnpgPurgeNeeded(purgedCondition, freshlyRecovered) { // Gate the destructive flow on the source cluster having shipped // the backup's required WALs to object storage. archive_command runs // on the source primary; once we delete the Cluster + PVCs, any @@ -1126,11 +1162,15 @@ func (r *RestoreJobReconciler) cnpgClusterHealthy(ctx context.Context, namespace return cluster.Status.Phase == cnpgClusterHealthyPhase, nil } -// clusterHasRecoveryBootstrap returns true when the live cnpg.io Cluster's -// spec.bootstrap.recovery is populated - the signal that the chart has -// re-rendered with our restore-shaped values and the operator is using the -// recovery bootstrap path. Treats a missing Cluster as "not yet" rather -// than an error so the caller can keep polling while HelmRelease catches up. +// recoveryBootstrapClusterState fetches the live cnpg.io Cluster and reports +// whether its spec.bootstrap.recovery is populated - the signal that the chart +// has re-rendered with our restore-shaped values and the operator is using the +// recovery bootstrap path - together with the Cluster's creationTimestamp. The +// caller pairs the timestamp with the RestoreJob's StartedAt (via +// cnpgClusterFreshlyRecovered) to tell a Cluster this restore just produced +// apart from one left over by an earlier completed restore. Treats a missing +// Cluster as "not yet" rather than an error so the caller can keep polling +// while HelmRelease catches up. // // A Cluster carrying DeletionTimestamp is also treated as "not yet": that is // the in-flight purge case, where r.Delete has fired but cnpg.io's @@ -1143,18 +1183,22 @@ func (r *RestoreJobReconciler) cnpgClusterHealthy(ctx context.Context, namespace // drops the change and the cluster ends up with the original initdb spec. // Holding here forces the caller to requeue until the old CR is fully GC'd // and the chart re-creates a fresh one. -func (r *RestoreJobReconciler) clusterHasRecoveryBootstrap(ctx context.Context, namespace, clusterName string) (bool, error) { +func (r *RestoreJobReconciler) recoveryBootstrapClusterState(ctx context.Context, namespace, clusterName string) (hasRecovery bool, createdAt *metav1.Time, err error) { cluster := &cnpgtypes.Cluster{} if err := r.Get(ctx, types.NamespacedName{Namespace: namespace, Name: clusterName}, cluster); err != nil { if apierrors.IsNotFound(err) { - return false, nil + return false, nil, nil } - return false, err + return false, nil, err } if !cluster.DeletionTimestamp.IsZero() { - return false, nil + return false, nil, nil + } + if cluster.Spec.Bootstrap == nil || cluster.Spec.Bootstrap.Recovery == nil { + return false, nil, nil } - return cluster.Spec.Bootstrap != nil && cluster.Spec.Bootstrap.Recovery != nil, nil + created := cluster.CreationTimestamp + return true, &created, nil } // --------------------------------------------------------------------------- diff --git a/internal/backupcontroller/cnpgstrategy_controller_test.go b/internal/backupcontroller/cnpgstrategy_controller_test.go index f8ee1c7b93..da82dfa536 100644 --- a/internal/backupcontroller/cnpgstrategy_controller_test.go +++ b/internal/backupcontroller/cnpgstrategy_controller_test.go @@ -549,43 +549,122 @@ func TestBuildPostgresAppRestorePatch_NoSecretRefIsSkipped(t *testing.T) { // freshly-recovered Cluster on a status-update failure. The controller used // to rely solely on a Status condition: if the post-purge Status().Update // raced or failed, the next reconcile would re-purge the just-restored -// Cluster, destroying recovery progress. Cross-checking the live Cluster's -// bootstrap.recovery makes the second purge a no-op. +// Cluster, destroying recovery progress. Cross-checking that the live +// Cluster is a *freshly-recovered* one (bootstrap.recovery + created after +// the job started) makes that second purge a no-op - while still purging a +// stale recovery Cluster left over from an earlier completed restore, so a +// repeat in-place restore is not a silent no-op. func TestCNPGPurgeNeeded(t *testing.T) { cases := []struct { - name string - purgedCondition bool - hasRecovery bool - want bool + name string + purgedCondition bool + liveClusterFreshlyRecovered bool + want bool }{ { - name: "fresh restore, old cluster still in place: purge", - purgedCondition: false, - hasRecovery: false, - want: true, + name: "fresh restore, old cluster still in place: purge", + purgedCondition: false, + liveClusterFreshlyRecovered: false, + want: true, }, { - name: "purge already recorded: skip", - purgedCondition: true, - hasRecovery: false, - want: false, + name: "purge already recorded: skip", + purgedCondition: true, + liveClusterFreshlyRecovered: false, + want: false, }, { - name: "live cluster already recovered (status write raced): skip - the bug fix", - purgedCondition: false, - hasRecovery: true, - want: false, + name: "live cluster freshly recovered by this restore (status write raced): skip", + purgedCondition: false, + liveClusterFreshlyRecovered: true, + want: false, }, { - name: "both true: skip (post-purge steady state)", - purgedCondition: true, - hasRecovery: true, - want: false, + name: "stale recovery cluster from a previous completed restore: purge (repeat-restore fix)", + purgedCondition: false, + liveClusterFreshlyRecovered: false, + want: true, + }, + { + name: "both true: skip (post-purge steady state)", + purgedCondition: true, + liveClusterFreshlyRecovered: true, + want: false, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := cnpgPurgeNeeded(tc.purgedCondition, tc.liveClusterFreshlyRecovered) + if got != tc.want { + t.Errorf("got %v want %v", got, tc.want) + } + }) + } +} + +// TestCNPGClusterFreshlyRecovered locks in the signal that separates a +// recovery Cluster THIS restore just produced from one left over by an +// earlier completed restore. The repeat-in-place-restore bug was exactly a +// stale recovery Cluster (created before StartedAt) being mistaken for a +// freshly-recovered one and skipping the purge. +func TestCNPGClusterFreshlyRecovered(t *testing.T) { + started := metav1.NewTime(time.Now()) + after := metav1.NewTime(started.Add(time.Minute)) + before := metav1.NewTime(started.Add(-time.Hour)) + + cases := []struct { + name string + hasRecovery bool + createdAt *metav1.Time + startedAt *metav1.Time + want bool + }{ + { + name: "no recovery bootstrap: not fresh", + hasRecovery: false, + createdAt: &after, + startedAt: &started, + want: false, + }, + { + name: "recovery cluster created after start (our own purge re-render): fresh", + hasRecovery: true, + createdAt: &after, + startedAt: &started, + want: true, + }, + { + name: "recovery cluster created before start (leftover from previous restore): not fresh", + hasRecovery: true, + createdAt: &before, + startedAt: &started, + want: false, + }, + { + name: "recovery cluster created exactly at start: fresh (boundary)", + hasRecovery: true, + createdAt: &started, + startedAt: &started, + want: true, + }, + { + name: "missing cluster creation timestamp: not fresh", + hasRecovery: true, + createdAt: nil, + startedAt: &started, + want: false, + }, + { + name: "missing job start timestamp: not fresh", + hasRecovery: true, + createdAt: &after, + startedAt: nil, + want: false, }, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - got := cnpgPurgeNeeded(tc.purgedCondition, tc.hasRecovery) + got := cnpgClusterFreshlyRecovered(tc.hasRecovery, tc.createdAt, tc.startedAt) if got != tc.want { t.Errorf("got %v want %v", got, tc.want) } @@ -898,7 +977,7 @@ func TestApplyClusterBarmanObjectStore_NotFoundOnMissingCluster(t *testing.T) { } } -// TestClusterHasRecoveryBootstrap_TerminatingCluster locks in the +// TestRecoveryBootstrapClusterState_TerminatingCluster locks in the // DeletionTimestamp guard. Without it, a Cluster CR mid-deletion (after // purgeExistingCluster fired r.Delete but cnpg.io's finalizers haven't // drained yet) would get treated as "still has bootstrap" by the @@ -907,7 +986,7 @@ func TestApplyClusterBarmanObjectStore_NotFoundOnMissingCluster(t *testing.T) { // might SSA-merge the chart's bootstrap.recovery onto the terminating // CR and cnpg-operator's bootstrap-immutability check would drop the // change, leaving the cluster on the original initdb spec. -func TestClusterHasRecoveryBootstrap_TerminatingCluster(t *testing.T) { +func TestRecoveryBootstrapClusterState_TerminatingCluster(t *testing.T) { now := metav1.Now() t.Run("terminating cluster reports not-yet-recovered", func(t *testing.T) { cluster := &cnpgtypes.Cluster{ @@ -925,18 +1004,22 @@ func TestClusterHasRecoveryBootstrap_TerminatingCluster(t *testing.T) { } c := newCNPGStrategyTestClient(t, cluster) r := &RestoreJobReconciler{Client: c} - got, err := r.clusterHasRecoveryBootstrap(context.Background(), "tenant", "postgres-app") + got, createdAt, err := r.recoveryBootstrapClusterState(context.Background(), "tenant", "postgres-app") if err != nil { t.Fatalf("unexpected error: %v", err) } if got { t.Fatalf("expected false for terminating cluster, got true") } + if createdAt != nil { + t.Fatalf("expected nil createdAt for terminating cluster, got %v", createdAt) + } }) - t.Run("live recovery cluster reports recovered", func(t *testing.T) { + t.Run("live recovery cluster reports recovered with creation timestamp", func(t *testing.T) { + created := metav1.NewTime(time.Now().Add(-time.Hour)) cluster := &cnpgtypes.Cluster{ - ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "postgres-app"}, + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "postgres-app", CreationTimestamp: created}, Spec: cnpgtypes.ClusterSpec{ Bootstrap: &cnpgtypes.BootstrapConfiguration{ Recovery: &cnpgtypes.RecoverySource{Source: "pg-src"}, @@ -945,25 +1028,31 @@ func TestClusterHasRecoveryBootstrap_TerminatingCluster(t *testing.T) { } c := newCNPGStrategyTestClient(t, cluster) r := &RestoreJobReconciler{Client: c} - got, err := r.clusterHasRecoveryBootstrap(context.Background(), "tenant", "postgres-app") + got, createdAt, err := r.recoveryBootstrapClusterState(context.Background(), "tenant", "postgres-app") if err != nil { t.Fatalf("unexpected error: %v", err) } if !got { t.Fatalf("expected true for live recovery cluster, got false") } + if createdAt == nil { + t.Fatalf("expected non-nil createdAt for live recovery cluster") + } }) t.Run("missing cluster reports not-yet", func(t *testing.T) { c := newCNPGStrategyTestClient(t) r := &RestoreJobReconciler{Client: c} - got, err := r.clusterHasRecoveryBootstrap(context.Background(), "tenant", "postgres-app") + got, createdAt, err := r.recoveryBootstrapClusterState(context.Background(), "tenant", "postgres-app") if err != nil { t.Fatalf("unexpected error: %v", err) } if got { t.Fatalf("expected false for missing cluster, got true") } + if createdAt != nil { + t.Fatalf("expected nil createdAt for missing cluster, got %v", createdAt) + } }) } From 711f09f8d174d320cd7198342bfc59b05bbc8ca1 Mon Sep 17 00:00:00 2001 From: Ivan Okhotnikov Date: Sat, 4 Jul 2026 15:57:49 +0300 Subject: [PATCH 30/69] fix(cert-manager): raise cainjector memory limit to unblock caBundle injection cainjector loads all ValidatingWebhookConfigurations, APIServices and CRDs into its informer caches at startup. On a full cozystack install that working set exceeds the 128Mi limit, so the single leader-elected cainjector Pod is OOMKilled during cache population and CrashLoops before it injects any caBundle. Webhook ValidatingWebhookConfigurations then keep an empty caBundle and every admission call fails with "x509: certificate signed by unknown authority" (failurePolicy: Fail), intermittently blocking Ingress creation and stalling tenant installs. Raise the cainjector memory limit to 512Mi (matching the controller) and the request to 128Mi, and add a helm-unittest that pins the sizing so a blanket resource change or a `make update` cannot silently lower it again. Signed-off-by: Ivan Okhotnikov (cherry picked from commit c3afc63cb3a84c8085e991223ebd3f18a70d5183) (cherry picked from commit 9f312d7a216c34724848142415d2de7c086a711e) --- packages/system/cert-manager/Makefile | 3 ++ .../tests/cainjector_resources_test.yaml | 30 +++++++++++++ packages/system/cert-manager/values.yaml | 42 +++++++++++++++++++ 3 files changed, 75 insertions(+) create mode 100644 packages/system/cert-manager/tests/cainjector_resources_test.yaml diff --git a/packages/system/cert-manager/Makefile b/packages/system/cert-manager/Makefile index c3ff574b72..a0885bf9aa 100644 --- a/packages/system/cert-manager/Makefile +++ b/packages/system/cert-manager/Makefile @@ -3,6 +3,9 @@ export NAMESPACE=cozy-$(NAME) include ../../../hack/package.mk +test: + helm unittest . + update: rm -rf charts helm repo add jetstack https://charts.jetstack.io diff --git a/packages/system/cert-manager/tests/cainjector_resources_test.yaml b/packages/system/cert-manager/tests/cainjector_resources_test.yaml new file mode 100644 index 0000000000..5dab84b0d8 --- /dev/null +++ b/packages/system/cert-manager/tests/cainjector_resources_test.yaml @@ -0,0 +1,30 @@ +suite: cert-manager cainjector resources + +# Pins the cainjector memory sizing that keeps CA injection working on a full +# cozystack install. cainjector loads all ValidatingWebhookConfigurations, +# APIServices and CRDs into its informer caches at startup; on a large cluster +# that working set exceeds the old 128Mi limit, so the single leader-elected +# cainjector Pod is OOMKilled during cache population and CrashLoops before it +# ever injects a caBundle. Every webhook then keeps an empty caBundle and +# admission calls fail with "x509: certificate signed by unknown authority" +# (failurePolicy: Fail). This guards the override in values.yaml against a +# blanket resource change or a `make update` silently lowering it again. + +templates: + - charts/cert-manager/templates/cainjector-deployment.yaml + +release: + name: cert-manager + namespace: cozy-cert-manager + +tests: + - it: sizes the cainjector memory limit above the OOM floor + asserts: + - isKind: + of: Deployment + - equal: + path: spec.template.spec.containers[0].resources.limits.memory + value: 512Mi + - equal: + path: spec.template.spec.containers[0].resources.requests.memory + value: 128Mi diff --git a/packages/system/cert-manager/values.yaml b/packages/system/cert-manager/values.yaml index e69de29bb2..2ce6734e51 100644 --- a/packages/system/cert-manager/values.yaml +++ b/packages/system/cert-manager/values.yaml @@ -0,0 +1,42 @@ +cert-manager: + resources: + limits: + cpu: 500m + memory: 512Mi + requests: + cpu: 50m + memory: 128Mi + webhook: + resources: + limits: + cpu: 200m + memory: 128Mi + requests: + cpu: 10m + memory: 32Mi + cainjector: + # cainjector loads all ValidatingWebhookConfigurations, APIServices and CRDs + # into its informer caches at startup. On a full cozystack install that + # working set exceeds 128Mi, so cainjector is OOMKilled during cache + # population and CrashLoops before it ever injects a caBundle. Because it + # runs as a single leader-elected replica, that stalls CA injection for the + # whole cluster: webhook ValidatingWebhookConfigurations keep an empty + # caBundle and every admission call fails with + # "x509: certificate signed by unknown authority" (failurePolicy: Fail). + # Size the limit to the controller's 512Mi and lift the request off the + # OOM floor so injection completes deterministically. + resources: + limits: + cpu: 200m + memory: 512Mi + requests: + cpu: 10m + memory: 128Mi + startupapicheck: + resources: + limits: + cpu: 100m + memory: 64Mi + requests: + cpu: 10m + memory: 16Mi From 629a972a118dfbd1a452fda1687df9a3c4b99398 Mon Sep 17 00:00:00 2001 From: Ivan Okhotnikov Date: Wed, 15 Jul 2026 17:38:42 +0300 Subject: [PATCH 31/69] test(backups/postgres): reconcile-level repeat-restore regression + conservative freshness tie Address review feedback on the repeat in-place restore fix: - Make the freshness discriminator strict (creationTimestamp > StartedAt) so an exact tie resolves to "not fresh" and the caller purges, matching the function's stated conservative default; a fresh Cluster is always created well after StartedAt so this never misclassifies the status-write-race path. - Document why a wall-clock timestamp comparison was chosen over recording the purged Cluster's UID on the TargetPurged condition (the UID approach leans on the same status write the race path assumes can fail). - Add a reconcile-level regression test that drives reconcileCNPGRestore end to end with a fake client: a stale leftover recovery Cluster from a prior restore is purged, and a Cluster this restore just re-created is not re-purged. This covers the actual call site where #3311 lived, not just the pure helpers. Signed-off-by: Ivan Okhotnikov (cherry picked from commit a8494d874faca726b3a93190f0e8887ee0420e06) (cherry picked from commit 0fdf3b5a6152db8912a094493f2ff5ea122ac7bb) (cherry picked from commit 2d6083ccb6f418a35ba1ae058b3a91f7edcc89fd) --- .../cnpgstrategy_controller.go | 22 ++- .../cnpgstrategy_controller_test.go | 147 +++++++++++++++++- 2 files changed, 164 insertions(+), 5 deletions(-) diff --git a/internal/backupcontroller/cnpgstrategy_controller.go b/internal/backupcontroller/cnpgstrategy_controller.go index 51d20c0691..a17737b54b 100644 --- a/internal/backupcontroller/cnpgstrategy_controller.go +++ b/internal/backupcontroller/cnpgstrategy_controller.go @@ -336,8 +336,9 @@ func cnpgPurgeNeeded(purgedCondition, liveClusterFreshlyRecovered bool) bool { // cnpgClusterFreshlyRecovered reports whether a live cnpg.io Cluster that // carries spec.bootstrap.recovery was produced by THIS RestoreJob's own purge -// + chart re-render (its creationTimestamp is at or after the job's StartedAt) -// rather than being left over from an earlier, already-completed restore. +// + chart re-render (its creationTimestamp is strictly after the job's +// StartedAt) rather than being left over from an earlier, already-completed +// restore. // // Only the former is safe to skip re-purging. A leftover recovery Cluster from // a prior restore predates StartedAt and still holds the prior data, so it must @@ -346,6 +347,21 @@ func cnpgPurgeNeeded(purgedCondition, liveClusterFreshlyRecovered bool) bool { // job sets StartedAt on its first reconcile, long before it purges and the // chart re-renders), so the timestamp comparison cleanly separates the two. // +// The comparison is strict (created > started), so an exact tie resolves to +// "not fresh" and the caller purges. That matches the conservative default +// below: the only classification that must never be wrong is calling a stale +// leftover "fresh" (which reintroduces the silent no-op), and a fresh Cluster +// is always created well after StartedAt (see above), never exactly at it. +// +// An identity-based alternative was considered - recording the purged Cluster's +// UID on the TargetPurged condition and comparing UIDs on later reconciles - +// which is immune to clock skew. It was rejected because it leans on the same +// status write that the status-write-race path (the whole reason this skip +// exists) assumes can fail, so it cannot cover that case; the timestamp +// comparison needs no extra persisted state and the fresh-vs-stale gap +// (a full purge + re-render cycle) always dwarfs any plausible control-plane +// clock skew. +// // Returns false when freshness cannot be determined (no recovery bootstrap, or // a missing timestamp): the caller then proceeds to purge, which is the safe // default for any pre-existing, non-fresh Cluster. @@ -353,7 +369,7 @@ func cnpgClusterFreshlyRecovered(hasRecovery bool, clusterCreatedAt, restoreStar if !hasRecovery || clusterCreatedAt == nil || restoreStartedAt == nil { return false } - return !clusterCreatedAt.Time.Before(restoreStartedAt.Time) + return clusterCreatedAt.After(restoreStartedAt.Time) } // applyClusterBarmanObjectStore SSA-patches the live CNPG Cluster's diff --git a/internal/backupcontroller/cnpgstrategy_controller_test.go b/internal/backupcontroller/cnpgstrategy_controller_test.go index da82dfa536..3504e72780 100644 --- a/internal/backupcontroller/cnpgstrategy_controller_test.go +++ b/internal/backupcontroller/cnpgstrategy_controller_test.go @@ -641,11 +641,11 @@ func TestCNPGClusterFreshlyRecovered(t *testing.T) { want: false, }, { - name: "recovery cluster created exactly at start: fresh (boundary)", + name: "recovery cluster created exactly at start: not fresh (conservative tie -> purge)", hasRecovery: true, createdAt: &started, startedAt: &started, - want: true, + want: false, }, { name: "missing cluster creation timestamp: not fresh", @@ -1598,6 +1598,149 @@ func TestReconcileCNPGRestore_MissingStrategyFailsClosedAfterDeadline(t *testing } } +// TestReconcileCNPGRestore_RepeatInPlacePurgesStaleRecoveryCluster is the +// reconcile-level regression test for #3311. The pure-function tests +// (TestCNPGPurgeNeeded / TestCNPGClusterFreshlyRecovered) lock in the truth +// table, but the bug lived at the call site: a repeat in-place restore fed the +// purge decision the wrong signal and skipped the destructive purge, so the +// job reported Succeeded against untouched data. This drives the real +// reconcileCNPGRestore path with a fake client and asserts the destructive +// purge actually fires for a stale leftover recovery Cluster - and, in the +// mirror case, that a Cluster this restore just re-created is NOT re-purged +// (the status-write-race protection the guard was originally built for). +func TestReconcileCNPGRestore_RepeatInPlacePurgesStaleRecoveryCluster(t *testing.T) { + const ( + ns = "tenant" + appName = "app" + clusterName = "postgres-app" + cnpgBkName = "cnpgbk" + ) + apiGroup := backupsv1alpha1.DefaultApplicationAPIGroup + strategyGroup := strategyv1alpha1.GroupVersion.Group + ctx := context.Background() + + // startedAt anchors the discriminator: a stale leftover Cluster predates + // it, a freshly-recovered one postdates it. + startedAt := metav1.NewTime(time.Now()) + + mkBackupArtifact := func(t *testing.T) *backupsv1alpha1.Backup { + t.Helper() + snap, err := marshalCNPGBackupSnapshot(newPostgresApp(appName, ns), nil) + if err != nil { + t.Fatalf("marshal snapshot: %v", err) + } + return &backupsv1alpha1.Backup{ + ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: "bk"}, + Spec: backupsv1alpha1.BackupSpec{ + ApplicationRef: corev1.TypedLocalObjectReference{APIGroup: &apiGroup, Kind: postgresAppKind, Name: appName}, + StrategyRef: corev1.TypedLocalObjectReference{APIGroup: &strategyGroup, Kind: strategyv1alpha1.CNPGStrategyKind, Name: "cnpg-strategy"}, + DriverMetadata: map[string]string{ + cnpgServerNameKey: appName, + cnpgDestinationPathKey: "s3://bucket/" + appName + "/", + cnpgBackupNameKey: cnpgBkName, + }, + }, + Status: backupsv1alpha1.BackupStatus{UnderlyingResources: snap}, + } + } + strategy := &strategyv1alpha1.CNPG{ + ObjectMeta: metav1.ObjectMeta{Name: "cnpg-strategy"}, + Spec: strategyv1alpha1.CNPGSpec{ + Template: strategyv1alpha1.CNPGTemplate{ + BarmanObjectStore: strategyv1alpha1.BarmanObjectStoreTemplate{DestinationPath: "s3://bucket/"}, + }, + }, + } + // A completed cnpg.io/Backup with endWal set clears the WAL-archive gate + // so the reconcile can reach the purge step. + cnpgBackup := &cnpgtypes.Backup{ + ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: cnpgBkName}, + Spec: cnpgtypes.BackupSpec{Cluster: cnpgtypes.ClusterReference{Name: clusterName}}, + Status: cnpgtypes.BackupStatus{Phase: cnpgBackupPhaseComplete, EndWal: "000000010000000000000003"}, + } + mkRecoveryCluster := func(created metav1.Time) *cnpgtypes.Cluster { + return &cnpgtypes.Cluster{ + ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: clusterName, CreationTimestamp: created}, + Spec: cnpgtypes.ClusterSpec{ + Bootstrap: &cnpgtypes.BootstrapConfiguration{ + Recovery: &cnpgtypes.RecoverySource{Source: appName}, + }, + }, + } + } + mkClusterPVC := func() *corev1.PersistentVolumeClaim { + return &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: clusterName + "-1", Labels: map[string]string{cnpgClusterLabel: clusterName}}, + } + } + mkRestoreJob := func() *backupsv1alpha1.RestoreJob { + sa := startedAt + return &backupsv1alpha1.RestoreJob{ + ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: "rj"}, + Spec: backupsv1alpha1.RestoreJobSpec{BackupRef: corev1.LocalObjectReference{Name: "bk"}}, + Status: backupsv1alpha1.RestoreJobStatus{StartedAt: &sa, Phase: backupsv1alpha1.RestoreJobPhaseRunning}, + } + } + + t.Run("stale leftover recovery cluster from a prior restore is purged", func(t *testing.T) { + backup := mkBackupArtifact(t) + // creationTimestamp an hour before StartedAt: leftover from a prior restore. + stale := metav1.NewTime(startedAt.Add(-time.Hour)) + c := newCNPGStrategyTestClient(t, backup, mkRestoreJob(), strategy, cnpgBackup, + newPostgresApp(appName, ns), mkRecoveryCluster(stale), mkClusterPVC()) + r := &RestoreJobReconciler{Client: c, Interface: dynamicfake.NewSimpleDynamicClient(testCNPGScheme(t))} + + rj := &backupsv1alpha1.RestoreJob{} + if err := c.Get(ctx, client.ObjectKey{Namespace: ns, Name: "rj"}, rj); err != nil { + t.Fatalf("get seeded RestoreJob: %v", err) + } + if _, err := r.reconcileCNPGRestore(ctx, rj, backup); err != nil { + t.Fatalf("reconcileCNPGRestore: %v", err) + } + + // The stale Cluster (and its labelled PVC) must have been purged. + err := c.Get(ctx, client.ObjectKey{Namespace: ns, Name: clusterName}, &cnpgtypes.Cluster{}) + if !apierrors.IsNotFound(err) { + t.Fatalf("expected stale Cluster to be purged (NotFound), got err=%v", err) + } + pvcs := &corev1.PersistentVolumeClaimList{} + if err := c.List(ctx, pvcs, client.InNamespace(ns), client.MatchingLabels{cnpgClusterLabel: clusterName}); err != nil { + t.Fatalf("list PVCs: %v", err) + } + if len(pvcs.Items) != 0 { + t.Fatalf("expected cluster PVCs purged, still have %d", len(pvcs.Items)) + } + }) + + t.Run("freshly-recovered cluster from this restore is not re-purged", func(t *testing.T) { + backup := mkBackupArtifact(t) + // creationTimestamp a minute after StartedAt: this restore's own re-render. + fresh := metav1.NewTime(startedAt.Add(time.Minute)) + c := newCNPGStrategyTestClient(t, backup, mkRestoreJob(), strategy, cnpgBackup, + newPostgresApp(appName, ns), mkRecoveryCluster(fresh)) + r := &RestoreJobReconciler{Client: c, Interface: dynamicfake.NewSimpleDynamicClient(testCNPGScheme(t))} + + rj := &backupsv1alpha1.RestoreJob{} + if err := c.Get(ctx, client.ObjectKey{Namespace: ns, Name: "rj"}, rj); err != nil { + t.Fatalf("get seeded RestoreJob: %v", err) + } + if _, err := r.reconcileCNPGRestore(ctx, rj, backup); err != nil { + t.Fatalf("reconcileCNPGRestore: %v", err) + } + + // The freshly-recovered Cluster must survive - re-purging it would + // destroy the recovery this restore just started (the status-write-race + // protection the guard was built for). + got := &cnpgtypes.Cluster{} + if err := c.Get(ctx, client.ObjectKey{Namespace: ns, Name: clusterName}, got); err != nil { + t.Fatalf("expected freshly-recovered Cluster to survive, got err=%v", err) + } + if !got.DeletionTimestamp.IsZero() { + t.Fatalf("freshly-recovered Cluster must not be marked for deletion") + } + }) +} + // newCNPGStrategyTestClient returns a fake client.Client wired up with // the schemes the CNPG-strategy reconciler needs. func newCNPGStrategyTestClient(t *testing.T, objs ...client.Object) client.Client { From d2cf83ccc99ad15ceae56d25a30bb275c7e7d5e1 Mon Sep 17 00:00:00 2001 From: myasnikovdaniil <60174387+myasnikovdaniil@users.noreply.github.com> Date: Mon, 20 Jul 2026 11:11:22 +0500 Subject: [PATCH 32/69] fix(mariadb): widen startup probe budget so bootstrap cannot be killed Backport of #3344 onto release-1.5. The bot's cherry-pick imported hack/e2e-chainsaw/mariadb/chainsaw-test.yaml whole, because release-1.5 has no such file: the app suite here is still BATS, and the Chainsaw port landed with #2826 on main only. That file is dropped and its widening applied to the gate that actually runs on this branch instead. That gate needed the change more than the chainsaw one did. In hack/e2e-apps/mariadb.bats both tests wait 80s for an endpoint address to appear, and an address appears only once a replica has cleared its startup then readiness probe. This commit raises the startup budget to 310s, so an 80s ceiling would fail runs the probe was still willing to wait for -- importing the chainsaw file and leaving the BATS suite alone would have turned the fix into an e2e failure. Both waits go to 600s, matching the 10m the upstream chainsaw assert uses and for the same reason. packages/apps/mariadb/Makefile gains the `test:` target. It is absent on release-1.5 -- it arrived with #3071, which was not backported -- and hack/helm-unit-tests.sh discovers suites by probing for that target, so without it the backported tests/startup_probe_test.yaml would never be executed by `make unit-tests`. Verified locally: 4 assertions in tests/startup_probe_test.yaml pass. Assisted-By: Claude Signed-off-by: Myasnikov Daniil (cherry picked from commit a37715062e5da24e54956f075dd2d6417c4dff2e) --- hack/e2e-apps/mariadb.bats | 17 +++++- packages/apps/mariadb/Makefile | 4 ++ packages/apps/mariadb/templates/mariadb.yaml | 22 ++++++++ .../mariadb/tests/startup_probe_test.yaml | 52 +++++++++++++++++++ 4 files changed, 93 insertions(+), 2 deletions(-) create mode 100644 packages/apps/mariadb/tests/startup_probe_test.yaml diff --git a/hack/e2e-apps/mariadb.bats b/hack/e2e-apps/mariadb.bats index e099018b10..94506ebb6c 100644 --- a/hack/e2e-apps/mariadb.bats +++ b/hack/e2e-apps/mariadb.bats @@ -40,7 +40,16 @@ EOF timeout 60 sh -ec "until kubectl -n tenant-test get hr mariadb-$name >/dev/null 2>&1; do sleep 2; done" kubectl -n tenant-test wait hr mariadb-$name --timeout=5m --for=condition=ready timeout 80 sh -ec "until kubectl -n tenant-test get svc mariadb-$name -o jsonpath='{.spec.ports[0].port}' | grep -q '3306'; do sleep 10; done" - timeout 80 sh -ec "until kubectl -n tenant-test get endpoints mariadb-$name -o jsonpath='{.subsets[*].addresses[*].ip}' | grep -q '[0-9]'; do sleep 10; done" + # The chart hands the operator a MariaDB CR and ships no built-in workload, so + # the HelmRelease above goes Ready as soon as helm applies the manifests — it + # never waits for a pod. This is the first gate that does: an endpoint address + # appears only once a replica has passed its startup and then its readiness + # probe. The startup probe grants each replica 310s of first-boot budget (see + # packages/apps/mariadb/templates/mariadb.yaml for why), so a ceiling under + # that fails runs the probe was still willing to wait for. One ready address + # satisfies this, so it has to clear one budget, not two, plus the operator + # reconcile, PVC bind and image pull that precede it. + timeout 600 sh -ec "until kubectl -n tenant-test get endpoints mariadb-$name -o jsonpath='{.subsets[*].addresses[*].ip}' | grep -q '[0-9]'; do sleep 10; done" timeout 60 sh -ec "until kubectl -n tenant-test get statefulset.apps/mariadb-$name >/dev/null 2>&1; do sleep 2; done" kubectl -n tenant-test wait statefulset.apps/mariadb-$name --timeout=110s --for=jsonpath='{.status.replicas}'=2 timeout 80 sh -ec "until kubectl -n tenant-test get svc mariadb-$name-metrics -o jsonpath='{.spec.ports[0].port}' | grep -q '9104'; do sleep 10; done" @@ -99,7 +108,11 @@ EOF # With replicas=1 the operator provisions the bare service (no # -primary/-secondary): assert it exists and has an endpoint. timeout 80 sh -ec "until kubectl -n tenant-test get svc mariadb-$name -o jsonpath='{.spec.ports[0].port}' | grep -q '3306'; do sleep 10; done" - timeout 80 sh -ec "until kubectl -n tenant-test get endpoints mariadb-$name -o jsonpath='{.subsets[*].addresses[*].ip}' | grep -q '[0-9]'; do sleep 10; done" + # Same gate as the replicated case above — the HelmRelease does not wait for a + # pod, so this is where first boot is actually covered. One replica means one + # 310s startup budget, plus the operator reconcile, PVC bind and image pull + # ahead of it. + timeout 600 sh -ec "until kubectl -n tenant-test get endpoints mariadb-$name -o jsonpath='{.subsets[*].addresses[*].ip}' | grep -q '[0-9]'; do sleep 10; done" timeout 60 sh -ec "until kubectl -n tenant-test get statefulset.apps/mariadb-$name >/dev/null 2>&1; do sleep 2; done" kubectl -n tenant-test wait statefulset.apps/mariadb-$name --timeout=110s --for=jsonpath='{.status.replicas}'=1 timeout 80 sh -ec "until kubectl -n tenant-test get svc mariadb-$name-metrics -o jsonpath='{.spec.ports[0].port}' | grep -q '9104'; do sleep 10; done" diff --git a/packages/apps/mariadb/Makefile b/packages/apps/mariadb/Makefile index eb12cf44a2..58eb927441 100644 --- a/packages/apps/mariadb/Makefile +++ b/packages/apps/mariadb/Makefile @@ -11,6 +11,10 @@ update: hack/update-versions.sh make generate +.PHONY: test +test: + helm unittest . + image: docker buildx build images/mariadb-backup \ $(call image-tags,mariadb-backup,$(MARIADB_BACKUP_TAG)) \ diff --git a/packages/apps/mariadb/templates/mariadb.yaml b/packages/apps/mariadb/templates/mariadb.yaml index 8a530134cd..d04f427407 100644 --- a/packages/apps/mariadb/templates/mariadb.yaml +++ b/packages/apps/mariadb/templates/mariadb.yaml @@ -12,6 +12,28 @@ spec: port: 3306 + # The operator leaves failureThreshold unset on the startup probe, so it + # defaults to 3. The kubelet only begins probing once initialDelaySeconds has + # elapsed and kills the container on the Nth consecutive failure, so the + # budget is initialDelay + (failureThreshold - 1) * period, here just 40s -- + # the same budget as the liveness probe the startup probe exists to defer. + # A fresh datadir bootstrap (mysql_install_db, then the entrypoint's + # temporary server applying the root password) does not reliably finish + # inside that budget under a constrained CPU limit or slow replicated + # storage. Overrunning it is not merely a slow start: the kubelet kills the + # container mid-bootstrap, leaving a datadir that already has the mysql + # system tables but never received the root grant. On restart the entrypoint + # keys DATABASE_ALREADY_EXISTS off that directory, skips initialization for + # good, and the server comes up with the install-time root account -- local + # only and passwordless -- instead of the credentials the operator holds. So + # every later probe fails with "Access denied for user 'root'" and the pod + # crash-loops permanently with no path back. A threshold of 30 lifts the + # budget to 310s. Widen only the startup budget; the liveness and readiness + # probes keep the operator defaults, so a server that hangs after a + # successful start is still caught as quickly as before. + startupProbe: + failureThreshold: 30 + replicas: {{ .Values.replicas }} replicasAllowEvenNumber: true affinity: diff --git a/packages/apps/mariadb/tests/startup_probe_test.yaml b/packages/apps/mariadb/tests/startup_probe_test.yaml new file mode 100644 index 0000000000..d6df7e0f41 --- /dev/null +++ b/packages/apps/mariadb/tests/startup_probe_test.yaml @@ -0,0 +1,52 @@ +suite: MariaDB startup probe budget + +# Regression cover for the widened startup budget. The rationale lives with +# the field itself, in templates/mariadb.yaml. + +templates: + - templates/mariadb.yaml + +tests: + - it: widens the startup probe budget for a single-replica instance + release: + name: mariadb-test + namespace: tenant-test + set: + replicas: 1 + asserts: + - equal: + path: spec.startupProbe.failureThreshold + value: 30 + + - it: widens the startup probe budget for a replicated instance + release: + name: mariadb-test + namespace: tenant-test + set: + replicas: 2 + asserts: + - equal: + path: spec.startupProbe.failureThreshold + value: 30 + + - it: leaves liveness and readiness probes to the operator defaults + release: + name: mariadb-test + namespace: tenant-test + asserts: + - notExists: + path: spec.livenessProbe + - notExists: + path: spec.readinessProbe + + - it: overrides only thresholds so the operator keeps its own probe handler + release: + name: mariadb-test + namespace: tenant-test + asserts: + - notExists: + path: spec.startupProbe.exec + - notExists: + path: spec.startupProbe.httpGet + - notExists: + path: spec.startupProbe.tcpSocket From 7abb7eccd6b382a3cc2cdf2de8376a593f9c717a Mon Sep 17 00:00:00 2001 From: Andrey Kolkov Date: Tue, 21 Jul 2026 19:30:38 +0400 Subject: [PATCH 33/69] fix(platform): forward backupStorage overrides to the backupstrategy-controller Package Backport of #3333 onto release-1.5. spec.components.platform.values.backupStorage on the cozystack.cozystack-platform Package -- the documented override path for the cozy-default BackupClass -- never reached the backupstrategy-controller HelmRelease, so endpoint/region/bucketName overrides for an external S3 were silently dropped (#3245). The platform chart now forwards the block into the emitted child Package the same way bundles/iaas.yaml forwards kubevirt and gpu-operator values. The 4-arg cozystack.platform.package helper the change needs already exists on this branch. Two conflicts, both from unrelated drift in the touched files rather than from the fix: * platform values.yaml -- main's hunk carries `monitoring.rootEnabled` alongside `backupStorage`. That key belongs to the tenant-root monitoring gate (#3181), which is not on release-1.5 and whose absence nothing here depends on, so only the backupStorage block is taken. * docs/operations/backup-classes.md -- the conflicting hunk rewrites the Postgres upgrade note for the CNPG barman plugin migration (spec.plugins + ObjectStore), which is 1.6 behaviour. release-1.5 still renders spec.backup.barmanObjectStore, so this branch keeps its own prose. The rest of the doc update applied cleanly and was checked claim-by-claim against this branch rather than assumed: the helmrelease-interval flag, the cozy-backup-controller namespace, the cozystack_backup_credentials_projection_* metric names and the provisionBucket / namespace / bucketName knobs the new table rows describe all exist here. Assisted-By: Claude Signed-off-by: Myasnikov Daniil (cherry picked from commit 13aa3cb31e71f89e6ffe2a5fcd0fcd9e3f80e970) --- docs/operations/backup-classes.md | 25 ++--- .../platform/templates/bundles/system.yaml | 16 +++- .../bundles_backupstorage_wiring_test.yaml | 91 +++++++++++++++++++ packages/core/platform/values.yaml | 12 +++ 4 files changed, 132 insertions(+), 12 deletions(-) create mode 100644 packages/core/platform/tests/bundles_backupstorage_wiring_test.yaml diff --git a/docs/operations/backup-classes.md b/docs/operations/backup-classes.md index f60cba0ad4..271012dde5 100644 --- a/docs/operations/backup-classes.md +++ b/docs/operations/backup-classes.md @@ -1,6 +1,6 @@ # Backup Classes -Cozystack ships a single platform-managed `BackupClass` named `cozy-default`. It is provisioned automatically when the `backupstrategy-controller` package is installed and references the system-managed S3 bucket `cozy-backups` in the `tenant-root` namespace. +Cozystack ships a single platform-managed `BackupClass` named `cozy-default`. It is provisioned automatically when the `backupstrategy-controller` package is installed and references the system-managed bucket provisioned through the `apps.cozystack.io/Bucket` CR `cozy-backups` in the `tenant-root` namespace (the real S3 bucket name is the COSI-assigned one from `BucketClaim.status.bucketName`). Tenants reference `cozy-default` from `BackupJob`, `Plan`, and `RestoreJob` resources — they do **not** supply S3 credentials, endpoints, or paths. The platform projects the system-managed credentials Secret into the tenant namespace per BackupJob (or, for long-lived references like Velero's `BackupStorageLocation`, into a fixed list of system namespaces on a periodic tick), and the default strategy templates encode `/` into every S3 path so two tenants with the same application name never collide. @@ -46,7 +46,7 @@ VM-driven (Velero) backups land in the same `cozy-backups` bucket under the `vel The strategy CR `cozy-default-foundationdb` is shipped, but it is **not** bound by `cozy-default` yet. Restore runs `fdbrestore` from inside the `cozy-foundationdb-operator` Deployment, which does not yet mount `cozy-backups-creds`. Until the operator deployment is updated to mount the projected Secret, FDB platform-default restore silently fails — admins who need it today should keep using a per-app `Bucket` plus a custom `BackupClass`, or wire the credentials file into the operator deployment themselves. -**Cleanup gotcha (zombie backup_agent).** Unlike CNPG/MariaDB/Altinity (one-shot operator-side Backup CRs), the FoundationDB driver creates a `foundationdb.org/FoundationDBBackup` CR that drives a **long-lived** `backup_agent` Deployment streaming continuously to S3. Deleting a Cozystack `Backup` (e.g. via retention sweeping) does NOT stop that Deployment — the agent keeps writing until the next BackupJob's `stopOtherFoundationDBBackups` call swaps it out, until an admin invokes `examples/backups/foundationdb/cleanup.sh`, or until the operator-side CR is deleted by hand. If a tenant deletes their last Cozystack Backup and never submits another BackupJob, the agent pods will continue running indefinitely and accumulate S3 PUTs. This is intentional today (the driver has no RBAC verb to stop the operator-side CR on Cozystack-Backup deletion) but admins should be aware of it. +**Cleanup gotcha (zombie backup_agent).** Unlike CNPG/MariaDB/Altinity (one-shot operator-side Backup CRs), the FoundationDB driver creates an `apps.foundationdb.org/FoundationDBBackup` CR that drives a **long-lived** `backup_agent` Deployment streaming continuously to S3. Deleting a Cozystack `Backup` (e.g. via retention sweeping) does NOT stop that Deployment — the agent keeps writing until the next BackupJob's `stopOtherFoundationDBBackups` call swaps it out, until an admin invokes `examples/backups/foundationdb/cleanup.sh`, or until the operator-side CR is deleted by hand. If a tenant deletes their last Cozystack Backup and never submits another BackupJob, the agent pods will continue running indefinitely and accumulate S3 PUTs. This is intentional today (the driver has no RBAC verb to stop the operator-side CR on Cozystack-Backup deletion) but admins should be aware of it. ## ClickHouse: opt-in to the system bucket @@ -76,7 +76,7 @@ kubectl -n tenant-root get secret bucket-cozy-backups-system-credentials kubectl -n cozy-velero get backupstoragelocation cozy-default ``` -The bucket lives in `tenant-root` and is provisioned through the `apps.cozystack.io/Bucket` CR. The system-managed credentials Secret never leaves that namespace. The backupstrategy-controller projects a copy under the name `cozy-backups-creds` into a tenant namespace right before each BackupJob runs, and refreshes the same Secret in `cozy-velero` (and any other namespace listed in `backupStorage.systemNamespaces`) on a 1-minute tick. The projected Secret carries multiple key formats so each driver finds what it needs in one place: +The bucket lives in `tenant-root` and is provisioned through the `apps.cozystack.io/Bucket` CR. The system-managed credentials Secret never leaves that namespace. The backupstrategy-controller projects a copy under the name `cozy-backups-creds` into a tenant namespace right before each BackupJob or RestoreJob runs, and refreshes the same Secret in `cozy-velero` (and any other namespace listed in `backupStorage.systemNamespaces`) on a 1-minute tick. The projected Secret carries multiple key formats so each driver finds what it needs in one place: | Key | Consumer | |-----------------------------------------------|-------------------------------------------| @@ -87,15 +87,15 @@ The bucket lives in `tenant-root` and is provisioned through the `apps.cozystack ### Bootstrap window -On a fresh-cluster install, the Velero `BackupStorageLocation` `cozy-default` is rendered before the credentials projector has had a chance to copy `cozy-backups-creds` into `cozy-velero`. The BSL reports `Unavailable` until the projector's first synchronous round completes (which happens immediately when the `backupstrategy-controller` Pod becomes Ready — typically tens of seconds after `helm install` returns, not minutes). Velero rejects new `Backup` AND `Restore` requests against `storageLocation: cozy-default` during that window. Plan VM backup automation accordingly, or wait for `kubectl -n cozy-velero get bsl cozy-default -o jsonpath='{.status.phase}' = Available` before submitting backups. +On a fresh-cluster install, the Velero `BackupStorageLocation` `cozy-default` is rendered before the credentials projector has had a chance to copy `cozy-backups-creds` into `cozy-velero`. The BSL reports `Unavailable` until the projector's first synchronous round completes (which runs as soon as the `backupstrategy-controller` acquires leadership — in practice moments after the Pod becomes Ready, typically tens of seconds after `helm install` returns, not minutes). Velero rejects new `Backup` AND `Restore` requests against `storageLocation: cozy-default` during that window. Plan VM backup automation accordingly, or wait for the BSL to become ready before submitting backups: `kubectl -n cozy-velero wait backupstoragelocation cozy-default --for=jsonpath='{.status.phase}'=Available --timeout=5m`. **Note on controller restarts.** The BSL flickers `Unavailable` on every `backupstrategy-controller` pod restart while the projector replays its first synchronous round. The window is short (single-digit seconds) but operators who alert on BSL availability should suppress alerts during the controller's `kube_pod_container_status_restarts_total{container=backupstrategy-controller}` events or use a longer evaluation window than the projector tick (60s). ### Cozy-default Bucket bootstrap -`cozy-default` ships an `apps.cozystack.io/Bucket cozy-backups` CR in `tenant-root`, which the bucket-application chart turns into a `BucketClaim`; the COSI driver then assigns the real S3 bucket name and writes it to the BucketClaim's `.status.bucketName`. The strategy templates and the Velero BSL all read that real bucket name (Helm `lookup` against the BucketClaim). On a fresh install the BucketClaim takes a short reconcile cycle to populate its status — until it does, the strategy templates render empty and only the `Bucket` CR + `BackupClass` are present in the cluster. Flux re-renders the HelmRelease on its standard interval (default 10 minutes), at which point the populated BucketClaim status causes the missing strategy templates to materialise. +`cozy-default` ships an `apps.cozystack.io/Bucket cozy-backups` CR in `tenant-root`, which the bucket-application chart turns into a `BucketClaim`; the COSI driver then assigns the real S3 bucket name and writes it to the BucketClaim's `.status.bucketName`. The strategy templates and the Velero BSL all read that real bucket name (Helm `lookup` against the BucketClaim). On a fresh install the BucketClaim takes a short reconcile cycle to populate its status — until it does, the strategy templates render empty and only the `Bucket` CR + `BackupClass` are present in the cluster. The HelmRelease re-reconciles on its interval (5 minutes by default — set by the cozystack operator's `helmrelease-interval` flag, not a Flux default), at which point the populated BucketClaim status causes the missing strategy templates to materialise. -If you need the BackupClass functional immediately (e.g. an e2e), trigger a Flux reconcile (`flux reconcile helmrelease backupstrategy-controller`) once you see `kubectl get bucketclaim -n tenant-root bucket-cozy-backups -o jsonpath='{.status.bucketName}'` non-empty. +If you need the BackupClass functional immediately (e.g. an e2e), trigger a Flux reconcile (`flux reconcile helmrelease backupstrategy-controller -n cozy-backup-controller`) once you see `kubectl get bucketclaim -n tenant-root bucket-cozy-backups -o jsonpath='{.status.bucketName}'` non-empty. ### Observability @@ -104,11 +104,11 @@ The credentials projector emits two Prometheus counters labelled by `namespace` - `cozystack_backup_credentials_projection_successes_total` - `cozystack_backup_credentials_projection_failures_total` -Alert on `rate(failures_total) > 0` or `absent_over_time(successes_total[10m])` to catch a stale BSL credential or a malformed source Secret without log scraping. +Alert on `rate(cozystack_backup_credentials_projection_failures_total[5m]) > 0` or `absent_over_time(cozystack_backup_credentials_projection_successes_total[10m])` to catch a stale BSL credential or a malformed source Secret without log scraping. ## Admin overrides for `cozy-default` -`cozy-default` is rendered by the `backupstrategy-controller` chart and owned by Flux's helm-controller. **Direct `kubectl edit backupclass cozy-default` is overwritten on the next helm reconcile** — the same applies to its companion `strategy.backups.cozystack.io/*` CRs (`cozy-default-cnpg`, `cozy-default-etcd`, `cozy-default-mariadb`, `cozy-default-altinity`, `cozy-default-foundationdb`, the two `cozy-default-velero-*`). The supported override path is the cozystack `Package` CR, which lets admins inject Helm values into platform components: +`cozy-default` is rendered by the `backupstrategy-controller` chart and owned by Flux's helm-controller. **Direct `kubectl edit backupclass cozy-default` is overwritten on the next helm reconcile** — the same applies to its companion `strategy.backups.cozystack.io/*` CRs (`cozy-default-cnpg`, `cozy-default-etcd`, `cozy-default-mariadb`, `cozy-default-altinity`, `cozy-default-foundationdb`, the two `cozy-default-velero-*`). The supported override path is the `backupStorage` block on the **`platform` component** of the `cozystack.cozystack-platform` Package CR: ```yaml apiVersion: cozystack.io/v1alpha1 @@ -117,7 +117,7 @@ metadata: name: cozystack.cozystack-platform spec: components: - backupstrategy-controller: + platform: values: backupStorage: provisionBucket: true # default; set false for external S3 @@ -130,10 +130,13 @@ spec: - cozy-velero ``` +The platform chart forwards this block into the child `Package cozystack.backupstrategy-controller` as `components.backupstrategy-controller.values.backupStorage` (`packages/core/platform/templates/bundles/system.yaml`), from where the cozystack operator merges it into the `backupstrategy-controller` HelmRelease over the chart defaults. Two paths that look plausible do **not** work: `spec.components.backupstrategy-controller` on the `cozystack.cozystack-platform` Package is silently ignored (the only component under that PackageSource is `platform`), and patching the child `Package cozystack.backupstrategy-controller` directly is reverted whenever the platform helm-reconcile re-renders it. + | Knob | Effect | |---|---| | `provisionBucket` | Toggle creation of the in-cluster `apps.cozystack.io/Bucket` CR. Set `false` for external S3 (see [Disabling the platform-managed bucket](#disabling-the-platform-managed-bucket)). | -| `bucketName` | K8s name of the Bucket CR + lookup key for the COSI BucketClaim. The actual S3 bucket name is the COSI-assigned UUID, surfaced through `BucketClaim.status.bucketName`. | +| `bucketName` | Two modes. With `provisionBucket: true` (default): K8s name of the Bucket CR + lookup key for the COSI BucketClaim — the actual S3 bucket name is the COSI-assigned UUID, surfaced through `BucketClaim.status.bucketName`. With `provisionBucket: false`: taken **verbatim as the real S3 bucket name** and baked into every strategy CR + the Velero BSL. | +| `namespace` | Namespace the Bucket CR (and its system-credentials Secret) lives in — `tenant-root` by default. Must be a tenant namespace (`tenant-*`): the Bucket chart's RBAC helper fails the Helm render for any other prefix. | | `bucketNameOverride` | Escape hatch for offline `helm template` renders — bypasses the live-cluster BucketClaim lookup. Leave empty in production. | | `endpoint` | S3 endpoint baked into every default strategy CR + the Velero BSL. Switching to `https://` silently enables TLS in the MariaDB strategy — ensure the CA bundle is reachable to the relevant operator/driver Pods before flipping it. | | `region` | Re-projected into `cozy-backups-creds` on the next reconcile. Pod-restart required for chart-emitted clients consuming the region via env (ClickHouse sidecar today). | @@ -158,7 +161,7 @@ The system-managed credentials Secret is the **only** way for in-cluster strateg ## Disabling the platform-managed bucket -If a deployment runs against an external S3 (no SeaweedFS), set `backupStorage.provisionBucket: false` in the `backupstrategy-controller` values and create the source credentials Secret in `tenant-root` manually (flat-key format: `accessKey` / `secretKey` / `endpoint` / `bucketName`; or the raw COSI `BucketInfo` JSON). Update `backupStorage.endpoint`, `backupStorage.region`, and (for VM backups) the chart's Velero BSL settings to point at the external S3. +If a deployment runs against an external S3 (no SeaweedFS), set `backupStorage.provisionBucket: false` via the `platform` component override described above and create the source credentials Secret in `tenant-root` manually (flat-key format: `accessKey` / `secretKey` / `endpoint` / `bucketName`; or the raw COSI `BucketInfo` JSON). In the same `backupStorage` block, update `endpoint`, `region`, **and `bucketName`**: with `provisionBucket: false` the strategies and the Velero BSL take `bucketName` verbatim as the real S3 bucket name (no COSI lookup), so it must name the actual bucket on the external S3 — the `bucketName` key inside the Secret alone is not enough. The Velero `BackupStorageLocation` picks the same values up automatically (the chart renders it from the same `backupStorage` block), so no separate BSL configuration is needed. Note that disabling the cluster-default BSL itself (the chart's `velero.bslEnabled` value) is **not** carried by the `backupStorage` override path — the platform Package forwards only the `backupStorage` block. ## Upgrade notes from chart-managed backups diff --git a/packages/core/platform/templates/bundles/system.yaml b/packages/core/platform/templates/bundles/system.yaml index 84aff47ac7..faf1484dbe 100644 --- a/packages/core/platform/templates/bundles/system.yaml +++ b/packages/core/platform/templates/bundles/system.yaml @@ -195,7 +195,21 @@ {{include "cozystack.platform.package.default" (list "cozystack.monitoring-application" $) }} {{include "cozystack.platform.package.default" (list "cozystack.etcd-application" $) }} {{include "cozystack.platform.package.default" (list "cozystack.cozystack-basics" $) }} -{{include "cozystack.platform.package.default" (list "cozystack.backupstrategy-controller" $) }} +{{- /* + Forward admin backupStorage overrides (endpoint, region, systemSecretName, …) + into the emitted cozystack.backupstrategy-controller Package CR's + components.backupstrategy-controller.values, so the supported override path + on the cozystack-platform Package — spec.components.platform.values.backupStorage — + actually reaches the backupstrategy-controller HelmRelease (#3245). Same + pattern as the kubevirt / gpu-operator forwarding in bundles/iaas.yaml. The + components block is only rendered when the admin set something, so an unset + backupStorage keeps the emitted Package identical to the historical render. +*/ -}} +{{- $backupStrategyComponents := dict -}} +{{- if .Values.backupStorage -}} +{{- $_ := set $backupStrategyComponents "backupstrategy-controller" (dict "values" (dict "backupStorage" .Values.backupStorage)) -}} +{{- end -}} +{{include "cozystack.platform.package" (list "cozystack.backupstrategy-controller" "default" $ $backupStrategyComponents) }} {{include "cozystack.platform.package.default" (list "cozystack.backup-controller" $) }} {{- /* velero must be a default package: backupstrategy-controller (default) hard-depends on cozystack.velero, so an optional velero leaves the diff --git a/packages/core/platform/tests/bundles_backupstorage_wiring_test.yaml b/packages/core/platform/tests/bundles_backupstorage_wiring_test.yaml new file mode 100644 index 0000000000..3828bd0acf --- /dev/null +++ b/packages/core/platform/tests/bundles_backupstorage_wiring_test.yaml @@ -0,0 +1,91 @@ +suite: bundles.system backupStorage → backupstrategy-controller wiring +templates: + - templates/bundles/system.yaml +release: + name: cozystack + namespace: cozy-system +tests: + # Regression for #3245: the documented admin override for the cozy-default + # backup S3 coordinates rides on spec.components.platform.values.backupStorage + # of the cozystack-platform Package. The platform chart must forward that + # block into the emitted cozystack.backupstrategy-controller Package CR's + # components, otherwise the override silently never reaches the + # backupstrategy-controller HelmRelease. + - it: emits cozystack.backupstrategy-controller without a components block when backupStorage is unset + set: + bundles: + system: + enabled: true + variant: isp-full + documentSelector: + path: metadata.name + value: cozystack.backupstrategy-controller + asserts: + - equal: + path: kind + value: Package + - equal: + path: spec.variant + value: default + - notExists: + path: spec.components + + - it: forwards backupStorage overrides into the backupstrategy-controller component values + set: + bundles: + system: + enabled: true + variant: isp-full + backupStorage: + provisionBucket: false + endpoint: https://s3.example.com + region: eu-central-1 + systemSecretName: external-s3-credentials + documentSelector: + path: metadata.name + value: cozystack.backupstrategy-controller + asserts: + - equal: + path: spec.components.backupstrategy-controller.values.backupStorage.provisionBucket + value: false + - equal: + path: spec.components.backupstrategy-controller.values.backupStorage.endpoint + value: https://s3.example.com + - equal: + path: spec.components.backupstrategy-controller.values.backupStorage.region + value: eu-central-1 + - equal: + path: spec.components.backupstrategy-controller.values.backupStorage.systemSecretName + value: external-s3-credentials + + - it: forwards backupStorage overrides on the isp-full-generic variant too + set: + bundles: + system: + enabled: true + variant: isp-full-generic + backupStorage: + endpoint: https://s3.example.com + documentSelector: + path: metadata.name + value: cozystack.backupstrategy-controller + asserts: + - equal: + path: spec.components.backupstrategy-controller.values.backupStorage.endpoint + value: https://s3.example.com + + - it: forwards backupStorage overrides on the isp-hosted variant too + set: + bundles: + system: + enabled: true + variant: isp-hosted + backupStorage: + endpoint: https://s3.example.com + documentSelector: + path: metadata.name + value: cozystack.backupstrategy-controller + asserts: + - equal: + path: spec.components.backupstrategy-controller.values.backupStorage.endpoint + value: https://s3.example.com diff --git a/packages/core/platform/values.yaml b/packages/core/platform/values.yaml index 47598fe319..985b8c7b2f 100644 --- a/packages/core/platform/values.yaml +++ b/packages/core/platform/values.yaml @@ -295,6 +295,18 @@ gateway: - cozy-monitoring - cozy-linstor-gui - default +# Backup storage overrides for the platform-managed cozy-default BackupClass. +# Everything set here is forwarded verbatim into the emitted +# cozystack.backupstrategy-controller Package CR as +# components.backupstrategy-controller.values.backupStorage, so it lands on the +# backupstrategy-controller HelmRelease and deep-merges over that chart's own +# backupStorage defaults (packages/system/backupstrategy-controller/values.yaml +# documents every knob: provisionBucket, bucketName, endpoint, region, +# forcePathStyle, systemSecretName, systemNamespaces, …). Admins set this via +# spec.components.platform.values.backupStorage on the +# cozystack.cozystack-platform Package — see docs/operations/backup-classes.md. +# Empty (the default) forwards nothing and keeps the child chart defaults. +backupStorage: {} # Pod scheduling configuration scheduling: globalAppTopologySpreadConstraints: "" From 982d6d4b3139ffb4bbcb5c4d8c292463c0109fde Mon Sep 17 00:00:00 2001 From: myasnikovdaniil <60174387+myasnikovdaniil@users.noreply.github.com> Date: Mon, 20 Jul 2026 12:26:14 +0500 Subject: [PATCH 34/69] fix(cert-manager): move the webhook off the kubelet's port MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Backport of #3359 onto release-1.5. The cert-manager webhook listened on 10250, the port the kubelet also serves. When a connection to the webhook Service resolved to a node IP rather than a Pod IP it reached the kubelet, which completed the TLS handshake with its own node serving certificate, and the API server then rejected every cert-manager admission call cluster-wide: failed calling webhook "webhook.cert-manager.io": ... x509: certificate is valid for srv3, not cert-manager-webhook.cozy-cert-manager.svc The certificate is valid — it is simply the wrong server's — so the error reads as a cert-manager fault and hides the misroute that caused it. Moving the webhook to 10260 is upstream's own remedy for this signature. It does not fix the misroute; it removes the collision that turns a rare transient one into a cluster-wide outage attributed to the wrong component. The override lives in the package-root values file, which `make update` leaves alone, so it survives re-vendoring. Stacked on the #3199 backport (#3202): release-1.5 ships an EMPTY packages/system/cert-manager/values.yaml, so every resource override in that file arrived after v1.5.2. The bot's cherry-pick of this fix conflicted for that reason alone — nothing to merge into — and applies cleanly once the cainjector backport is in place. Upgrade safety is unchanged from the original: one replica, no explicit strategy, so maxUnavailable 0 / maxSurge 1 apply and the replacement Pod must be Ready before the old one goes. The Service targets a NAMED port, so during the overlap each Pod is reached on the port it actually serves and there is no window with zero ready endpoints. Closes #3355 (cherry picked from commit 3affec992041394f8c6edeb63583a89790b248b3) Assisted-By: Claude Signed-off-by: Myasnikov Daniil --- .../tests/webhook_secure_port_test.yaml | 64 +++++++++++++++++++ packages/system/cert-manager/values.yaml | 32 ++++++++++ 2 files changed, 96 insertions(+) create mode 100644 packages/system/cert-manager/tests/webhook_secure_port_test.yaml diff --git a/packages/system/cert-manager/tests/webhook_secure_port_test.yaml b/packages/system/cert-manager/tests/webhook_secure_port_test.yaml new file mode 100644 index 0000000000..6546b4bf53 --- /dev/null +++ b/packages/system/cert-manager/tests/webhook_secure_port_test.yaml @@ -0,0 +1,64 @@ +suite: cert-manager webhook secure port + +# Keeps the webhook off the kubelet's port. The values.yaml override this +# guards carries the reasoning; it is not repeated here. +# +# The first test is the regression guard: it fails if the override is dropped +# or if `make update` re-vendors a chart that ignores it, and it asserts the +# listener flag together with the advertised containerPort so the two cannot +# drift apart. The other two pin preconditions the fix depends on and already +# hold on their own — that the port lives in the Pod's network namespace +# rather than the node's, and that the Service reaches it by name, which is +# what lets a port change roll out without dropping an endpoint. +# +# An upstream rename of the key needs no test: the vendored values.schema.json +# sets additionalProperties: false, so the override would fail helm outright +# rather than be silently ignored. + +templates: + - charts/cert-manager/templates/webhook-deployment.yaml + - charts/cert-manager/templates/webhook-service.yaml + +release: + name: cert-manager + namespace: cozy-cert-manager + +tests: + - it: serves the webhook on 10260, not the kubelet's 10250 + template: charts/cert-manager/templates/webhook-deployment.yaml + asserts: + - isKind: + of: Deployment + - contains: + path: spec.template.spec.containers[0].args + content: --secure-port=10260 + - equal: + path: spec.template.spec.containers[0].ports[0] + value: + name: https + protocol: TCP + containerPort: 10260 + + - it: keeps the webhook off the host network, so the port lives in the Pod netns + template: charts/cert-manager/templates/webhook-deployment.yaml + asserts: + # The template emits the key only when the value is truthy, so absence + # is the only shape a disabled hostNetwork can take here. isNullOrEmpty + # would not do: it fails on a path that does not exist. + - notExists: + path: spec.template.spec.hostNetwork + + - it: resolves the Service to the webhook container by name, not by number + template: charts/cert-manager/templates/webhook-service.yaml + asserts: + - isKind: + of: Service + - equal: + path: spec.ports[0].name + value: https + - equal: + path: spec.ports[0].port + value: 443 + - equal: + path: spec.ports[0].targetPort + value: https diff --git a/packages/system/cert-manager/values.yaml b/packages/system/cert-manager/values.yaml index 2ce6734e51..48e7884b18 100644 --- a/packages/system/cert-manager/values.yaml +++ b/packages/system/cert-manager/values.yaml @@ -7,6 +7,38 @@ cert-manager: cpu: 50m memory: 128Mi webhook: + # Upstream defaults securePort to 10250, the port the kubelet serves on. + # That default is chosen for GKE private clusters, where apiservers may + # reach nodes only on 443 and 10250; it costs nothing there and is unsafe + # here. When a connection to the webhook Service resolves to a node IP + # rather than a Pod IP, it reaches the kubelet, which completes the TLS + # handshake with its node serving certificate, and the apiserver rejects + # every cert-manager admission call cluster-wide with "x509: certificate + # is valid for , not cert-manager-webhook.cozy-cert-manager.svc". + # The certificate is valid — it is simply the wrong server's — so the + # error reads as a cert-manager fault and hides the misroute that caused + # it. Serving on a port nothing else answers on turns that same misroute + # into a connection refusal, which names the real problem. This does not + # prevent the misroute; it removes the collision that makes one + # catastrophic and misattributed. + # + # What has to hold for 10260 to be an improvement is that nothing answers + # on it in the node's network namespace, since the misroute lands on a + # node IP. Nothing does, in either cluster this chart is installed into. + # On management nodes the only host-network workload of the kind that + # might is local-ccm, which links none of the cloud-provider framework + # that binds 10258 and 10260 by default, and binds no TCP port at all. + # No cloud-controller-manager runs on a tenant node either: the kubevirt + # one does build on that framework, but it is a pod-networked Deployment + # in the management cluster, so whatever it binds stays in a Pod's + # namespace. Anything host-network added later to either cluster has to + # keep clear of the port. + # + # Under webhook.hostNetwork the webhook itself would join that namespace, + # where the clusterwide policy in packages/system/cilium-networkpolicy + # denies 10250 from the world but has no entry for 10260. Enabling + # hostNetwork means revisiting that policy. + securePort: 10260 resources: limits: cpu: 200m From f716e2f81da04221a21bf15a84da8f523fc66797 Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Tue, 4 Aug 2026 12:07:29 +0500 Subject: [PATCH 35/69] fix(postgres-operator): align CNPG CRDs with 1.27.3 The operator image is pinned to 1.27.3 while the vendored chart and CRDs still advertise 1.27.1. Kubernetes prunes Backup status.instanceID.sessionID, causing false instance-manager restart failures. Import the official 1.27.3 CRDs, align the chart appVersion, and preserve the overlay across make update. Add render and API round-trip regression coverage. Assisted-By: GPT-5 Signed-off-by: Myasnikov Daniil --- hack/e2e-apps/postgres.bats | 19 + packages/system/postgres-operator/Makefile | 1 + .../charts/cloudnative-pg/Chart.yaml | 2 +- .../cloudnative-pg/templates/crds/crds.yaml | 150 +++++-- .../patches/cloudnative-pg-1.27.3.patch | 369 ++++++++++++++++++ .../tests/cnpg-version_test.yaml | 34 ++ 6 files changed, 550 insertions(+), 25 deletions(-) create mode 100644 packages/system/postgres-operator/patches/cloudnative-pg-1.27.3.patch create mode 100644 packages/system/postgres-operator/tests/cnpg-version_test.yaml diff --git a/hack/e2e-apps/postgres.bats b/hack/e2e-apps/postgres.bats index 44c0c5bd77..a4988c1da4 100644 --- a/hack/e2e-apps/postgres.bats +++ b/hack/e2e-apps/postgres.bats @@ -53,6 +53,25 @@ EOF # for some reason it takes longer for the read-only endpoint to be ready #timeout 120 sh -ec "until kubectl -n tenant-test get endpoints postgres-$name-ro -o jsonpath='{.subsets[*].addresses[*].ip}' | grep -q '[0-9]'; do sleep 10; done" timeout 120 sh -ec "until kubectl -n tenant-test get endpoints postgres-$name-rw -o jsonpath='{.subsets[*].addresses[*].ip}' | grep -q '[0-9]'; do sleep 10; done" + backup_name="$name-session-id-schema" + kubectl -n tenant-test delete backups.postgresql.cnpg.io "$backup_name" --ignore-not-found --timeout=2m + kubectl -n tenant-test apply -f - < Date: Wed, 24 Jun 2026 19:44:45 +0300 Subject: [PATCH 36/69] fix(objectstorage-controller): requeue BucketClaim until backend Bucket is ready The dynamic-provisioning path re-reads the live Bucket after create and copies its readiness onto the BucketClaim, but only during a reconcile. Nothing re-drives that reconcile afterwards: the controller does not watch Buckets, and the generic controller drops no-op resync deltas (reflect.DeepEqual), so once the BucketClaim stops changing the listener is never re-invoked. The Bucket's false->true transition, performed asynchronously by the sidecar on a different object, is therefore reflected onto the claim only if the Bucket happened to be ready during that single reconcile -- a race the sidecar usually loses, leaving bucketReady frozen at false and BucketAccess never granted. Return an error while the backend Bucket is not ready so the work queue requeues the claim with exponential backoff; each retry re-reads the live Bucket until its readiness has been propagated, converging deterministically instead of depending on a resync that never fires. Add a unit test pinning the contract (requeue while not ready, propagate true once ready) and adjust the existing status-update conflict-retry test, which pinned the old never-requeue behavior. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin (cherry picked from commit 4437cccaf0bdd5c031f5dd5607bca7ded0618cf4) --- .../92-bucketclaim-propagate-ready.diff | 148 +++++++++++++++++- 1 file changed, 145 insertions(+), 3 deletions(-) diff --git a/packages/system/objectstorage-controller/images/objectstorage/patches/92-bucketclaim-propagate-ready.diff b/packages/system/objectstorage-controller/images/objectstorage/patches/92-bucketclaim-propagate-ready.diff index e5fc92431d..3a4e691a17 100644 --- a/packages/system/objectstorage-controller/images/objectstorage/patches/92-bucketclaim-propagate-ready.diff +++ b/packages/system/objectstorage-controller/images/objectstorage/patches/92-bucketclaim-propagate-ready.diff @@ -1,11 +1,11 @@ diff --git a/controller/pkg/bucketclaim/bucketclaim.go b/controller/pkg/bucketclaim/bucketclaim.go -index 2b55c49..038cf28 100644 +index 2b55c49..e554761 100644 --- a/controller/pkg/bucketclaim/bucketclaim.go +++ b/controller/pkg/bucketclaim/bucketclaim.go @@ -213,8 +213,23 @@ func (b *BucketClaimListener) provisionBucketClaimOperation(ctx context.Context, return b.recordError(inputBucketClaim, v1.EventTypeWarning, v1alpha1.FailedCreateBucket, err) } - + + // The Bucket for this claim may already exist from an earlier reconcile, + // in which case the sidecar may have provisioned the backend and flipped + // the Bucket to ready in the meantime. The controller does not watch @@ -25,5 +25,147 @@ index 2b55c49..038cf28 100644 - bucketClaim.Status.BucketReady = false + bucketClaim.Status.BucketReady = createdBucket.Status.BucketReady } - + // Update status with retry logic for conflict errors +@@ -282,6 +297,18 @@ func (b *BucketClaimListener) provisionBucketClaimOperation(ctx context.Context, + return b.recordError(inputBucketClaim, v1.EventTypeWarning, v1alpha1.FailedCreateBucket, err) + } + ++ // The backend Bucket is provisioned asynchronously by the sidecar, which ++ // flips Bucket.status.bucketReady on an object this controller does not ++ // watch. The periodic informer resync cannot re-drive convergence either: ++ // the generic controller drops no-op resync deltas (reflect.DeepEqual in ++ // internal/runtime/controller.go) and nothing mutates the BucketClaim after ++ // this point. So while the Bucket is not ready, return an error to force a ++ // rate-limited requeue; each retry re-reads the live Bucket above until its ++ // readiness has been propagated onto the claim. ++ if !statusBucketReady { ++ return fmt.Errorf("backend Bucket %q for BucketClaim %q not ready yet; requeuing", bucketName, bucketClaim.ObjectMeta.Name) ++ } ++ + klog.V(3).Infof("Finished creating Bucket %v", bucketName) + return nil + } +diff --git a/controller/pkg/bucketclaim/bucketclaim_test.go b/controller/pkg/bucketclaim/bucketclaim_test.go +index fea6f16..2a8149c 100644 +--- a/controller/pkg/bucketclaim/bucketclaim_test.go ++++ b/controller/pkg/bucketclaim/bucketclaim_test.go +@@ -379,6 +379,18 @@ func TestRetryOnConflictStatusUpdate(t *testing.T) { + t.Fatalf("Error occurred when creating BucketClaim: %v", err) + } + ++ // Pre-create the backend Bucket already flipped to ready so readiness ++ // propagation lands true and Add completes. This test exercises the ++ // status-update conflict-retry path, not the not-ready requeue path ++ // (the latter is covered by TestProvisionRequeuesUntilBucketReady). ++ readyBucket := &v1alpha1.Bucket{ ++ ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("bucket-%s", bucketClaim.UID)}, ++ Status: v1alpha1.BucketStatus{BucketReady: true}, ++ } ++ if _, err := client.ObjectstorageV1alpha1().Buckets().Create(ctx, readyBucket, metav1.CreateOptions{}); err != nil { ++ t.Fatalf("Error occurred when pre-creating ready Bucket: %v", err) ++ } ++ + // Cleanup + defer util.DeleteObjects(ctx, client, *bucketClaim, *bucketclass) + +@@ -436,8 +448,8 @@ func TestRetryOnConflictStatusUpdate(t *testing.T) { + t.Errorf("Expected BucketName %s, got %s", expectedBucketName, updatedClaim.Status.BucketName) + } + +- if updatedClaim.Status.BucketReady != false { +- t.Errorf("Expected BucketReady to be false, got %v", updatedClaim.Status.BucketReady) ++ if updatedClaim.Status.BucketReady != true { ++ t.Errorf("Expected BucketReady to be true, got %v", updatedClaim.Status.BucketReady) + } + + // Verify finalizer was added +@@ -445,3 +457,88 @@ func TestRetryOnConflictStatusUpdate(t *testing.T) { + t.Errorf("Expected finalizer to be added, but it was not found") + } + } ++ ++// TestProvisionRequeuesUntilBucketReady pins the readiness-propagation contract. ++// The controller does not watch Bucket objects and the generic controller drops ++// no-op resync deltas, so the BucketClaim must be driven to ready by the ++// provision path itself: Add returns an error (forcing a rate-limited requeue) ++// while the backend Bucket is not ready, and returns nil with BucketReady=true ++// once the Bucket's readiness has propagated. Without the requeue the claim ++// would stay BucketReady=false forever and BucketAccess would never be granted. ++func TestProvisionRequeuesUntilBucketReady(t *testing.T) { ++ newListener := func(client *fakebucketclientset.Clientset) *BucketClaimListener { ++ l := NewBucketClaimListener() ++ l.InitializeKubeClient(fakekubeclientset.NewSimpleClientset()) ++ l.InitializeBucketClient(client) ++ l.InitializeEventRecorder(record.NewFakeRecorder(3)) ++ return l ++ } ++ ++ t.Run("requeues while backend Bucket is not ready", func(t *testing.T) { ++ ctx, cancel := context.WithCancel(context.Background()) ++ defer cancel() ++ ++ client := fakebucketclientset.NewSimpleClientset() ++ if _, err := util.CreateBucketClass(ctx, client, &goldClass); err != nil { ++ t.Fatalf("Error occurred when creating BucketClass: %v", err) ++ } ++ bucketClaim, err := util.CreateBucketClaim(ctx, client, &bucketClaim1) ++ if err != nil { ++ t.Fatalf("Error occurred when creating BucketClaim: %v", err) ++ } ++ ++ // The controller creates the Bucket with BucketReady=false and nothing ++ // flips it here, so Add must report an error to force a requeue. ++ if err := newListener(client).Add(ctx, bucketClaim); err == nil { ++ t.Fatal("Add should return an error to requeue while the backend Bucket is not ready") ++ } ++ ++ updated, err := client.ObjectstorageV1alpha1().BucketClaims(bucketClaim.Namespace).Get(ctx, bucketClaim.Name, metav1.GetOptions{}) ++ if err != nil { ++ t.Fatalf("Error occurred when reading BucketClaim: %v", err) ++ } ++ if updated.Status.BucketReady { ++ t.Errorf("Expected BucketReady to stay false while the backend Bucket is not ready") ++ } ++ // Status (name + finalizer) is still persisted before the requeue. ++ if updated.Status.BucketName != fmt.Sprintf("bucket-%s", bucketClaim.UID) { ++ t.Errorf("Expected BucketName to be set even while not ready, got %q", updated.Status.BucketName) ++ } ++ }) ++ ++ t.Run("converges once the backend Bucket is ready", func(t *testing.T) { ++ ctx, cancel := context.WithCancel(context.Background()) ++ defer cancel() ++ ++ client := fakebucketclientset.NewSimpleClientset() ++ if _, err := util.CreateBucketClass(ctx, client, &goldClass); err != nil { ++ t.Fatalf("Error occurred when creating BucketClass: %v", err) ++ } ++ bucketClaim, err := util.CreateBucketClaim(ctx, client, &bucketClaim1) ++ if err != nil { ++ t.Fatalf("Error occurred when creating BucketClaim: %v", err) ++ } ++ ++ // Simulate the sidecar having already provisioned the backend and ++ // flipped the Bucket to ready before this reconcile. ++ readyBucket := &v1alpha1.Bucket{ ++ ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("bucket-%s", bucketClaim.UID)}, ++ Status: v1alpha1.BucketStatus{BucketReady: true}, ++ } ++ if _, err := client.ObjectstorageV1alpha1().Buckets().Create(ctx, readyBucket, metav1.CreateOptions{}); err != nil { ++ t.Fatalf("Error occurred when pre-creating ready Bucket: %v", err) ++ } ++ ++ if err := newListener(client).Add(ctx, bucketClaim); err != nil { ++ t.Fatalf("Add should succeed once the backend Bucket is ready: %v", err) ++ } ++ ++ updated, err := client.ObjectstorageV1alpha1().BucketClaims(bucketClaim.Namespace).Get(ctx, bucketClaim.Name, metav1.GetOptions{}) ++ if err != nil { ++ t.Fatalf("Error occurred when reading BucketClaim: %v", err) ++ } ++ if !updated.Status.BucketReady { ++ t.Errorf("Expected BucketReady to propagate true once the backend Bucket is ready") ++ } ++ }) ++} From bded448d5629263efcb98e8d5f7229d90e487ff7 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Wed, 24 Jun 2026 19:44:58 +0300 Subject: [PATCH 37/69] fix(objectstorage-controller): shorten COSI leader-election lease The controller and the per-driver sidecar both run single-replica, so leader election provides no high availability -- it only governs how quickly a freshly recreated pod takes over. With the upstream 150s lease duration a recreated provisioner pod stalls reconciliation for up to 150s while it waits out the stale lease still held by the previous, now-deleted pod. Lower the lease timings to the standard Kubernetes values (15s/10s/2s) so a recreated pod resumes reconciliation in roughly 15s. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin (cherry picked from commit 250d92323dac3ea9af46633f58ac22b9318bf465) --- .../93-shorten-leaderelection-lease.diff | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 packages/system/objectstorage-controller/images/objectstorage/patches/93-shorten-leaderelection-lease.diff diff --git a/packages/system/objectstorage-controller/images/objectstorage/patches/93-shorten-leaderelection-lease.diff b/packages/system/objectstorage-controller/images/objectstorage/patches/93-shorten-leaderelection-lease.diff new file mode 100644 index 0000000000..1181f62e67 --- /dev/null +++ b/packages/system/objectstorage-controller/images/objectstorage/patches/93-shorten-leaderelection-lease.diff @@ -0,0 +1,23 @@ +diff --git a/internal/runtime/controller.go b/internal/runtime/controller.go +index 902116a..4cbe4c3 100644 +--- a/internal/runtime/controller.go ++++ b/internal/runtime/controller.go +@@ -171,10 +171,14 @@ func NewObjectStorageControllerWithClientset(identity string, leaderLockName str + threadiness: threads, + + ResyncPeriod: 30 * time.Second, +- // leader election +- LeaseDuration: 150 * time.Second, +- RenewDeadline: 120 * time.Second, +- RetryPeriod: 60 * time.Second, ++ // Leader election. The COSI controller and the per-driver sidecar both ++ // run single-replica, so leader election provides no HA — it only gates ++ // how fast a freshly recreated pod takes over. Use the standard ++ // Kubernetes leader-election timings instead of the upstream 150/120/60s ++ // so a recreated pod resumes reconciliation in ~15s rather than ~150s. ++ LeaseDuration: 15 * time.Second, ++ RenewDeadline: 10 * time.Second, ++ RetryPeriod: 2 * time.Second, + + opMap: &sync.Map{}, + }, nil From 400856250df9acd17843698fba9f46ddf37a5ac5 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Wed, 24 Jun 2026 19:44:58 +0300 Subject: [PATCH 38/69] test(bucket,harbor): assert BucketClaim converges to bucketReady promptly The bucket suite waited on '{.status.bucketReady}' without an expected value; a bare jsonpath match is satisfied by the literal "false", so the wait passed against an unready claim. Both suites also allowed 300s/600s, long enough to tolerate the propagation race rather than catch it, so they stayed green whether or not the BucketClaim ever converged. Assert bucketReady=true on a 120s bound. Convergence is now controller-driven (the controller requeues the claim until the backend Bucket's readiness propagates) and completes within tens of seconds; a claim frozen by the propagation race never converges and now fails the wait fast instead of being masked. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin (cherry picked from commit 63d14e7487f1490d46dcf02a27b747fafc75666a) --- hack/e2e-apps/bucket.bats | 13 ++++++++++++- hack/e2e-apps/harbor.bats | 19 ++++++++++++++----- 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/hack/e2e-apps/bucket.bats b/hack/e2e-apps/bucket.bats index e51053a601..10a4867420 100644 --- a/hack/e2e-apps/bucket.bats +++ b/hack/e2e-apps/bucket.bats @@ -21,7 +21,18 @@ EOF # Wait for the bucket to be ready kubectl -n tenant-test wait hr bucket-${name} --timeout=5m --for=condition=ready timeout 60 sh -ec "until kubectl -n tenant-test get bucketclaims.objectstorage.k8s.io bucket-${name} >/dev/null 2>&1; do sleep 2; done" - kubectl -n tenant-test wait bucketclaims.objectstorage.k8s.io bucket-${name} --timeout=300s --for=jsonpath='{.status.bucketReady}' + # The COSI controller requeues the BucketClaim until the backend Bucket's + # readiness propagates, so this converges within tens of seconds. Assert + # bucketReady=true explicitly: a bare jsonpath match also matches the literal + # "false", so the unqualified form passed against an unready claim. The tight + # bound makes a propagation-race regression fail fast instead of hiding. + kubectl -n tenant-test wait bucketclaims.objectstorage.k8s.io bucket-${name} --timeout=120s --for=jsonpath='{.status.bucketReady}'=true || { + echo "=== BucketClaim did not converge to bucketReady=true ===" + kubectl -n tenant-test get bucketclaims.objectstorage.k8s.io bucket-${name} -o yaml 2>&1 || true + echo "=== backend Buckets (cluster-scoped) ===" + kubectl get buckets.objectstorage.k8s.io -o wide 2>&1 || true + false + } timeout 60 sh -ec "until kubectl -n tenant-test get bucketaccesses.objectstorage.k8s.io bucket-${name}-admin >/dev/null 2>&1; do sleep 2; done" kubectl -n tenant-test wait bucketaccesses.objectstorage.k8s.io bucket-${name}-admin --timeout=300s --for=jsonpath='{.status.accessGranted}' timeout 60 sh -ec "until kubectl -n tenant-test get bucketaccesses.objectstorage.k8s.io bucket-${name}-viewer >/dev/null 2>&1; do sleep 2; done" diff --git a/hack/e2e-apps/harbor.bats b/hack/e2e-apps/harbor.bats index 0648b10373..8fe300939e 100644 --- a/hack/e2e-apps/harbor.bats +++ b/hack/e2e-apps/harbor.bats @@ -43,13 +43,22 @@ EOF timeout 60 sh -ec "until kubectl -n tenant-test get hr $release >/dev/null 2>&1; do sleep 2; done" kubectl -n tenant-test wait hr $release --timeout=5m --for=condition=ready - # Wait for COSI to provision bucket. The driver creates the Bucket and grants - # access quickly, but the central COSI controller's propagation of the Bucket's - # readiness back onto the namespaced BucketClaim can lag several minutes on a - # loaded runner, so allow the same 10m budget the dependent HelmRelease gets. + # Wait for COSI to provision the bucket. The driver creates the backend Bucket + # and grants access; the central COSI controller requeues the BucketClaim until + # the Bucket's readiness has propagated, so it converges within tens of seconds. + # A frozen claim (the propagation race this guards against) never converges, so + # the tight bound surfaces that regression instead of masking it. timeout 60 sh -ec "until kubectl -n tenant-test get bucketclaims.objectstorage.k8s.io $release-registry >/dev/null 2>&1; do sleep 2; done" kubectl -n tenant-test wait bucketclaims.objectstorage.k8s.io $release-registry \ - --timeout=600s --for=jsonpath='{.status.bucketReady}'=true + --timeout=120s --for=jsonpath='{.status.bucketReady}'=true || { + echo "=== BucketClaim did not converge to bucketReady=true ===" + kubectl -n tenant-test get bucketclaims.objectstorage.k8s.io $release-registry -o yaml 2>&1 || true + echo "=== backend Buckets (cluster-scoped) ===" + kubectl get buckets.objectstorage.k8s.io -o wide 2>&1 || true + echo "=== objectstorage-controller ===" + kubectl -n cozy-objectstorage-controller get pods 2>&1 || true + false + } timeout 60 sh -ec "until kubectl -n tenant-test get bucketaccesses.objectstorage.k8s.io $release-registry >/dev/null 2>&1; do sleep 2; done" kubectl -n tenant-test wait bucketaccesses.objectstorage.k8s.io $release-registry \ --timeout=60s --for=jsonpath='{.status.accessGranted}'=true From fa921b90b70cdd5f96b64596baebfca1624dd1da Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Wed, 24 Jun 2026 19:58:57 +0300 Subject: [PATCH 39/69] ci(pr-labeler): map objectstorage-controller scope to area/storage The objectstorage-controller package ships the COSI control plane; its commits use the objectstorage-controller scope, which had no area mapping and fell back to area/uncategorized. Map it to area/storage alongside the other COSI/storage scopes. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin (cherry picked from commit 4f5dcea4326ba8a18d3a2153462faf572770421f) --- .github/workflows/pr-labeler.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pr-labeler.yaml b/.github/workflows/pr-labeler.yaml index 63696fddfd..25c97005dc 100644 --- a/.github/workflows/pr-labeler.yaml +++ b/.github/workflows/pr-labeler.yaml @@ -103,6 +103,7 @@ jobs: // area/storage 'seaweedfs': 'area/storage', 'seaweedfs-cosi-driver': 'area/storage', + 'objectstorage-controller': 'area/storage', 'bucket': 'area/storage', 'linstor': 'area/storage', 'velero': 'area/storage', From 85dc51c48a2a32436f9f9735b608f0b89b2a9821 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Wed, 24 Jun 2026 23:28:15 +0300 Subject: [PATCH 40/69] test(e2e): assert bucketaccess accessGranted=true and dump controller pods on bucket failure The two bucketaccesses waits used the bare jsonpath form, which also matches the literal "false"; require =true so a never-granted access fails the wait instead of passing against an ungranted object. Also print the objectstorage-controller pod status in the bucketReady failure block so controller-side issues are visible from the test log. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin (cherry picked from commit 0855ff2d1b598c59d307541e324c4a29d2c3e4a3) --- hack/e2e-apps/bucket.bats | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/hack/e2e-apps/bucket.bats b/hack/e2e-apps/bucket.bats index 10a4867420..9c885b4d4a 100644 --- a/hack/e2e-apps/bucket.bats +++ b/hack/e2e-apps/bucket.bats @@ -31,12 +31,14 @@ EOF kubectl -n tenant-test get bucketclaims.objectstorage.k8s.io bucket-${name} -o yaml 2>&1 || true echo "=== backend Buckets (cluster-scoped) ===" kubectl get buckets.objectstorage.k8s.io -o wide 2>&1 || true + echo "=== objectstorage-controller ===" + kubectl -n cozy-objectstorage-controller get pods 2>&1 || true false } timeout 60 sh -ec "until kubectl -n tenant-test get bucketaccesses.objectstorage.k8s.io bucket-${name}-admin >/dev/null 2>&1; do sleep 2; done" - kubectl -n tenant-test wait bucketaccesses.objectstorage.k8s.io bucket-${name}-admin --timeout=300s --for=jsonpath='{.status.accessGranted}' + kubectl -n tenant-test wait bucketaccesses.objectstorage.k8s.io bucket-${name}-admin --timeout=300s --for=jsonpath='{.status.accessGranted}'=true timeout 60 sh -ec "until kubectl -n tenant-test get bucketaccesses.objectstorage.k8s.io bucket-${name}-viewer >/dev/null 2>&1; do sleep 2; done" - kubectl -n tenant-test wait bucketaccesses.objectstorage.k8s.io bucket-${name}-viewer --timeout=300s --for=jsonpath='{.status.accessGranted}' + kubectl -n tenant-test wait bucketaccesses.objectstorage.k8s.io bucket-${name}-viewer --timeout=300s --for=jsonpath='{.status.accessGranted}'=true # Get admin (readwrite) credentials kubectl -n tenant-test get secret bucket-${name}-admin -ojsonpath='{.data.BucketInfo}' | base64 -d > bucket-admin-credentials.json From 18105d2e69dd32fbf34bfcee24fac009eded0fb9 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Wed, 24 Jun 2026 23:28:24 +0300 Subject: [PATCH 41/69] fix(objectstorage): emit a Warning event while a BucketClaim awaits backend Bucket readiness While the backend Bucket is not ready the controller returns an error to force a requeue, but that error is only logged at klog.V(3) and is suppressed at default verbosity, so an operator cannot see why the claim is stuck. Record a WaitingForBucket Warning event on the BucketClaim on each requeue (event aggregation folds the repetition) and still return the error so the requeue happens as before. Pin the requeue contract in the test by asserting the error carries the stable "not ready yet" substring, not merely that it is non-nil. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin (cherry picked from commit 5991808d28782ed0eb60b81550d70e558f759ec8) --- .../92-bucketclaim-propagate-ready.diff | 37 ++++++++++++++----- 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/packages/system/objectstorage-controller/images/objectstorage/patches/92-bucketclaim-propagate-ready.diff b/packages/system/objectstorage-controller/images/objectstorage/patches/92-bucketclaim-propagate-ready.diff index 3a4e691a17..ae245287db 100644 --- a/packages/system/objectstorage-controller/images/objectstorage/patches/92-bucketclaim-propagate-ready.diff +++ b/packages/system/objectstorage-controller/images/objectstorage/patches/92-bucketclaim-propagate-ready.diff @@ -1,5 +1,5 @@ diff --git a/controller/pkg/bucketclaim/bucketclaim.go b/controller/pkg/bucketclaim/bucketclaim.go -index 2b55c49..e554761 100644 +index 2b55c49..9195195 100644 --- a/controller/pkg/bucketclaim/bucketclaim.go +++ b/controller/pkg/bucketclaim/bucketclaim.go @@ -213,8 +213,23 @@ func (b *BucketClaimListener) provisionBucketClaimOperation(ctx context.Context, @@ -27,7 +27,7 @@ index 2b55c49..e554761 100644 } // Update status with retry logic for conflict errors -@@ -282,6 +297,18 @@ func (b *BucketClaimListener) provisionBucketClaimOperation(ctx context.Context, +@@ -282,6 +297,22 @@ func (b *BucketClaimListener) provisionBucketClaimOperation(ctx context.Context, return b.recordError(inputBucketClaim, v1.EventTypeWarning, v1alpha1.FailedCreateBucket, err) } @@ -38,19 +38,31 @@ index 2b55c49..e554761 100644 + // internal/runtime/controller.go) and nothing mutates the BucketClaim after + // this point. So while the Bucket is not ready, return an error to force a + // rate-limited requeue; each retry re-reads the live Bucket above until its -+ // readiness has been propagated onto the claim. ++ // readiness has been propagated onto the claim. Surface a Warning event so ++ // an operator can see why the claim is still pending (the returned error is ++ // logged at klog.V(3), suppressed at default verbosity); event aggregation ++ // folds the per-requeue repetition into a single counted entry. + if !statusBucketReady { -+ return fmt.Errorf("backend Bucket %q for BucketClaim %q not ready yet; requeuing", bucketName, bucketClaim.ObjectMeta.Name) ++ err := fmt.Errorf("backend Bucket %q for BucketClaim %q not ready yet; requeuing", bucketName, bucketClaim.ObjectMeta.Name) ++ return b.recordError(inputBucketClaim, v1.EventTypeWarning, v1alpha1.WaitingForBucket, err) + } + klog.V(3).Infof("Finished creating Bucket %v", bucketName) return nil } diff --git a/controller/pkg/bucketclaim/bucketclaim_test.go b/controller/pkg/bucketclaim/bucketclaim_test.go -index fea6f16..2a8149c 100644 +index fea6f16..837f7e5 100644 --- a/controller/pkg/bucketclaim/bucketclaim_test.go +++ b/controller/pkg/bucketclaim/bucketclaim_test.go -@@ -379,6 +379,18 @@ func TestRetryOnConflictStatusUpdate(t *testing.T) { +@@ -3,6 +3,7 @@ package bucketclaim + import ( + "context" + "fmt" ++ "strings" + "sync" + "testing" + +@@ -379,6 +380,18 @@ func TestRetryOnConflictStatusUpdate(t *testing.T) { t.Fatalf("Error occurred when creating BucketClaim: %v", err) } @@ -69,7 +81,7 @@ index fea6f16..2a8149c 100644 // Cleanup defer util.DeleteObjects(ctx, client, *bucketClaim, *bucketclass) -@@ -436,8 +448,8 @@ func TestRetryOnConflictStatusUpdate(t *testing.T) { +@@ -436,8 +449,8 @@ func TestRetryOnConflictStatusUpdate(t *testing.T) { t.Errorf("Expected BucketName %s, got %s", expectedBucketName, updatedClaim.Status.BucketName) } @@ -80,7 +92,7 @@ index fea6f16..2a8149c 100644 } // Verify finalizer was added -@@ -445,3 +457,88 @@ func TestRetryOnConflictStatusUpdate(t *testing.T) { +@@ -445,3 +458,95 @@ func TestRetryOnConflictStatusUpdate(t *testing.T) { t.Errorf("Expected finalizer to be added, but it was not found") } } @@ -116,9 +128,16 @@ index fea6f16..2a8149c 100644 + + // The controller creates the Bucket with BucketReady=false and nothing + // flips it here, so Add must report an error to force a requeue. -+ if err := newListener(client).Add(ctx, bucketClaim); err == nil { ++ err = newListener(client).Add(ctx, bucketClaim) ++ if err == nil { + t.Fatal("Add should return an error to requeue while the backend Bucket is not ready") + } ++ // Pin the stable substring the production requeue error carries. A ++ // refactor that drops the requeue (returning nil or a different error) ++ // would silently regress convergence; this keeps the contract loud. ++ if !strings.Contains(err.Error(), "not ready yet") { ++ t.Fatalf("Expected requeue error to contain %q, got: %v", "not ready yet", err) ++ } + + updated, err := client.ObjectstorageV1alpha1().BucketClaims(bucketClaim.Namespace).Get(ctx, bucketClaim.Name, metav1.GetOptions{}) + if err != nil { From 46cef7ced025d7044b2c2481b4b766968af88a2b Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Mon, 29 Jun 2026 23:32:34 +0300 Subject: [PATCH 42/69] fix(velero): add startupProbe so slow startup does not crashloop the install gate Velero binds its metrics/health endpoint (:8085, http-monitoring) only after the server loads its plugins and connects to the API server. Under a heavy parallel platform install that can take minutes, but the upstream liveness probe begins checking /metrics at initialDelaySeconds=10 and kills the container after failureThreshold=5 (~130s total) before :8085 is bound. The kubelet SIGTERMs velero (exitCode 0/Completed), it BackOff-restarts, and never escapes the loop inside the install-gate window. Because backupstrategy-controller hard-depends on velero, this blocks the gate. Add a startupProbe to the velero server container. While it is failing the kubelet holds off liveness and readiness entirely, granting failureThreshold * periodSeconds = 30 * 10s = 300s of startup grace, then hands over to the unchanged, tight upstream liveness (10s / 30s / 5) for steady-state crash detection. This is strictly better than loosening liveness, which would also blunt steady-state detection. The upstream velero chart (12.0.3) does not render a startupProbe, so the package patches the deployment template via patches/add-startup-probe.patch (re-applied after re-vendor by the Makefile update target, matching the seaweedfs/ingress-nginx convention) and supplies the startupProbe through the package values override. Pin the startupProbe budget and the unchanged liveness with a helm-unittest regression guard. Backport adaptations for release-1.5, which vendors velero chart 11.0.0 (appVersion 1.17.0) rather than main's 12.0.3 (1.18.1): - The chart version pin stays at the version this branch already vendors; only the patch re-application is added to `update`. The `helm pull` was previously unpinned, which would have let `update` float to a chart the patch does not apply to, so it is pinned to 11.0.0 to keep the new patch step deterministic. - Added the `test:` target, which exists on main but not here, so the shipped regression guard actually runs. - The image assertions pin this branch's own vendored versions (velero v1.17.0, plugin-for-aws v1.12.1, kubevirt plugin v0.8.0), and the upgrade-crds rationale is restated for chart 11.0.0, where that Job still runs a kubectl image. The startupProbe hunk applies to chart 11.0.0's deployment template unchanged, in the same position inside the metrics.enabled gate. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin (cherry picked from commit f7271b80111d98e8772d97939fc45279a55036d3) Signed-off-by: Myasnikov Daniil --- packages/system/velero/Makefile | 8 +- .../charts/velero/templates/deployment.yaml | 3 + .../velero/patches/add-startup-probe.patch | 14 +++ packages/system/velero/tests/velero_test.yaml | 108 ++++++++++++++++++ packages/system/velero/values.yaml | 24 ++++ 5 files changed, 156 insertions(+), 1 deletion(-) create mode 100644 packages/system/velero/patches/add-startup-probe.patch create mode 100644 packages/system/velero/tests/velero_test.yaml diff --git a/packages/system/velero/Makefile b/packages/system/velero/Makefile index 44eba95114..749efca032 100644 --- a/packages/system/velero/Makefile +++ b/packages/system/velero/Makefile @@ -3,9 +3,15 @@ export NAMESPACE=cozy-$(NAME) include ../../../hack/package.mk +test: + helm unittest . + update: rm -rf charts # Velero helm repo add tanzu https://vmware-tanzu.github.io/helm-charts helm repo update tanzu - helm pull tanzu/velero --untar --untardir charts + helm pull tanzu/velero --untar --untardir charts --version 11.0.0 + # Render a startupProbe from .Values.startupProbe (upstream chart omits it), + # so a slow server start under heavy parallel install does not trip liveness. + patch --no-backup-if-mismatch -p4 < patches/add-startup-probe.patch diff --git a/packages/system/velero/charts/velero/templates/deployment.yaml b/packages/system/velero/charts/velero/templates/deployment.yaml index d2a72639ad..a46d732006 100644 --- a/packages/system/velero/charts/velero/templates/deployment.yaml +++ b/packages/system/velero/charts/velero/templates/deployment.yaml @@ -199,6 +199,9 @@ spec: {{- toYaml . | nindent 12 }} {{- end }} {{- if .Values.metrics.enabled }} + {{- with .Values.startupProbe }} + startupProbe: {{- toYaml . | nindent 12 }} + {{- end }} {{- with .Values.livenessProbe }} livenessProbe: {{- toYaml . | nindent 12 }} {{- end }} diff --git a/packages/system/velero/patches/add-startup-probe.patch b/packages/system/velero/patches/add-startup-probe.patch new file mode 100644 index 0000000000..78c8945a8c --- /dev/null +++ b/packages/system/velero/patches/add-startup-probe.patch @@ -0,0 +1,14 @@ +diff --git a/packages/system/velero/charts/velero/templates/deployment.yaml b/packages/system/velero/charts/velero/templates/deployment.yaml +index d2a72639a..a46d73200 100644 +--- a/packages/system/velero/charts/velero/templates/deployment.yaml ++++ b/packages/system/velero/charts/velero/templates/deployment.yaml +@@ -199,6 +199,9 @@ spec: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- if .Values.metrics.enabled }} ++ {{- with .Values.startupProbe }} ++ startupProbe: {{- toYaml . | nindent 12 }} ++ {{- end }} + {{- with .Values.livenessProbe }} + livenessProbe: {{- toYaml . | nindent 12 }} + {{- end }} diff --git a/packages/system/velero/tests/velero_test.yaml b/packages/system/velero/tests/velero_test.yaml new file mode 100644 index 0000000000..8a03e1acc5 --- /dev/null +++ b/packages/system/velero/tests/velero_test.yaml @@ -0,0 +1,108 @@ +suite: cozy-velero chart rendering invariants +release: + name: velero + namespace: cozy-velero +tests: + - it: renders the velero server Deployment on the pinned upstream image + template: charts/velero/templates/deployment.yaml + documentSelector: + path: metadata.name + value: velero + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: velero/velero:v1.17.0 + + - it: pins the backup-plugin initContainers to velero-1.17-compatible releases + template: charts/velero/templates/deployment.yaml + documentSelector: + path: metadata.name + value: velero + asserts: + - equal: + path: spec.template.spec.initContainers[0].image + value: velero/velero-plugin-for-aws:v1.12.1 + - equal: + path: spec.template.spec.initContainers[1].image + value: quay.io/kubevirt/kubevirt-velero-plugin:v0.8.0 + + # Cozystack disables the upgrade-crds Job (upgradeCRDs: false) because CRDs + # ship via the chart's crds/ directory, making the Job redundant, and because + # on this chart (11.0.0) the Job runs a kubectl image whose tag defaults to the + # chart's Kubernetes version and need not match the cluster. Pin that + # invariant — it silently regressed once. + - it: does not emit the upgrade-crds Job when upgradeCRDs is disabled + template: charts/velero/templates/upgrade-crds/upgrade-crds.yaml + asserts: + - hasDocuments: + count: 0 + + # Velero binds :8085 (http-monitoring) late under a heavy parallel install, so + # the upstream liveness probe (initialDelaySeconds 10, ~130s total budget) would + # kill it mid-startup and crashloop it out of the install-gate window. A + # startupProbe (patched into the upstream chart, which omits one) holds liveness + # and readiness off until the server is up, granting 30 * 10s = 300s of startup + # grace while leaving steady-state liveness at the tight upstream default. Pin + # the startupProbe budget AND that liveness was NOT loosened, so neither + # silently regresses. + - it: gates startup with a startupProbe and keeps liveness tight + template: charts/velero/templates/deployment.yaml + documentSelector: + path: metadata.name + value: velero + asserts: + - equal: + path: spec.template.spec.containers[0].startupProbe.periodSeconds + value: 10 + - equal: + path: spec.template.spec.containers[0].startupProbe.failureThreshold + value: 30 + - equal: + path: spec.template.spec.containers[0].startupProbe.timeoutSeconds + value: 5 + - equal: + path: spec.template.spec.containers[0].startupProbe.successThreshold + value: 1 + - equal: + path: spec.template.spec.containers[0].startupProbe.httpGet.path + value: /metrics + - equal: + path: spec.template.spec.containers[0].startupProbe.httpGet.port + value: http-monitoring + # No initialDelaySeconds: the first startup check fires immediately, so the + # full 300s budget is spent probing. Pin its absence so a future change + # cannot silently re-introduce a delay that eats into the grace window. + - notExists: + path: spec.template.spec.containers[0].startupProbe.initialDelaySeconds + # Liveness stays at the tight upstream default — the startupProbe owns the + # grace, so loosening liveness would only weaken steady-state crash + # detection. Pin the full budget (initialDelay/period/failureThreshold) so a + # future change cannot silently re-delay or loosen liveness. + - equal: + path: spec.template.spec.containers[0].livenessProbe.initialDelaySeconds + value: 10 + - equal: + path: spec.template.spec.containers[0].livenessProbe.periodSeconds + value: 30 + - equal: + path: spec.template.spec.containers[0].livenessProbe.failureThreshold + value: 5 + + # All three probes live behind {{- if .Values.metrics.enabled }} — :8085 only + # exists when metrics are on. Pin that the startupProbe follows the same gate as + # liveness/readiness, so disabling metrics drops all probes together and never + # leaves a startupProbe pointed at an unbound port. + - it: omits all probes (including the startupProbe) when metrics are disabled + template: charts/velero/templates/deployment.yaml + documentSelector: + path: metadata.name + value: velero + set: + velero.metrics.enabled: false + asserts: + - notExists: + path: spec.template.spec.containers[0].startupProbe + - notExists: + path: spec.template.spec.containers[0].livenessProbe + - notExists: + path: spec.template.spec.containers[0].readinessProbe diff --git a/packages/system/velero/values.yaml b/packages/system/velero/values.yaml index 7c94b768c4..9bf2200aaa 100644 --- a/packages/system/velero/values.yaml +++ b/packages/system/velero/values.yaml @@ -30,3 +30,27 @@ velero: # Increase timeout for item operations to 24 hours to prevent timeouts # during backups of very large volumes. The Velero default is 4 hours. defaultItemOperationTimeout: 24h + + # Velero binds its metrics/health endpoint (:8085, http-monitoring) only after + # the server finishes loading plugins and connecting to the API server. Under a + # heavy parallel platform install that can take minutes, but the upstream + # liveness probe starts checking /metrics at initialDelaySeconds=10 and kills + # the container after failureThreshold=5 (~130s total) before :8085 is bound. + # The kubelet SIGTERMs velero (exitCode 0/Completed), it BackOff-restarts, and + # never escapes the loop inside the install-gate window, blocking the gate. + # + # Give the server an explicit startup budget. While the startupProbe is failing + # the kubelet holds off the liveness and readiness probes entirely, so steady- + # state liveness stays at the tight upstream default (10s / 30s / 5 failures) + # once startup completes. failureThreshold * periodSeconds = 30 * 10s = 300s of + # grace. The upstream chart does not render a startupProbe, so the velero + # package patches it in (patches/add-startup-probe.patch, re-applied on update). + startupProbe: + httpGet: + path: /metrics + port: http-monitoring + scheme: HTTP + periodSeconds: 10 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 30 From 55d6a5d9682bfc677657aa6427cb9b52fe65acd0 Mon Sep 17 00:00:00 2001 From: IvanHunters Date: Fri, 12 Jun 2026 18:09:25 +0300 Subject: [PATCH 43/69] fix(tenant): emit full ancestor label chain on tenant namespaces The tenant.cozystack.io/ namespace labels drive the -egress CiliumClusterwideNetworkPolicy, which lets an ancestor tenant reach a descendant namespace only when the descendant carries tenant.cozystack.io/. namespace.yaml derived that chain by splitting .Release.Namespace, which only reached one level up: a direct child of tenant-root got tenant-root, but a grandchild (tenant-ktj-htdev under tenant-ktj) got only tenant-ktj and dropped tenant-root entirely. The root ingress controller in tenant-root then could not reach nested tenants, surfacing as upstream timeouts. Derive the chain deterministically from the namespace name instead. Every tenant descends from tenant-root, and tenant.name builds each child as -, so every progressive dash-prefix of the name is a real ancestor namespace. This needs no cluster lookup, renders offline (so it is unit-testable), and converges on every reconcile regardless of parent/child HelmRelease ordering. The ownerReference lookup is made non-fatal so the template still renders when the parent namespace lookup is empty; in a real reconcile the parent always exists and the ownerReference is always emitted. Signed-off-by: IvanHunters (cherry picked from commit 1ac9398a4b284a2c0bb00f4b29a6663071777968) (cherry picked from commit 6ce09079b7b84ed124268addfd21881d64976541) (cherry picked from commit ac7176257b517cbb09fc1792f90ce1f2a18d89be) --- packages/apps/tenant/templates/_helpers.tpl | 32 +++++++ packages/apps/tenant/templates/namespace.yaml | 24 ++--- .../tests/namespace_ancestor_labels_test.yaml | 91 +++++++++++++++++++ 3 files changed, 136 insertions(+), 11 deletions(-) create mode 100644 packages/apps/tenant/tests/namespace_ancestor_labels_test.yaml diff --git a/packages/apps/tenant/templates/_helpers.tpl b/packages/apps/tenant/templates/_helpers.tpl index 6d687cb863..0f7d064d94 100644 --- a/packages/apps/tenant/templates/_helpers.tpl +++ b/packages/apps/tenant/templates/_helpers.tpl @@ -53,6 +53,38 @@ typing, so the "key absent" form is what distinguishes "unset" from explicit `false`. */}} +{{/* + tenant.ancestorTenantLabels emits the full set of + `tenant.cozystack.io/: ""` namespace labels for the tenant whose + namespace name is passed as the single argument (e.g. "tenant-ktj-htdev"). + + Every tenant descends from tenant-root, so that label is always emitted. + The remaining ancestors are encoded in the namespace name itself: tenant.name + constructs each child namespace as `-`, so every + progressive dash-prefix of the name is a real ancestor namespace + (tenant-ktj-htdev -> tenant-ktj, tenant-ktj-htdev). The last prefix is the + tenant's own name, so its self-label is included too. + + Deriving the chain from the name (rather than from a lookup of the parent + namespace's labels) is deterministic: it needs no cluster state, renders + identically offline, and converges on every reconcile regardless of the + order in which parent and child HelmReleases reconcile. It replaces an + earlier splitList over `.Release.Namespace` that only ever reached one level + up and therefore dropped tenant-root for tenants at depth >= 2 — breaking the + `-egress` CiliumClusterwideNetworkPolicy that grants an ancestor + reachability to its descendants via the `tenant.cozystack.io/` + namespace label. +*/}} +{{- define "tenant.ancestorTenantLabels" -}} +{{- $parts := splitList "-" . -}} +tenant.cozystack.io/tenant-root: "" +{{- range $i, $v := $parts }} +{{- if ne $i 0 }} +{{ printf "tenant.cozystack.io/%s: \"\"" (join "-" (slice $parts 0 (add $i 1))) }} +{{- end }} +{{- end }} +{{- end -}} + {{- define "tenant.gatewayEffective" -}} {{- if kindIs "invalid" .Values.gateway -}} false diff --git a/packages/apps/tenant/templates/namespace.yaml b/packages/apps/tenant/templates/namespace.yaml index 6e4ea1cb5c..92ceacab73 100644 --- a/packages/apps/tenant/templates/namespace.yaml +++ b/packages/apps/tenant/templates/namespace.yaml @@ -1,8 +1,14 @@ -{{/* Lookup for namespace uid (needed for ownerReferences) */}} +{{/* + Lookup the parent namespace, used below to stamp the child namespace's + ownerReference uid. During a real helm-controller reconcile the parent + namespace (.Release.Namespace) always exists, so the ownerReference is + always emitted. The lookup returns empty only when rendering offline + (helm template / helm unittest), where the ownerReference is simply omitted + so the rest of the template — notably the ancestor labels below, which are + derived purely from the namespace name and need no cluster state — still + renders and can be unit-tested. +*/}} {{- $existingNS := lookup "v1" "Namespace" "" .Release.Namespace }} -{{- if not $existingNS }} -{{- fail (printf "error lookup existing namespace: %s" .Release.Namespace) }} -{{- end }} {{- if ne (include "tenant.name" .) "tenant-root" }} {{/* Compute namespace values once for use in both Secret and labels */}} @@ -75,13 +81,7 @@ metadata: name: {{ $tenantName }} {{- if hasPrefix "tenant-" .Release.Namespace }} labels: - tenant.cozystack.io/{{ $tenantName }}: "" - {{- $parts := splitList "-" .Release.Namespace }} - {{- range $i, $v := $parts }} - {{- if ne $i 0 }} - tenant.cozystack.io/{{ join "-" (slice $parts 0 (add $i 1)) }}: "" - {{- end }} - {{- end }} + {{- include "tenant.ancestorTenantLabels" $tenantName | nindent 4 }} {{/* Labels for network policies */}} namespace.cozystack.io/etcd: {{ $etcd | quote }} namespace.cozystack.io/ingress: {{ $ingress | quote }} @@ -93,6 +93,7 @@ metadata: scheduler.cozystack.io/scheduling-class: {{ . | quote }} {{- end }} alpha.kubevirt.io/auto-memory-limits-ratio: "1.0" + {{- if $existingNS }} ownerReferences: - apiVersion: v1 blockOwnerDeletion: true @@ -101,6 +102,7 @@ metadata: name: {{ .Release.Namespace }} uid: {{ $existingNS.metadata.uid }} {{- end }} + {{- end }} --- apiVersion: v1 kind: Secret diff --git a/packages/apps/tenant/tests/namespace_ancestor_labels_test.yaml b/packages/apps/tenant/tests/namespace_ancestor_labels_test.yaml new file mode 100644 index 0000000000..773f7a86c9 --- /dev/null +++ b/packages/apps/tenant/tests/namespace_ancestor_labels_test.yaml @@ -0,0 +1,91 @@ +suite: tenant namespace ancestor labels +# Regression cover for the tenant.cozystack.io/ namespace labels +# emitted by namespace.yaml. These labels drive the -egress +# CiliumClusterwideNetworkPolicy (networkpolicy.yaml): an ancestor tenant is +# allowed to reach any namespace carrying tenant.cozystack.io/, so +# every tenant namespace MUST carry the label of each of its ancestors, +# tenant-root included, at every depth. +# +# The labels are derived from the namespace name by tenant.ancestorTenantLabels +# (see _helpers.tpl) with no cluster lookup, so helm-unittest can render them +# offline — the ownerReference lookup is the only cluster-coupled part of +# namespace.yaml and is skipped when the parent namespace lookup is empty. +templates: + - templates/namespace.yaml +set: + _cluster: + root-host: example.com +tests: + - it: depth-1 tenant (direct child of root) carries tenant-root and its own label + release: + name: tenant-ktj + namespace: tenant-root + asserts: + - documentIndex: 0 + isKind: + of: Namespace + - documentIndex: 0 + equal: + path: metadata.name + value: tenant-ktj + - documentIndex: 0 + equal: + path: metadata.labels["tenant.cozystack.io/tenant-root"] + value: "" + - documentIndex: 0 + equal: + path: metadata.labels["tenant.cozystack.io/tenant-ktj"] + value: "" + + - it: depth-2 tenant carries the full ancestor chain including tenant-root + release: + name: tenant-htdev + namespace: tenant-ktj + asserts: + - documentIndex: 0 + equal: + path: metadata.name + value: tenant-ktj-htdev + - documentIndex: 0 + equal: + path: metadata.labels["tenant.cozystack.io/tenant-root"] + value: "" + - documentIndex: 0 + equal: + path: metadata.labels["tenant.cozystack.io/tenant-ktj"] + value: "" + - documentIndex: 0 + equal: + path: metadata.labels["tenant.cozystack.io/tenant-ktj-htdev"] + value: "" + + - it: depth-3 tenant carries every ancestor from root down to itself + release: + name: tenant-foo + namespace: tenant-ktj-htdev + asserts: + - documentIndex: 0 + equal: + path: metadata.name + value: tenant-ktj-htdev-foo + - documentIndex: 0 + equal: + path: metadata.labels["tenant.cozystack.io/tenant-root"] + value: "" + - documentIndex: 0 + equal: + path: metadata.labels["tenant.cozystack.io/tenant-ktj"] + value: "" + - documentIndex: 0 + equal: + path: metadata.labels["tenant.cozystack.io/tenant-ktj-htdev"] + value: "" + - documentIndex: 0 + equal: + path: metadata.labels["tenant.cozystack.io/tenant-ktj-htdev-foo"] + value: "" + # A tenant must NOT claim descendants it does not have: the depth-3 + # namespace is not an ancestor of any deeper tenant, so no stray label. + - documentIndex: 0 + notExists: + path: metadata.labels["tenant.cozystack.io/tenant-ktj-htdev-foo-bar"] From 5427bcab4a957cac67307f545102d685d2f56f8f Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Mon, 17 Aug 2026 14:57:42 +0500 Subject: [PATCH 44/69] fix(migrations): pin KubeadmConfigTemplate in the 1.5 slot 45 migration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Migration slot 45 holds two different migrations on the two branches. On release-1.5 it is the SeaweedFS db-split repair (#3370 backport, which also bumped targetVersion to 46); on main and release-1.6 it is the helm.sh/resource-policy=keep pin on KubeadmConfigTemplate / KubeadmConfig that stops Helm pruning them while CAPI still references them from the kubeadm-backed MachineSet. v1.5.0 through v1.5.3 all ship targetVersion 45, so no released 1.5 cluster is stamped past 45 and every one of them still runs 1.6's slot 45 on the way up. Cutting v1.5.4 off release-1.5 as it stands is what first ships targetVersion 46: those clusters get stamped 46, a later 1.5.4 -> 1.6 upgrade runs `seq 46 53`, and 1.6's slot 45 never executes. The keep-pin is then silently skipped, the 1.6 kubernetes chart upgrade prunes KubeadmConfigTemplate while the live kubeadm-backed MachineSet still references it, and its bootstrap.configRef dangles through the Talos worker rollover. Tenant Kubernetes only; a noisy broken upgrade rather than data loss, but not self-healing. Fold the pin into release-1.5's slot 45 so that slot does both jobs. A v1.5.4 cluster stamped 46 then skips 1.6's slot 45 harmlessly, because the annotation is already on the objects. Renumbering release-1.5's repair to 1.6's slot 53 was the alternative and does not work: run-migrations.sh hard-fails on a missing slot file, so it would need no-op stubs for 46-52, and a cluster stamped 54 would run `seq 54 53` and skip every real 1.6 migration. The pin logic is ported into release-1.5's idiom rather than copied from main: this branch's slots are #!/bin/sh (busybox ash in the migrations image) and have no lib/cozystack-version.sh, so the helper is POSIX-clean with prefixed globals instead of `local`, and slot 45 keeps its existing inline labelled version stamp. The selector is a LABEL, app.kubernetes.io/managed-by=Helm, which Helm injects into everything it applies; meta.helm.sh/release-name is an annotation and would match nothing while reporting a clean run. Confirmed against a live v1.5 stand stamped 45: the chart's KubeadmConfigTemplate carries that label and no resource-policy, and the KubeadmConfig children CAPI spawns carry no managed-by at all, so they are correctly left alone. Fails closed but aggregates: every object that can be pinned is pinned before a non-zero return aborts ahead of the version stamp, so the Job retries the whole slot and reports all failures at once. The single per-object exception is an object that disappears between the fleet scan and the annotate — an app being deleted concurrently with the hook. Helm cannot prune what no longer exists, so that is a skip rather than a failed pre-upgrade hook blocking the upgrade. "not found" is accepted only there, never for the fleet scan, where a list never answers NotFound and accepting it would let a real failure read as an empty fleet. The SeaweedFS repair runs first and the pin second. Both fail closed ahead of the stamp and the Job retries the whole slot, so a half-completed attempt is safe either way; but on an attempt where only one half gets to run it should be the one whose failure is irreversible, and running the pin first would stop the SeaweedFS repair from being attempted at all on that pass. targetVersion stays 46 and no slot is added or renumbered. Signed-off-by: Myasnikov Daniil --- .../platform/images/migrations/migrations/45 | 30 ++- .../migrations/lib/kubeadm-keep-pin.sh | 199 ++++++++++++++++++ 2 files changed, 227 insertions(+), 2 deletions(-) create mode 100644 packages/core/platform/images/migrations/migrations/lib/kubeadm-keep-pin.sh diff --git a/packages/core/platform/images/migrations/migrations/45 b/packages/core/platform/images/migrations/migrations/45 index 8243aea577..38c11899e0 100755 --- a/packages/core/platform/images/migrations/migrations/45 +++ b/packages/core/platform/images/migrations/migrations/45 @@ -1,7 +1,26 @@ #!/bin/sh # Migration 45 --> 46 -# Repair the 1.5.0 db-split hand-over for SeaweedFS instances whose name is not -# the default `seaweedfs`. +# This slot does TWO independent jobs. Both are protective, both are idempotent, +# and neither may be reached by way of the other failing. +# +# 1. Repair the 1.5.0 db-split hand-over for SeaweedFS instances whose name is +# not the default `seaweedfs` (the original contents of this slot, below). +# 2. Pin the CAPI kubeadm bootstrap objects with helm.sh/resource-policy=keep, +# which is 1.6's OWN migration 45 folded in here — see +# lib/kubeadm-keep-pin.sh for why it has to be, and ORDER below for why it +# runs second. +# +# ORDER. The SeaweedFS repair runs first and the keep-pin second, because the two +# failure modes are not equally bad: a missed SeaweedFS hand-over loses a tenant's +# filer metadata, hence all of its S3, while a missed keep-pin gives a noisy +# broken tenant-Kubernetes worker rollover that is recoverable by hand. Both halves +# fail closed ahead of the version stamp at the bottom and the Job retries the +# whole slot, so a half-completed attempt is safe either way — but on an attempt +# where only one half gets to run, it should be the one whose failure is +# irreversible. Running the pin first would mean a pin failure stops the SeaweedFS +# repair from being attempted at all on that pass, which is strictly worse. +# +# --- 1. SeaweedFS db-split repair ------------------------------------------- # # Migration 43 performs the hand-over of Cluster/seaweedfs-db from the # -system release to the -db release introduced by the split (PR @@ -42,9 +61,16 @@ set -euo pipefail # shellcheck source=lib/seaweedfs-db-adopt.sh . "$(dirname "$0")/lib/seaweedfs-db-adopt.sh" +# shellcheck source=lib/kubeadm-keep-pin.sh +. "$(dirname "$0")/lib/kubeadm-keep-pin.sh" +# Called bare, so a non-zero return trips errexit and aborts before the stamp +# below. Both helpers report their own reason on stderr first. adopt_seaweedfs_db_clusters +# --- 2. kubeadm bootstrap keep-pin ------------------------------------------ +pin_kubeadm_bootstrap_objects + # Stamp version. # Mirror migration 42's labeled manifest. A label-less apply by the same field # manager (kubectl create | kubectl apply) would strip the diff --git a/packages/core/platform/images/migrations/migrations/lib/kubeadm-keep-pin.sh b/packages/core/platform/images/migrations/migrations/lib/kubeadm-keep-pin.sh new file mode 100644 index 0000000000..fad2a28a12 --- /dev/null +++ b/packages/core/platform/images/migrations/migrations/lib/kubeadm-keep-pin.sh @@ -0,0 +1,199 @@ +# shellcheck shell=sh +# Shared helper for pinning the CAPI kubeadm bootstrap objects with +# helm.sh/resource-policy=keep, so the 1.6 Talos worker migration cannot let Helm +# prune them while CAPI still references them. +# +# 1.6 drops KubeadmConfigTemplate from the tenant `kubernetes` app chart +# entirely — workers move to TalosConfigTemplate. On that upgrade Helm sees the +# resource present in the previous release manifest and absent from the new one, +# and issues a delete. The kubeadm-backed MachineSet is still around until CAPI +# finishes rolling workers onto Talos, so its bootstrap.configRef then points at +# a missing object: controller-manager floods with reconcile errors, and where the +# Talos image fetch from factory.talos.dev is slow or a MachineHealthCheck +# remediates, workers can hang pending with nothing to bootstrap from. Tenant +# Kubernetes only, and a noisy broken upgrade rather than data loss — but it is +# not self-healing. +# +# WHY THIS LIVES ON release-1.5 AT ALL. 1.6 ships the same pin as its own +# migration 45, and every released 1.5 (v1.5.0 through v1.5.3, all stamped +# targetVersion 45) reaches it on the way to 1.6. What breaks that is v1.5.4: the +# #3370 backport put the SeaweedFS db-split repair in release-1.5's slot 45 and +# bumped targetVersion to 46, so a v1.5.4 cluster is stamped 46 and its later 1.6 +# upgrade runs `seq 46 53` — skipping 1.6's slot 45, and with it the pin. Folding +# the pin into the slot that IS run leaves the annotation already in place by the +# time 1.6 skips it. Renumbering release-1.5's repair to 1.6's slot 53 instead is +# not an option: run-migrations.sh hard-fails on a missing slot file, so it would +# need no-op stubs for 46-52, and a cluster stamped 54 would then run `seq 54 53` +# and skip every real 1.6 migration. +# +# WHY THE managed-by SELECTOR, AND WHY A LABEL. Helm injects the +# app.kubernetes.io/managed-by=Helm LABEL into every resource it applies. +# meta.helm.sh/release-name is an ANNOTATION and cannot be used as a --selector: +# doing so silently matches nothing and the pin is applied to nothing at all, +# which looks exactly like a clean run. Verified on a live v1.5 stand stamped 45: +# the chart's KubeadmConfigTemplate carries managed-by=Helm and no +# resource-policy, while the KubeadmConfig children CAPI spawns from the template +# carry no managed-by at all — they are owned by their Machine, Helm never prunes +# them, and this selector correctly leaves them alone. +# +# FAILS CLOSED, AND AGGREGATES. Every object that can be pinned is pinned before a +# non-zero return aborts the migration ahead of its version stamp, so the Job +# retries the whole slot and reports all failures at once instead of one per +# attempt. Best-effort (returning 0 on partial failure) is not an option: +# migrations never re-run, and any KubeadmConfigTemplate left un-pinned is one +# Helm prune away from breaking a live MachineSet's bootstrap.configRef +# mid-rollover, which is recovered only by recreating the template by hand with +# the right Helm ownership metadata. +# +# Idempotent: an object already carrying keep is skipped WITHOUT a write, so +# re-running is a no-op on a cluster that has already been pinned. +# +# COST OF A KEEP THAT WAS NOT NEEDED, since `keep` is stamped on every +# Helm-managed template rather than only on those a 1.6 upgrade would prune: +# within 1.5 the chart still renders these objects and `keep` only suppresses +# DELETION, so updates are unaffected. What it does leave behind is an orphan +# where a nodeGroup is removed or the whole `kubernetes` app is deleted while its +# namespace survives. That orphan keeps its Helm ownership annotations, so a +# same-named app is adopted rather than blocked on re-install, and the asymmetry +# is the same one the SeaweedFS hand-over accepts: an orphan is cleaned up by +# hand, a pruned bootstrap reference breaks a live worker rollover. +# +# Sourced, not executed: +# . "$(dirname "$0")/lib/kubeadm-keep-pin.sh" + +_KKP_KEEP_ANNOTATION="helm.sh/resource-policy=keep" +_KKP_HELM_MANAGED_SELECTOR="app.kubernetes.io/managed-by=Helm" + +# Fully qualified so a same-named CRD in another API group can never be matched +# instead. KubeadmConfig is included for parity with 1.6's slot 45: the 1.5 chart +# renders only the template, and the managed-by selector leaves the Machine-owned +# children alone, so this is a no-op on a stock cluster and covers a Helm-managed +# KubeadmConfig if one is ever rendered. +_KKP_KINDS="kubeadmconfigtemplates.bootstrap.cluster.x-k8s.io +kubeadmconfigs.bootstrap.cluster.x-k8s.io" + +# _kkp_is_absent_err +# True when a kubectl error means the resource type simply is not served, as +# opposed to the API being unreachable, forbidden, throttled, or not yet +# established: a cluster with no CAPI bootstrap provider has nothing to pin. +# Deliberately narrow — anything unrecognised is fatal — and deliberately NOT +# shared with lib/seaweedfs-db-adopt.sh, because migration 43 sources that lib +# alone and neither may depend on the other having been loaded. +_kkp_is_absent_err() { + grep -qiE "server doesn't have a resource type|server could not find the requested resource|could not find the requested resource|no matches for kind" "$1" +} + +# _kkp_is_gone_err +# True when a PER-OBJECT kubectl error means that one object is no longer there: +# an app being deleted concurrently with this hook can take its +# KubeadmConfigTemplate away between the fleet scan and the annotate. Nothing is at +# risk in that case — Helm cannot prune an object that does not exist — so it is a +# skip, not a failure, and must not fail a pre-upgrade hook and block the platform +# upgrade over an object nobody needs any more. +# +# Separate from _kkp_is_absent_err, and wider than it by exactly one phrase, +# because "not found" must NOT be accepted for the fleet SCAN: a list never +# answers NotFound, so accepting it there would let some genuine failure read as +# an empty fleet and leave every template un-pinned. +_kkp_is_gone_err() { + _kkp_is_absent_err "$1" || grep -qiE "not found" "$1" +} + +# _kkp_pin_one +# Stamp keep on one object, counting the outcome. Always returns 0: a failure is +# recorded in _kkp_failures and reported by the caller once the whole fleet has +# been walked, so one bad object cannot abort the loop before the rest are pinned. +_kkp_pin_one() { + _kkp_p_kind="$1" + _kkp_p_ns="$2" + _kkp_p_name="$3" + + # An unreadable current value is deliberately NOT fatal here: the annotate below + # is the operation that has to succeed and it IS checked, so a failed read costs + # only a redundant write. There is no fail-open hiding in this `|| true`. + _kkp_p_current=$(kubectl get "$_kkp_p_kind" --namespace "$_kkp_p_ns" "$_kkp_p_name" \ + --output 'jsonpath={.metadata.annotations.helm\.sh/resource-policy}' 2>/dev/null) \ + || _kkp_p_current="" + + if [ "$_kkp_p_current" = "keep" ]; then + echo "$_kkp_p_kind $_kkp_p_ns/$_kkp_p_name already carries $_KKP_KEEP_ANNOTATION — nothing to do" + _kkp_skipped=$((_kkp_skipped + 1)) + return 0 + fi + + if kubectl annotate "$_kkp_p_kind" --namespace "$_kkp_p_ns" "$_kkp_p_name" \ + "$_KKP_KEEP_ANNOTATION" --overwrite >/dev/null 2>"$_kkp_err"; then + echo "Pinned $_kkp_p_kind $_kkp_p_ns/$_kkp_p_name with $_KKP_KEEP_ANNOTATION" + _kkp_patched=$((_kkp_patched + 1)) + elif _kkp_is_gone_err "$_kkp_err"; then + echo "$_kkp_p_kind $_kkp_p_ns/$_kkp_p_name disappeared between the scan and the pin — skipping" + _kkp_skipped=$((_kkp_skipped + 1)) + else + echo "WARNING: failed to pin $_kkp_p_kind $_kkp_p_ns/$_kkp_p_name:" >&2 + cat "$_kkp_err" >&2 + _kkp_failures=$((_kkp_failures + 1)) + fi + return 0 +} + +# pin_kubeadm_bootstrap_objects +# Stamp helm.sh/resource-policy=keep on every Helm-managed kubeadm bootstrap +# object. Returns non-zero (aborting the migration before its version stamp) on +# any error that is not "this resource type is not served". +pin_kubeadm_bootstrap_objects() { + _kkp_err=$(mktemp) + _kkp_items=$(mktemp) + _kkp_patched=0 + _kkp_skipped=0 + _kkp_failures=0 + + for _kkp_kind in $_KKP_KINDS; do + echo "==> Pinning Helm-managed $_kkp_kind with $_KKP_KEEP_ANNOTATION" + + # Redirecting to a file inside `if !` keeps errexit from firing so the exit + # status can be inspected. `for x in $(kubectl ...)` would not trip errexit + # either: on failure it iterates zero times and lets the caller stamp the + # version regardless, which is the fail-open this guard exists for. + if ! kubectl get "$_kkp_kind" --all-namespaces \ + --selector "$_KKP_HELM_MANAGED_SELECTOR" \ + --output 'jsonpath={range .items[*]}{.metadata.namespace}{"/"}{.metadata.name}{"\n"}{end}' \ + >"$_kkp_items" 2>"$_kkp_err"; then + if _kkp_is_absent_err "$_kkp_err"; then + echo "$_kkp_kind is not served on this cluster — nothing to pin" + continue + fi + echo "FATAL: cannot list $_kkp_kind across namespaces; refusing to stamp past an unverified fleet:" >&2 + cat "$_kkp_err" >&2 + rm -f "$_kkp_err" "$_kkp_items" + return 1 + fi + + # Read from a file, not a pipe: `kubectl ... | while read` runs the loop body + # in a subshell, so the counters would be discarded on exit and a failed pin + # would be reported as a clean run. A namespace and an object name are both + # DNS labels and cannot contain "/", so the separator is unambiguous. + while IFS= read -r _kkp_item; do + [ -n "$_kkp_item" ] || continue + _kkp_ns="${_kkp_item%%/*}" + _kkp_name="${_kkp_item#*/}" + if [ -z "$_kkp_ns" ] || [ -z "$_kkp_name" ] || [ "$_kkp_ns" = "$_kkp_item" ]; then + echo "WARNING: cannot parse '$_kkp_item' as / for $_kkp_kind — refusing to guess" >&2 + _kkp_failures=$((_kkp_failures + 1)) + continue + fi + _kkp_pin_one "$_kkp_kind" "$_kkp_ns" "$_kkp_name" + done <"$_kkp_items" + done + + rm -f "$_kkp_err" "$_kkp_items" + + echo "==> kubeadm keep-pin summary: pinned=$_kkp_patched already-pinned=$_kkp_skipped failures=$_kkp_failures" + + if [ "$_kkp_failures" -gt 0 ]; then + echo "ERROR: $_kkp_failures kubeadm bootstrap object(s) could not be pinned; refusing to stamp the version so this migration retries on the next platform upgrade." >&2 + echo "ERROR: an un-pinned KubeadmConfigTemplate can be pruned by the 1.6 kubernetes chart upgrade, breaking the bootstrap.configRef of the kubeadm-backed MachineSet that is still rolling workers over to Talos." >&2 + return 1 + fi + + return 0 +} From f9713b8c981bb817894227ead883e0c377fee1c5 Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Mon, 17 Aug 2026 14:58:00 +0500 Subject: [PATCH 45/69] test(migrations): cover the kubeadm keep-pin folded into slot 45 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Drives the pin end-to-end through the migration script, in the harness the SeaweedFS half already uses: the real slot 45 executed by path inside the migrations image's own pinned alpine base, so the interpreter under test is the busybox ash that run-migrations.sh actually gives it, with only the cluster boundary faked. The property asserted is the annotation landing on the object, never a function being callable. Thirteen cases: an unannotated Helm-managed KubeadmConfigTemplate gets pinned and the slot stamps 46; several templates are pinned each in its own namespace; an already-pinned one is skipped WITHOUT a write; zero matching objects is a clean run; a CAPI-spawned KubeadmConfig is left alone because it is not Helm-managed; a cluster with no kubeadm bootstrap provider stamps cleanly; an object that vanishes between the scan and the pin is skipped rather than fatal; a failing fleet scan and a failed pin both abort without stamping; a partial failure still pins the rest and then aborts; both halves run in one pass with the SeaweedFS hand-over first; a SeaweedFS failure aborts before the pin is attempted; and migration 43 does not pin, since the two libs must stay independently sourceable. The fake kubectl MODELS THE LABEL SELECTOR rather than ignoring it, which is what makes the characteristic form of this bug detectable. Helm injects app.kubernetes.io/managed-by=Helm as a LABEL; meta.helm.sh/release-name is an ANNOTATION and can never be a selector, so using it matches nothing, the pin silently applies to nothing, and the run reports pinned=0 failures=0 and exits 0. A fake that answered every selector identically could not tell that apart from a working pin. An ABSENT selector lists everything, exactly as kubectl does, so dropping the selector shows up as the pin stamping keep on Machine-owned objects instead of as a quiet no-op. Also rewrites the nine pre-existing SeaweedFS "this did not happen" assertions, which could not fail. POSIX and bash both exempt a !-negated pipeline from errexit — "the -e setting shall be ignored ... if the command's return value is being inverted with !" — so `! grep -q X f` runs, returns 1, and the test carries on to report success. Measured in this harness, not inferred: with the two halves of slot 45 deliberately swapped, a `! grep -q 'PIN '` assertion stayed green while the command log plainly contained the PIN line. The replacement, [ "$(grep -c X f)" -eq 0 ], puts the result inside `[`, whose non-zero status does trip errexit, and is the form already used elsewhere in the file. Their claims turn out to have been true all along, so nothing about the migrations changes; what changes is that they are now checked — with the hand-over mutated to re-annotate a Cluster that already carries keep, "idempotent: a Cluster already owned by -db AND carrying keep is left alone" now goes red where before it passed. Every case was confirmed against a deliberately broken pin before being made to pass: annotation-as-selector, no selector at all, best-effort instead of fail-closed, an unconditional write instead of read-then-skip, a vanished object treated as fatal, the halves swapped, and the pin never called. Signed-off-by: Myasnikov Daniil --- hack/migration-seaweedfs-db-adopt.bats | 298 ++++++++++++++++++- hack/testdata/migration-seaweedfs-db/kubectl | 110 ++++++- 2 files changed, 394 insertions(+), 14 deletions(-) diff --git a/hack/migration-seaweedfs-db-adopt.bats b/hack/migration-seaweedfs-db-adopt.bats index 47af1d2d27..6d8aa889cb 100644 --- a/hack/migration-seaweedfs-db-adopt.bats +++ b/hack/migration-seaweedfs-db-adopt.bats @@ -1,7 +1,20 @@ #!/usr/bin/env bats # ----------------------------------------------------------------------------- -# Unit tests for the SeaweedFS db-split hand-over shared by platform migrations -# 43 (original) and 45 (repair) — lib/seaweedfs-db-adopt.sh. +# Unit tests for the platform migrations in slots 43 and 45 — driven end-to-end +# against a fake kubectl, so what is under test is the migration scripts rather +# than their helpers in isolation. +# +# Slot 45 does TWO independent jobs on release-1.5, and both are covered here +# because they run in one script and share one version stamp: +# +# 1. lib/seaweedfs-db-adopt.sh — the SeaweedFS db-split hand-over, shared with +# migration 43. Sections 1-3 below. +# 2. lib/kubeadm-keep-pin.sh — the helm.sh/resource-policy=keep pin on the CAPI +# kubeadm bootstrap objects, which is 1.6's own migration 45 folded into this +# slot because a v1.5.4 cluster is stamped 46 and its later 1.6 upgrade runs +# `seq 46 53`, skipping 1.6's slot 45 entirely. Section 4 below. +# +# --- 1-3: the SeaweedFS hand-over --- # # The 1.5.0 split (PR #2601) moved Cluster/seaweedfs-db out of the -system # release into a new -db release. The hand-over must, before -system @@ -97,6 +110,11 @@ run_migration() { -e FAKE_LIST_FAIL="${FAKE_LIST_FAIL-}" \ -e FAKE_GET_FAIL="${FAKE_GET_FAIL-}" \ -e FAKE_ANNOTATE_FAIL="${FAKE_ANNOTATE_FAIL-}" \ + -e FAKE_KCTS="${FAKE_KCTS-}" \ + -e FAKE_KCS="${FAKE_KCS-}" \ + -e FAKE_KUBEADM_LIST_FAIL="${FAKE_KUBEADM_LIST_FAIL-}" \ + -e FAKE_KUBEADM_ANNOTATE_FAIL="${FAKE_KUBEADM_ANNOTATE_FAIL-}" \ + -e FAKE_KUBEADM_ANNOTATE_FAIL_NS="${FAKE_KUBEADM_ANNOTATE_FAIL_NS-}" \ "$ALPINE" "/migrations/$1" || _run_migration_rc=$? return "$_run_migration_rc" } @@ -116,7 +134,11 @@ prep() { : > "$FAKE_CMDLOG" export NAMESPACE=cozy-system export FAKE_CLUSTERS="" + export FAKE_KCTS="" + export FAKE_KCS="" unset FAKE_LIST_FAIL FAKE_GET_FAIL FAKE_ANNOTATE_FAIL || true + unset FAKE_KUBEADM_LIST_FAIL FAKE_KUBEADM_ANNOTATE_FAIL \ + FAKE_KUBEADM_ANNOTATE_FAIL_NS || true } # --- 1. instance name ------------------------------------------------------- @@ -217,7 +239,7 @@ tenant-named foo-db keep" run_migration 45 >"$WORK/out" 2>&1 || rc=$? cat "$WORK/out"; cat "$FAKE_CMDLOG" [ "$rc" -eq 0 ] - ! grep -q 'ANNOTATE' "$FAKE_CMDLOG" + [ "$(grep -c 'ANNOTATE' "$FAKE_CMDLOG")" -eq 0 ] grep -qF -- "STAMP 46" "$FAKE_CMDLOG" rm -rf "$WORK" } @@ -231,7 +253,7 @@ tenant-named foo-db keep" run_migration 45 >"$WORK/out" 2>&1 || rc=$? cat "$WORK/out"; cat "$FAKE_CMDLOG" [ "$rc" -eq 0 ] - ! grep -q 'ANNOTATE' "$FAKE_CMDLOG" + [ "$(grep -c 'ANNOTATE' "$FAKE_CMDLOG")" -eq 0 ] grep -qF -- "carries no meta.helm.sh/release-name" "$WORK/out" rm -rf "$WORK" } @@ -243,7 +265,7 @@ tenant-named foo-db keep" run_migration 45 >"$WORK/out" 2>&1 || rc=$? cat "$WORK/out"; cat "$FAKE_CMDLOG" [ "$rc" -eq 0 ] - ! grep -q 'ANNOTATE' "$FAKE_CMDLOG" + [ "$(grep -c 'ANNOTATE' "$FAKE_CMDLOG")" -eq 0 ] grep -qF -- "owned by unrelated release" "$WORK/out" rm -rf "$WORK" } @@ -258,7 +280,7 @@ tenant-named foo-db keep" run_migration 45 >"$WORK/out" 2>&1 || rc=$? cat "$WORK/out"; cat "$FAKE_CMDLOG" [ "$rc" -eq 0 ] - ! grep -q 'ANNOTATE' "$FAKE_CMDLOG" + [ "$(grep -c 'ANNOTATE' "$FAKE_CMDLOG")" -eq 0 ] grep -qF -- "no instance name" "$WORK/out" rm -rf "$WORK" } @@ -275,7 +297,7 @@ tenant-named foo-db keep" # Must propagate: the Job retries rather than advancing the version. [ "$rc" -ne 0 ] grep -qF -- "refusing to stamp past an unverified fleet" "$WORK/out" - ! grep -q 'STAMP' "$FAKE_CMDLOG" + [ "$(grep -c 'STAMP' "$FAKE_CMDLOG")" -eq 0 ] rm -rf "$WORK" } @@ -288,7 +310,7 @@ tenant-named foo-db keep" cat "$WORK/out"; cat "$FAKE_CMDLOG" [ "$rc" -ne 0 ] grep -qF -- "cannot read the Helm owner" "$WORK/out" - ! grep -q 'STAMP' "$FAKE_CMDLOG" + [ "$(grep -c 'STAMP' "$FAKE_CMDLOG")" -eq 0 ] rm -rf "$WORK" } @@ -300,7 +322,7 @@ tenant-named foo-db keep" run_migration 45 >"$WORK/out" 2>&1 || rc=$? cat "$WORK/out"; cat "$FAKE_CMDLOG" [ "$rc" -ne 0 ] - ! grep -q 'STAMP' "$FAKE_CMDLOG" + [ "$(grep -c 'STAMP' "$FAKE_CMDLOG")" -eq 0 ] rm -rf "$WORK" } @@ -315,7 +337,7 @@ tenant-named foo-db keep" cat "$WORK/out"; cat "$FAKE_CMDLOG" [ "$rc" -eq 0 ] grep -qF -- "resource type is not served" "$WORK/out" - ! grep -q 'ANNOTATE' "$FAKE_CMDLOG" + [ "$(grep -c 'ANNOTATE' "$FAKE_CMDLOG")" -eq 0 ] grep -qF -- "STAMP 46" "$FAKE_CMDLOG" rm -rf "$WORK" } @@ -327,7 +349,261 @@ tenant-named foo-db keep" run_migration 45 >"$WORK/out" 2>&1 || rc=$? cat "$WORK/out"; cat "$FAKE_CMDLOG" [ "$rc" -eq 0 ] - ! grep -q 'ANNOTATE' "$FAKE_CMDLOG" + [ "$(grep -c 'ANNOTATE' "$FAKE_CMDLOG")" -eq 0 ] + grep -qF -- "STAMP 46" "$FAKE_CMDLOG" + rm -rf "$WORK" +} + +# --- 4. kubeadm bootstrap keep-pin ----------------------------------------- +# +# THE THREAT. 1.6 drops KubeadmConfigTemplate from the tenant `kubernetes` chart +# (workers move to TalosConfigTemplate), so its upgrade sees the resource in the +# previous release manifest and absent from the new one and deletes it — while the +# kubeadm-backed MachineSet is still around mid-rollover with its +# bootstrap.configRef pointing at it. 1.6 guards that with its own migration 45. +# A v1.5.4 cluster is stamped 46 and runs `seq 46 53`, so it never executes that +# slot; the pin has to already be on the objects, which is what this half of +# release-1.5's slot 45 does. +# +# So the property under test is the ANNOTATION LANDING ON THE OBJECT, not a +# function being callable. Every assertion below reads the PIN records the fake +# kubectl wrote, and the fake models the label selector rather than ignoring it: +# selecting on meta.helm.sh/release-name (an annotation, and therefore never a +# valid selector) returns no rows, so the classic silent-no-op shape of this bug +# fails these tests instead of passing them. +# +# PIN, not ANNOTATE, is the fake's verb for this half — the SeaweedFS assertions +# above count ANNOTATE lines, and a shared verb would couple the two halves' +# tests to each other. +# +# "DID NOT HAPPEN" IS ASSERTED AS `[ "$(grep -c ...)" -eq 0 ]`, NEVER `! grep -q`. +# POSIX and bash both exempt a !-negated pipeline from errexit — "the -e setting +# shall be ignored ... if the command's return value is being inverted with !" — +# so `! grep -q X file` cannot fail a cozytest test no matter what the file +# contains. It reads like an assertion and is a no-op. Measured, not assumed: with +# the two halves of slot 45 deliberately swapped, a `! grep -q 'PIN '` test stayed +# green while the cmdlog plainly contained the PIN line. The counting form puts the +# result in `[`, whose non-zero status does trip errexit. + +@test "pins an unannotated Helm-managed KubeadmConfigTemplate, and stamps 46" { + prep + export FAKE_KCTS="tenant-root kubernetes-md0 - Helm" + rc=0 + run_migration 45 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -eq 0 ] + grep -qF -- "PIN kubeadmconfigtemplate tenant-root/kubernetes-md0 resource-policy=keep" "$FAKE_CMDLOG" + grep -qF -- "STAMP 46" "$FAKE_CMDLOG" + rm -rf "$WORK" +} + +@test "pins every Helm-managed KubeadmConfigTemplate, each in its own namespace" { + prep + export FAKE_KCTS="tenant-root kubernetes-md0 - Helm +tenant-root kubernetes-gpu-md1 - Helm +tenant-a other-cluster-md0 - Helm" + rc=0 + run_migration 45 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -eq 0 ] + grep -qF -- "PIN kubeadmconfigtemplate tenant-root/kubernetes-md0 resource-policy=keep" "$FAKE_CMDLOG" + grep -qF -- "PIN kubeadmconfigtemplate tenant-root/kubernetes-gpu-md1 resource-policy=keep" "$FAKE_CMDLOG" + grep -qF -- "PIN kubeadmconfigtemplate tenant-a/other-cluster-md0 resource-policy=keep" "$FAKE_CMDLOG" + [ "$(grep -c 'PIN ' "$FAKE_CMDLOG")" -eq 3 ] + rm -rf "$WORK" +} + +# Re-running must not write. The pin reads helm.sh/resource-policy first and skips +# on "keep", so an already-pinned fleet produces no annotate call at all — an +# unconditional `kubectl annotate --overwrite` would still be correct on the +# cluster but would make "no-op" unobservable, and this is the assertion that +# keeps it observable. +@test "idempotent: an already-pinned KubeadmConfigTemplate is skipped without a write" { + prep + export FAKE_KCTS="tenant-root kubernetes-md0 keep Helm" + rc=0 + run_migration 45 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -eq 0 ] + [ "$(grep -c 'PIN ' "$FAKE_CMDLOG")" -eq 0 ] + grep -qF -- "already carries helm.sh/resource-policy=keep" "$WORK/out" + grep -qF -- "already-pinned=1" "$WORK/out" + grep -qF -- "STAMP 46" "$FAKE_CMDLOG" + rm -rf "$WORK" +} + +# A cluster with no tenant Kubernetes clusters at all. Zero matching objects is a +# clean run, not a failure: the CRDs are served (CAPI is installed platform-wide) +# and the selector simply matches nothing. +@test "zero Helm-managed kubeadm objects is success, not failure" { + prep + export FAKE_KCTS="" + export FAKE_KCS="" + rc=0 + run_migration 45 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -eq 0 ] + [ "$(grep -c 'PIN ' "$FAKE_CMDLOG")" -eq 0 ] + grep -qF -- "pinned=0 already-pinned=0 failures=0" "$WORK/out" + grep -qF -- "STAMP 46" "$FAKE_CMDLOG" + rm -rf "$WORK" +} + +# The KubeadmConfig children CAPI spawns from the template are owned by their +# Machine, not by Helm: no app.kubernetes.io/managed-by label, so Helm never +# prunes them and the pin must not touch them. Pinning one would leave keep on an +# object whose lifecycle belongs to CAPI. Verified against a live v1.5 stand, +# where the spawned KubeadmConfig carries no managed-by at all. +@test "leaves a CAPI-spawned KubeadmConfig alone: it is not Helm-managed" { + prep + export FAKE_KCS="tenant-root kubernetes-md0-lw46d-z2wqn - -" + rc=0 + run_migration 45 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -eq 0 ] + [ "$(grep -c 'PIN ' "$FAKE_CMDLOG")" -eq 0 ] + grep -qF -- "STAMP 46" "$FAKE_CMDLOG" + rm -rf "$WORK" +} + +# The fail-open that IS load-bearing, mirroring the CNPG case above: a cluster +# without the CAPI bootstrap provider has nothing to pin and must not be blocked +# from upgrading. +@test "a cluster with no kubeadm bootstrap provider stamps cleanly without pinning" { + prep + export FAKE_KUBEADM_LIST_FAIL="error: the server doesn't have a resource type \"kubeadmconfigtemplates\" in group \"bootstrap.cluster.x-k8s.io\"" + rc=0 + run_migration 45 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -eq 0 ] + grep -qF -- "is not served on this cluster" "$WORK/out" + [ "$(grep -c 'PIN ' "$FAKE_CMDLOG")" -eq 0 ] grep -qF -- "STAMP 46" "$FAKE_CMDLOG" rm -rf "$WORK" } + +@test "a failing kubeadm fleet scan aborts instead of stamping past it" { + prep + export FAKE_KCTS="tenant-root kubernetes-md0 - Helm" + export FAKE_KUBEADM_LIST_FAIL="Error from server (Timeout): the server was unable to return a response in the time allotted" + rc=0 + run_migration 45 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + # Must propagate: the Job retries rather than advancing the version, because + # nothing later will pin what this pass could not see. + [ "$rc" -ne 0 ] + grep -qF -- "refusing to stamp past an unverified fleet" "$WORK/out" + [ "$(grep -c 'STAMP' "$FAKE_CMDLOG")" -eq 0 ] + rm -rf "$WORK" +} + +# The one per-object failure that is NOT a failure: an app deleted concurrently +# with this hook takes its KubeadmConfigTemplate away between the fleet scan and +# the annotate. Helm cannot prune what no longer exists, so nothing is at risk, and +# failing the pre-upgrade hook over it would block the platform upgrade on an +# object nobody needs. "not found" is accepted HERE and deliberately not for the +# fleet scan, where a list never answers NotFound and accepting it would let a real +# failure read as an empty fleet. +@test "an object that disappears between the scan and the pin is skipped, not fatal" { + prep + export FAKE_KCTS="tenant-doomed kubernetes-md0 - Helm" + export FAKE_KUBEADM_ANNOTATE_FAIL="Error from server (NotFound): kubeadmconfigtemplates.bootstrap.cluster.x-k8s.io \"kubernetes-md0\" not found" + rc=0 + run_migration 45 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -eq 0 ] + grep -qF -- "disappeared between the scan and the pin" "$WORK/out" + grep -qF -- "pinned=0 already-pinned=1 failures=0" "$WORK/out" + grep -qF -- "STAMP 46" "$FAKE_CMDLOG" + rm -rf "$WORK" +} + +@test "a failed pin aborts rather than stamping a half-pinned fleet" { + prep + export FAKE_KCTS="tenant-root kubernetes-md0 - Helm" + export FAKE_KUBEADM_ANNOTATE_FAIL="Error from server (Conflict): the object has been modified" + rc=0 + run_migration 45 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -ne 0 ] + grep -qF -- "could not be pinned" "$WORK/out" + [ "$(grep -c 'STAMP' "$FAKE_CMDLOG")" -eq 0 ] + rm -rf "$WORK" +} + +# Aggregation, and the reason it is worth having: one unpinnable object must not +# stop the others from being pinned, because the version is not stamped either way +# and the next attempt starts from the same place. The failing namespace is listed +# FIRST so a loop that aborted on the first failure would leave tenant-b unpinned +# and fail this test. +@test "a partial pin failure still pins the rest, then aborts without stamping" { + prep + export FAKE_KCTS="tenant-a broken-md0 - Helm +tenant-b healthy-md0 - Helm" + export FAKE_KUBEADM_ANNOTATE_FAIL="Error from server (Forbidden): kubeadmconfigtemplates.bootstrap.cluster.x-k8s.io is forbidden" + export FAKE_KUBEADM_ANNOTATE_FAIL_NS="tenant-a" + rc=0 + run_migration 45 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -ne 0 ] + grep -qF -- "PIN kubeadmconfigtemplate tenant-b/healthy-md0 resource-policy=keep" "$FAKE_CMDLOG" + [ "$(grep -cF -- "tenant-a/broken-md0 resource-policy=keep" "$FAKE_CMDLOG")" -eq 0 ] + grep -qF -- "pinned=1 already-pinned=0 failures=1" "$WORK/out" + [ "$(grep -c 'STAMP' "$FAKE_CMDLOG")" -eq 0 ] + rm -rf "$WORK" +} + +# --- 5. the two halves, in one slot ---------------------------------------- + +@test "runs both halves in one pass: SeaweedFS hand-over first, then the pin" { + prep + export FAKE_CLUSTERS="tenant-named foo-system -" + export FAKE_KCTS="tenant-root kubernetes-md0 - Helm" + rc=0 + run_migration 45 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -eq 0 ] + grep -qF -- "ANNOTATE tenant-named release-name=foo-db resource-policy=keep" "$FAKE_CMDLOG" + grep -qF -- "PIN kubeadmconfigtemplate tenant-root/kubernetes-md0 resource-policy=keep" "$FAKE_CMDLOG" + grep -qF -- "STAMP 46" "$FAKE_CMDLOG" + # Order is deliberate, not incidental: a missed hand-over loses a tenant's filer + # metadata, a missed pin gives a recoverable broken worker rollover. On a pass + # where only one half gets to run, it must be the irreversible one. + sw_line=$(grep -n 'ANNOTATE tenant-named' "$FAKE_CMDLOG" | head -1 | cut -d: -f1) + pin_line=$(grep -n 'PIN kubeadmconfigtemplate' "$FAKE_CMDLOG" | head -1 | cut -d: -f1) + [ "$sw_line" -lt "$pin_line" ] + rm -rf "$WORK" +} + +# The other side of that order: the SeaweedFS half failing must abort before the +# pin is attempted at all, and must not stamp. +@test "a SeaweedFS failure aborts before the pin half runs" { + prep + export FAKE_CLUSTERS="tenant-named foo-system -" + export FAKE_KCTS="tenant-root kubernetes-md0 - Helm" + export FAKE_LIST_FAIL="Error from server (Timeout): the server was unable to return a response in the time allotted" + rc=0 + run_migration 45 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -ne 0 ] + [ "$(grep -c 'PIN ' "$FAKE_CMDLOG")" -eq 0 ] + [ "$(grep -c 'STAMP' "$FAKE_CMDLOG")" -eq 0 ] + rm -rf "$WORK" +} + +# Migration 43 sources ONLY lib/seaweedfs-db-adopt.sh. The pin must not leak into +# it: a cluster running 43 is mid-upgrade to 44 and 1.6's slot 45 will still run +# for it, and the two libs must stay independently sourceable (neither may call +# the other's private helpers). +@test "migration 43 does not pin: the keep-pin belongs to slot 45 alone" { + prep + export FAKE_CLUSTERS="tenant-root seaweedfs-system -" + export FAKE_KCTS="tenant-root kubernetes-md0 - Helm" + rc=0 + run_migration 43 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -eq 0 ] + [ "$(grep -c 'PIN ' "$FAKE_CMDLOG")" -eq 0 ] + grep -qF -- "STAMP 44" "$FAKE_CMDLOG" + rm -rf "$WORK" +} diff --git a/hack/testdata/migration-seaweedfs-db/kubectl b/hack/testdata/migration-seaweedfs-db/kubectl index dba02f54bf..d005b52fcf 100755 --- a/hack/testdata/migration-seaweedfs-db/kubectl +++ b/hack/testdata/migration-seaweedfs-db/kubectl @@ -1,8 +1,9 @@ #!/bin/sh # Fake kubectl for hack/migration-seaweedfs-db-adopt.bats. It serves only the -# calls lib/seaweedfs-db-adopt.sh makes; behaviour is driven by FAKE_* env and -# every invocation is appended to $FAKE_CMDLOG so the test can assert on which -# namespaces were acted on and with which annotations. +# calls the two halves of migration 45 make — lib/seaweedfs-db-adopt.sh and +# lib/kubeadm-keep-pin.sh; behaviour is driven by FAKE_* env and every invocation +# is appended to $FAKE_CMDLOG so the test can assert on which objects were acted +# on and with which annotations. # # FAKE_CMDLOG file to append a one-line record of each call to # FAKE_CLUSTERS newline list of " " — one @@ -13,6 +14,24 @@ # FAKE_GET_FAIL non-empty => the per-namespace get exits 1 with this stderr # FAKE_ANNOTATE_FAIL non-empty => `annotate` exits 1 (a failed hand-over) # +# For the keep-pin half: +# +# FAKE_KCTS newline list of " " — one +# per KubeadmConfigTemplate. "-" models the annotation/label being +# ABSENT; managed-by is "Helm" for a chart-rendered object. +# FAKE_KCS the same, for KubeadmConfig. A CAPI-spawned child is modelled with +# managed-by "-": it is owned by its Machine, Helm never prunes it, +# and the pin must leave it alone. +# FAKE_KUBEADM_LIST_FAIL non-empty => the fleet scan for either kubeadm +# kind exits 1 with this as stderr +# FAKE_KUBEADM_ANNOTATE_FAIL non-empty => `annotate` on a kubeadm object +# exits 1 with this as stderr +# FAKE_KUBEADM_ANNOTATE_FAIL_NS restrict the above to one namespace, so a +# partial failure can be told apart from a total +# one — that is what proves the pin keeps going +# and reports every failure at the end rather +# than aborting on the first. +# # The failure knobs exist because the point of the helper is that a kubectl error # must ABORT the migration rather than let it stamp the version and never run # again. A fake that can only succeed cannot test that. @@ -39,6 +58,24 @@ field() { done } +# kubeadm_rows -- echo the row list for whichever kubeadm kind names. +# "kubeadmconfigs" is not a substring of "kubeadmconfigtemplates", so the two +# patterns cannot both match; templates is tried first regardless. +kubeadm_rows() { + case "$1" in + *kubeadmconfigtemplates*) printf '%s\n' "${FAKE_KCTS:-}" ;; + *kubeadmconfigs*) printf '%s\n' "${FAKE_KCS:-}" ;; + esac +} + +# kubeadm_short -- singular kind name for the PIN log line. +kubeadm_short() { + case "$1" in + *kubeadmconfigtemplates*) printf 'kubeadmconfigtemplate' ;; + *kubeadmconfigs*) printf 'kubeadmconfig' ;; + esac +} + case "$args" in # Version stamp: kubectl apply --filename - *"apply --filename -"*) @@ -95,6 +132,73 @@ case "$args" in log "ANNOTATE $ns release-name=${rel:-} resource-policy=${keep:-}" exit 0 ;; + + # --- kubeadm bootstrap objects: migration 45's keep-pin half -------------- + # Fleet scan for one kubeadm kind. + # + # THE LABEL SELECTOR IS MODELLED, NOT IGNORED. The pin selects on + # app.kubernetes.io/managed-by=Helm, which Helm injects as a LABEL into every + # resource it applies. meta.helm.sh/release-name is an ANNOTATION and can never + # be a selector: passing it matches nothing, the pin silently applies to nothing, + # and the run looks clean. A fake that answered every selector identically could + # not tell that apart from a working pin, so a row is returned only when the + # selector actually names its managed-by value. + *"get kubeadmconfig"*" --all-namespaces"*) + if [ -n "${FAKE_KUBEADM_LIST_FAIL:-}" ]; then + printf '%s\n' "$FAKE_KUBEADM_LIST_FAIL" >&2 + exit 1 + fi + sel=$(arg_after --selector "$@") + kubeadm_rows "$args" | while read -r ns name policy mgr; do + [ -n "${ns:-}" ] || continue + # NO --selector LISTS EVERYTHING, exactly as kubectl does. Modelling an + # absent selector as "matches nothing" would be backwards and would hide the + # over-pinning bug: dropping the selector would then read as a quiet no-op + # instead of as the pin stamping keep on Machine-owned objects whose + # lifecycle belongs to CAPI. + if [ -n "$sel" ]; then + [ "app.kubernetes.io/managed-by=${mgr:--}" = "$sel" ] || continue + fi + printf '%s/%s\n' "$ns" "$name" + done + exit 0 + ;; + + # Per-object read of helm.sh/resource-policy, which is what makes the pin skip + # an already-pinned object without a write. + *"get kubeadmconfig"*) + ns=$(arg_after --namespace "$@") + name=$(arg_after "$ns" "$@") + kubeadm_rows "$args" | while read -r n nm policy mgr; do + if [ "$n" = "$ns" ] && [ "$nm" = "$name" ]; then + [ "${policy:--}" = "-" ] || printf '%s' "$policy" + break + fi + done + exit 0 + ;; + + # The pin itself. Logged as PIN rather than ANNOTATE on purpose: the SeaweedFS + # assertions above count ANNOTATE lines, and reusing the verb would make them + # pass or fail for the other half's reasons. + *"annotate kubeadmconfig"*) + ns=$(arg_after --namespace "$@") + name=$(arg_after "$ns" "$@") + if [ -n "${FAKE_KUBEADM_ANNOTATE_FAIL:-}" ] && + { [ -z "${FAKE_KUBEADM_ANNOTATE_FAIL_NS:-}" ] || + [ "$ns" = "${FAKE_KUBEADM_ANNOTATE_FAIL_NS:-}" ]; }; then + printf '%s\n' "$FAKE_KUBEADM_ANNOTATE_FAIL" >&2 + exit 1 + fi + keep="" + for a in "$@"; do + case "$a" in + helm.sh/resource-policy=*) keep="${a#helm.sh/resource-policy=}" ;; + esac + done + log "PIN $(kubeadm_short "$args") $ns/$name resource-policy=${keep:-}" + exit 0 + ;; esac log "UNHANDLED $args" From 3311cc3510b5585078c5e6339c7570e19995c968 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Mon, 20 Jul 2026 11:44:32 +0300 Subject: [PATCH 46/69] fix(keycloak-configure): patch HelmRelease in release namespace on teardown The pre-delete teardown Job clears the HelmRelease finalizers as its final step, but targeted a hardcoded namespace that does not match the namespace where the release and its RBAC live. The Job's Role and RoleBinding are created in the release namespace and grant patch on the HelmRelease by release name, so the ServiceAccount was Forbidden to patch the HelmRelease in the hardcoded namespace. The Job errored and retried forever, the HelmRelease stuck in Terminating, and the Helm release wedged in "uninstalling", blocking any teardown or reinstall. Target the release name and namespace so the patch matches the RBAC that grants it. Add a helm-unittest pinning the rendered namespace. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin (cherry picked from commit fd568dee87cd190c36d76a89b54a5ba88925bffc) --- .../keycloak-configure/templates/delete.yaml | 2 +- .../keycloak-configure/tests/delete_test.yaml | 58 +++++++++++++++++++ 2 files changed, 59 insertions(+), 1 deletion(-) create mode 100644 packages/system/keycloak-configure/tests/delete_test.yaml diff --git a/packages/system/keycloak-configure/templates/delete.yaml b/packages/system/keycloak-configure/templates/delete.yaml index 99abd6e67e..f499c77f6d 100644 --- a/packages/system/keycloak-configure/templates/delete.yaml +++ b/packages/system/keycloak-configure/templates/delete.yaml @@ -39,7 +39,7 @@ spec: done done - kubectl patch hr keycloak-configure -n cozy-system --type=merge -p '{"metadata":{"finalizers":[]}}' + kubectl patch hr {{ .Release.Name }} -n {{ .Release.Namespace }} --type=merge -p '{"metadata":{"finalizers":[]}}' --- diff --git a/packages/system/keycloak-configure/tests/delete_test.yaml b/packages/system/keycloak-configure/tests/delete_test.yaml new file mode 100644 index 0000000000..9e740d5b83 --- /dev/null +++ b/packages/system/keycloak-configure/tests/delete_test.yaml @@ -0,0 +1,58 @@ +suite: flux teardown job patches the HelmRelease in the release namespace + +# The pre-delete teardown Job clears the HelmRelease finalizers as its last +# step. Its RBAC (Role + RoleBinding, resourceNames: [.Release.Name]) is created +# in .Release.Namespace, so the kubectl patch MUST target that same namespace and +# release name. A hardcoded namespace that differs from the install namespace +# makes the ServiceAccount Forbidden to patch the HelmRelease, the Job retries +# forever, and the Helm release wedges in "uninstalling". +release: + name: keycloak-configure + namespace: cozy-keycloak +tests: + - it: teardown patches the HelmRelease using the release name and namespace + template: templates/delete.yaml + documentSelector: + path: kind + value: Job + set: + _cluster: + root-host: example.com + kube-root-ca: "" + asserts: + - matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: kubectl patch hr keycloak-configure -n cozy-keycloak + + - it: teardown never targets a namespace other than the release namespace + template: templates/delete.yaml + documentSelector: + path: kind + value: Job + set: + _cluster: + root-host: example.com + kube-root-ca: "" + asserts: + - notMatchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: patch hr \S+ -n cozy-system + + # Prove the patch is parameterized on .Release.Name / .Release.Namespace + # rather than matching the default install namespace by coincidence. + - it: teardown tracks a non-default release name and namespace + template: templates/delete.yaml + documentSelector: + path: kind + value: Job + release: + name: custom-release + namespace: custom-ns + set: + _cluster: + root-host: example.com + kube-root-ca: "" + asserts: + - matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: kubectl patch hr custom-release -n custom-ns From 11ffa32419926d9cf69078a56dbadb11726883b2 Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Wed, 22 Jul 2026 23:25:01 +0500 Subject: [PATCH 47/69] fix(seaweedfs): make naming audit fail closed on kubectl and payload errors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit hack/seaweedfs-naming-audit.sh was fail-open: any kubectl failure produced an empty table indistinguishable from an honestly clean fleet (namespace LIST failure = zero namespaces walked, secret/pvc/sts LIST failures = zero findings, all under 2>/dev/null with no error handling). The runbook uses this output as the gate before PVC deletion, as post-deletion verification, and as an upgrade precondition, so a transient API error could green-light destroying data (#3431). Route every kubectl call through a run_kubectl helper: non-zero exit prints a FATAL line naming the failed query and propagates the code up the whole chain (enumerations restructured to capture-then-check, since an exit inside $(...) dies with the subshell). By-name GETs distinguish legitimate absence from real errors via --ignore-not-found on a separate existence check; a Secret that exists but has no decodable release payload, or one that decodes without a chart name, is corrupt state and fails loudly instead of silently dropping the tenant. Incomplete PV-age evidence now marks the generation incomplete and falls to the safe no-direction branch, so a failed GET can no longer flip OVERLAP into a wrong Step-3 deletion candidate. A successful run with zero findings still prints the same bytes as before. Tests: 14 new cozytest cases — failure injection for every LIST and by-name GET site, corrupt-payload variants, and full-output golden diffs for the success paths (verified byte-identical to the pre-change script). 25/25 green. Fixes #3431 Assisted-By: Claude Assisted-By: GPT-5 Signed-off-by: Myasnikov Daniil (cherry picked from commit 3dc6cbd4f45aaafea795daf10b7de8162b39ee04) --- hack/seaweedfs-naming-audit.bats | 369 +++++++++++++++++++++++++++++++ hack/seaweedfs-naming-audit.sh | 204 ++++++++++++++--- 2 files changed, 540 insertions(+), 33 deletions(-) diff --git a/hack/seaweedfs-naming-audit.bats b/hack/seaweedfs-naming-audit.bats index eb6356e025..424de20a35 100644 --- a/hack/seaweedfs-naming-audit.bats +++ b/hack/seaweedfs-naming-audit.bats @@ -138,3 +138,372 @@ EOF done < "$chart/rendered" rm -rf "$chart" } + +# ----------------------------------------------------------------------------- +# Fail-closed tests (issue #3431). +# +# The classification tests above mock nothing below the classifier. These drive +# the KUBECTL layer, where the fail-open bug lived: a kubectl call that failed +# used to return empty stdout, byte-identical to a genuinely clean fleet, so the +# audit printed an empty table and exited 0 -- the exact false "nothing to do" the +# operator is told to trust before deleting PVCs. They shim `kubectl` on PATH with +# a fake and assert the audit fails LOUDLY (non-zero exit + a FATAL naming the +# query) on any real error, while still treating a genuinely-absent object as +# clean. Two families: +# +# * enumerating LISTs (get ns / secret / pvc / sts) -- a failure is always fatal; +# * by-NAME GETs (a release secret in system_releases, a PVC/PV in pv_epoch) -- +# `--ignore-not-found` splits a real error (fatal) from a legitimate absence: +# an absent release revision is a clean skip; an absent PV degrades to the +# safe "cannot establish direction" note and must NOT become a deletion +# candidate (the range-narrowing flip Codex reproduced). +# +# The two golden tests guard the reverse: on a healthy cluster the output stays +# byte-for-byte what it was on origin/main, including the MIXED path that exercises +# every pv_epoch GET. +# +# The fake is a real executable on PATH, so it exercises the actual exit-status +# handling in run_kubectl and the propagation up through every caller -- a for +# loop over `$(...)` swallows the status, so this is where a regression would hide. + +# _release_blob [chart-name] -- the value kubectl returns for a Helm release +# secret's `.data.release`: base64(base64(gzip(json))). release_json base64-decodes +# twice and gunzips it, and system_releases reads the chart name from +# chart.metadata.name (as real Helm payloads carry it). Defaults to a cozy-seaweedfs +# release so the audit confirms the tenant; pass another chart name for a non- +# SeaweedFS release, or the literal EMPTY for a chartless {} payload. +_release_blob() { + _rb_name=${1:-cozy-seaweedfs} + if [ "$_rb_name" = EMPTY ]; then + _rb_json='{}' + else + _rb_json='{"name":"seaweedfs-system","chart":{"metadata":{"name":"'"$_rb_name"'"}}}' + fi + printf '%s' "$_rb_json" | gzip | base64 | base64 | tr -d '\n' +} + +# _write_fake_kubectl -- drop a fake kubectl +# into . +# one of ns | secretlist | secretget | secretget_absent | +# secret_missing_field | secret_bad_payload | pvclist | pvcget | +# sts | pvget | "" -- the single call to make behave badly. Real- +# error targets exit non-zero with a stderr message; secretget_absent +# mirrors an absent Secret (--ignore-not-found: exit 0, empty -o +# name); secret_missing_field / secret_bad_payload keep the Secret +# present but return an empty / undecodable .data.release payload. +# a PV name (pv-legacy|pv-legacy2|pv-renamed) to report as absent +# (exit 0, empty), exercising the "bound claim, PV gone" path. +# newline-separated `get pvc -o name` LIST output. +# It otherwise walks one confirmed seaweedfs-system tenant. The by-name PVC->PV and +# PV->timestamp maps are fixed here; timestamps are chosen so the two legacy PVs +# straddle the single renamed PV (true answer OVERLAP), which is what makes the +# range-narrowing flip observable. Kept POSIX and free of a column-0 `}` so +# cozytest.sh's awk converter passes the heredoc through untouched. +_write_fake_kubectl() { + # Grouped redirect (one open of the file). The closing brace is indented, so + # cozytest.sh's awk -- which only rewrites a `}` in column 0 -- leaves it and the + # heredoc alone. + { + printf '#!/bin/sh\n' + printf "FAIL='%s'\n" "$2" + printf "ABSENT_PV='%s'\n" "$3" + printf "BLOB='%s'\n" "$4" + printf "PVCS='%s'\n" "$5" + cat <<'FAKE' +verb=${1:-}; res=${2:-}; args="$*" +fail() { echo "fake kubectl: $1 (real error)" >&2; exit 1; } +if [ "$verb $res" = "get ns" ]; then + [ "$FAIL" = ns ] && fail "get ns" + printf 'namespace/tenant-test\n'; exit 0 +fi +if [ "$verb $res" = "get secret" ]; then + case "$args" in + *sh.helm.release.v1*) + # release_json now asks two questions: existence (-o name) then payload + # (jsonpath .data.release). Mirror that split so absence, real error, and + # corrupt-payload are all reachable independently. + case "$args" in + *"-o name"*) + [ "$FAIL" = secretget ] && fail "release secret existence GET" + [ "$FAIL" = secretget_absent ] && exit 0 + printf 'secret/sh.helm.release.v1.seaweedfs-system.v1\n'; exit 0 ;; + *) + [ "$FAIL" = secret_missing_field ] && exit 0 + if [ "$FAIL" = secret_bad_payload ]; then printf '@@@not-base64@@@\n'; exit 0; fi + printf '%s\n' "$BLOB"; exit 0 ;; + esac ;; + *owner=helm*) printf '1\n'; exit 0 ;; + *) [ "$FAIL" = secretlist ] && fail "namespace secret LIST" + printf 'seaweedfs-system\n'; exit 0 ;; + esac +fi +if [ "$verb $res" = "get pvc" ]; then + case "$args" in + *"-o name"*) + [ "$FAIL" = pvclist ] && fail "pvc LIST" + [ -n "$PVCS" ] && printf '%s\n' "$PVCS" + exit 0 ;; + *data1-seaweedfs-system-volume-0*) [ "$FAIL" = pvcget ] && fail "pvc GET"; printf 'pv-renamed\n'; exit 0 ;; + *data1-seaweedfs-volume-1*) [ "$FAIL" = pvcget ] && fail "pvc GET"; printf 'pv-legacy2\n'; exit 0 ;; + *data1-seaweedfs-volume-0*) [ "$FAIL" = pvcget ] && fail "pvc GET"; printf 'pv-legacy\n'; exit 0 ;; + esac + exit 0 +fi +if [ "$verb $res" = "get sts" ]; then + [ "$FAIL" = sts ] && fail "sts LIST" + exit 0 +fi +if [ "$verb $res" = "get pv" ]; then + [ "$FAIL" = pvget ] && fail "get pv" + case "$args" in + *pv-legacy2*) [ "$ABSENT_PV" = pv-legacy2 ] && exit 0; printf '2099-01-01T00:00:00Z\n'; exit 0 ;; + *pv-legacy*) [ "$ABSENT_PV" = pv-legacy ] && exit 0; printf '2020-01-01T00:00:00Z\n'; exit 0 ;; + *pv-renamed*) [ "$ABSENT_PV" = pv-renamed ] && exit 0; printf '2020-06-01T00:00:00Z\n'; exit 0 ;; + esac + exit 0 +fi +exit 0 +FAKE + } > "$1/kubectl" + chmod +x "$1/kubectl" +} + +# _pvcs_mixed -- the `get pvc -o name` LIST for a MIXED tenant: two legacy claims +# and one release-named claim, whose PV vintages truly overlap. +_pvcs_mixed() { + printf '%s\n' \ + persistentvolumeclaim/data1-seaweedfs-volume-0 \ + persistentvolumeclaim/data1-seaweedfs-volume-1 \ + persistentvolumeclaim/data1-seaweedfs-system-volume-0 +} + +# _expected_L / _expected_mixed_overlap -- golden output built with the SAME printf +# contract the script uses, so a drift in row count, text, or padding fails the diff. +_expected_L() { + printf '%-24s %-14s %-8s %s\n' NAMESPACE RELEASE CLASS NOTE + printf '%-24s %-14s %-8s %s\n' --------- ------- ----- ---- + printf '%-24s %-14s %-8s %s\n' tenant-test seaweedfs-system L 'chart-named only; the upgrade adopts it, nothing to do' +} +_expected_mixed_overlap() { + printf '%-24s %-14s %-8s %s\n' NAMESPACE RELEASE CLASS NOTE + printf '%-24s %-14s %-8s %s\n' --------- ------- ----- ---- + printf '%-24s %-14s %-8s %s\n' tenant-test seaweedfs-system MIXED 'both generations; the chart REFUSES until one is removed' + printf '%-24s %-14s %-8s %s\n' '' '' '' 'PV vintages OVERLAP => no candidate. An interrupted Step 2 re-bind looks exactly like this (both generations on original PVs). Finish Step 2 if one is in progress; otherwise escalate. Do NOT run Step 2a.' + printf '%-24s %-14s %-8s %s\n' '' '' '' 'CANDIDATE ONLY: "original" does not mean the other set is EMPTY. A duplicate that' + printf '%-24s %-14s %-8s %s\n' '' '' '' 'served writes and later crashed looks identical here. Verify emptiness before deleting.' +} + +@test "fails closed when 'kubectl get ns' fails (whole-cluster mode)" { + # No namespace args -> main enumerates namespaces itself. If that LIST fails, + # the OLD code audited zero namespaces and still printed an empty clean table. + d=$(mktemp -d) + _write_fake_kubectl "$d" ns "" "$(_release_blob)" "" + rc=0 + out=$(PATH="$d:$PATH" SEAWEEDFS_AUDIT_LIB=0 sh "$PWD/hack/seaweedfs-naming-audit.sh" 2>&1) || rc=$? + rm -rf "$d" + echo "rc=$rc"; echo "$out" + [ "$rc" -ne 0 ] + printf '%s\n' "$out" | grep -q 'FATAL' + printf '%s\n' "$out" | grep -q 'list namespaces' +} + +@test "fails closed when the namespace-wide secret LIST fails" { + # system_releases enumerates Helm releases with a namespace-wide secret LIST -- + # the exact call that timed out in the field and reported a false clean. + d=$(mktemp -d) + _write_fake_kubectl "$d" secretlist "" "$(_release_blob)" "" + rc=0 + out=$(PATH="$d:$PATH" SEAWEEDFS_AUDIT_LIB=0 sh "$PWD/hack/seaweedfs-naming-audit.sh" tenant-test 2>&1) || rc=$? + rm -rf "$d" + echo "rc=$rc"; echo "$out" + [ "$rc" -ne 0 ] + printf '%s\n' "$out" | grep -q 'FATAL' + printf '%s\n' "$out" | grep -q 'list Helm release secrets' +} + +@test "fails closed when the by-name release-secret existence GET hits a real error" { + # Codex Finding 1: system_releases uses release_json to decide whether a release + # IS SeaweedFS. Both LISTs succeed, then a transient/forbidden by-name Secret GET + # used to leave `chart` empty -> the tenant was silently skipped -> exit 0, empty + # table. A real error here must now be fatal, not a false clean. + d=$(mktemp -d) + _write_fake_kubectl "$d" secretget "" "$(_release_blob)" "" + rc=0 + out=$(PATH="$d:$PATH" SEAWEEDFS_AUDIT_LIB=0 sh "$PWD/hack/seaweedfs-naming-audit.sh" tenant-test 2>&1) || rc=$? + rm -rf "$d" + echo "rc=$rc"; echo "$out" + [ "$rc" -ne 0 ] + printf '%s\n' "$out" | grep -q 'FATAL' + printf '%s\n' "$out" | grep -q 'Helm release secret' +} + +@test "an absent release-secret revision is a clean skip, not a failure" { + # The one legitimately-empty case for release_json: the revision secret does not + # exist (pruned by Helm history limit). The existence check (--ignore-not-found -o + # name) returns empty + exit 0, so the release is simply not confirmed as + # SeaweedFS. No error, no crash -- the audit completes and reports nothing here. + d=$(mktemp -d) + _write_fake_kubectl "$d" secretget_absent "" "$(_release_blob)" "" + rc=0 + out=$(PATH="$d:$PATH" SEAWEEDFS_AUDIT_LIB=0 sh "$PWD/hack/seaweedfs-naming-audit.sh" tenant-test 2>&1) || rc=$? + rm -rf "$d" + echo "rc=$rc"; echo "$out" + [ "$rc" -eq 0 ] + [ "$(printf '%s\n' "$out" | grep -c 'FATAL')" -eq 0 ] +} + +@test "fails closed when a present release secret has no .data.release payload" { + # Codex re-review round 2: the existence check passes (Secret EXISTS), but its + # .data.release field is empty/missing. That is CORRUPT state, not an absence -- + # and because the earlier fix used --ignore-not-found -o jsonpath, which returns + # empty+0 for BOTH, it used to read as a clean skip. It must be a loud stop. + d=$(mktemp -d) + _write_fake_kubectl "$d" secret_missing_field "" "$(_release_blob)" "" + rc=0 + out=$(PATH="$d:$PATH" SEAWEEDFS_AUDIT_LIB=0 sh "$PWD/hack/seaweedfs-naming-audit.sh" tenant-test 2>&1) || rc=$? + rm -rf "$d" + echo "rc=$rc"; echo "$out" + [ "$rc" -ne 0 ] + printf '%s\n' "$out" | grep -q 'FATAL' + printf '%s\n' "$out" | grep -q 'no .data.release payload' +} + +@test "fails closed when a present release secret has an undecodable payload" { + # Existence passes, .data.release is non-empty but is not base64(base64(gzip(...))), + # so every decode fails. The old `|| return 0` converted that decode failure into + # a clean 0 return -> silent skip. A corrupt payload must be fatal. + d=$(mktemp -d) + _write_fake_kubectl "$d" secret_bad_payload "" "$(_release_blob)" "" + rc=0 + out=$(PATH="$d:$PATH" SEAWEEDFS_AUDIT_LIB=0 sh "$PWD/hack/seaweedfs-naming-audit.sh" tenant-test 2>&1) || rc=$? + rm -rf "$d" + echo "rc=$rc"; echo "$out" + [ "$rc" -ne 0 ] + printf '%s\n' "$out" | grep -q 'FATAL' + printf '%s\n' "$out" | grep -q 'not decodable' +} + +@test "fails closed when a present release secret decodes to a chartless payload" { + # Codex round 3: the payload fetches and DECODES cleanly (valid JSON {}), so the + # decode guards all pass -- but it carries no chart name. system_releases then + # extracted chart='' and silently skipped the tenant -> exit 0, clean table. A + # decoded helm.sh/release.v1 release without a chart name is corrupt: fatal. + d=$(mktemp -d) + _write_fake_kubectl "$d" "" "" "$(_release_blob EMPTY)" "" + rc=0 + out=$(PATH="$d:$PATH" SEAWEEDFS_AUDIT_LIB=0 sh "$PWD/hack/seaweedfs-naming-audit.sh" tenant-test 2>&1) || rc=$? + rm -rf "$d" + echo "rc=$rc"; echo "$out" + [ "$rc" -ne 0 ] + printf '%s\n' "$out" | grep -q 'FATAL' + printf '%s\n' "$out" | grep -q 'no chart name' +} + +@test "a present release secret for a non-SeaweedFS chart is a silent legitimate skip" { + # The counterpart the guard must NOT break: a real, well-formed release whose + # chart is simply not cozy-seaweedfs. It has a chart name (so it is not corrupt), + # but the wrong one, so it is filtered out exactly as before -- no row, no error. + d=$(mktemp -d) + _write_fake_kubectl "$d" "" "" "$(_release_blob cozy-postgres)" "" + rc=0 + out=$(PATH="$d:$PATH" SEAWEEDFS_AUDIT_LIB=0 sh "$PWD/hack/seaweedfs-naming-audit.sh" tenant-test 2>&1) || rc=$? + rm -rf "$d" + echo "rc=$rc"; echo "$out" + [ "$rc" -eq 0 ] + [ "$(printf '%s\n' "$out" | grep -c 'FATAL')" -eq 0 ] + # Not confirmed as SeaweedFS -> no classification row for the release. + [ "$(printf '%s\n' "$out" | grep -c 'seaweedfs-system')" -eq 0 ] +} + +@test "fails closed when 'kubectl get pvc' LIST fails" { + # The secret enumeration succeeds and confirms a seaweedfs-system tenant, so the + # audit reaches the per-namespace PVC LIST; a failure there must abort, not read + # as "this tenant has no claims". + d=$(mktemp -d) + _write_fake_kubectl "$d" pvclist "" "$(_release_blob)" "" + rc=0 + out=$(PATH="$d:$PATH" SEAWEEDFS_AUDIT_LIB=0 sh "$PWD/hack/seaweedfs-naming-audit.sh" tenant-test 2>&1) || rc=$? + rm -rf "$d" + echo "rc=$rc"; echo "$out" + [ "$rc" -ne 0 ] + printf '%s\n' "$out" | grep -q 'FATAL' + printf '%s\n' "$out" | grep -q 'list PVCs' +} + +@test "fails closed when 'kubectl get sts' LIST fails" { + # PVC LIST succeeds (empty), so the audit reaches the StatefulSet LIST; a + # failure there must abort rather than read as "no StatefulSets". + d=$(mktemp -d) + _write_fake_kubectl "$d" sts "" "$(_release_blob)" "" + rc=0 + out=$(PATH="$d:$PATH" SEAWEEDFS_AUDIT_LIB=0 sh "$PWD/hack/seaweedfs-naming-audit.sh" tenant-test 2>&1) || rc=$? + rm -rf "$d" + echo "rc=$rc"; echo "$out" + [ "$rc" -ne 0 ] + printf '%s\n' "$out" | grep -q 'FATAL' + printf '%s\n' "$out" | grep -q 'StatefulSets' +} + +@test "fails closed when the by-name PV GET hits a real error" { + # Codex Finding 2, error half: pv_epoch reads each bound PV's age by name. A real + # error (RBAC/timeout) must abort, not silently drop that PV from the range. + d=$(mktemp -d) + _write_fake_kubectl "$d" pvget "" "$(_release_blob)" "$(_pvcs_mixed)" + rc=0 + out=$(PATH="$d:$PATH" SEAWEEDFS_AUDIT_LIB=0 sh "$PWD/hack/seaweedfs-naming-audit.sh" tenant-test 2>&1) || rc=$? + rm -rf "$d" + echo "rc=$rc"; echo "$out" + [ "$rc" -ne 0 ] + printf '%s\n' "$out" | grep -q 'FATAL' + printf '%s\n' "$out" | grep -q "get PV '" +} + +@test "an absent PV degrades to the safe fallback, never a wrong candidate" { + # Codex Finding 2, absence half. True vintages: legacy PVs 2020 + 2099 straddle + # the single renamed PV 2020-06 => OVERLAP. Report the newest legacy PV (2099) as + # ABSENT: the observed legacy range collapses to {2020} and, unguarded, "every + # release-named PV is strictly newer" would fire -- naming the release-named set a + # deletion candidate on incomplete evidence. The generation must instead read as + # incomplete and fall to "cannot establish direction". + d=$(mktemp -d) + _write_fake_kubectl "$d" "" pv-legacy2 "$(_release_blob)" "$(_pvcs_mixed)" + rc=0 + out=$(PATH="$d:$PATH" SEAWEEDFS_AUDIT_LIB=0 sh "$PWD/hack/seaweedfs-naming-audit.sh" tenant-test 2>&1) || rc=$? + rm -rf "$d" + echo "rc=$rc"; echo "$out" + [ "$rc" -eq 0 ] + [ "$(printf '%s\n' "$out" | grep -c 'FATAL')" -eq 0 ] + printf '%s\n' "$out" | grep -q 'direction cannot be established from PV ages' + # The whole point: incomplete evidence must NOT be reported as a deletion candidate. + [ "$(printf '%s\n' "$out" | grep -c 'candidate duplicate')" -eq 0 ] +} + +@test "success path (L) is byte-identical to the expected fixture" { + # A single chart-named claim, no release-named one, no StatefulSets => L, adopt in + # place. Full-output compare, so an extra row / warning / duplicate line fails. + d=$(mktemp -d) + _write_fake_kubectl "$d" "" "" "$(_release_blob)" "persistentvolumeclaim/data1-seaweedfs-volume-0" + rc=0 + out=$(PATH="$d:$PATH" SEAWEEDFS_AUDIT_LIB=0 sh "$PWD/hack/seaweedfs-naming-audit.sh" tenant-test 2>&1) || rc=$? + printf '%s\n' "$out" > "$d/got" + _expected_L > "$d/want" + echo "rc=$rc"; echo "--- got ---"; cat "$d/got"; echo "--- want ---"; cat "$d/want" + [ "$rc" -eq 0 ] + diff "$d/want" "$d/got" + rm -rf "$d" +} + +@test "success path (MIXED/overlap) is byte-identical and exercises pv_epoch" { + # Both generations present with all PVs readable => the MIXED path runs every + # pv_epoch GET and lands on OVERLAP. Full-output compare against the golden. + d=$(mktemp -d) + _write_fake_kubectl "$d" "" "" "$(_release_blob)" "$(_pvcs_mixed)" + rc=0 + out=$(PATH="$d:$PATH" SEAWEEDFS_AUDIT_LIB=0 sh "$PWD/hack/seaweedfs-naming-audit.sh" tenant-test 2>&1) || rc=$? + printf '%s\n' "$out" > "$d/got" + _expected_mixed_overlap > "$d/want" + echo "rc=$rc"; echo "--- got ---"; cat "$d/got"; echo "--- want ---"; cat "$d/want" + [ "$rc" -eq 0 ] + diff "$d/want" "$d/got" + rm -rf "$d" +} diff --git a/hack/seaweedfs-naming-audit.sh b/hack/seaweedfs-naming-audit.sh index a1e28c4c41..b01138a276 100755 --- a/hack/seaweedfs-naming-audit.sh +++ b/hack/seaweedfs-naming-audit.sh @@ -44,7 +44,15 @@ # # Usage: hack/seaweedfs-naming-audit.sh [namespace...] # KUBECONFIG= hack/seaweedfs-naming-audit.sh -# Exit: 0 always (audit only). Nothing is mutated. +# Exit: 0 the audit RAN TO COMPLETION; the table is the whole answer +# (an empty table then means a genuinely clean fleet). +# non-zero a kubectl query FAILED, so the audit is INCOMPLETE and the +# table must NOT be trusted -- a message naming the failed query +# is on stderr. This is fail-CLOSED by design (see run_kubectl): +# the output gates a destructive runbook step, so an unreachable +# API aborts loudly rather than printing an empty table that +# reads identically to "nothing to do". +# Nothing is ever mutated on either path -- every kubectl call is read-only. # # POSIX sh, deliberately. hack/seaweedfs-naming-audit.bats sources this file, and # cozytest.sh sources the converted test into its own /bin/sh — which is dash on @@ -55,6 +63,48 @@ # particular is not POSIX and dash 0.5.12 (Ubuntu) rejects it outright. set -u +# run_kubectl -- run ONE read-only kubectl query +# fail-CLOSED and print its stdout. This audit is the documented gate in front of +# a destructive runbook step (deleting a tenant's PVCs), so a query that FAILS +# must never be mistaken for a query that found nothing: an enumerating LIST +# returns empty stdout on a timeout or an RBAC denial, byte-identical to the same +# LIST succeeding on a genuinely clean fleet, and the old `2>/dev/null` on each +# call turned an unreachable API into a silent false "nothing to do". Every +# enumeration is routed through here so any failure is loud and terminal instead. +# +# On success prints kubectl's stdout verbatim -- which may legitimately be empty, +# because a LIST that matched nothing is honestly clean; keeping that case +# distinct from failure is the whole point. On a non-zero exit prints the failed +# query to stderr and returns that exit code; kubectl's own stderr stays on fd 2 +# and reaches the operator unfiltered. Callers MUST propagate the non-zero status +# (`|| return`/`|| exit`) -- a bare `for x in $(run_kubectl ...)` would swallow +# it, because a for-loop ignores the exit status of its word-list command, and an +# `exit` inside the `$(...)` subshell would only leave that subshell. +run_kubectl() { + _rk_what="$1"; shift + if _rk_out=$(kubectl "$@"); then + printf '%s' "$_rk_out" + return 0 + else + _rk_rc=$? + printf 'seaweedfs-naming-audit: FATAL: %s failed (kubectl %s -> exit %s). Refusing to report a clean fleet on an unreachable API.\n' \ + "$_rk_what" "$*" "$_rk_rc" >&2 + return "$_rk_rc" + fi +} + +# audit_fatal -- the counterpart to run_kubectl for corruption the API +# call itself did NOT report: the query succeeded, but what it returned is +# impossible for a healthy object (a helm.sh/release.v1 Secret with no decodable +# release payload). Same fail-CLOSED contract -- print a FATAL line naming the +# object to stderr and return non-zero; callers MUST propagate it. A silent skip +# here would be the identical false-clean this whole script guards against, just +# one field deeper than an unreachable API. +audit_fatal() { + printf 'seaweedfs-naming-audit: FATAL: %s Refusing to report a clean fleet on corrupt state.\n' "$1" >&2 + return 1 +} + # renamed_volume_prefix -- reconstruct the name 4.31 gives the volume # component of . Mirrors seaweedfs.fullname + seaweedfs.componentName # (see packages/system/seaweedfs/templates/_naming.tpl, which the chart's guard @@ -71,18 +121,51 @@ renamed_volume_prefix() { printf '%s-volume' "$(printf '%s' "$full" | cut -c1-56 | sed 's/-$//')" } -# release_json [rev] -- decode a Helm release secret. -# Helm stores base64(gzip(json)) in Secret.data.release, and Kubernetes base64s -# the data value again, hence the two decodes. +# release_json [rev] -- decode a Helm release secret's payload, or +# "" when that revision's secret does not exist. Helm stores base64(gzip(json)) in +# Secret.data.release, and Kubernetes base64s the data value again, hence the two +# decodes. +# +# This is a by-NAME GET, but it is NOT best-effort: system_releases decides whether +# a release IS SeaweedFS from the result, so a swallowed failure silently drops a +# real tenant -- the false-clean this script exists to prevent. EXISTENCE and +# EXTRACTION are therefore two separate questions, because `--ignore-not-found +# -o jsonpath` cannot tell them apart (it returns empty + exit 0 both for an absent +# Secret AND for a present Secret whose .data.release is missing): +# +# 1. Existence, via `--ignore-not-found -o name`. Empty + exit 0 = the revision +# was pruned by Helm's history limit -- a legitimate absence; return "" and +# let rev1_scheme / first_deployed treat it as "pruned". A real error (RBAC, +# timeout, apiserver down) stays non-zero and run_kubectl makes it fatal. +# 2. The Secret EXISTS, so its payload MUST decode. A helm.sh/release.v1 Secret +# with no .data.release, a payload that is not base64(base64(gzip(...))), or an +# empty JSON after decoding is CORRUPT state, not an absence -- and here that +# is safety-critical, so every such anomaly is audit_fatal, never a silent skip +# (contrast pv_epoch, where an empty field is a benign "no evidence"). +# +# The extraction GET drops --ignore-not-found deliberately: the Secret existed a +# moment ago, so a NotFound now is a mid-audit deletion race, and failing closed on +# it is correct. release_json() { - kubectl get secret -n "$1" "sh.helm.release.v1.$2.v${3:-1}" -o jsonpath='{.data.release}' 2>/dev/null \ - | base64 -d 2>/dev/null | base64 -d 2>/dev/null | gunzip 2>/dev/null + _rj_secret="sh.helm.release.v1.$2.v${3:-1}" + _rj_exists=$(run_kubectl "check Helm release secret '$_rj_secret' in namespace '$1'" \ + get secret -n "$1" "$_rj_secret" --ignore-not-found -o name) || return $? + [ -n "$_rj_exists" ] || return 0 + _rj_field=$(run_kubectl "read .data.release of secret '$_rj_secret' in namespace '$1'" \ + get secret -n "$1" "$_rj_secret" -o jsonpath='{.data.release}') || return $? + [ -n "$_rj_field" ] || { audit_fatal "secret '$_rj_secret' in namespace '$1' exists but has no .data.release payload (corrupt Helm release)."; return 1; } + _rj_json=$(printf '%s' "$_rj_field" | base64 -d 2>/dev/null | base64 -d 2>/dev/null | gunzip 2>/dev/null) \ + || { audit_fatal "secret '$_rj_secret' in namespace '$1': .data.release is not decodable base64(base64(gzip(json))) (corrupt Helm release)."; return 1; } + [ -n "$_rj_json" ] || { audit_fatal "secret '$_rj_secret' in namespace '$1': release payload decoded to nothing (corrupt Helm release)."; return 1; } + printf '%s' "$_rj_json" } # revisions -- retained revision numbers, oldest first. revisions() { - kubectl get secret -n "$1" -l "name=$2,owner=helm" \ - -o jsonpath='{range .items[*]}{.metadata.labels.version}{"\n"}{end}' 2>/dev/null | sort -n + _rev_out=$(run_kubectl "list Helm revision secrets for '$2' in namespace '$1'" \ + get secret -n "$1" -l "name=$2,owner=helm" \ + -o jsonpath='{range .items[*]}{.metadata.labels.version}{"\n"}{end}') || return $? + printf '%s\n' "$_rev_out" | sort -n } # system_releases -- the SeaweedFS -system Helm releases in a namespace. @@ -92,12 +175,29 @@ revisions() { # unrelated release in a namespace that happens to run SeaweedFS would be reported # as a SeaweedFS tenant. Confirm the chart. system_releases() { - for rel in $(kubectl get secret -n "$1" \ - -o jsonpath='{range .items[?(@.type=="helm.sh/release.v1")]}{.metadata.labels.name}{"\n"}{end}' 2>/dev/null \ - | grep -E -- '-system$' | sort -u); do - for rev in $(revisions "$1" "$rel"); do - chart=$(release_json "$1" "$rel" "$rev" | sed -n 's/.*"name":"\(cozy-seaweedfs\)".*/\1/p' | head -1) - if [ -n "$chart" ]; then printf '%s\n' "$rel"; fi + _sr_secrets=$(run_kubectl "list Helm release secrets in namespace '$1'" \ + get secret -n "$1" \ + -o jsonpath='{range .items[?(@.type=="helm.sh/release.v1")]}{.metadata.labels.name}{"\n"}{end}') || return $? + for rel in $(printf '%s\n' "$_sr_secrets" | grep -E -- '-system$' | sort -u); do + _sr_revs=$(revisions "$1" "$rel") || return $? + for rev in $_sr_revs; do + _sr_json=$(release_json "$1" "$rel" "$rev") || return $? + # Empty "" here means the revision secret was pruned (release_json's absence + # path) -- a legitimate skip. A NON-empty payload, however, was fetched and + # decoded successfully, so it MUST name its chart: a helm.sh/release.v1 release + # always carries chart.metadata.name. A payload without one ({} , or any valid + # JSON lacking it) is corrupt/unexpected, and since "no chart name" is + # indistinguishable downstream from "not SeaweedFS", silently skipping it is + # the very false-clean this guard exists to stop -- so it is FATAL. A present + # but different chart name is a real, non-SeaweedFS release and stays a + # legitimate skip, filtered exactly as before by the cozy-seaweedfs test. + # ("chart" is the release struct's only such key -- vendored subcharts nest + # under "dependencies" -- so this match is unambiguous.) + if [ -n "$_sr_json" ]; then + _sr_chart=$(printf '%s' "$_sr_json" | sed -n 's/.*"chart":{"metadata":{"name":"\([^"]*\)".*/\1/p' | head -1) + [ -n "$_sr_chart" ] || { audit_fatal "secret 'sh.helm.release.v1.$rel.v$rev' in namespace '$1' decoded to a payload with no chart name (corrupt Helm release)."; return 1; } + if [ "$_sr_chart" = cozy-seaweedfs ]; then printf '%s\n' "$rel"; fi + fi break done done @@ -106,28 +206,52 @@ system_releases() { # first_deployed -- epoch seconds of the release's first install, # from any retained revision (the field is identical on all of them). first_deployed() { - for rev in $(revisions "$1" "$2"); do - ts=$(release_json "$1" "$2" "$rev" | sed -n 's/.*"first_deployed":"\([^"]*\)".*/\1/p' | head -1) + _fd_revs=$(revisions "$1" "$2") || return $? + for rev in $_fd_revs; do + _fd_json=$(release_json "$1" "$2" "$rev") || return $? + ts=$(printf '%s' "$_fd_json" | sed -n 's/.*"first_deployed":"\([^"]*\)".*/\1/p' | head -1) if [ -n "$ts" ]; then date -u -d "$(printf '%s' "$ts" | cut -c1-19)" +%s 2>/dev/null; return; fi done } # rev1_scheme -- "legacy" | "renamed" | "" (revision 1 pruned). rev1_scheme() { - m=$(release_json "$1" "$2" 1) + m=$(release_json "$1" "$2" 1) || return $? [ -n "$m" ] || return 0 if printf '%s' "$m" | grep -q 'name: seaweedfs-master'; then printf 'legacy' elif printf '%s' "$m" | grep -qE "name: $2(-seaweedfs)?-master"; then printf 'renamed' fi } -# pv_epoch -- creation epoch of the PV the claim is BOUND to. +# pv_epoch -- creation epoch of the PV the claim is BOUND to, or "" when +# there is no usable PV age. Two by-NAME GETs. +# +# A real API error must abort (an epoch silently missing from a range can invert +# the strict-newer comparison and name the WRONG deletion candidate), so both GETs +# go through run_kubectl. But UNLIKE release_json, an EMPTY field here is NOT +# corruption and is NOT fatal -- because the consequence differs. When release_json +# returns empty for a present Secret the tenant is dropped from the report: a +# false-clean. When pv_epoch returns "" the claim merely contributes no epoch, +# which marks its whole generation INCOMPLETE (see audit_ns) and forces the safe +# "direction cannot be established -> classify by hand" branch. That degradation is +# conservative by construction: it never yields a false-clean and never names a +# candidate. So the two empty-field cases are deliberately kept benign: +# * PVC .spec.volumeName empty -- a Pending / unbound claim, a routine state; +# * PV .metadata.creationTimestamp empty -- shouldn't happen (the apiserver always +# stamps it), but if it ever did the only effect is one missing epoch, i.e. the +# same safe incomplete fallback, so a hard stop is not warranted here. +# `--ignore-not-found` keeps a genuinely-absent PVC/PV (NotFound) on that same +# benign path rather than turning it into a run_kubectl failure, and the final +# `|| return 0` keeps an unparseable timestamp there too; only a real run_kubectl +# error propagates. pv_epoch() { - pv=$(kubectl get pvc -n "$1" "$2" -o jsonpath='{.spec.volumeName}' 2>/dev/null) + pv=$(run_kubectl "get PVC '$2' in namespace '$1'" \ + get pvc -n "$1" "$2" --ignore-not-found -o jsonpath='{.spec.volumeName}') || return $? [ -n "$pv" ] || return 0 - t=$(kubectl get pv "$pv" -o jsonpath='{.metadata.creationTimestamp}' 2>/dev/null) + t=$(run_kubectl "get PV '$pv' (bound by PVC '$2' in namespace '$1')" \ + get pv "$pv" --ignore-not-found -o jsonpath='{.metadata.creationTimestamp}') || return $? [ -n "$t" ] || return 0 - date -u -d "$(printf '%s' "$t" | cut -c1-19)" +%s 2>/dev/null + date -u -d "$(printf '%s' "$t" | cut -c1-19)" +%s 2>/dev/null || return 0 } # classify_mixed_direction -- direction of a MIXED @@ -153,17 +277,21 @@ classify_mixed_direction() { audit_ns() { ns="$1" - for rel in $(system_releases "$ns"); do + _rels=$(system_releases "$ns") || return $? + for rel in $_rels; do prefix=$(renamed_volume_prefix "$rel") legacy_pvcs=""; renamed_pvcs="" - for pvc in $(kubectl get pvc -n "$ns" -o name 2>/dev/null | sed 's|persistentvolumeclaim/||'); do + _pvcs=$(run_kubectl "list PVCs in namespace '$ns'" get pvc -n "$ns" -o name) || return $? + for pvc in $(printf '%s\n' "$_pvcs" | sed 's|persistentvolumeclaim/||'); do case "$pvc" in "data1-${prefix}"*) renamed_pvcs="$renamed_pvcs $pvc" ;; data1-seaweedfs-volume*) legacy_pvcs="$legacy_pvcs $pvc" ;; esac done legacy_sts=""; renamed_sts="" - for sts in $(kubectl get sts -n "$ns" -l app.kubernetes.io/name=seaweedfs -o name 2>/dev/null | sed 's|statefulset.apps/||'); do + _sts_list=$(run_kubectl "list SeaweedFS StatefulSets in namespace '$ns'" \ + get sts -n "$ns" -l app.kubernetes.io/name=seaweedfs -o name) || return $? + for sts in $(printf '%s\n' "$_sts_list" | sed 's|statefulset.apps/||'); do case "$sts" in "${prefix}"*) renamed_sts="$renamed_sts $sts" ;; seaweedfs-volume*) legacy_sts="$legacy_sts $sts" ;; @@ -184,21 +312,30 @@ audit_ns() { fi # Both generations. Report which is ORIGINAL from durable evidence. - fd=$(first_deployed "$ns" "$rel") - scheme=$(rev1_scheme "$ns" "$rel") + fd=$(first_deployed "$ns" "$rel") || return $? + scheme=$(rev1_scheme "$ns" "$rel") || return $? printf '%-24s %-14s %-8s %s\n' "$ns" "$rel" "MIXED" "both generations; the chart REFUSES until one is removed" if [ -n "$scheme" ]; then printf '%-24s %-14s %-8s revision 1 was installed with the %s names => the %s generation is ORIGINAL\n' \ "" "" "" "$scheme" "$scheme" fi - lmin=""; lmax=""; rmin=""; rmax="" + # The direction rule is "EVERY PV of one generation strictly newer than every + # PV of the other". That holds only if we saw EVERY bound PV: a claim whose PV + # age we could not read (unbound, or PV gone) leaves the range unbounded on one + # side, and a silently-narrowed range can flip OVERLAP into a confident (wrong) + # candidate. So a generation with any unreadable claim is INCOMPLETE + # and forces the safe "cannot establish" branch -- distinct from a real API + # error, which pv_epoch has already turned into a hard failure above. + lmin=""; lmax=""; rmin=""; rmax=""; l_incomplete=0; r_incomplete=0 for p in $legacy_pvcs; do - e=$(pv_epoch "$ns" "$p"); [ -n "$e" ] || continue + e=$(pv_epoch "$ns" "$p") || return $? + if [ -z "$e" ]; then l_incomplete=1; continue; fi { [ -z "$lmin" ] || [ "$e" -lt "$lmin" ]; } && lmin=$e { [ -z "$lmax" ] || [ "$e" -gt "$lmax" ]; } && lmax=$e done for p in $renamed_pvcs; do - e=$(pv_epoch "$ns" "$p"); [ -n "$e" ] || continue + e=$(pv_epoch "$ns" "$p") || return $? + if [ -z "$e" ]; then r_incomplete=1; continue; fi { [ -z "$rmin" ] || [ "$e" -lt "$rmin" ]; } && rmin=$e { [ -z "$rmax" ] || [ "$e" -gt "$rmax" ]; } && rmax=$e done @@ -206,7 +343,7 @@ audit_ns() { printf '%-24s %-14s %-8s oldest PV vs first_deployed: chart-named +%ss, release-named +%ss (context only, not the rule)\n' \ "" "" "" "$((lmin - fd))" "$((rmin - fd))" fi - if [ -n "$lmin" ] && [ -n "$rmin" ]; then + if [ -n "$lmin" ] && [ -n "$rmin" ] && [ "$l_incomplete" = 0 ] && [ "$r_incomplete" = 0 ]; then case $(classify_mixed_direction "$lmin" "$lmax" "$rmin" "$rmax") in legacy-original) printf '%-24s %-14s %-8s every release-named PV is strictly newer => chart-named is ORIGINAL, the release-named set is the candidate duplicate (Step 3)\n' "" "" "" ;; @@ -216,7 +353,7 @@ audit_ns() { printf '%-24s %-14s %-8s PV vintages OVERLAP => no candidate. An interrupted Step 2 re-bind looks exactly like this (both generations on original PVs). Finish Step 2 if one is in progress; otherwise escalate. Do NOT run Step 2a.\n' "" "" "" ;; esac else - printf '%-24s %-14s %-8s a generation has no bound PVs (Pending claims, or StatefulSets only) => direction cannot be established from PV ages. Resolve the Pending claims or classify by hand.\n' "" "" "" + printf '%-24s %-14s %-8s direction cannot be established from PV ages: a generation has no bound PVs (Pending/unbound claims, or StatefulSets only) or a bound PV age could not be read. Resolve those claims or classify by hand. Do NOT run Step 2a.\n' "" "" "" fi printf '%-24s %-14s %-8s CANDIDATE ONLY: "original" does not mean the other set is EMPTY. A duplicate that\n' "" "" "" printf '%-24s %-14s %-8s served writes and later crashed looks identical here. Verify emptiness before deleting.\n' "" "" "" @@ -227,9 +364,10 @@ main() { printf '%-24s %-14s %-8s %s\n' NAMESPACE RELEASE CLASS NOTE printf '%-24s %-14s %-8s %s\n' --------- ------- ----- ---- if [ "$#" -gt 0 ]; then - for ns in "$@"; do audit_ns "$ns"; done + for ns in "$@"; do audit_ns "$ns" || exit $?; done else - for ns in $(kubectl get ns -o name 2>/dev/null | sed 's|namespace/||'); do audit_ns "$ns"; done + _ns_list=$(run_kubectl 'list namespaces (whole-cluster mode)' get ns -o name) || exit $? + for ns in $(printf '%s\n' "$_ns_list" | sed 's|namespace/||'); do audit_ns "$ns" || exit $?; done fi } From 6b7024590b78791ccc99cf489e4df70d0278d6c7 Mon Sep 17 00:00:00 2001 From: IvanHunters Date: Wed, 5 Aug 2026 15:04:35 +0300 Subject: [PATCH 48/69] fix(kubeovn-webhook): reload serving certificate on cert-manager renewal The webhook loaded its TLS key pair once at startup and served it as a static tls.Config.Certificates entry for the lifetime of the process. A cert-manager renewal of the backing Secret was never picked up by a running pod, so once the certificate expired (about a year after install) the pod kept presenting the expired certificate. The kube-apiserver's TLS call to the webhook then failed and, because the MutatingWebhookConfiguration uses failurePolicy: Fail, every pod creation in tenant namespaces was rejected (including virt-launcher pods, blocking all VMIs). Serve the certificate through a reloading tls.Config.GetCertificate callback that re-reads the key pair when the mounted files change and keeps serving the last good certificate on a failed reload. Add a regression test that fails against the static implementation and passes once the certificate is reloaded. Signed-off-by: IvanHunters (cherry picked from commit 44d9f534a9bef729a9650b61c929828236b2a510) --- .../images/kubeovn-webhook/cert.go | 97 ++++++++++ .../images/kubeovn-webhook/cert_test.go | 181 ++++++++++++++++++ .../images/kubeovn-webhook/main.go | 11 +- 3 files changed, 282 insertions(+), 7 deletions(-) create mode 100644 packages/system/kubeovn-webhook/images/kubeovn-webhook/cert.go create mode 100644 packages/system/kubeovn-webhook/images/kubeovn-webhook/cert_test.go diff --git a/packages/system/kubeovn-webhook/images/kubeovn-webhook/cert.go b/packages/system/kubeovn-webhook/images/kubeovn-webhook/cert.go new file mode 100644 index 0000000000..9e15ec302b --- /dev/null +++ b/packages/system/kubeovn-webhook/images/kubeovn-webhook/cert.go @@ -0,0 +1,97 @@ +package main + +import ( + "crypto/tls" + "fmt" + "log" + "os" + "sync" + "time" +) + +// certReloader keeps the webhook serving certificate in sync with the key pair +// files written by cert-manager. +// +// The certificate is served through tls.Config.GetCertificate, so a cert-manager +// renewal of the backing Secret is picked up by the running process without a +// restart. Without this, the key pair is read exactly once at startup and cached +// for the lifetime of the process: after the certificate expires (roughly one year +// after install) the pod keeps presenting the expired certificate, the +// kube-apiserver's TLS call to the webhook fails, and because the +// MutatingWebhookConfiguration uses failurePolicy: Fail, every pod creation in +// tenant namespaces is rejected. +type certReloader struct { + certFile string + keyFile string + + mu sync.RWMutex + cert *tls.Certificate + modTime time.Time +} + +// newCertReloader loads the initial key pair and returns a reloader for it. +func newCertReloader(certFile, keyFile string) (*certReloader, error) { + cr := &certReloader{certFile: certFile, keyFile: keyFile} + if err := cr.reload(); err != nil { + return nil, err + } + return cr, nil +} + +// reload reads the key pair from disk and atomically swaps the cached certificate. +func (cr *certReloader) reload() error { + cert, err := tls.LoadX509KeyPair(cr.certFile, cr.keyFile) + if err != nil { + return err + } + + var modTime time.Time + if fi, statErr := os.Stat(cr.certFile); statErr == nil { + modTime = fi.ModTime() + } + + cr.mu.Lock() + cr.cert = &cert + cr.modTime = modTime + cr.mu.Unlock() + return nil +} + +// changed reports whether the certificate file was modified since the last load. +func (cr *certReloader) changed() bool { + fi, err := os.Stat(cr.certFile) + if err != nil { + return false + } + cr.mu.RLock() + defer cr.mu.RUnlock() + return fi.ModTime().After(cr.modTime) +} + +// GetCertificate is a tls.Config.GetCertificate callback. It reloads the key pair +// when the file changes and keeps serving the last good certificate if a reload +// fails (for example a torn read while the mounted Secret is being updated). +func (cr *certReloader) GetCertificate(*tls.ClientHelloInfo) (*tls.Certificate, error) { + if cr.changed() { + if err := cr.reload(); err != nil { + log.Printf("certificate reload failed, keeping previous certificate: %v", err) + } + } + + cr.mu.RLock() + defer cr.mu.RUnlock() + if cr.cert == nil { + return nil, fmt.Errorf("no certificate loaded") + } + return cr.cert, nil +} + +// newReloadingTLSConfig builds a tls.Config that serves the key pair via a +// certReloader, so cert-manager renewals are honoured without a restart. +func newReloadingTLSConfig(certFile, keyFile string) (*tls.Config, error) { + cr, err := newCertReloader(certFile, keyFile) + if err != nil { + return nil, err + } + return &tls.Config{GetCertificate: cr.GetCertificate}, nil +} diff --git a/packages/system/kubeovn-webhook/images/kubeovn-webhook/cert_test.go b/packages/system/kubeovn-webhook/images/kubeovn-webhook/cert_test.go new file mode 100644 index 0000000000..1a670e4fd8 --- /dev/null +++ b/packages/system/kubeovn-webhook/images/kubeovn-webhook/cert_test.go @@ -0,0 +1,181 @@ +package main + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "math/big" + "os" + "path/filepath" + "testing" + "time" +) + +// genSelfSigned returns a PEM-encoded self-signed cert/key pair carrying the given +// serial number, so tests can tell two generations of the certificate apart. +func genSelfSigned(t *testing.T, serial int64) (certPEM, keyPEM []byte) { + t.Helper() + + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatalf("generate key: %v", err) + } + + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(serial), + Subject: pkix.Name{CommonName: "kube-ovn-webhook-test"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + DNSNames: []string{"localhost"}, + BasicConstraintsValid: true, + } + + der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) + if err != nil { + t.Fatalf("create certificate: %v", err) + } + keyDER, err := x509.MarshalPKCS8PrivateKey(key) + if err != nil { + t.Fatalf("marshal key: %v", err) + } + + certPEM = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}) + keyPEM = pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: keyDER}) + return certPEM, keyPEM +} + +// writeKeyPair writes the cert/key files and bumps their mtime strictly forward, +// mirroring how cert-manager replaces the mounted Secret on renewal. +func writeKeyPair(t *testing.T, certFile, keyFile string, certPEM, keyPEM []byte, mtime time.Time) { + t.Helper() + if err := os.WriteFile(certFile, certPEM, 0o600); err != nil { + t.Fatalf("write cert: %v", err) + } + if err := os.WriteFile(keyFile, keyPEM, 0o600); err != nil { + t.Fatalf("write key: %v", err) + } + if err := os.Chtimes(certFile, mtime, mtime); err != nil { + t.Fatalf("chtimes cert: %v", err) + } + if err := os.Chtimes(keyFile, mtime, mtime); err != nil { + t.Fatalf("chtimes key: %v", err) + } +} + +// servedSerial completes a TLS handshake against addr and returns the serial number +// of the leaf certificate the server actually presented. +func servedSerial(t *testing.T, addr string) int64 { + t.Helper() + conn, err := tls.Dial("tcp", addr, &tls.Config{InsecureSkipVerify: true}) //nolint:gosec // test-only, inspecting the served cert + if err != nil { + t.Fatalf("dial: %v", err) + } + defer conn.Close() + certs := conn.ConnectionState().PeerCertificates + if len(certs) == 0 { + t.Fatalf("server presented no certificate") + } + return certs[0].SerialNumber.Int64() +} + +// TestReloadingTLSConfigServesRenewedCertificate is the regression test for the +// cert-manager renewal not being picked up: it fails against the old code that +// loads the key pair once into tls.Config.Certificates, and passes once the +// certificate is served via a reloading GetCertificate callback. +func TestReloadingTLSConfigServesRenewedCertificate(t *testing.T) { + dir := t.TempDir() + certFile := filepath.Join(dir, "tls.crt") + keyFile := filepath.Join(dir, "tls.key") + + certA, keyA := genSelfSigned(t, 1) + writeKeyPair(t, certFile, keyFile, certA, keyA, time.Now().Add(-2*time.Second)) + + tlsConfig, err := newReloadingTLSConfig(certFile, keyFile) + if err != nil { + t.Fatalf("newReloadingTLSConfig: %v", err) + } + + ln, err := tls.Listen("tcp", "127.0.0.1:0", tlsConfig) + if err != nil { + t.Fatalf("listen: %v", err) + } + defer ln.Close() + go func() { + for { + conn, err := ln.Accept() + if err != nil { + return + } + go func() { + if tc, ok := conn.(*tls.Conn); ok { + _ = tc.Handshake() + } + conn.Close() + }() + } + }() + + addr := ln.Addr().String() + + if got := servedSerial(t, addr); got != 1 { + t.Fatalf("before renewal: expected serial 1, got %d", got) + } + + // cert-manager renews the Secret: the mounted files are replaced in place. + certB, keyB := genSelfSigned(t, 2) + writeKeyPair(t, certFile, keyFile, certB, keyB, time.Now().Add(2*time.Second)) + + if got := servedSerial(t, addr); got != 2 { + t.Fatalf("after renewal: expected renewed serial 2, got %d (certificate was not reloaded)", got) + } +} + +// TestCertReloaderKeepsLastGoodCertificate verifies that a failed reload (e.g. a +// torn read while the volume is updated) does not take the webhook down: it keeps +// serving the previously loaded certificate instead of erroring the handshake. +func TestCertReloaderKeepsLastGoodCertificate(t *testing.T) { + dir := t.TempDir() + certFile := filepath.Join(dir, "tls.crt") + keyFile := filepath.Join(dir, "tls.key") + + certA, keyA := genSelfSigned(t, 1) + writeKeyPair(t, certFile, keyFile, certA, keyA, time.Now().Add(-2*time.Second)) + + cr, err := newCertReloader(certFile, keyFile) + if err != nil { + t.Fatalf("newCertReloader: %v", err) + } + + // Corrupt the cert file and advance its mtime so a reload is attempted and fails. + if err := os.WriteFile(certFile, []byte("not a certificate"), 0o600); err != nil { + t.Fatalf("corrupt cert: %v", err) + } + if err := os.Chtimes(certFile, time.Now().Add(2*time.Second), time.Now().Add(2*time.Second)); err != nil { + t.Fatalf("chtimes: %v", err) + } + + got, err := cr.GetCertificate(&tls.ClientHelloInfo{}) + if err != nil { + t.Fatalf("GetCertificate returned error instead of serving last good cert: %v", err) + } + if got == nil || got.Leaf == nil { + // Leaf may be nil depending on the Go version; fall back to parsing. + if got == nil || len(got.Certificate) == 0 { + t.Fatalf("GetCertificate returned no certificate") + } + leaf, perr := x509.ParseCertificate(got.Certificate[0]) + if perr != nil { + t.Fatalf("parse served cert: %v", perr) + } + got.Leaf = leaf + } + if got.Leaf.SerialNumber.Int64() != 1 { + t.Fatalf("expected to keep serving serial 1 after failed reload, got %d", got.Leaf.SerialNumber.Int64()) + } +} diff --git a/packages/system/kubeovn-webhook/images/kubeovn-webhook/main.go b/packages/system/kubeovn-webhook/images/kubeovn-webhook/main.go index 70185961d7..a85776c605 100644 --- a/packages/system/kubeovn-webhook/images/kubeovn-webhook/main.go +++ b/packages/system/kubeovn-webhook/images/kubeovn-webhook/main.go @@ -1,7 +1,6 @@ package main import ( - "crypto/tls" "flag" "log" "net/http" @@ -28,17 +27,15 @@ func main() { mux := http.NewServeMux() mux.HandleFunc("/mutate-pods", HandleMutatePods) - tlsCert, err := tls.LoadX509KeyPair(tlsCertFile, tlsKeyFile) + tlsConfig, err := newReloadingTLSConfig(tlsCertFile, tlsKeyFile) if err != nil { log.Fatalf("Failed to load key pair: %v", err) } server := &http.Server{ - Addr: ":8443", - TLSConfig: &tls.Config{ - Certificates: []tls.Certificate{tlsCert}, - }, - Handler: mux, + Addr: ":8443", + TLSConfig: tlsConfig, + Handler: mux, } log.Printf("Starting webhook server on %s", server.Addr) From 46f3f4cd9228cc68bb270017d92f57d0a404575c Mon Sep 17 00:00:00 2001 From: IvanHunters Date: Wed, 5 Aug 2026 15:04:43 +0300 Subject: [PATCH 49/69] fix(kubeovn-webhook): widen serving cert renewBefore to 720h renewBefore: 24h on a one-year certificate leaves a one-day window for cert-manager to rotate the Secret and for the webhook to pick it up. Widen it to 720h (30 days) so renewal happens well ahead of expiry. Signed-off-by: IvanHunters (cherry picked from commit 791d99d8e5afda091719005b742cbbd43532f66d) --- packages/system/kubeovn-webhook/templates/certmanager.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/system/kubeovn-webhook/templates/certmanager.yaml b/packages/system/kubeovn-webhook/templates/certmanager.yaml index f8eee740e4..cb2770360a 100644 --- a/packages/system/kubeovn-webhook/templates/certmanager.yaml +++ b/packages/system/kubeovn-webhook/templates/certmanager.yaml @@ -38,7 +38,7 @@ metadata: spec: secretName: {{ include "namespace-annotation-webhook.fullname" . }}-tls duration: 8760h - renewBefore: 24h + renewBefore: 720h issuerRef: name: {{ include "namespace-annotation-webhook.fullname" . }}-ca-issuer commonName: {{ include "namespace-annotation-webhook.fullname" . }}-tls From f2af08196bb4a576e5e50219e154290e89a9a976 Mon Sep 17 00:00:00 2001 From: IvanHunters Date: Wed, 5 Aug 2026 15:24:17 +0300 Subject: [PATCH 50/69] fix(kubeovn-webhook): harden certificate reloader Address review of the reload path: - Observe the cert file mtime before reading the key pair, so a Secret swap racing the read is retried on the next handshake instead of being cached under a newer mtime and missed. - Record the load attempt regardless of outcome, so a persistently unreadable file is retried only once its mtime advances, not on every handshake. - Detect change with mtime inequality instead of a strictly-newer comparison, catching equal-mtime replacements and backward clock steps. - Log stat failures instead of swallowing them, so a broken mount is not invisible until expiry. Signed-off-by: IvanHunters (cherry picked from commit a537bb647bf3b20a0a7bd7d1fd9b2dfcab076cf0) --- .../images/kubeovn-webhook/cert.go | 67 ++++++++++--------- 1 file changed, 37 insertions(+), 30 deletions(-) diff --git a/packages/system/kubeovn-webhook/images/kubeovn-webhook/cert.go b/packages/system/kubeovn-webhook/images/kubeovn-webhook/cert.go index 9e15ec302b..56abdbb877 100644 --- a/packages/system/kubeovn-webhook/images/kubeovn-webhook/cert.go +++ b/packages/system/kubeovn-webhook/images/kubeovn-webhook/cert.go @@ -24,56 +24,63 @@ type certReloader struct { certFile string keyFile string - mu sync.RWMutex - cert *tls.Certificate - modTime time.Time + mu sync.RWMutex + cert *tls.Certificate + loadedModTime time.Time } // newCertReloader loads the initial key pair and returns a reloader for it. func newCertReloader(certFile, keyFile string) (*certReloader, error) { cr := &certReloader{certFile: certFile, keyFile: keyFile} - if err := cr.reload(); err != nil { + if err := cr.reload(cr.certModTime()); err != nil { return nil, err } return cr, nil } -// reload reads the key pair from disk and atomically swaps the cached certificate. -func (cr *certReloader) reload() error { - cert, err := tls.LoadX509KeyPair(cr.certFile, cr.keyFile) +// certModTime returns the modification time of the certificate file, or the zero +// time if it cannot be stat'd (logged so a broken mount is not silently invisible). +func (cr *certReloader) certModTime() time.Time { + fi, err := os.Stat(cr.certFile) if err != nil { - return err + log.Printf("certificate stat failed, keeping previous certificate: %v", err) + return time.Time{} } + return fi.ModTime() +} - var modTime time.Time - if fi, statErr := os.Stat(cr.certFile); statErr == nil { - modTime = fi.ModTime() - } +// reload reads the key pair from disk and atomically swaps the cached certificate. +// +// modTime is the certificate file's modification time observed BEFORE the read, so a +// Secret swap racing the read leaves modTime older than the file on disk and is caught +// on the next handshake instead of being cached under a newer mtime and missed. The +// attempt is recorded regardless of outcome, so a persistently unreadable file is +// retried only once its mtime advances, not on every handshake. +func (cr *certReloader) reload(modTime time.Time) error { + cert, err := tls.LoadX509KeyPair(cr.certFile, cr.keyFile) cr.mu.Lock() - cr.cert = &cert - cr.modTime = modTime + cr.loadedModTime = modTime + if err == nil { + cr.cert = &cert + } cr.mu.Unlock() - return nil + return err } -// changed reports whether the certificate file was modified since the last load. -func (cr *certReloader) changed() bool { - fi, err := os.Stat(cr.certFile) - if err != nil { - return false - } +// GetCertificate is a tls.Config.GetCertificate callback. It reloads the key pair when +// the certificate file's modification time changes and keeps serving the last good +// certificate if a reload fails (for example a torn read while the mounted Secret is +// being updated). +func (cr *certReloader) GetCertificate(*tls.ClientHelloInfo) (*tls.Certificate, error) { + modTime := cr.certModTime() + cr.mu.RLock() - defer cr.mu.RUnlock() - return fi.ModTime().After(cr.modTime) -} + stale := !modTime.IsZero() && !modTime.Equal(cr.loadedModTime) + cr.mu.RUnlock() -// GetCertificate is a tls.Config.GetCertificate callback. It reloads the key pair -// when the file changes and keeps serving the last good certificate if a reload -// fails (for example a torn read while the mounted Secret is being updated). -func (cr *certReloader) GetCertificate(*tls.ClientHelloInfo) (*tls.Certificate, error) { - if cr.changed() { - if err := cr.reload(); err != nil { + if stale { + if err := cr.reload(modTime); err != nil { log.Printf("certificate reload failed, keeping previous certificate: %v", err) } } From 9ac6e0b3773839ec39ccc08bca5d3da09843c281 Mon Sep 17 00:00:00 2001 From: IvanHunters Date: Wed, 5 Aug 2026 15:52:30 +0300 Subject: [PATCH 51/69] fix(kubeovn-webhook): retry certificate reload after a transient failure A previous revision recorded the certificate file mtime even when the reload failed, to avoid re-reading a persistently bad file on every handshake. That conflated two failure modes: a transient, content- independent error (fd exhaustion, a torn read) on a VALID renewed file advanced the recorded mtime, so the staleness check never fired again and the renewed certificate was never loaded. The cached certificate then expired within renewBefore, and failurePolicy: Fail blocked all tenant pod creation - exactly the outage this reloader prevents. Advance the recorded mtime only on a successful load, and bound retries of a failed load with a wall-clock interval instead. A transient failure is now retried until it succeeds, while a persistently unreadable file is still retried only once per interval, not on every handshake. Keep a single reload in flight to avoid a thundering herd, and rate-limit the failure log. Add a regression test that fails when the mtime is advanced on failure. Detect change with mtime inequality, so a backward clock step is also picked up. Signed-off-by: IvanHunters (cherry picked from commit 5b6bb47caf7aa879f75d0297c4cdb979bff500c3) --- .../images/kubeovn-webhook/cert.go | 121 +++++++++++++----- .../images/kubeovn-webhook/cert_test.go | 120 +++++++++++++---- 2 files changed, 178 insertions(+), 63 deletions(-) diff --git a/packages/system/kubeovn-webhook/images/kubeovn-webhook/cert.go b/packages/system/kubeovn-webhook/images/kubeovn-webhook/cert.go index 56abdbb877..ebc3c3e654 100644 --- a/packages/system/kubeovn-webhook/images/kubeovn-webhook/cert.go +++ b/packages/system/kubeovn-webhook/images/kubeovn-webhook/cert.go @@ -24,66 +24,117 @@ type certReloader struct { certFile string keyFile string + // retryInterval bounds how often a failed reload (or a repeated stat failure) is + // retried, so a persistently broken file does not read/parse/log on every handshake. + retryInterval time.Duration + mu sync.RWMutex cert *tls.Certificate loadedModTime time.Time + reloading bool + nextRetry time.Time + nextLog time.Time } +const defaultReloadRetryInterval = 30 * time.Second + // newCertReloader loads the initial key pair and returns a reloader for it. func newCertReloader(certFile, keyFile string) (*certReloader, error) { - cr := &certReloader{certFile: certFile, keyFile: keyFile} - if err := cr.reload(cr.certModTime()); err != nil { + cr := &certReloader{ + certFile: certFile, + keyFile: keyFile, + retryInterval: defaultReloadRetryInterval, + } + modTime, err := fileModTime(certFile) + if err != nil { + return nil, err + } + if err := cr.load(modTime); err != nil { return nil, err } return cr, nil } -// certModTime returns the modification time of the certificate file, or the zero -// time if it cannot be stat'd (logged so a broken mount is not silently invisible). -func (cr *certReloader) certModTime() time.Time { - fi, err := os.Stat(cr.certFile) +// fileModTime returns the modification time of the given file. +func fileModTime(name string) (time.Time, error) { + fi, err := os.Stat(name) if err != nil { - log.Printf("certificate stat failed, keeping previous certificate: %v", err) - return time.Time{} + return time.Time{}, err } - return fi.ModTime() + return fi.ModTime(), nil } -// reload reads the key pair from disk and atomically swaps the cached certificate. -// -// modTime is the certificate file's modification time observed BEFORE the read, so a -// Secret swap racing the read leaves modTime older than the file on disk and is caught -// on the next handshake instead of being cached under a newer mtime and missed. The -// attempt is recorded regardless of outcome, so a persistently unreadable file is -// retried only once its mtime advances, not on every handshake. -func (cr *certReloader) reload(modTime time.Time) error { +// load reads the key pair and, on success, caches it tagged with modTime. On failure +// the cached certificate and its recorded mtime are left unchanged, so a transient +// failure on a valid file is retried rather than the renewal being abandoned. +func (cr *certReloader) load(modTime time.Time) error { cert, err := tls.LoadX509KeyPair(cr.certFile, cr.keyFile) - + if err != nil { + return err + } cr.mu.Lock() + cr.cert = &cert cr.loadedModTime = modTime - if err == nil { - cr.cert = &cert - } cr.mu.Unlock() - return err + return nil } -// GetCertificate is a tls.Config.GetCertificate callback. It reloads the key pair when -// the certificate file's modification time changes and keeps serving the last good -// certificate if a reload fails (for example a torn read while the mounted Secret is -// being updated). -func (cr *certReloader) GetCertificate(*tls.ClientHelloInfo) (*tls.Certificate, error) { - modTime := cr.certModTime() +// refresh reloads the key pair when the certificate file's modification time changes. +// +// A failed reload keeps the last good certificate and does NOT advance the recorded +// mtime, so a transient error (fd exhaustion, a torn read) is retried, at most once per +// retryInterval, until it succeeds: a transient failure must never permanently pin an +// expiring certificate. A single reload is kept in flight so a burst of handshakes does +// not trigger a thundering herd of reads. +func (cr *certReloader) refresh() { + modTime, err := fileModTime(cr.certFile) + if err != nil { + cr.logRateLimited("certificate stat failed, keeping previous certificate: %v", err) + return + } - cr.mu.RLock() - stale := !modTime.IsZero() && !modTime.Equal(cr.loadedModTime) - cr.mu.RUnlock() + cr.mu.Lock() + if modTime.Equal(cr.loadedModTime) || cr.reloading || time.Now().Before(cr.nextRetry) { + cr.mu.Unlock() + return + } + cr.reloading = true + cr.mu.Unlock() + + loadErr := cr.load(modTime) + + cr.mu.Lock() + cr.reloading = false + if loadErr == nil { + cr.nextRetry = time.Time{} + } else { + cr.nextRetry = time.Now().Add(cr.retryInterval) + } + cr.mu.Unlock() - if stale { - if err := cr.reload(modTime); err != nil { - log.Printf("certificate reload failed, keeping previous certificate: %v", err) - } + if loadErr != nil { + cr.logRateLimited("certificate reload failed, keeping previous certificate: %v", loadErr) } +} + +// logRateLimited emits a log line at most once per retryInterval, so a persistently +// broken mount does not spam the log on every TLS handshake. +func (cr *certReloader) logRateLimited(format string, args ...any) { + cr.mu.Lock() + if time.Now().Before(cr.nextLog) { + cr.mu.Unlock() + return + } + cr.nextLog = time.Now().Add(cr.retryInterval) + cr.mu.Unlock() + log.Printf(format, args...) +} + +// GetCertificate is a tls.Config.GetCertificate callback. It refreshes the key pair +// when the certificate file changes and keeps serving the last good certificate if a +// reload fails. +func (cr *certReloader) GetCertificate(*tls.ClientHelloInfo) (*tls.Certificate, error) { + cr.refresh() cr.mu.RLock() defer cr.mu.RUnlock() diff --git a/packages/system/kubeovn-webhook/images/kubeovn-webhook/cert_test.go b/packages/system/kubeovn-webhook/images/kubeovn-webhook/cert_test.go index 1a670e4fd8..c3b1d4baf9 100644 --- a/packages/system/kubeovn-webhook/images/kubeovn-webhook/cert_test.go +++ b/packages/system/kubeovn-webhook/images/kubeovn-webhook/cert_test.go @@ -50,8 +50,8 @@ func genSelfSigned(t *testing.T, serial int64) (certPEM, keyPEM []byte) { return certPEM, keyPEM } -// writeKeyPair writes the cert/key files and bumps their mtime strictly forward, -// mirroring how cert-manager replaces the mounted Secret on renewal. +// writeKeyPair writes the cert/key files and sets their mtime, mirroring how +// cert-manager replaces the mounted Secret on renewal. func writeKeyPair(t *testing.T, certFile, keyFile string, certPEM, keyPEM []byte, mtime time.Time) { t.Helper() if err := os.WriteFile(certFile, certPEM, 0o600); err != nil { @@ -60,12 +60,36 @@ func writeKeyPair(t *testing.T, certFile, keyFile string, certPEM, keyPEM []byte if err := os.WriteFile(keyFile, keyPEM, 0o600); err != nil { t.Fatalf("write key: %v", err) } - if err := os.Chtimes(certFile, mtime, mtime); err != nil { - t.Fatalf("chtimes cert: %v", err) + setModTime(t, certFile, mtime) + setModTime(t, keyFile, mtime) +} + +func setModTime(t *testing.T, name string, mtime time.Time) { + t.Helper() + if err := os.Chtimes(name, mtime, mtime); err != nil { + t.Fatalf("chtimes %s: %v", name, err) } - if err := os.Chtimes(keyFile, mtime, mtime); err != nil { - t.Fatalf("chtimes key: %v", err) +} + +// certSerial returns the serial number of a *tls.Certificate leaf without mutating the +// shared value returned by GetCertificate (it parses into a local when Leaf is unset). +func certSerial(t *testing.T, c *tls.Certificate) int64 { + t.Helper() + if c == nil { + t.Fatal("nil certificate") + } + leaf := c.Leaf + if leaf == nil { + if len(c.Certificate) == 0 { + t.Fatal("certificate has no DER data") + } + parsed, err := x509.ParseCertificate(c.Certificate[0]) + if err != nil { + t.Fatalf("parse served cert: %v", err) + } + leaf = parsed } + return leaf.SerialNumber.Int64() } // servedSerial completes a TLS handshake against addr and returns the serial number @@ -84,10 +108,9 @@ func servedSerial(t *testing.T, addr string) int64 { return certs[0].SerialNumber.Int64() } -// TestReloadingTLSConfigServesRenewedCertificate is the regression test for the -// cert-manager renewal not being picked up: it fails against the old code that -// loads the key pair once into tls.Config.Certificates, and passes once the -// certificate is served via a reloading GetCertificate callback. +// TestReloadingTLSConfigServesRenewedCertificate is the core regression test: it fails +// against the old code that loads the key pair once into tls.Config.Certificates, and +// passes once the certificate is served via a reloading GetCertificate callback. func TestReloadingTLSConfigServesRenewedCertificate(t *testing.T) { dir := t.TempDir() certFile := filepath.Join(dir, "tls.crt") @@ -136,9 +159,9 @@ func TestReloadingTLSConfigServesRenewedCertificate(t *testing.T) { } } -// TestCertReloaderKeepsLastGoodCertificate verifies that a failed reload (e.g. a -// torn read while the volume is updated) does not take the webhook down: it keeps -// serving the previously loaded certificate instead of erroring the handshake. +// TestCertReloaderKeepsLastGoodCertificate verifies that a failed reload (e.g. a torn +// read while the volume is updated) does not take the webhook down: it keeps serving +// the previously loaded certificate instead of erroring the handshake. func TestCertReloaderKeepsLastGoodCertificate(t *testing.T) { dir := t.TempDir() certFile := filepath.Join(dir, "tls.crt") @@ -156,26 +179,67 @@ func TestCertReloaderKeepsLastGoodCertificate(t *testing.T) { if err := os.WriteFile(certFile, []byte("not a certificate"), 0o600); err != nil { t.Fatalf("corrupt cert: %v", err) } - if err := os.Chtimes(certFile, time.Now().Add(2*time.Second), time.Now().Add(2*time.Second)); err != nil { - t.Fatalf("chtimes: %v", err) - } + setModTime(t, certFile, time.Now().Add(2*time.Second)) got, err := cr.GetCertificate(&tls.ClientHelloInfo{}) if err != nil { t.Fatalf("GetCertificate returned error instead of serving last good cert: %v", err) } - if got == nil || got.Leaf == nil { - // Leaf may be nil depending on the Go version; fall back to parsing. - if got == nil || len(got.Certificate) == 0 { - t.Fatalf("GetCertificate returned no certificate") - } - leaf, perr := x509.ParseCertificate(got.Certificate[0]) - if perr != nil { - t.Fatalf("parse served cert: %v", perr) - } - got.Leaf = leaf + if certSerial(t, got) != 1 { + t.Fatalf("expected to keep serving serial 1 after failed reload, got %d", certSerial(t, got)) + } +} + +// TestCertReloaderRetriesAfterTransientFailure is the regression test for the reloader +// abandoning a renewal on a transient error. A failed reload must NOT advance the +// recorded mtime, so once the (unchanged-mtime) file becomes readable the renewed +// certificate is still picked up. Against the code that advanced the mtime on failure, +// the renewal is permanently missed and this test fails. +func TestCertReloaderRetriesAfterTransientFailure(t *testing.T) { + dir := t.TempDir() + certFile := filepath.Join(dir, "tls.crt") + keyFile := filepath.Join(dir, "tls.key") + + certA, keyA := genSelfSigned(t, 1) + writeKeyPair(t, certFile, keyFile, certA, keyA, time.Now().Add(-2*time.Second)) + + cr, err := newCertReloader(certFile, keyFile) + if err != nil { + t.Fatalf("newCertReloader: %v", err) + } + cr.retryInterval = 0 // retry immediately, no backoff wait in the test + + // Renewal advances the mtime, but the first load fails (simulating a transient, + // content-independent error) by writing an unreadable cert at the new mtime. + renewMod := time.Now().Add(2 * time.Second) + if err := os.WriteFile(certFile, []byte("transiently unreadable"), 0o600); err != nil { + t.Fatalf("write bad cert: %v", err) + } + setModTime(t, certFile, renewMod) + + if got, gerr := cr.GetCertificate(&tls.ClientHelloInfo{}); gerr != nil { + t.Fatalf("GetCertificate errored during transient failure: %v", gerr) + } else if certSerial(t, got) != 1 { + t.Fatalf("during transient failure: expected to keep serial 1, got %d", certSerial(t, got)) + } + + // The renewed, valid certificate is now readable at the SAME mtime as the failed + // attempt. The reloader must still pick it up on retry. + certB, keyB := genSelfSigned(t, 2) + if err := os.WriteFile(certFile, certB, 0o600); err != nil { + t.Fatalf("write renewed cert: %v", err) + } + if err := os.WriteFile(keyFile, keyB, 0o600); err != nil { + t.Fatalf("write renewed key: %v", err) + } + setModTime(t, certFile, renewMod) + setModTime(t, keyFile, renewMod) + + got, gerr := cr.GetCertificate(&tls.ClientHelloInfo{}) + if gerr != nil { + t.Fatalf("GetCertificate errored after retry: %v", gerr) } - if got.Leaf.SerialNumber.Int64() != 1 { - t.Fatalf("expected to keep serving serial 1 after failed reload, got %d", got.Leaf.SerialNumber.Int64()) + if certSerial(t, got) != 2 { + t.Fatalf("after transient failure: expected renewed serial 2 on retry, got %d (mtime advanced on failure, renewal permanently missed)", certSerial(t, got)) } } From f6981e05558912535265b6b2b843167a27fe7895 Mon Sep 17 00:00:00 2001 From: IvanHunters Date: Fri, 7 Aug 2026 16:36:42 +0300 Subject: [PATCH 52/69] refactor(kube-ovn): reload webhook cert via a plain GetCertificate closure Collapse the certReloader (mtime tracking, serialized refresh, retry backoff, rate-limited logging, last-good-certificate fallback) into a GetCertificate callback that simply re-reads the key pair from disk on every handshake. The signature is unchanged, so main.go is untouched. The extra machinery guarded scenarios this deployment cannot produce: - Torn reads are unreachable: the cert/key files come from a plain Secret volume with no subPath, and Kubernetes' atomic writer swaps the whole ..data directory via a single symlink flip, so a reader always sees a complete old or complete new generation of both files. - The per-handshake LoadX509KeyPair cost is negligible next to the asymmetric crypto the handshake already performs, so no cache is needed. - The webhook configures no mTLS, so GetCertificate not refreshing client CA pools does not apply. If the mounted files genuinely become unreadable (an operator error, not a renewal) the handshake now fails loudly instead of silently serving a stale certificate. The core regression test is retained and remains non-vacuous: it fails against a static tls.Config.Certificates and passes with the reloading callback. Signed-off-by: IvanHunters (cherry picked from commit 8baa449f53f6410a348ce2e7e7ea89393f1c431d) --- .../images/kubeovn-webhook/cert.go | 182 ++++-------------- .../images/kubeovn-webhook/cert_test.go | 106 ---------- 2 files changed, 36 insertions(+), 252 deletions(-) diff --git a/packages/system/kubeovn-webhook/images/kubeovn-webhook/cert.go b/packages/system/kubeovn-webhook/images/kubeovn-webhook/cert.go index ebc3c3e654..1fcbf573b4 100644 --- a/packages/system/kubeovn-webhook/images/kubeovn-webhook/cert.go +++ b/packages/system/kubeovn-webhook/images/kubeovn-webhook/cert.go @@ -1,155 +1,45 @@ package main -import ( - "crypto/tls" - "fmt" - "log" - "os" - "sync" - "time" -) +import "crypto/tls" -// certReloader keeps the webhook serving certificate in sync with the key pair -// files written by cert-manager. +// newReloadingTLSConfig returns a tls.Config that reloads the certificate and +// key from disk on every TLS handshake, so a cert-manager renewal of the mounted +// Secret is picked up by the running process without a restart. // -// The certificate is served through tls.Config.GetCertificate, so a cert-manager -// renewal of the backing Secret is picked up by the running process without a -// restart. Without this, the key pair is read exactly once at startup and cached -// for the lifetime of the process: after the certificate expires (roughly one year -// after install) the pod keeps presenting the expired certificate, the -// kube-apiserver's TLS call to the webhook fails, and because the -// MutatingWebhookConfiguration uses failurePolicy: Fail, every pod creation in -// tenant namespaces is rejected. -type certReloader struct { - certFile string - keyFile string - - // retryInterval bounds how often a failed reload (or a repeated stat failure) is - // retried, so a persistently broken file does not read/parse/log on every handshake. - retryInterval time.Duration - - mu sync.RWMutex - cert *tls.Certificate - loadedModTime time.Time - reloading bool - nextRetry time.Time - nextLog time.Time -} - -const defaultReloadRetryInterval = 30 * time.Second - -// newCertReloader loads the initial key pair and returns a reloader for it. -func newCertReloader(certFile, keyFile string) (*certReloader, error) { - cr := &certReloader{ - certFile: certFile, - keyFile: keyFile, - retryInterval: defaultReloadRetryInterval, - } - modTime, err := fileModTime(certFile) - if err != nil { - return nil, err - } - if err := cr.load(modTime); err != nil { - return nil, err - } - return cr, nil -} - -// fileModTime returns the modification time of the given file. -func fileModTime(name string) (time.Time, error) { - fi, err := os.Stat(name) - if err != nil { - return time.Time{}, err - } - return fi.ModTime(), nil -} - -// load reads the key pair and, on success, caches it tagged with modTime. On failure -// the cached certificate and its recorded mtime are left unchanged, so a transient -// failure on a valid file is retried rather than the renewal being abandoned. -func (cr *certReloader) load(modTime time.Time) error { - cert, err := tls.LoadX509KeyPair(cr.certFile, cr.keyFile) - if err != nil { - return err - } - cr.mu.Lock() - cr.cert = &cert - cr.loadedModTime = modTime - cr.mu.Unlock() - return nil -} - -// refresh reloads the key pair when the certificate file's modification time changes. +// Without this, the key pair is read exactly once at startup and cached for the +// lifetime of the process: after the certificate expires (roughly one year after +// install) the pod keeps presenting the expired certificate, the kube-apiserver's +// TLS call to the webhook fails, and because the MutatingWebhookConfiguration uses +// failurePolicy: Fail, every pod creation in tenant namespaces is rejected. // -// A failed reload keeps the last good certificate and does NOT advance the recorded -// mtime, so a transient error (fd exhaustion, a torn read) is retried, at most once per -// retryInterval, until it succeeds: a transient failure must never permanently pin an -// expiring certificate. A single reload is kept in flight so a burst of handshakes does -// not trigger a thundering herd of reads. -func (cr *certReloader) refresh() { - modTime, err := fileModTime(cr.certFile) - if err != nil { - cr.logRateLimited("certificate stat failed, keeping previous certificate: %v", err) - return - } - - cr.mu.Lock() - if modTime.Equal(cr.loadedModTime) || cr.reloading || time.Now().Before(cr.nextRetry) { - cr.mu.Unlock() - return - } - cr.reloading = true - cr.mu.Unlock() - - loadErr := cr.load(modTime) - - cr.mu.Lock() - cr.reloading = false - if loadErr == nil { - cr.nextRetry = time.Time{} - } else { - cr.nextRetry = time.Now().Add(cr.retryInterval) - } - cr.mu.Unlock() - - if loadErr != nil { - cr.logRateLimited("certificate reload failed, keeping previous certificate: %v", loadErr) - } -} - -// logRateLimited emits a log line at most once per retryInterval, so a persistently -// broken mount does not spam the log on every TLS handshake. -func (cr *certReloader) logRateLimited(format string, args ...any) { - cr.mu.Lock() - if time.Now().Before(cr.nextLog) { - cr.mu.Unlock() - return - } - cr.nextLog = time.Now().Add(cr.retryInterval) - cr.mu.Unlock() - log.Printf(format, args...) -} - -// GetCertificate is a tls.Config.GetCertificate callback. It refreshes the key pair -// when the certificate file changes and keeps serving the last good certificate if a -// reload fails. -func (cr *certReloader) GetCertificate(*tls.ClientHelloInfo) (*tls.Certificate, error) { - cr.refresh() - - cr.mu.RLock() - defer cr.mu.RUnlock() - if cr.cert == nil { - return nil, fmt.Errorf("no certificate loaded") - } - return cr.cert, nil -} - -// newReloadingTLSConfig builds a tls.Config that serves the key pair via a -// certReloader, so cert-manager renewals are honoured without a restart. +// tls.Config.GetCertificate is invoked on every handshake whenever Certificates is +// left unset, so re-reading the files there is enough; no watcher, mtime tracking +// or cache is needed: +// - The cert/key files are mounted from a plain Secret volume with no subPath. +// Kubernetes' atomic writer swaps the whole ..data directory via a single +// symlink flip, so a reader always sees a complete old or complete new +// generation of both files, never a torn mid-write mixture. A per-handshake +// LoadX509KeyPair therefore cannot observe a partial renewal. +// - The per-handshake cost (a few KB of file I/O plus a PEM/key parse) is +// negligible next to the asymmetric crypto the handshake already performs. +// - The webhook configures no mTLS (no ClientCAs), so GetCertificate's +// limitation of not refreshing client CA pools does not apply. +// +// If the mounted files genuinely become unreadable (Secret deleted, permissions +// broken: an operator error, not a normal renewal) the handshake fails loudly +// rather than silently serving a stale certificate. func newReloadingTLSConfig(certFile, keyFile string) (*tls.Config, error) { - cr, err := newCertReloader(certFile, keyFile) - if err != nil { + // Fail fast at startup if the initial key pair is missing or malformed. + if _, err := tls.LoadX509KeyPair(certFile, keyFile); err != nil { return nil, err } - return &tls.Config{GetCertificate: cr.GetCertificate}, nil + return &tls.Config{ + GetCertificate: func(*tls.ClientHelloInfo) (*tls.Certificate, error) { + cert, err := tls.LoadX509KeyPair(certFile, keyFile) + if err != nil { + return nil, err + } + return &cert, nil + }, + }, nil } diff --git a/packages/system/kubeovn-webhook/images/kubeovn-webhook/cert_test.go b/packages/system/kubeovn-webhook/images/kubeovn-webhook/cert_test.go index c3b1d4baf9..2d39fe5999 100644 --- a/packages/system/kubeovn-webhook/images/kubeovn-webhook/cert_test.go +++ b/packages/system/kubeovn-webhook/images/kubeovn-webhook/cert_test.go @@ -71,27 +71,6 @@ func setModTime(t *testing.T, name string, mtime time.Time) { } } -// certSerial returns the serial number of a *tls.Certificate leaf without mutating the -// shared value returned by GetCertificate (it parses into a local when Leaf is unset). -func certSerial(t *testing.T, c *tls.Certificate) int64 { - t.Helper() - if c == nil { - t.Fatal("nil certificate") - } - leaf := c.Leaf - if leaf == nil { - if len(c.Certificate) == 0 { - t.Fatal("certificate has no DER data") - } - parsed, err := x509.ParseCertificate(c.Certificate[0]) - if err != nil { - t.Fatalf("parse served cert: %v", err) - } - leaf = parsed - } - return leaf.SerialNumber.Int64() -} - // servedSerial completes a TLS handshake against addr and returns the serial number // of the leaf certificate the server actually presented. func servedSerial(t *testing.T, addr string) int64 { @@ -158,88 +137,3 @@ func TestReloadingTLSConfigServesRenewedCertificate(t *testing.T) { t.Fatalf("after renewal: expected renewed serial 2, got %d (certificate was not reloaded)", got) } } - -// TestCertReloaderKeepsLastGoodCertificate verifies that a failed reload (e.g. a torn -// read while the volume is updated) does not take the webhook down: it keeps serving -// the previously loaded certificate instead of erroring the handshake. -func TestCertReloaderKeepsLastGoodCertificate(t *testing.T) { - dir := t.TempDir() - certFile := filepath.Join(dir, "tls.crt") - keyFile := filepath.Join(dir, "tls.key") - - certA, keyA := genSelfSigned(t, 1) - writeKeyPair(t, certFile, keyFile, certA, keyA, time.Now().Add(-2*time.Second)) - - cr, err := newCertReloader(certFile, keyFile) - if err != nil { - t.Fatalf("newCertReloader: %v", err) - } - - // Corrupt the cert file and advance its mtime so a reload is attempted and fails. - if err := os.WriteFile(certFile, []byte("not a certificate"), 0o600); err != nil { - t.Fatalf("corrupt cert: %v", err) - } - setModTime(t, certFile, time.Now().Add(2*time.Second)) - - got, err := cr.GetCertificate(&tls.ClientHelloInfo{}) - if err != nil { - t.Fatalf("GetCertificate returned error instead of serving last good cert: %v", err) - } - if certSerial(t, got) != 1 { - t.Fatalf("expected to keep serving serial 1 after failed reload, got %d", certSerial(t, got)) - } -} - -// TestCertReloaderRetriesAfterTransientFailure is the regression test for the reloader -// abandoning a renewal on a transient error. A failed reload must NOT advance the -// recorded mtime, so once the (unchanged-mtime) file becomes readable the renewed -// certificate is still picked up. Against the code that advanced the mtime on failure, -// the renewal is permanently missed and this test fails. -func TestCertReloaderRetriesAfterTransientFailure(t *testing.T) { - dir := t.TempDir() - certFile := filepath.Join(dir, "tls.crt") - keyFile := filepath.Join(dir, "tls.key") - - certA, keyA := genSelfSigned(t, 1) - writeKeyPair(t, certFile, keyFile, certA, keyA, time.Now().Add(-2*time.Second)) - - cr, err := newCertReloader(certFile, keyFile) - if err != nil { - t.Fatalf("newCertReloader: %v", err) - } - cr.retryInterval = 0 // retry immediately, no backoff wait in the test - - // Renewal advances the mtime, but the first load fails (simulating a transient, - // content-independent error) by writing an unreadable cert at the new mtime. - renewMod := time.Now().Add(2 * time.Second) - if err := os.WriteFile(certFile, []byte("transiently unreadable"), 0o600); err != nil { - t.Fatalf("write bad cert: %v", err) - } - setModTime(t, certFile, renewMod) - - if got, gerr := cr.GetCertificate(&tls.ClientHelloInfo{}); gerr != nil { - t.Fatalf("GetCertificate errored during transient failure: %v", gerr) - } else if certSerial(t, got) != 1 { - t.Fatalf("during transient failure: expected to keep serial 1, got %d", certSerial(t, got)) - } - - // The renewed, valid certificate is now readable at the SAME mtime as the failed - // attempt. The reloader must still pick it up on retry. - certB, keyB := genSelfSigned(t, 2) - if err := os.WriteFile(certFile, certB, 0o600); err != nil { - t.Fatalf("write renewed cert: %v", err) - } - if err := os.WriteFile(keyFile, keyB, 0o600); err != nil { - t.Fatalf("write renewed key: %v", err) - } - setModTime(t, certFile, renewMod) - setModTime(t, keyFile, renewMod) - - got, gerr := cr.GetCertificate(&tls.ClientHelloInfo{}) - if gerr != nil { - t.Fatalf("GetCertificate errored after retry: %v", gerr) - } - if certSerial(t, got) != 2 { - t.Fatalf("after transient failure: expected renewed serial 2 on retry, got %d (mtime advanced on failure, renewal permanently missed)", certSerial(t, got)) - } -} From 35fb07912ea9c484202fc8845972580a1e103bed Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Mon, 27 Jul 2026 13:24:05 +0500 Subject: [PATCH 53/69] fix(seaweedfs): read the chart name first-match, fail the fake on unmodelled calls Review findings from #3436, plus what an adversarial re-check of the first attempt at them turned up. The chart-name extraction was a greedy sed, i.e. LAST match. Helm marshals "config" (the release`s values) after "chart", so a values subtree that spells chart.metadata.name shadowed the real chart name: the release then read as non-SeaweedFS, the tenant silently vanished from the report, and the script exited 0 -- the precise false clean this PR exists to prevent, reachable without any corruption at all. It now takes the FIRST match. The path stays adjacent on purpose: a looser "any name after metadata" matches chart.templates[].name, which Helm emits immediately after metadata on every healthy release, so the review`s suggested relaxation would have returned a template path for every tenant. Over-strictness fails loudly and recoverably; over-looseness deletes data quietly. Newlines are now folded before matching, so a pretty-printed payload parses at all rather than aborting, and whitespace around the punctuation is tolerated. first_deployed got the same treatment -- it had the identical greedy, single- line-only shape, and a spaced payload silently dropped the PV-vintage row from the report. The FATAL message names the path it read and the shape it expected, so a future Helm format change is diagnosable instead of looking like real corruption. Tests: three new payload shapes (spaced, pretty multi-line, and a values decoy) each byte-compared against the same golden as the compact payload, so the shape must make no difference to the report. Mutation-checked -- restoring last-match extraction fails the decoy test on its own, restoring the single-line matcher fails the pretty test. The test fake answered any unmodelled kubectl invocation with exit 0 and empty stdout, the fail-open shape this script was rewritten to reject, and enough to let a newly added query pass the goldens unnoticed. Unmodelled calls now exit 97 naming the invocation. Verified inert first by instrumenting the fake: 88 invocations across the suite, none unmodelled. Also added the missing test for the by-name PVC GET fatal path, whose FAIL mode existed with nothing behind it. The runbook now tells the operator to read the exit code: a non-zero exit means the table is incomplete and no step may be taken on it. That contract lived only in the script header, while the runbook is what the operator follows. 29/29 tests green; sh -n clean. Assisted-By: Claude Signed-off-by: Myasnikov Daniil (cherry picked from commit e02adad38f8339848344260a7cbeb9d2f58105d5) --- .../seaweedfs-431-rename-recovery.md | 4 +- hack/seaweedfs-naming-audit.bats | 114 ++++++++++++++++-- hack/seaweedfs-naming-audit.sh | 41 ++++++- 3 files changed, 144 insertions(+), 15 deletions(-) diff --git a/docs/operations/seaweedfs-431-rename-recovery.md b/docs/operations/seaweedfs-431-rename-recovery.md index ed4b092441..cca31f2e87 100644 --- a/docs/operations/seaweedfs-431-rename-recovery.md +++ b/docs/operations/seaweedfs-431-rename-recovery.md @@ -55,6 +55,8 @@ hack/seaweedfs-naming-audit.sh # whole cluster hack/seaweedfs-naming-audit.sh tenant-foo # or named namespaces ``` +**Read the exit code, not just the table.** The audit fails closed: any error it cannot interpret — a kubectl call that fails, a Helm release payload it cannot decode — makes it print `FATAL` and exit non-zero, and the table it printed up to that point is incomplete. A non-zero exit means you do not yet know the state of the fleet, so none of the steps below may be taken on the strength of it. Only a zero exit means the table is the whole answer; an empty table with exit 0 is a genuinely clean fleet. + It mutates nothing. Earlier revisions of this runbook inlined the classification as a shell snippet here; it is a tested script now (`hack/seaweedfs-naming-audit.bats`), because it is what the chart's refusal hands you to and acting on it deletes PVCs. Two inline versions shipped wrong — one whose selector matched both generations at once and so inverted its own primary rule, one that could not see a long instance name at all — so it is not a snippet any more. It reports one class per SeaweedFS instance, matching exactly what the chart's guard decides: @@ -298,7 +300,7 @@ Do **not** start by deleting PVCs. If the duplicate ever served, they may hold o Never delete `data1-seaweedfs-volume-*` here — those are the live data PVCs of the authoritative set. (For an `S-damaged` tenant it is the other way round; that is Step 2a's job, and it has its own precondition check.) -5. **Re-run the audit.** The tenant must now read `L` (or `S`, if you are on the Step 2 path). Only then upgrade. +5. **Re-run the audit.** The tenant must now read `L` (or `S`, if you are on the Step 2 path), and the audit must exit zero — a `FATAL` here means the re-check never completed, not that the tenant is fine. Only then upgrade. ```sh hack/seaweedfs-naming-audit.sh "$ns" diff --git a/hack/seaweedfs-naming-audit.bats b/hack/seaweedfs-naming-audit.bats index 424de20a35..eb605c5297 100644 --- a/hack/seaweedfs-naming-audit.bats +++ b/hack/seaweedfs-naming-audit.bats @@ -172,13 +172,34 @@ EOF # chart.metadata.name (as real Helm payloads carry it). Defaults to a cozy-seaweedfs # release so the audit confirms the tenant; pass another chart name for a non- # SeaweedFS release, or the literal EMPTY for a chartless {} payload. +# +# Three payload SHAPES beyond the default compact one, because the chart-name +# extraction is now fatal on a miss and every shape below is something a +# re-serializer or a user's values could produce: +# SPACED whitespace around the JSON punctuation (json.dumps' default) +# PRETTY indented and MULTI-LINE (jq . / yq -o=json), which a line-based +# matcher cannot read at all unless newlines are folded first +# DECOY a real cozy-seaweedfs chart PLUS a values subtree that spells +# chart.metadata.name with a different value. Helm marshals "config" +# (the values) AFTER "chart", so a last-match extraction returns the +# decoy and silently declares the tenant non-SeaweedFS. _release_blob() { _rb_name=${1:-cozy-seaweedfs} - if [ "$_rb_name" = EMPTY ]; then - _rb_json='{}' - else - _rb_json='{"name":"seaweedfs-system","chart":{"metadata":{"name":"'"$_rb_name"'"}}}' - fi + case "$_rb_name" in + EMPTY) _rb_json='{}' ;; + SPACED) _rb_json='{"name": "seaweedfs-system", "chart": {"metadata": {"name": "cozy-seaweedfs", "version": "1.0.0"}}}' ;; + PRETTY) _rb_json='{ + "name": "seaweedfs-system", + "chart": { + "metadata": { + "name": "cozy-seaweedfs", + "version": "1.0.0" + } + } +}' ;; + DECOY) _rb_json='{"name":"seaweedfs-system","chart":{"metadata":{"name":"cozy-seaweedfs"}},"config":{"chart":{"metadata":{"name":"decoy-not-seaweedfs"}}}}' ;; + *) _rb_json='{"name":"seaweedfs-system","chart":{"metadata":{"name":"'"$_rb_name"'"}}}' ;; + esac printf '%s' "$_rb_json" | gzip | base64 | base64 | tr -d '\n' } @@ -212,6 +233,10 @@ _write_fake_kubectl() { cat <<'FAKE' verb=${1:-}; res=${2:-}; args="$*" fail() { echo "fake kubectl: $1 (real error)" >&2; exit 1; } +# Anything this fake does not model must NOT look like a successful empty +# answer: that is the fail-open shape the audited script exists to reject, and +# it would let a new query added to the script pass these goldens unnoticed. +unmodelled() { echo "fake kubectl: unmodelled invocation: $args" >&2; exit 97; } if [ "$verb $res" = "get ns" ]; then [ "$FAIL" = ns ] && fail "get ns" printf 'namespace/tenant-test\n'; exit 0 @@ -247,7 +272,7 @@ if [ "$verb $res" = "get pvc" ]; then *data1-seaweedfs-volume-1*) [ "$FAIL" = pvcget ] && fail "pvc GET"; printf 'pv-legacy2\n'; exit 0 ;; *data1-seaweedfs-volume-0*) [ "$FAIL" = pvcget ] && fail "pvc GET"; printf 'pv-legacy\n'; exit 0 ;; esac - exit 0 + unmodelled fi if [ "$verb $res" = "get sts" ]; then [ "$FAIL" = sts ] && fail "sts LIST" @@ -260,9 +285,9 @@ if [ "$verb $res" = "get pv" ]; then *pv-legacy*) [ "$ABSENT_PV" = pv-legacy ] && exit 0; printf '2020-01-01T00:00:00Z\n'; exit 0 ;; *pv-renamed*) [ "$ABSENT_PV" = pv-renamed ] && exit 0; printf '2020-06-01T00:00:00Z\n'; exit 0 ;; esac - exit 0 + unmodelled fi -exit 0 +unmodelled FAKE } > "$1/kubectl" chmod +x "$1/kubectl" @@ -396,7 +421,10 @@ _expected_mixed_overlap() { echo "rc=$rc"; echo "$out" [ "$rc" -ne 0 ] printf '%s\n' "$out" | grep -q 'FATAL' - printf '%s\n' "$out" | grep -q 'no chart name' + printf '%s\n' "$out" | grep -q 'could not read the chart name' + # The diagnostic must name the path it looked at, so a future Helm payload + # format change is diagnosable as such instead of reading as real corruption. + printf '%s\n' "$out" | grep -q '\.chart\.metadata\.name' } @test "a present release secret for a non-SeaweedFS chart is a silent legitimate skip" { @@ -507,3 +535,71 @@ _expected_mixed_overlap() { diff "$d/want" "$d/got" rm -rf "$d" } + +@test "a whitespace-spaced Helm payload still classifies the tenant" { + # The chart-name read is FATAL on a miss, so any payload shape a re-serializer + # can produce must parse. Byte-compare against the same golden as the compact + # payload: the shape must make no difference to the report at all. + d=$(mktemp -d) + _write_fake_kubectl "$d" "" "" "$(_release_blob SPACED)" "persistentvolumeclaim/data1-seaweedfs-volume-0" + rc=0 + out=$(PATH="$d:$PATH" SEAWEEDFS_AUDIT_LIB=0 sh "$PWD/hack/seaweedfs-naming-audit.sh" tenant-test 2>&1) || rc=$? + printf '%s\n' "$out" > "$d/got" + _expected_L > "$d/want" + echo "rc=$rc"; echo "--- got ---"; cat "$d/got" + [ "$rc" -eq 0 ] + diff "$d/want" "$d/got" + rm -rf "$d" +} + +@test "a pretty-printed multi-line Helm payload still classifies the tenant" { + # Line-based sed/grep cannot see across newlines at all, so this shape is the + # one that fails hardest without the newline fold -- and jq/yq re-serialization + # is exactly how a payload would arrive indented. + d=$(mktemp -d) + _write_fake_kubectl "$d" "" "" "$(_release_blob PRETTY)" "persistentvolumeclaim/data1-seaweedfs-volume-0" + rc=0 + out=$(PATH="$d:$PATH" SEAWEEDFS_AUDIT_LIB=0 sh "$PWD/hack/seaweedfs-naming-audit.sh" tenant-test 2>&1) || rc=$? + printf '%s\n' "$out" > "$d/got" + _expected_L > "$d/want" + echo "rc=$rc"; echo "--- got ---"; cat "$d/got" + [ "$rc" -eq 0 ] + diff "$d/want" "$d/got" + rm -rf "$d" +} + +@test "a values subtree spelling chart.metadata.name cannot shadow the real chart" { + # Helm marshals "config" (the user's values) AFTER "chart", so a LAST-match + # extraction returns the decoy, the release reads as non-SeaweedFS, and the + # tenant vanishes from the report with exit 0 -- a false clean, the failure this + # whole script exists to prevent. First-match extraction is what stops it. + d=$(mktemp -d) + _write_fake_kubectl "$d" "" "" "$(_release_blob DECOY)" "persistentvolumeclaim/data1-seaweedfs-volume-0" + rc=0 + out=$(PATH="$d:$PATH" SEAWEEDFS_AUDIT_LIB=0 sh "$PWD/hack/seaweedfs-naming-audit.sh" tenant-test 2>&1) || rc=$? + printf '%s\n' "$out" > "$d/got" + _expected_L > "$d/want" + echo "rc=$rc"; echo "--- got ---"; cat "$d/got" + [ "$rc" -eq 0 ] + # The decoy name must never reach the report. + [ "$(printf '%s\n' "$out" | grep -c 'decoy-not-seaweedfs')" -eq 0 ] + diff "$d/want" "$d/got" + rm -rf "$d" +} + +@test "fails closed when the by-name PVC GET hits a real error" { + # pv_epoch resolves each claim's bound PV by name before reading the PV's age. + # A real error there (RBAC/timeout) must abort: silently dropping the claim + # shrinks the observed vintage range, which is what produces a wrong "candidate + # duplicate" verdict. The PV half of this pair was already covered; this is the + # PVC half, whose fake FAIL mode existed with no test behind it. + d=$(mktemp -d) + _write_fake_kubectl "$d" pvcget "" "$(_release_blob)" "$(_pvcs_mixed)" + rc=0 + out=$(PATH="$d:$PATH" SEAWEEDFS_AUDIT_LIB=0 sh "$PWD/hack/seaweedfs-naming-audit.sh" tenant-test 2>&1) || rc=$? + rm -rf "$d" + echo "rc=$rc"; echo "$out" + [ "$rc" -ne 0 ] + printf '%s\n' "$out" | grep -q 'FATAL' + printf '%s\n' "$out" | grep -q "get PVC '" +} diff --git a/hack/seaweedfs-naming-audit.sh b/hack/seaweedfs-naming-audit.sh index b01138a276..b66e91939e 100755 --- a/hack/seaweedfs-naming-audit.sh +++ b/hack/seaweedfs-naming-audit.sh @@ -191,11 +191,33 @@ system_releases() { # the very false-clean this guard exists to stop -- so it is FATAL. A present # but different chart name is a real, non-SeaweedFS release and stays a # legitimate skip, filtered exactly as before by the cozy-seaweedfs test. - # ("chart" is the release struct's only such key -- vendored subcharts nest - # under "dependencies" -- so this match is unambiguous.) + # + # Two properties make this match trustworthy, and both are load-bearing now + # that a miss is FATAL: + # + # * FIRST match, not last. Helm's Release marshals "chart" before + # "config" (the user's values), so a values subtree that happens to + # spell chart.metadata.name sits LATER in the payload -- and a greedy + # `sed 's/.*"chart"...'` would return that decoy instead of the real + # chart. A wrong name reads downstream as "not SeaweedFS" and drops a + # real release from the report: the exact silent false-clean this guard + # exists to stop. `grep -o | head -1` takes the leftmost match. + # * The chart -> metadata -> name key path stays ADJACENT. A looser "any + # 'name' after 'metadata'" matches chart.templates[].name, which Helm + # serializes immediately after metadata on EVERY healthy release, so it + # would return a template path for every tenant. + # + # Newlines are folded first so a pretty-printed payload parses at all (sed + # and grep are line-based), and whitespace around the punctuation is + # tolerated. Failing loudly on a payload this cannot read is the safe + # direction -- over-strictness stops the runbook, over-looseness lets it + # delete data -- so the message says what shape was expected. if [ -n "$_sr_json" ]; then - _sr_chart=$(printf '%s' "$_sr_json" | sed -n 's/.*"chart":{"metadata":{"name":"\([^"]*\)".*/\1/p' | head -1) - [ -n "$_sr_chart" ] || { audit_fatal "secret 'sh.helm.release.v1.$rel.v$rev' in namespace '$1' decoded to a payload with no chart name (corrupt Helm release)."; return 1; } + _sr_chart=$(printf '%s' "$_sr_json" | tr '\n' ' ' \ + | grep -o '"chart"[[:space:]]*:[[:space:]]*{[[:space:]]*"metadata"[[:space:]]*:[[:space:]]*{[[:space:]]*"name"[[:space:]]*:[[:space:]]*"[^"]*"' \ + | head -1 \ + | sed -n 's/.*"\([^"]*\)"$/\1/p') + [ -n "$_sr_chart" ] || { audit_fatal "could not read the chart name of secret 'sh.helm.release.v1.$rel.v$rev' in namespace '$1' at .chart.metadata.name (expected 'name' as metadata's first key; a corrupt Helm release, or a payload format this parser does not handle)."; return 1; } if [ "$_sr_chart" = cozy-seaweedfs ]; then printf '%s\n' "$rel"; fi fi break @@ -209,7 +231,16 @@ first_deployed() { _fd_revs=$(revisions "$1" "$2") || return $? for rev in $_fd_revs; do _fd_json=$(release_json "$1" "$2" "$rev") || return $? - ts=$(printf '%s' "$_fd_json" | sed -n 's/.*"first_deployed":"\([^"]*\)".*/\1/p' | head -1) + # Same extraction discipline as the chart name above: fold newlines so a + # pretty-printed payload parses at all, tolerate whitespace around the + # punctuation, and take the FIRST match (.info precedes .config, so a values + # subtree cannot shadow the real timestamp). A miss here is not fatal -- the + # caller degrades to the documented safe fallback -- but a WRONG timestamp + # would silently change a vintage verdict, which is worse than none. + ts=$(printf '%s' "$_fd_json" | tr '\n' ' ' \ + | grep -o '"first_deployed"[[:space:]]*:[[:space:]]*"[^"]*"' \ + | head -1 \ + | sed -n 's/.*"\([^"]*\)"$/\1/p') if [ -n "$ts" ]; then date -u -d "$(printf '%s' "$ts" | cut -c1-19)" +%s 2>/dev/null; return; fi done } From 2298d1f26c767d177853d647b44c61c01ed15599 Mon Sep 17 00:00:00 2001 From: IvanHunters Date: Fri, 7 Aug 2026 16:36:49 +0300 Subject: [PATCH 54/69] fix(kube-ovn): bound webhook server request timeouts The webhook http.Server set no read, write, or idle timeout, so a client that opens a connection and sends headers slowly could hold a goroutine and file descriptor indefinitely. Add ReadHeaderTimeout, ReadTimeout, WriteTimeout, and IdleTimeout. Signed-off-by: IvanHunters (cherry picked from commit 4151885871d917f20d3840616045f9c4d45d1f4c) --- .../kubeovn-webhook/images/kubeovn-webhook/main.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/packages/system/kubeovn-webhook/images/kubeovn-webhook/main.go b/packages/system/kubeovn-webhook/images/kubeovn-webhook/main.go index a85776c605..7f1d8ec7ab 100644 --- a/packages/system/kubeovn-webhook/images/kubeovn-webhook/main.go +++ b/packages/system/kubeovn-webhook/images/kubeovn-webhook/main.go @@ -4,6 +4,7 @@ import ( "flag" "log" "net/http" + "time" ) var ( @@ -33,9 +34,13 @@ func main() { } server := &http.Server{ - Addr: ":8443", - TLSConfig: tlsConfig, - Handler: mux, + Addr: ":8443", + TLSConfig: tlsConfig, + Handler: mux, + ReadHeaderTimeout: 10 * time.Second, + ReadTimeout: 30 * time.Second, + WriteTimeout: 30 * time.Second, + IdleTimeout: 60 * time.Second, } log.Printf("Starting webhook server on %s", server.Addr) From 4759a9bb11da51e057b7764516d185b03d6fbd7b Mon Sep 17 00:00:00 2001 From: IvanHunters Date: Tue, 4 Aug 2026 23:07:10 +0300 Subject: [PATCH 55/69] fix(flux-shard-operator): strip proxy env from cloned helm-controller A sharded helm-controller inherits HTTP_PROXY/HTTPS_PROXY/NO_PROXY from the flux-aio helm-controller container. A standalone shard needs no external egress (source-controller fetches artifacts), so behind an unreachable corporate proxy it makes a blocking startup HTTPS call that never completes, the manager never serves /healthz, and the liveness probe crashloops the pod forever, freezing every HelmRelease carrying a sharding key. flux-aio itself survives only because it starts once and never restarts. Drop the proxy env (and the now-pointless NO_PROXY) when cloning, alongside the existing localhost/KubePrism sanitisation, so the shard talks to the in-cluster apiserver directly. Signed-off-by: IvanHunters (cherry picked from commit c475faae0fbe1b72961f8aa62149531024e0fe10) --- internal/fluxshardoperator/provisioner.go | 11 +++++++++++ internal/fluxshardoperator/provisioner_test.go | 14 ++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/internal/fluxshardoperator/provisioner.go b/internal/fluxshardoperator/provisioner.go index a0acb9400c..fdbe24e864 100644 --- a/internal/fluxshardoperator/provisioner.go +++ b/internal/fluxshardoperator/provisioner.go @@ -316,6 +316,17 @@ func BuildShardDeployment(flux *appsv1.Deployment, idx int, cfg *Config) (*appsv // is not hostNetwork, so it must fall back to the in-cluster defaults. case "KUBERNETES_SERVICE_HOST", "KUBERNETES_SERVICE_PORT": continue + // Corporate-proxy env inherited from flux-aio. A standalone shard needs + // no external egress (source-controller fetches artifacts), so the proxy + // only harms it: a startup HTTPS call that stalls through an unreachable + // proxy blocks the manager before it serves /healthz, and the liveness + // probe then crashloops the pod indefinitely. flux-aio survives only + // because it starts once and never restarts. Drop the proxy env (and the + // now-pointless NO_PROXY) so the shard talks to the in-cluster apiserver + // directly. + case "HTTP_PROXY", "HTTPS_PROXY", "NO_PROXY", + "http_proxy", "https_proxy", "no_proxy": + continue } env = append(env, e) } diff --git a/internal/fluxshardoperator/provisioner_test.go b/internal/fluxshardoperator/provisioner_test.go index 678f93833c..96f742a08e 100644 --- a/internal/fluxshardoperator/provisioner_test.go +++ b/internal/fluxshardoperator/provisioner_test.go @@ -92,6 +92,15 @@ func fluxAIODeployment() *appsv1.Deployment { FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.namespace"}, }}, {Name: "TUF_ROOT", Value: "/tmp/.sigstore"}, + // Corporate-proxy env the installer puts on + // flux-aio; harmless there but hangs a + // standalone shard at startup. + {Name: "HTTP_PROXY", Value: "http://proxy.example:3128"}, + {Name: "HTTPS_PROXY", Value: "http://proxy.example:3128"}, + {Name: "NO_PROXY", Value: ".svc"}, + {Name: "http_proxy", Value: "http://proxy.example:3128"}, + {Name: "https_proxy", Value: "http://proxy.example:3128"}, + {Name: "no_proxy", Value: ".svc"}, }, VolumeMounts: []corev1.VolumeMount{{Name: "tmp", MountPath: "/tmp"}}, }, @@ -188,6 +197,11 @@ func TestBuildShardDeployment(t *testing.T) { // A non-hostNetwork pod dialing the node-local KubePrism endpoint // crashloops on "dial tcp [::1]:7445: connect: connection refused". t.Fatalf("node-local apiserver endpoint env leaked through: %s", e.Name) + case "HTTP_PROXY", "HTTPS_PROXY", "NO_PROXY", + "http_proxy", "https_proxy", "no_proxy": + // A standalone shard behind an unreachable proxy hangs at startup + // and crashloops before it ever serves /healthz. + t.Fatalf("corporate-proxy env leaked through: %s", e.Name) } } envNames := make([]string, 0, len(hc.Env)) From ee3e162d6c76236eddc63fc42ed34ac15dc88693 Mon Sep 17 00:00:00 2001 From: IvanHunters Date: Tue, 4 Aug 2026 23:08:39 +0300 Subject: [PATCH 56/69] fix(flux-shard-operator): add startupProbe to cloned helm-controller The cloned shard Deployment inherits only a liveness probe (~30s window, no startupProbe), so a controller still syncing caches on startup is killed before it serves /healthz and never recovers. Derive a startupProbe from the liveness handler with a generous budget so a slow start is not fatal, while liveness still catches a wedged running pod. Signed-off-by: IvanHunters (cherry picked from commit d44c87d9394652b688d2776c320db82f49ccc71f) --- internal/fluxshardoperator/provisioner.go | 16 ++++++++++++++ .../fluxshardoperator/provisioner_test.go | 21 +++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/internal/fluxshardoperator/provisioner.go b/internal/fluxshardoperator/provisioner.go index fdbe24e864..d6ad8aa426 100644 --- a/internal/fluxshardoperator/provisioner.go +++ b/internal/fluxshardoperator/provisioner.go @@ -338,6 +338,22 @@ func BuildShardDeployment(flux *appsv1.Deployment, idx int, cfg *Config) (*appsv mergeResourceList(&hc.Resources.Requests, cfg.ShardResources.Requests) mergeResourceList(&hc.Resources.Limits, cfg.ShardResources.Limits) + // Guard startup with a startupProbe derived from the liveness handler. + // Without it the inherited ~30s liveness window kills a controller that is + // still syncing caches (a slow start on a large cluster, or a transient + // dependency), turning any slow start into an unrecoverable crashloop. The + // generous startup budget defers liveness until the manager is serving, + // then liveness still catches a wedged running pod. + if hc.LivenessProbe != nil && hc.StartupProbe == nil { + sp := hc.LivenessProbe.DeepCopy() + sp.InitialDelaySeconds = 0 + sp.PeriodSeconds = 10 + sp.TimeoutSeconds = 1 + sp.SuccessThreshold = 1 + sp.FailureThreshold = 30 + hc.StartupProbe = sp + } + mounted := map[string]bool{} for _, m := range hc.VolumeMounts { mounted[m.Name] = true diff --git a/internal/fluxshardoperator/provisioner_test.go b/internal/fluxshardoperator/provisioner_test.go index 96f742a08e..9f4a0e1cf9 100644 --- a/internal/fluxshardoperator/provisioner_test.go +++ b/internal/fluxshardoperator/provisioner_test.go @@ -8,6 +8,7 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" ) // fluxAIODeployment models the relevant shape of the flux-aio "flux" @@ -103,6 +104,16 @@ func fluxAIODeployment() *appsv1.Deployment { {Name: "no_proxy", Value: ".svc"}, }, VolumeMounts: []corev1.VolumeMount{{Name: "tmp", MountPath: "/tmp"}}, + LivenessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/healthz", + Port: intstr.FromString("healthz-hc"), + }, + }, + PeriodSeconds: 10, + FailureThreshold: 3, + }, }, {Name: "notification-controller", Image: "ghcr.io/fluxcd/notification-controller:v1.8.0"}, }, @@ -215,6 +226,16 @@ func TestBuildShardDeployment(t *testing.T) { if hc.Resources.Limits.Memory().String() != "1Gi" { t.Fatalf("resources override not applied: %v", hc.Resources) } + + if hc.StartupProbe == nil { + t.Fatal("startupProbe must be added so a slow start is not liveness-killed into a crashloop") + } + if hc.StartupProbe.HTTPGet == nil || hc.StartupProbe.HTTPGet.Path != "/healthz" { + t.Fatalf("startupProbe must reuse the liveness /healthz handler: %+v", hc.StartupProbe) + } + if hc.StartupProbe.FailureThreshold < 10 { + t.Fatalf("startupProbe budget too small to cover a slow cache sync: %d", hc.StartupProbe.FailureThreshold) + } } func TestBuildShardDeploymentInheritsResourcesWhenUnset(t *testing.T) { From 2713cccb462597fe559feee4a9b2722997480b8b Mon Sep 17 00:00:00 2001 From: IvanHunters Date: Tue, 4 Aug 2026 23:22:59 +0300 Subject: [PATCH 57/69] fix(flux-shard-operator): inherit probe timeout, correct comments Review follow-ups: - Do not force the cloned startupProbe TimeoutSeconds to 1: keep the value inherited from the liveness handler, otherwise a future flux-aio with a larger liveness timeout would get a stricter startup probe and could recreate the crashloop. Covered by a regression guard (liveness timeout 5s must be inherited). - Soften the proxy-strip rationale to the A/B-confirmed mechanism (blocking external HTTPS call at startup) and note the remote spec.kubeConfig exception that does not apply to cozystack in-cluster guest apiservers. - Correct the test fixture comment: the proxy env is operator/patch-level, not installer-injected (the installer injects only KUBERNETES_SERVICE_*). Signed-off-by: IvanHunters (cherry picked from commit affdd70e9d38f4560613e8f41003e1b6f2426757) --- internal/fluxshardoperator/provisioner.go | 25 ++++++++++++------- .../fluxshardoperator/provisioner_test.go | 12 ++++++--- 2 files changed, 25 insertions(+), 12 deletions(-) diff --git a/internal/fluxshardoperator/provisioner.go b/internal/fluxshardoperator/provisioner.go index d6ad8aa426..3d6a0bde59 100644 --- a/internal/fluxshardoperator/provisioner.go +++ b/internal/fluxshardoperator/provisioner.go @@ -317,13 +317,17 @@ func BuildShardDeployment(flux *appsv1.Deployment, idx int, cfg *Config) (*appsv case "KUBERNETES_SERVICE_HOST", "KUBERNETES_SERVICE_PORT": continue // Corporate-proxy env inherited from flux-aio. A standalone shard needs - // no external egress (source-controller fetches artifacts), so the proxy - // only harms it: a startup HTTPS call that stalls through an unreachable - // proxy blocks the manager before it serves /healthz, and the liveness - // probe then crashloops the pod indefinitely. flux-aio survives only - // because it starts once and never restarts. Drop the proxy env (and the - // now-pointless NO_PROXY) so the shard talks to the in-cluster apiserver - // directly. + // no external egress (source-controller does artifact fetching and + // cosign/TUF verification), so the proxy only harms it: a blocking + // external HTTPS call at startup stalls through an unreachable proxy, + // the manager never serves /healthz, and the liveness probe then + // crashloops the pod. (flux-aio itself is unaffected in practice: it is + // long-lived and does not restart, so it never re-hits this startup + // path.) Drop the proxy env (and the now-pointless NO_PROXY) so the + // shard reaches the in-cluster apiserver directly. A HelmRelease + // targeting a remote cluster via spec.kubeConfig reachable only through + // the proxy is the one theoretical exception; cozystack guest-cluster + // apiservers are in-cluster, so this does not apply here. case "HTTP_PROXY", "HTTPS_PROXY", "NO_PROXY", "http_proxy", "https_proxy", "no_proxy": continue @@ -343,12 +347,15 @@ func BuildShardDeployment(flux *appsv1.Deployment, idx int, cfg *Config) (*appsv // still syncing caches (a slow start on a large cluster, or a transient // dependency), turning any slow start into an unrecoverable crashloop. The // generous startup budget defers liveness until the manager is serving, - // then liveness still catches a wedged running pod. + // then liveness still catches a wedged running pod. Only the budget fields + // are normalised; the liveness handler and its TimeoutSeconds are inherited, + // so a controller whose /healthz is slow under load keeps the same tolerance + // at startup as at runtime (overriding TimeoutSeconds down could make the + // startup probe stricter than liveness and recreate the crashloop). if hc.LivenessProbe != nil && hc.StartupProbe == nil { sp := hc.LivenessProbe.DeepCopy() sp.InitialDelaySeconds = 0 sp.PeriodSeconds = 10 - sp.TimeoutSeconds = 1 sp.SuccessThreshold = 1 sp.FailureThreshold = 30 hc.StartupProbe = sp diff --git a/internal/fluxshardoperator/provisioner_test.go b/internal/fluxshardoperator/provisioner_test.go index 9f4a0e1cf9..0058941116 100644 --- a/internal/fluxshardoperator/provisioner_test.go +++ b/internal/fluxshardoperator/provisioner_test.go @@ -93,9 +93,10 @@ func fluxAIODeployment() *appsv1.Deployment { FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.namespace"}, }}, {Name: "TUF_ROOT", Value: "/tmp/.sigstore"}, - // Corporate-proxy env the installer puts on - // flux-aio; harmless there but hangs a - // standalone shard at startup. + // Corporate-proxy env present on flux-aio in + // proxied installs (operator/patch level, not the + // installer, which injects only KUBERNETES_SERVICE_*); + // harmless there, hangs a standalone shard at startup. {Name: "HTTP_PROXY", Value: "http://proxy.example:3128"}, {Name: "HTTPS_PROXY", Value: "http://proxy.example:3128"}, {Name: "NO_PROXY", Value: ".svc"}, @@ -112,6 +113,7 @@ func fluxAIODeployment() *appsv1.Deployment { }, }, PeriodSeconds: 10, + TimeoutSeconds: 5, FailureThreshold: 3, }, }, @@ -236,6 +238,10 @@ func TestBuildShardDeployment(t *testing.T) { if hc.StartupProbe.FailureThreshold < 10 { t.Fatalf("startupProbe budget too small to cover a slow cache sync: %d", hc.StartupProbe.FailureThreshold) } + if hc.StartupProbe.TimeoutSeconds != 5 { + t.Fatalf("startupProbe must inherit the liveness TimeoutSeconds (5), not force it stricter: got %d", + hc.StartupProbe.TimeoutSeconds) + } } func TestBuildShardDeploymentInheritsResourcesWhenUnset(t *testing.T) { From dd6a8a2416ddde17bf54802a6d44fb9d5481b1d5 Mon Sep 17 00:00:00 2001 From: IvanHunters Date: Fri, 7 Aug 2026 16:33:13 +0300 Subject: [PATCH 58/69] docs(flux-shard-operator): record proxy-env drop and startupProbe in sanitisation list The BuildShardDeployment doc comment and the operator README are the only record of how far the cloned shard container has diverged from flux-aio. Add the two steps this fix introduced (dropping the inherited corporate-proxy env and deriving a startupProbe) so the list stays the authoritative account and does not invite re-adding what was deliberately removed. Signed-off-by: IvanHunters (cherry picked from commit 036a31b550e080553da3fef1c6365ed9b579fe45) --- internal/fluxshardoperator/provisioner.go | 9 ++++++++- packages/system/flux-shard-operator/README.md | 2 +- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/internal/fluxshardoperator/provisioner.go b/internal/fluxshardoperator/provisioner.go index 3d6a0bde59..f430ba8947 100644 --- a/internal/fluxshardoperator/provisioner.go +++ b/internal/fluxshardoperator/provisioner.go @@ -263,7 +263,14 @@ func mergeResourceList(dst *corev1.ResourceList, overrides corev1.ResourceList) // - the required podAntiAffinity cloned from flux-aio (which keeps its own // replicas off one node) is dropped, since it targets // app.kubernetes.io/name=flux and would otherwise leave every shard -// Pending on a single-node cluster. +// Pending on a single-node cluster; +// - the corporate-proxy env (HTTP_PROXY/HTTPS_PROXY/NO_PROXY) inherited from +// flux-aio is dropped: a standalone shard needs no external egress, and an +// unreachable proxy stalls a blocking startup HTTPS call, so the manager +// never serves /healthz and the pod crashloops; +// - a startupProbe is derived from the liveness handler (generous failure +// budget, liveness handler and TimeoutSeconds inherited) so a slow but +// progressing start is not killed by the short inherited liveness window. func BuildShardDeployment(flux *appsv1.Deployment, idx int, cfg *Config) (*appsv1.Deployment, error) { var src *corev1.Container for i := range flux.Spec.Template.Spec.Containers { diff --git a/packages/system/flux-shard-operator/README.md b/packages/system/flux-shard-operator/README.md index 660a6542b6..9f06cb9b6a 100644 --- a/packages/system/flux-shard-operator/README.md +++ b/packages/system/flux-shard-operator/README.md @@ -6,7 +6,7 @@ Spreads tenant HelmReleases across multiple helm-controller shards so a noisy te The operator has three parts, all served by one Deployment (leader-elected controllers, webhook on every replica): -1. **Shard runtime.** Reconciles `shardCount` helm-controller Deployments (`helm-controller-shard`, `--watch-label-selector=sharding.fluxcd.io/key=shard`) in the flux namespace, cloned from the flux-aio `flux` Deployment's helm-controller container and sanitised (no host networking, no localhost cross-container wiring). The helm-controller image and feature-gates are inherited from flux-aio automatically. Deployments beyond `shardCount` are pruned once they drain, and the legacy hand-rolled `flux-tenants` Deployment is retired once no HelmRelease carries `sharding.fluxcd.io/key=tenants`. +1. **Shard runtime.** Reconciles `shardCount` helm-controller Deployments (`helm-controller-shard`, `--watch-label-selector=sharding.fluxcd.io/key=shard`) in the flux namespace, cloned from the flux-aio `flux` Deployment's helm-controller container and sanitised (no host networking, no localhost cross-container wiring, no inherited corporate-proxy env, and a startupProbe guarding a slow start). The helm-controller image and feature-gates are inherited from flux-aio automatically. Deployments beyond `shardCount` are pruned once they drain, and the legacy hand-rolled `flux-tenants` Deployment is retired once no HelmRelease carries `sharding.fluxcd.io/key=tenants`. 2. **Placement controller.** Owns the tenant→shard assignment. The unit of placement is the tenant: all HelmReleases of one tenant (parent `tenant-` plus everything in namespace `tenant-`) carry the same shard label, so a noisy tenant's blast radius is bounded to its shard's co-residents. Tenants are distributed greedy least-loaded, weighted by HelmRelease count (N tenants over N shards land exactly 1 per shard). The assignment is recorded as the `internal.cozystack.io/flux-shard` label on the tenant namespace; HelmRelease labels remain the source of truth on restarts. Moves are paced and deleting tenants are never moved. Watches are metadata-only, so the controller does not decode the helm-controller status-patch firehose. From a2632c9a086687a853387cb1bb2ad15719d33a96 Mon Sep 17 00:00:00 2001 From: IvanHunters Date: Fri, 7 Aug 2026 16:33:21 +0300 Subject: [PATCH 59/69] test(flux-shard-operator): assert full startupProbe contract and reject alias regression Model the source liveness probe as flux-aio actually ships it (a bare httpGet with no timing fields) and tighten the startupProbe assertions: - assert the complete normalised budget (delay/period/success/failure) instead of only FailureThreshold, so a later PeriodSeconds regression cannot silently restore a short crashloop window; - assert TimeoutSeconds is inherited from the source probe rather than a hardcoded value, so forcing it stricter is caught; - assert the startupProbe is a DeepCopy of liveness, not an alias, which would otherwise stamp the 30-failure startup budget onto liveness and produce a never-failing liveness probe. The alias assertion is non-vacuous: it fails when the DeepCopy is regressed to a plain alias. Signed-off-by: IvanHunters (cherry picked from commit a10fcb3e4a6b8074c25eec46dce68bc4db4384b4) --- .../fluxshardoperator/provisioner_test.go | 55 +++++++++++++++---- 1 file changed, 45 insertions(+), 10 deletions(-) diff --git a/internal/fluxshardoperator/provisioner_test.go b/internal/fluxshardoperator/provisioner_test.go index 0058941116..96ad34ff00 100644 --- a/internal/fluxshardoperator/provisioner_test.go +++ b/internal/fluxshardoperator/provisioner_test.go @@ -105,6 +105,11 @@ func fluxAIODeployment() *appsv1.Deployment { {Name: "no_proxy", Value: ".svc"}, }, VolumeMounts: []corev1.VolumeMount{{Name: "tmp", MountPath: "/tmp"}}, + // flux-aio ships helm-controller with a bare httpGet + // liveness probe: no timing fields at all, so the + // kube-apiserver defaults (~30s window) apply. The + // startupProbe must inherit this handler and its unset + // TimeoutSeconds and only normalise the startup budget. LivenessProbe: &corev1.Probe{ ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ @@ -112,9 +117,6 @@ func fluxAIODeployment() *appsv1.Deployment { Port: intstr.FromString("healthz-hc"), }, }, - PeriodSeconds: 10, - TimeoutSeconds: 5, - FailureThreshold: 3, }, }, {Name: "notification-controller", Image: "ghcr.io/fluxcd/notification-controller:v1.8.0"}, @@ -136,7 +138,18 @@ func TestBuildShardDeployment(t *testing.T) { }, } - dep, err := BuildShardDeployment(fluxAIODeployment(), 2, cfg) + flux := fluxAIODeployment() + // Capture the source liveness probe verbatim so the startupProbe assertions + // can check inheritance against what flux-aio actually ships, rather than + // against a hardcoded value baked into the test. + var srcLiveness *corev1.Probe + for i := range flux.Spec.Template.Spec.Containers { + if flux.Spec.Template.Spec.Containers[i].Name == "helm-controller" { + srcLiveness = flux.Spec.Template.Spec.Containers[i].LivenessProbe.DeepCopy() + } + } + + dep, err := BuildShardDeployment(flux, 2, cfg) if err != nil { t.Fatal(err) } @@ -232,15 +245,37 @@ func TestBuildShardDeployment(t *testing.T) { if hc.StartupProbe == nil { t.Fatal("startupProbe must be added so a slow start is not liveness-killed into a crashloop") } + // Handler inherited from liveness verbatim. if hc.StartupProbe.HTTPGet == nil || hc.StartupProbe.HTTPGet.Path != "/healthz" { t.Fatalf("startupProbe must reuse the liveness /healthz handler: %+v", hc.StartupProbe) } - if hc.StartupProbe.FailureThreshold < 10 { - t.Fatalf("startupProbe budget too small to cover a slow cache sync: %d", hc.StartupProbe.FailureThreshold) - } - if hc.StartupProbe.TimeoutSeconds != 5 { - t.Fatalf("startupProbe must inherit the liveness TimeoutSeconds (5), not force it stricter: got %d", - hc.StartupProbe.TimeoutSeconds) + // Assert the complete normalised startup budget, not just FailureThreshold: + // a threshold check alone still passes if PeriodSeconds later regresses to 1, + // which would silently restore a short crashloop window. + if hc.StartupProbe.InitialDelaySeconds != 0 || + hc.StartupProbe.PeriodSeconds != 10 || + hc.StartupProbe.SuccessThreshold != 1 || + hc.StartupProbe.FailureThreshold != 30 { + t.Fatalf("startupProbe budget not normalised to the expected contract "+ + "(delay=0 period=10 success=1 failure=30): %+v", hc.StartupProbe) + } + // TimeoutSeconds is inherited from the source liveness probe, never forced. + // flux-aio ships a bare probe (TimeoutSeconds 0), so a stray + // sp.TimeoutSeconds = N would diverge from the source and fail here. + if hc.StartupProbe.TimeoutSeconds != srcLiveness.TimeoutSeconds { + t.Fatalf("startupProbe must inherit the liveness TimeoutSeconds (%d), got %d", + srcLiveness.TimeoutSeconds, hc.StartupProbe.TimeoutSeconds) + } + // The startupProbe must be a DeepCopy of the liveness probe, never an alias. + // If it aliased, normalising the startup FailureThreshold to 30 would also + // stamp 30 onto liveness, producing exactly the never-failing liveness probe + // this change exists to avoid. + if hc.LivenessProbe == hc.StartupProbe { + t.Fatal("startupProbe must be a DeepCopy of liveness, not an alias sharing its backing probe") + } + if hc.LivenessProbe.FailureThreshold != srcLiveness.FailureThreshold { + t.Fatalf("liveness FailureThreshold was clobbered by the startup budget "+ + "(alias regression): source=%d live=%d", srcLiveness.FailureThreshold, hc.LivenessProbe.FailureThreshold) } } From d8c8276e264303e158c7302e3fba9f6c747e39bd Mon Sep 17 00:00:00 2001 From: IvanHunters Date: Mon, 10 Aug 2026 12:56:42 +0300 Subject: [PATCH 60/69] docs(flux-shard-operator): drop unproven mechanism from proxy comments Name all six proxy env spellings in the sanitisation list, and replace the unestablished 'external HTTPS call' / 'flux-aio does not restart' claims with the grounded in-cluster mechanism: after the KUBERNETES_SERVICE_HOST drop above, the shard talks to the management apiserver over the kubelet-injected ClusterIP, which NO_PROXY=.svc does not match, so that startup call is what stalls behind an unreachable proxy. Comment-only; no behaviour change. Signed-off-by: IvanHunters (cherry picked from commit 66df3b20e05f0b64a321ccb8866a80b4ac93b68a) --- internal/fluxshardoperator/provisioner.go | 34 +++++++++++++---------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/internal/fluxshardoperator/provisioner.go b/internal/fluxshardoperator/provisioner.go index f430ba8947..1a0066d59d 100644 --- a/internal/fluxshardoperator/provisioner.go +++ b/internal/fluxshardoperator/provisioner.go @@ -264,10 +264,11 @@ func mergeResourceList(dst *corev1.ResourceList, overrides corev1.ResourceList) // replicas off one node) is dropped, since it targets // app.kubernetes.io/name=flux and would otherwise leave every shard // Pending on a single-node cluster; -// - the corporate-proxy env (HTTP_PROXY/HTTPS_PROXY/NO_PROXY) inherited from -// flux-aio is dropped: a standalone shard needs no external egress, and an -// unreachable proxy stalls a blocking startup HTTPS call, so the manager -// never serves /healthz and the pod crashloops; +// - the corporate-proxy env inherited from flux-aio is dropped, in both the +// upper- and lower-case spellings (HTTP_PROXY/HTTPS_PROXY/NO_PROXY): a +// standalone shard needs no external egress, and behind an unreachable +// proxy a stalled startup call leaves the manager never serving /healthz, +// so the liveness probe crashloops the pod; // - a startupProbe is derived from the liveness handler (generous failure // budget, liveness handler and TimeoutSeconds inherited) so a slow but // progressing start is not killed by the short inherited liveness window. @@ -324,17 +325,20 @@ func BuildShardDeployment(flux *appsv1.Deployment, idx int, cfg *Config) (*appsv case "KUBERNETES_SERVICE_HOST", "KUBERNETES_SERVICE_PORT": continue // Corporate-proxy env inherited from flux-aio. A standalone shard needs - // no external egress (source-controller does artifact fetching and - // cosign/TUF verification), so the proxy only harms it: a blocking - // external HTTPS call at startup stalls through an unreachable proxy, - // the manager never serves /healthz, and the liveness probe then - // crashloops the pod. (flux-aio itself is unaffected in practice: it is - // long-lived and does not restart, so it never re-hits this startup - // path.) Drop the proxy env (and the now-pointless NO_PROXY) so the - // shard reaches the in-cluster apiserver directly. A HelmRelease - // targeting a remote cluster via spec.kubeConfig reachable only through - // the proxy is the one theoretical exception; cozystack guest-cluster - // apiservers are in-cluster, so this does not apply here. + // no external egress: source-controller does artifact fetching and + // cosign/TUF verification, and every hop the shard makes is in-cluster + // (artifacts advertise as flux.$(RUNTIME_NAMESPACE).svc, guest-cluster + // apiservers are reached over .svc kubeconfigs). NO_PROXY=.svc covers + // those, but not the management apiserver: the KUBERNETES_SERVICE_HOST + // case above makes the shard fall back to the kubelet-injected + // ClusterIP, a bare IP that no .svc suffix matches, so that startup + // call is the one that stalls through an unreachable proxy and leaves + // the manager never serving /healthz until the liveness probe + // crashloops the pod. Dropping the proxy env (and the then-pointless + // NO_PROXY) keeps every hop direct. A HelmRelease targeting a remote + // cluster via spec.kubeConfig reachable only through the proxy is the + // one theoretical exception; cozystack guest-cluster apiservers are + // in-cluster, so this does not apply here. case "HTTP_PROXY", "HTTPS_PROXY", "NO_PROXY", "http_proxy", "https_proxy", "no_proxy": continue From 509a756ece1b874b8f6c8532bec28b3ad6cc0535 Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Mon, 17 Aug 2026 15:24:15 +0500 Subject: [PATCH 61/69] fix(kubernetes): pin KubeadmConfigTemplate with resource-policy=keep MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Platform migration 45 pins the KubeadmConfigTemplate objects that exist when a cluster runs it. That leaves a hole: a template created AFTERWARDS — a new tenant Kubernetes cluster, or a nodeGroup added while on 1.5.4 — is born without the annotation, and 1.6's own slot 45 never runs for that cluster because a v1.5.4 cluster is stamped 46 and runs `seq 46 53`. The 1.6 kubernetes chart upgrade then prunes the unpinned template while the kubeadm-backed MachineSet still references it from bootstrap.configRef, which is the bug the migration exists to prevent. Render the annotation in the chart so every template is born pinned. The migration then covers only the objects that predate it, and the two together leave no window. KEEP IS SAFE ON THE UNINSTALL PATH, which is the only behaviour it changes here. CAPI stamps an ownerReference to the Cluster on this object — verified on a live v1.5 stand, where KubeadmConfigTemplate, MachineDeployment and KubevirtMachineTemplate all carry ownerReferences: [Cluster/] that the chart itself does not render — and the Cluster is Helm-managed with no keep of its own. So `helm uninstall` deletes the Cluster and Kubernetes garbage collection reclaims the template through that reference. keep suppresses Helm's own delete; it has no bearing on owner-driven GC, so uninstall does not leak. The name is deterministic, -, not content-hashed the way KubevirtMachineTemplate is, so keep cannot accumulate a copy per upgrade. The one residue is a removed nodeGroup, whose template Helm no longer prunes: it is inert because nothing references it, it is re-adopted in place if that nodeGroup returns, and GC reclaims it with the Cluster. Not added to KubeadmConfig: this chart renders no bare KubeadmConfig, and the ones CAPI spawns from the template are owned by their Machine and carry no app.kubernetes.io/managed-by label, so Helm never manages or prunes them and the annotation would have nothing to act on. Confirmed on the same live stand. Nothing else in this chart sets helm.sh/resource-policy, so no existing conditional is being overridden. Tests assert the annotation on the rendered object for a single nodeGroup and for every group when two are configured — the second because the annotation has to sit inside the per-nodeGroup range, and a group added later is exactly the one that would otherwise be left prunable. Both were confirmed red before being made to pass: with the annotation removed from the template they fail on `unknown path metadata.annotations["helm.sh/resource-policy"]`, which is the object being born unpinned rather than a broken assertion. Signed-off-by: Myasnikov Daniil --- .../apps/kubernetes/templates/cluster.yaml | 27 +++++ .../apps/kubernetes/tests/cluster_test.yaml | 99 +++++++++++++++++++ 2 files changed, 126 insertions(+) diff --git a/packages/apps/kubernetes/templates/cluster.yaml b/packages/apps/kubernetes/templates/cluster.yaml index fbc10a2f08..cd6884edf0 100644 --- a/packages/apps/kubernetes/templates/cluster.yaml +++ b/packages/apps/kubernetes/templates/cluster.yaml @@ -432,6 +432,33 @@ kind: KubeadmConfigTemplate metadata: name: {{ $.Release.Name }}-{{ $groupName }} namespace: {{ $.Release.Namespace }} + annotations: + {{- /* 1.6 drops this object from the chart entirely — workers move to + TalosConfigTemplate. On that upgrade Helm sees it in the previous + release manifest and absent from the new one and deletes it, while the + kubeadm-backed MachineSet is still mid-rollover with its + bootstrap.configRef pointing here: controller-manager floods with + reconcile errors and workers can hang pending with nothing to bootstrap + from. Being born with keep means the object is already protected no + matter which migration path a cluster takes to 1.6 — platform migration + 45 pins the templates that already exist, and this covers every one + created after that migration has run. + + Safe on the uninstall path, which is the only thing keep changes here: + CAPI stamps an ownerReference to the Cluster on this object (verified on + a live v1.5 stand, alongside MachineDeployment and + KubevirtMachineTemplate), and the Cluster itself is Helm-managed with no + keep. So `helm uninstall` deletes the Cluster and Kubernetes garbage + collection reclaims this template through that ownerReference. keep only + suppresses Helm's own delete; it has no bearing on owner-driven GC. + + The name is deterministic — -, not content-hashed + like KubevirtMachineTemplate above — so keep cannot accumulate a copy + per upgrade. The one residue is a removed nodeGroup, whose template + Helm no longer prunes; it is inert (nothing references it), it is + re-adopted in place if that nodeGroup comes back, and GC reclaims it + with the Cluster. */}} + helm.sh/resource-policy: keep spec: template: spec: diff --git a/packages/apps/kubernetes/tests/cluster_test.yaml b/packages/apps/kubernetes/tests/cluster_test.yaml index 52a49180f3..5ebff7cbd8 100644 --- a/packages/apps/kubernetes/tests/cluster_test.yaml +++ b/packages/apps/kubernetes/tests/cluster_test.yaml @@ -388,6 +388,105 @@ tests: - failedTemplate: errorMessage: 'nodeGroup "md0": ephemeralStorage is no longer supported and should have been automatically migrated to diskSize by platform migration 41. If you see this error after upgrading, the migration did not run — check the cozystack-migration-hook Job logs in cozy-system.' + ############################################### + # KubeadmConfigTemplate — Helm prune guard # + ############################################### + + # THE THREAT. 1.6 drops KubeadmConfigTemplate from this chart entirely — + # workers move to TalosConfigTemplate. On that upgrade Helm sees the object in + # the previous release manifest and absent from the new one and deletes it, + # while the kubeadm-backed MachineSet is still mid-rollover with its + # bootstrap.configRef pointing at it: controller-manager floods with reconcile + # errors and workers can hang pending with nothing to bootstrap from. + # + # Platform migration 45 pins the templates that already exist when a cluster + # upgrades. This annotation covers the ones created AFTERWARDS — a new tenant + # Kubernetes cluster, or a nodeGroup added on 1.5.4 — which that migration has + # already run past and which 1.6's own slot 45 will skip, because a 1.5.4 + # cluster is stamped 46 and runs `seq 46 53`. + # + # Safe on uninstall, the only path keep changes: CAPI stamps an ownerReference + # to the Cluster on this object, and the Cluster is Helm-managed with no keep, + # so `helm uninstall` deletes the Cluster and garbage collection reclaims this + # template through that reference. keep suppresses only Helm's own delete. + + - it: is born with helm.sh/resource-policy=keep so the 1.6 upgrade cannot prune it + release: + name: test-k8s + namespace: tenant-test + asserts: + - isKind: + of: KubeadmConfigTemplate + documentIndex: 4 + - equal: + path: metadata.name + value: test-k8s-md0 + documentIndex: 4 + - equal: + path: metadata.annotations["helm.sh/resource-policy"] + value: keep + documentIndex: 4 + + # The annotation must sit inside the per-nodeGroup range, not on one template by + # accident. With two groups every KubeadmConfigTemplate has to carry it, or the + # groups added later are exactly the ones left prunable. + - it: pins the KubeadmConfigTemplate of every node group, not just the first + release: + name: test-k8s + namespace: tenant-test + set: + _namespace: + etcd: etcd + ingress: nginx + host: example.com + _cluster: + cluster-domain: cozy.local + nodeGroups: + md0: + minReplicas: 0 + maxReplicas: 10 + instanceType: "" + diskSize: 20Gi + roles: + - worker + resources: {} + md1: + minReplicas: 1 + maxReplicas: 5 + instanceType: "" + diskSize: 40Gi + roles: + - worker + resources: + memory: "8Gi" + cpu: 4 + asserts: + # md0 at documentIndex 4; md1 at documentIndex 9, since each nodeGroup emits + # 5 documents (KubeadmConfigTemplate, KubevirtMachineTemplate, + # MachineDeployment, MachineHealthCheck, WorkloadMonitor). + - isKind: + of: KubeadmConfigTemplate + documentIndex: 4 + - equal: + path: metadata.name + value: test-k8s-md0 + documentIndex: 4 + - equal: + path: metadata.annotations["helm.sh/resource-policy"] + value: keep + documentIndex: 4 + - isKind: + of: KubeadmConfigTemplate + documentIndex: 9 + - equal: + path: metadata.name + value: test-k8s-md1 + documentIndex: 9 + - equal: + path: metadata.annotations["helm.sh/resource-policy"] + value: keep + documentIndex: 9 + ############################################### # KubeadmConfigTemplate — files block # ############################################### From 314aa5aa849e6b303d5689042de63b7067fa81c0 Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Mon, 17 Aug 2026 15:27:15 +0500 Subject: [PATCH 62/69] test(seaweedfs): make the guard-parity discriminator assertions able to fail MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The four assertions in "neither chart classifies on mutable claim timestamps or liveness" were written as `! grep -qF X "$f"`, which cannot fail: POSIX and bash both exempt a !-negated pipeline from errexit — "the -e setting shall be ignored ... if the command's return value is being inverted with !" — so the command runs, returns 1, and the test carries on reporting success. Those four were the entire body of the test, so it asserted nothing whatsoever. Converting them mechanically turns out to fail, and for a reason worth keeping: all four names DO appear in both files, in the prose that explains why each was rejected as a discriminator. `creationTimestamp` and `readyReplicas` are discussed at length in the comment blocks that record the decision. So a raw grep would fail on the documentation of the very invariant the test enforces. Assert against the template logic instead, with {{/* */}} comment blocks stripped by a small awk helper. Every comment opener in both files starts its own line, so dropping whole lines cannot take code with it. What the test now forbids is a LIVE reference, which is what it always meant. Proven in both directions: it passes with the explanatory prose in place, and it fails when a real `readyReplicas` lookup is injected into live template code in either file. The awk helper is deliberately a one-liner. cozytest.sh rewrites any bare `}` in column 0 into `return 0` + `}`, which would corrupt a multi-line awk body. Signed-off-by: Myasnikov Daniil --- hack/seaweedfs-guard-parity.bats | 33 ++++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/hack/seaweedfs-guard-parity.bats b/hack/seaweedfs-guard-parity.bats index cdca9311a4..38c194e494 100644 --- a/hack/seaweedfs-guard-parity.bats +++ b/hack/seaweedfs-guard-parity.bats @@ -37,6 +37,16 @@ detection_block() { sed -n '/\$renamedVol := include/,/\$systemGen := or/p' "$1" } +# strip_tpl_comments -- the file with every {{/* ... */}} block removed, so +# an assertion can be made about the template LOGIC without the prose around it +# matching. Every comment opener in both files starts its own line, so dropping +# whole lines cannot take code with it. Written as a single awk program: cozytest.sh +# rewrites any bare `}` in column 0 into `return 0` + `}`, which would corrupt a +# multi-line awk body. +strip_tpl_comments() { + awk '/\{\{-? *\/\*/{c=1} !c{print} /\*\/ *-?\}\}/{c=0}' "$1" +} + @test "both charts detect naming generations with byte-identical logic" { a=$(mktemp); b=$(mktemp) detection_block "$SYS" > "$a" @@ -103,11 +113,26 @@ detection_block() { # old classification pointed at deletes the claim Step 2 just re-bound. # readyReplicas is likewise only a snapshot, not proof a duplicate never # served. Neither may come back as a discriminator. + # + # Counted rather than written as `! grep -qF ...`: POSIX and bash both exempt a + # !-negated pipeline from errexit — "the -e setting shall be ignored ... if the + # command's return value is being inverted with !" — so the negated form runs, + # returns 1, and the test carries on reporting success no matter what the file + # contains. These four assertions were the entire body of this test, so it + # asserted nothing at all. Putting the count inside `[` gives it a status errexit + # acts on. + # + # Matched against the TEMPLATE LOGIC ONLY, with {{/* */}} comment blocks stripped. + # All four names legitimately appear in the prose of both files — in the passages + # that explain why they were rejected as discriminators — so grepping the raw file + # would fail on the very documentation that records the decision this test exists + # to enforce. What must not come back is a live reference. for f in "$SYS" "$EXTRA"; do - ! grep -qF -- 'creationTimestamp' "$f" - ! grep -qF -- 'readyReplicas' "$f" - ! grep -qF -- '$systemOldest' "$f" - ! grep -qF -- '$legacyOldest' "$f" + logic=$(strip_tpl_comments "$f") + [ "$(printf '%s\n' "$logic" | grep -cF -- 'creationTimestamp')" -eq 0 ] + [ "$(printf '%s\n' "$logic" | grep -cF -- 'readyReplicas')" -eq 0 ] + [ "$(printf '%s\n' "$logic" | grep -cF -- '$systemOldest')" -eq 0 ] + [ "$(printf '%s\n' "$logic" | grep -cF -- '$legacyOldest')" -eq 0 ] done } From 27d474dbe0e9158646ff64427385cc91eb2f182a Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Mon, 17 Aug 2026 17:49:19 +0500 Subject: [PATCH 63/69] test(keycloak-configure): run the chart's helm-unittest suite The chart ships tests/delete_test.yaml, but the package Makefile has no `test:` target, and hack/helm-unit-tests.sh runs a package's suite only when `make -C -n test` succeeds. The suite was therefore never executed on this branch, so the namespace assertion added alongside the teardown fix shipped as dead weight. Add the target in the same form the other packages use. On main the same three lines arrived via an unrelated login-theme commit rather than with the fix itself, which is why the backport landed without them. Verified by mutating the rendered patch back to the pre-fix hardcoded `-n cozy-system`: all three tests go red, so the suite asserts something. Signed-off-by: Myasnikov Daniil --- packages/system/keycloak-configure/Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/system/keycloak-configure/Makefile b/packages/system/keycloak-configure/Makefile index 4cd5bd2ed1..b0a357e692 100644 --- a/packages/system/keycloak-configure/Makefile +++ b/packages/system/keycloak-configure/Makefile @@ -3,3 +3,6 @@ export NAMESPACE=cozy-keycloak include ../../../hack/common-envs.mk include ../../../hack/package.mk + +test: + helm unittest . From 9de1f16cb084b6ec7574abf9f0926a9c820433f0 Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Mon, 17 Aug 2026 17:52:25 +0500 Subject: [PATCH 64/69] fix(cozystack-basics): gate the hostname VAP policies on the VAP API Backport of #3409 to release-1.5. gateway-hostname-policy.yaml and route-hostname-policy.yaml rendered their ValidatingAdmissionPolicy and binding documents unconditionally, so installing the chart against an API server that does not serve admissionregistration.k8s.io/v1/ValidatingAdmissionPolicy failed. Guard both on the capability, matching the existing pattern in packages/core/platform/templates/deletion-protection.yaml, and declare the capability in the two rendering test suites so they keep exercising the policies. hostname-policies-capability-gate_test.yaml pins the absent-API case at suite level, since helm-unittest merges per-test capabilities into the suite's rather than replacing them. The upstream change also gated a third template, ingress-hostname-policy.yaml. That legacy-Ingress hostname policy landed on main in d89ad7069 and was never backported, so it does not exist on release-1.5 and is deliberately excluded here: importing it would ship an unreviewed admission policy that starts rejecting tenant Ingress hostnames in a patch release. Signed-off-by: Myasnikov Daniil --- .../templates/gateway-hostname-policy.yaml | 3 +++ .../templates/route-hostname-policy.yaml | 3 +++ .../tests/gateway-hostname-policy_test.yaml | 7 ++++++ ...ostname-policies-capability-gate_test.yaml | 23 +++++++++++++++++++ .../tests/route-hostname-policy_test.yaml | 7 ++++++ 5 files changed, 43 insertions(+) create mode 100644 packages/system/cozystack-basics/tests/hostname-policies-capability-gate_test.yaml diff --git a/packages/system/cozystack-basics/templates/gateway-hostname-policy.yaml b/packages/system/cozystack-basics/templates/gateway-hostname-policy.yaml index 193ac9efab..f048a4fd10 100644 --- a/packages/system/cozystack-basics/templates/gateway-hostname-policy.yaml +++ b/packages/system/cozystack-basics/templates/gateway-hostname-policy.yaml @@ -1,3 +1,5 @@ +{{- /* Render only where the ValidatingAdmissionPolicy API is served (GA since Kubernetes 1.30; the management cluster requires 1.33+). Same guard as packages/core/platform/templates/deletion-protection.yaml. */}} +{{- if .Capabilities.APIVersions.Has "admissionregistration.k8s.io/v1/ValidatingAdmissionPolicy" }} --- apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingAdmissionPolicy @@ -154,3 +156,4 @@ metadata: spec: policyName: cozystack-namespace-host-label-policy validationActions: [Deny] +{{- end }} diff --git a/packages/system/cozystack-basics/templates/route-hostname-policy.yaml b/packages/system/cozystack-basics/templates/route-hostname-policy.yaml index cde87d2494..ec39b80942 100644 --- a/packages/system/cozystack-basics/templates/route-hostname-policy.yaml +++ b/packages/system/cozystack-basics/templates/route-hostname-policy.yaml @@ -36,6 +36,8 @@ finished labelling yet — in which case the caller should retry. */}} {{- $celValidator := `(namespaceObject == null || !has(namespaceObject.metadata.labels) || !("namespace.cozystack.io/host" in namespaceObject.metadata.labels)) ? false : (!has(object.spec.hostnames) || object.spec.hostnames.all(h, h == namespaceObject.metadata.labels["namespace.cozystack.io/host"] || h.endsWith("." + namespaceObject.metadata.labels["namespace.cozystack.io/host"])))` -}} +{{- /* Render only where the ValidatingAdmissionPolicy API is served (GA since Kubernetes 1.30; the management cluster requires 1.33+). Same guard as packages/core/platform/templates/deletion-protection.yaml. */}} +{{- if .Capabilities.APIVersions.Has "admissionregistration.k8s.io/v1/ValidatingAdmissionPolicy" }} --- apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingAdmissionPolicy @@ -104,3 +106,4 @@ metadata: spec: policyName: cozystack-route-hostname-policy-tls validationActions: [Deny] +{{- end }} diff --git a/packages/system/cozystack-basics/tests/gateway-hostname-policy_test.yaml b/packages/system/cozystack-basics/tests/gateway-hostname-policy_test.yaml index 50940dee43..80749c4813 100644 --- a/packages/system/cozystack-basics/tests/gateway-hostname-policy_test.yaml +++ b/packages/system/cozystack-basics/tests/gateway-hostname-policy_test.yaml @@ -19,6 +19,13 @@ release: # Both gates are dropped; inheritance now flows via the label # selector on Gateway.spec.listeners[].allowedRoutes. +# The template now gates on the ValidatingAdmissionPolicy API (GA since +# Kubernetes 1.30); helm-unittest's default capability set omits it, so every +# rendering test declares it, exactly as the platform deletion-protection suite. +capabilities: + apiVersions: + - admissionregistration.k8s.io/v1/ValidatingAdmissionPolicy + tests: - it: renders 3 VAPs + 3 Bindings (6 documents total) asserts: diff --git a/packages/system/cozystack-basics/tests/hostname-policies-capability-gate_test.yaml b/packages/system/cozystack-basics/tests/hostname-policies-capability-gate_test.yaml new file mode 100644 index 0000000000..7bd8e9424c --- /dev/null +++ b/packages/system/cozystack-basics/tests/hostname-policies-capability-gate_test.yaml @@ -0,0 +1,23 @@ +suite: cozystack-basics hostname VAP policies are gated on the ValidatingAdmissionPolicy API +templates: + - templates/route-hostname-policy.yaml + - templates/gateway-hostname-policy.yaml +release: + name: cozystack-basics + namespace: cozy-system +# Neither template reads a value that could independently suppress it, so the +# only reason either renders nothing here is the missing ValidatingAdmissionPolicy +# API declared below. helm-unittest's per-test capabilities merge rather than +# replace the suite's, so the absent case is a separate suite pinned at suite +# level (mirrors the platform notes gate suite). +capabilities: + apiVersions: [] +tests: + - it: neither hostname VAP policy renders when the API is unavailable + asserts: + - hasDocuments: + count: 0 + template: templates/route-hostname-policy.yaml + - hasDocuments: + count: 0 + template: templates/gateway-hostname-policy.yaml diff --git a/packages/system/cozystack-basics/tests/route-hostname-policy_test.yaml b/packages/system/cozystack-basics/tests/route-hostname-policy_test.yaml index 36f87c7a47..16ef82a67a 100644 --- a/packages/system/cozystack-basics/tests/route-hostname-policy_test.yaml +++ b/packages/system/cozystack-basics/tests/route-hostname-policy_test.yaml @@ -6,6 +6,13 @@ release: name: cozystack-basics namespace: cozy-system +# The template now gates on the ValidatingAdmissionPolicy API (GA since +# Kubernetes 1.30); helm-unittest's default capability set omits it, so every +# rendering test declares it, exactly as the platform deletion-protection suite. +capabilities: + apiVersions: + - admissionregistration.k8s.io/v1/ValidatingAdmissionPolicy + tests: - it: renders ValidatingAdmissionPolicy + Binding for HTTPRoute and TLSRoute (Layer 7) # Layer 7 (route-hostname VAP) is the only VAP rendered by THIS From d316cefde4f03abc16e05336fcb9a35595506699 Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Mon, 17 Aug 2026 18:09:41 +0500 Subject: [PATCH 65/69] fix(postgres): request S3 checksums only when required Since botocore ~1.36 (early 2025) the default RequestChecksumCalculation is when_supported, so barman-cloud's boto3 attaches a flexible checksum to every PutObject. AWS S3 accepts it, but non-AWS S3-compatible backends - Ceph RADOS Gateway, the platform's own SeaweedFS system bucket, some MinIO / Cloudflare R2 builds - reject it with "InvalidArgument: x-amz-content-sha256 must be UNSIGNED-PAYLOAD, ...", which fails every backup and WAL-archive upload. Set AWS_REQUEST_CHECKSUM_CALCULATION=when_required through the CNPG Cluster's spec.env, which reaches the instance pods and therefore the barman-cloud subprocess the instance manager execs. This is a release-1.5-native equivalent of #3417 rather than a cherry-pick of it. That change sets spec.instanceSidecarConfiguration.env on barmancloud.cnpg.io ObjectStore objects, and release-1.5 has no barman-cloud plugin: there is no ObjectStore CRD on the branch and the rendered Cluster carries no spec.plugins, so those objects would template cleanly and then fail to apply, referenced by nothing. release-1.5 still uses CNPG's native in-tree spec.backup.barmanObjectStore, which shells out to the same barman-cloud client, so the bug is reachable here - the plugin is the packaging difference, not the cause. One spec.env covers all three paths on this branch at once: the chart-rendered legacy barmanObjectStore, the same field SSA-patched by the CNPG backup driver in the useSystemBucket flow, and externalClusters recovery. No Go change is needed - the driver's patch type carries only spec.backup and spec.bootstrap, so it cannot express or remove the Helm-owned spec.env. keycloak is out of scope on this branch. Its chart renders a CNPG Cluster but configures no backup at all - no backup key in values, no backup template - so it has no barman-cloud path to fix, and the upstream hunk for it references values that do not exist here. Verified against CNPG 1.27.3: spec.env is accepted by the webhook and stored unmodified, appears in the instance manager's own environment, and botocore inside the instance image then resolves request_checksum_calculation as when_required instead of its when_supported default. Adding the field to an already-running Cluster converges back to healthy after CNPG recreates the instance pod, which is the in-place upgrade path for existing releases with backups enabled. Assisted-By: Claude Signed-off-by: Myasnikov Daniil --- packages/apps/postgres/templates/db.yaml | 38 ++++++++++ .../postgres/tests/backup_storage_test.yaml | 71 +++++++++++++++++++ 2 files changed, 109 insertions(+) diff --git a/packages/apps/postgres/templates/db.yaml b/packages/apps/postgres/templates/db.yaml index 4e0b468194..c6a0e29411 100644 --- a/packages/apps/postgres/templates/db.yaml +++ b/packages/apps/postgres/templates/db.yaml @@ -189,6 +189,44 @@ spec: key: {{ $credSecretKey }} {{- end }} + {{/* + Every S3 path on this chart runs `barman-cloud-*` inside the instance + pod, and barman-cloud uploads through boto3: + - the chart-rendered spec.backup.barmanObjectStore above (legacy flow); + - the same field SSA-patched onto the live Cluster by the CNPG backup + driver in the platform useSystemBucket=true flow; + - externalClusters[].barmanObjectStore above, used by + bootstrap.recovery and by the RestoreJob flow (which sets + bootstrap.enabled on the Postgres app and lets this chart re-render). + + Since botocore ~1.36 (early 2025) the default RequestChecksumCalculation + is when_supported, so a flexible checksum (and the matching + x-amz-content-sha256 handling) rides on every PutObject. AWS S3 accepts + it, but non-AWS S3-compatible backends - Ceph RADOS Gateway, the + platform's own SeaweedFS system bucket, some MinIO / Cloudflare R2 + builds - reject it with "InvalidArgument: x-amz-content-sha256 must be + UNSIGNED-PAYLOAD, ...", which fails every backup and WAL-archive upload. + when_required computes a checksum only when the operation mandates one; + AWS S3 accepts that on a plain PutObject too, so it is a safe default + everywhere. + + spec.env lands in the instance pods, so the barman-cloud subprocess the + instance manager execs inherits it - one setting covers all of the paths + above. It stays Helm-owned: the backup driver's SSA patch carries only + spec.backup (applyClusterBarmanObjectStore in + internal/backupcontroller/cnpgstrategy_controller.go), so the two field + managers do not contend. + + Gated on backup/bootstrap being enabled rather than set unconditionally: + a Postgres release with no S3 configured never invokes barman-cloud, and + injecting an AWS_* variable into its instance pods would be noise. + */}} + {{- if or .Values.backup.enabled .Values.bootstrap.enabled }} + env: + - name: AWS_REQUEST_CHECKSUM_CALCULATION + value: when_required + {{- end }} + resources: {{- include "cozy-lib.resources.defaultingSanitize" (list .Values.resourcesPreset .Values.resources $) | nindent 4 }} imageName: ghcr.io/cloudnative-pg/postgresql:{{ include "postgres.versionMap" $ | trimPrefix "v" }} diff --git a/packages/apps/postgres/tests/backup_storage_test.yaml b/packages/apps/postgres/tests/backup_storage_test.yaml index 8adab98b1e..af5990319f 100644 --- a/packages/apps/postgres/tests/backup_storage_test.yaml +++ b/packages/apps/postgres/tests/backup_storage_test.yaml @@ -35,6 +35,18 @@ tests: documentSelector: path: kind value: Cluster + # The platform flow's barmanObjectStore is SSA-patched onto the live + # Cluster later, but it runs barman-cloud in these same instance pods, + # so the non-AWS S3 request-checksum pin has to be on the Cluster now. + - contains: + path: spec.env + content: + name: AWS_REQUEST_CHECKSUM_CALCULATION + value: when_required + template: templates/db.yaml + documentSelector: + path: kind + value: Cluster - hasDocuments: count: 0 template: templates/backup-secret.yaml @@ -60,6 +72,18 @@ tests: documentSelector: path: kind value: Cluster + # Non-AWS S3 gateways (Ceph RGW, the platform's own SeaweedFS) reject + # botocore's default flexible request checksum, so barman-cloud's boto3 + # is pinned to when_required via the instance pods' environment. + - contains: + path: spec.env + content: + name: AWS_REQUEST_CHECKSUM_CALCULATION + value: when_required + template: templates/db.yaml + documentSelector: + path: kind + value: Cluster - hasDocuments: count: 1 template: templates/backup-secret.yaml @@ -112,3 +136,50 @@ tests: asserts: - failedTemplate: errorMessage: "postgres: bootstrap.enabled=true requires S3 credentials: set backup.s3AccessKey+backup.s3SecretKey, backup.s3CredentialsSecret.name, or backup.useSystemBucket=true (with a backups.cozystack.io/RestoreJob)." + + - it: "bootstrap recovery with backup disabled: request-checksum pin still rendered" + # externalClusters[].barmanObjectStore drives recovery through the same + # barman-cloud client, so the pin must not hang off backup.enabled alone. + # This is also the RestoreJob flow: the CNPG driver patches the Postgres + # app's bootstrap.enabled and lets this chart re-render. + set: + backup: + enabled: false + useSystemBucket: false + s3AccessKey: AKIAEXAMPLE + s3SecretKey: s3cr3t + bootstrap: + enabled: true + oldName: old + asserts: + - exists: + path: spec.externalClusters + template: templates/db.yaml + documentSelector: + path: kind + value: Cluster + - contains: + path: spec.env + content: + name: AWS_REQUEST_CHECKSUM_CALCULATION + value: when_required + template: templates/db.yaml + documentSelector: + path: kind + value: Cluster + + - it: "no backup and no bootstrap: no AWS_* env injected into instance pods" + # A release with no S3 configured never invokes barman-cloud, so it gets + # no spec.env at all - the pin is scoped, not unconditional. + set: + backup: + enabled: false + bootstrap: + enabled: false + asserts: + - notExists: + path: spec.env + template: templates/db.yaml + documentSelector: + path: kind + value: Cluster From 58b184e77d3ef120bfeededc39b9d38369545748 Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Wed, 19 Aug 2026 11:59:14 +0500 Subject: [PATCH 66/69] ci(tags): move Versioned Tag jobs off the dead self-hosted pool All three jobs in tags.yaml were pinned to `[self-hosted]`, which is the decommissioned cozy-runner-1 (#2937). No self-hosted runner answers that label any more: the repo has zero registered runners, and the workflow would sit in `Queued` forever rather than fail -- a tag push meant to cut v1.5.4 would appear to hang with nothing in CI explaining why. Repoint to what release-1.6's tags.yaml uses for the same three jobs: prepare-release -> oracle-vm-24cpu-96gb-x86-64 generate-changelog -> ubuntu-latest update-website-docs -> ubuntu-latest prepare-release runs a serial `make build` plus `make assets`, so it needs the large ephemeral Oracle shape -- the same one build-main.yaml on this branch already moved to, and which is confirmed live here (the release-1.5 "Build cache" run executed on it). The other two are git, node and CLI work against the GitHub and website APIs with no docker and no repo toolchain, so a GitHub-hosted runner is enough. Runner labels only. This branch is retired after one final release, so nothing else from release-1.6's newer tags.yaml is ported. Signed-off-by: Myasnikov Daniil --- .github/workflows/tags.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tags.yaml b/.github/workflows/tags.yaml index 4faf63c136..ab35d3c289 100644 --- a/.github/workflows/tags.yaml +++ b/.github/workflows/tags.yaml @@ -15,7 +15,7 @@ concurrency: jobs: prepare-release: name: Prepare Release - runs-on: [self-hosted] + runs-on: oracle-vm-24cpu-96gb-x86-64 outputs: skip: ${{ steps.check_release.outputs.skip }} permissions: @@ -245,7 +245,7 @@ jobs: generate-changelog: name: Generate Changelog - runs-on: [self-hosted] + runs-on: ubuntu-latest needs: [prepare-release] permissions: contents: write @@ -450,7 +450,7 @@ jobs: update-website-docs: name: Update Website Docs - runs-on: [self-hosted] + runs-on: ubuntu-latest needs: [generate-changelog, prepare-release] # generate-changelog is non-blocking — run as long as prepare-release succeeded. # `always()` is needed so this job runs even if generate-changelog failed/skipped. From 572467042aa53b78546bab2a84fe02880d211d43 Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Wed, 19 Aug 2026 12:14:23 +0500 Subject: [PATCH 67/69] ci(release): move Finalize Release off the dead self-hosted pool `finalize` is the job that turns a prepared release into a published one: it creates the tag on the merge commit, ensures the release-X.Y maintenance branch, and publishes the draft release. It was pinned to `[self-hosted]`, the decommissioned cozy-runner-1 (#2937), which nothing answers -- so a v1.5.4 cut would build, draft, open its release PR, and then stall silently on merge with the release left as a draft nobody notices. Repoint to `[oracle-vm-4cpu-16gb-x86-64]`, which both `main` and `release-1.6` use for this same job and which is confirmed answered (jobs executed on it in this repo today). This branch's finalize does git and GitHub API work only -- no skopeo, no helm, no docker -- so the small shape is ample. Signed-off-by: Myasnikov Daniil --- .github/workflows/pull-requests-release.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pull-requests-release.yaml b/.github/workflows/pull-requests-release.yaml index 1d284227e5..13beba491a 100644 --- a/.github/workflows/pull-requests-release.yaml +++ b/.github/workflows/pull-requests-release.yaml @@ -14,7 +14,11 @@ concurrency: jobs: finalize: name: Finalize Release - runs-on: [self-hosted] + # Was [self-hosted], the decommissioned cozy-runner-1 (#2937), which nothing + # answers -- this job would sit in Queued and never publish the release. It + # is git plus GitHub API calls only, so the small ephemeral shape is ample; + # it is also the one both live branches use for this job. + runs-on: [oracle-vm-4cpu-16gb-x86-64] permissions: contents: write From 316248403ca1b55beda391b8131ee068ff6b3003 Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Wed, 19 Aug 2026 12:14:59 +0500 Subject: [PATCH 68/69] ci(tags): install flux in prepare-release for the ephemeral runner `make build` ends at packages/core/installer, whose `image-packages` target shells out to `flux push artifact`. The decommissioned self-hosted runner had flux baked in; the ephemeral shape does not, so without this the release build works through all 28 packages and then fails on the last one -- roughly an hour in, with the tag already pushed. Port the step release-1.6's tags.yaml gained for this same job when it made the same migration. The command is byte-identical to 1.6's and to the equivalents already in build-main.yaml and pull-requests.yaml on this branch. Only the `if:` gate differs, and it has to: 1.6 renamed the check_release output to `release_exists`, while this branch's step sets `skip` -- the expression here matches its sibling steps. flux is the only tool 1.6 installs for this job, so nothing else is carried over. Signed-off-by: Myasnikov Daniil --- .github/workflows/tags.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/tags.yaml b/.github/workflows/tags.yaml index ab35d3c289..ebe792eff0 100644 --- a/.github/workflows/tags.yaml +++ b/.github/workflows/tags.yaml @@ -102,6 +102,14 @@ jobs: fetch-depth: 0 fetch-tags: true + # Ephemeral runners lack the flux CLI the installer's image-packages step + # shells out to; install if absent (idempotent). + - name: Set up build toolchain + if: steps.check_release.outputs.skip == 'false' + run: | + command -v flux >/dev/null \ + || curl -fsSL https://fluxcd.io/install.sh | sudo bash + - name: Login to GHCR if: steps.check_release.outputs.skip == 'false' uses: docker/login-action@v3 From 811e6fd27b7a42cefcd6d8b27a92e415f861ca00 Mon Sep 17 00:00:00 2001 From: "cozystack-ci[bot]" <274107086+cozystack-ci[bot]@users.noreply.github.com> Date: Wed, 19 Aug 2026 09:01:33 +0000 Subject: [PATCH 69/69] Prepare release v1.5.4 Signed-off-by: cozystack-ci[bot] <274107086+cozystack-ci[bot]@users.noreply.github.com> --- .../apps/clickhouse/images/altinity-clickhouse-backup.tag | 2 +- packages/apps/clickhouse/images/clickhouse-backup.tag | 2 +- packages/apps/http-cache/images/nginx-cache.tag | 2 +- packages/apps/kubernetes/images/cluster-autoscaler.tag | 2 +- packages/apps/kubernetes/images/kubevirt-cloud-provider.tag | 2 +- packages/apps/kubernetes/images/kubevirt-csi-driver.tag | 2 +- .../apps/kubernetes/images/ubuntu-container-disk-v1.30.tag | 2 +- .../apps/kubernetes/images/ubuntu-container-disk-v1.31.tag | 2 +- .../apps/kubernetes/images/ubuntu-container-disk-v1.32.tag | 2 +- .../apps/kubernetes/images/ubuntu-container-disk-v1.33.tag | 2 +- .../apps/kubernetes/images/ubuntu-container-disk-v1.34.tag | 2 +- .../apps/kubernetes/images/ubuntu-container-disk-v1.35.tag | 2 +- packages/apps/mariadb/images/mariadb-backup.tag | 2 +- packages/core/installer/values.yaml | 4 ++-- packages/core/platform/values.yaml | 2 +- packages/core/testing/values.yaml | 2 +- packages/extra/bootbox/images/matchbox.tag | 2 +- packages/extra/seaweedfs/images/objectstorage-sidecar.tag | 2 +- packages/system/backup-controller/values.yaml | 2 +- packages/system/backupstrategy-controller/values.yaml | 2 +- packages/system/bucket/images/s3manager.tag | 2 +- packages/system/cilium/values.yaml | 2 +- packages/system/cozystack-api/values.yaml | 2 +- packages/system/cozystack-controller/values.yaml | 2 +- packages/system/dashboard/values.yaml | 4 ++-- packages/system/flux-shard-operator/values.yaml | 2 +- .../system/grafana-operator/images/grafana-dashboards.tag | 2 +- packages/system/kamaji/values.yaml | 4 ++-- packages/system/kubeovn-plunger/values.yaml | 2 +- packages/system/kubeovn-webhook/values.yaml | 2 +- packages/system/kubevirt-csi-node/values.yaml | 2 +- packages/system/lineage-controller-webhook/values.yaml | 2 +- packages/system/linstor-gui/values.yaml | 2 +- packages/system/linstor/values.yaml | 4 ++-- packages/system/metallb/values.yaml | 4 ++-- packages/system/monitoring/images/grafana.tag | 2 +- packages/system/multus/templates/multus-daemonset-thick.yml | 4 ++-- packages/system/objectstorage-controller/values.yaml | 2 +- packages/system/seaweedfs/values.yaml | 2 +- 39 files changed, 45 insertions(+), 45 deletions(-) diff --git a/packages/apps/clickhouse/images/altinity-clickhouse-backup.tag b/packages/apps/clickhouse/images/altinity-clickhouse-backup.tag index 15cb7286de..dd5d71c9eb 100644 --- a/packages/apps/clickhouse/images/altinity-clickhouse-backup.tag +++ b/packages/apps/clickhouse/images/altinity-clickhouse-backup.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/altinity-clickhouse-backup:v1.5.2@sha256:65d15779d062680ddb69210c1dfaf8920ea4be2beb069d028bcb5934870a4d72 +ghcr.io/cozystack/cozystack/altinity-clickhouse-backup:v1.5.4@sha256:65d15779d062680ddb69210c1dfaf8920ea4be2beb069d028bcb5934870a4d72 diff --git a/packages/apps/clickhouse/images/clickhouse-backup.tag b/packages/apps/clickhouse/images/clickhouse-backup.tag index 1190edad5f..f4927f42bb 100644 --- a/packages/apps/clickhouse/images/clickhouse-backup.tag +++ b/packages/apps/clickhouse/images/clickhouse-backup.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/clickhouse-backup:v1.5.2@sha256:8efef62d59fb82d39544823f81a797c2b9ac130a50241fee48e701161ce2670d +ghcr.io/cozystack/cozystack/clickhouse-backup:v1.5.4@sha256:1714da5dc14b58866d1e21bed8934d385bfe4c9873c8840b3b4be4a1abfe5f5d diff --git a/packages/apps/http-cache/images/nginx-cache.tag b/packages/apps/http-cache/images/nginx-cache.tag index 6d34d56a48..245159e46f 100644 --- a/packages/apps/http-cache/images/nginx-cache.tag +++ b/packages/apps/http-cache/images/nginx-cache.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/nginx-cache:v1.5.2@sha256:479e1bb78cb7e0c2b9bf496d6ad1207dfceacbba62c028171c3f2548d1d74c5c +ghcr.io/cozystack/cozystack/nginx-cache:v1.5.4@sha256:1ec93d5ce6f30f3bfd21a24236162883ac6d77afce7a03d234bd893ce7f4ab86 diff --git a/packages/apps/kubernetes/images/cluster-autoscaler.tag b/packages/apps/kubernetes/images/cluster-autoscaler.tag index 75c24bf9c9..aaca7d5fd6 100644 --- a/packages/apps/kubernetes/images/cluster-autoscaler.tag +++ b/packages/apps/kubernetes/images/cluster-autoscaler.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/cluster-autoscaler:v1.5.2@sha256:1767a51cf78d6700cb0b39ea1562b1443421afa416cb6d85e724225499859616 +ghcr.io/cozystack/cozystack/cluster-autoscaler:v1.5.4@sha256:b4ac67d2f1ba9557a5e7d0d53ab4daf5efd2e78e441cf7cb5d99e8d54cd2f176 diff --git a/packages/apps/kubernetes/images/kubevirt-cloud-provider.tag b/packages/apps/kubernetes/images/kubevirt-cloud-provider.tag index ff00fb596c..9c217e54b8 100644 --- a/packages/apps/kubernetes/images/kubevirt-cloud-provider.tag +++ b/packages/apps/kubernetes/images/kubevirt-cloud-provider.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/kubevirt-cloud-provider:v1.5.2@sha256:c7224442a074a6a5e7af7ddec14be6cabf8ae9dd2569bfa38794a0dce9188ef9 +ghcr.io/cozystack/cozystack/kubevirt-cloud-provider:v1.5.4@sha256:88f04b08cda502fdcdd90f2bdc429a8300eb8580a202ad26bbeca51bb9bd6431 diff --git a/packages/apps/kubernetes/images/kubevirt-csi-driver.tag b/packages/apps/kubernetes/images/kubevirt-csi-driver.tag index 63a9710981..361582ee7c 100644 --- a/packages/apps/kubernetes/images/kubevirt-csi-driver.tag +++ b/packages/apps/kubernetes/images/kubevirt-csi-driver.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/kubevirt-csi-driver:v1.5.2@sha256:808553aa64adf49c7c555a05d16f68ba5fb68fbd266e27eeff497e4d97d6dea1 +ghcr.io/cozystack/cozystack/kubevirt-csi-driver:v1.5.4@sha256:51b9a6c0b59dcb6dbffcbdeec613f1106a0f4bf8e5009ee41e64fea6dab21f8d diff --git a/packages/apps/kubernetes/images/ubuntu-container-disk-v1.30.tag b/packages/apps/kubernetes/images/ubuntu-container-disk-v1.30.tag index 3e16716187..af83f18b57 100644 --- a/packages/apps/kubernetes/images/ubuntu-container-disk-v1.30.tag +++ b/packages/apps/kubernetes/images/ubuntu-container-disk-v1.30.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/ubuntu-container-disk:v1.30-v1.5.2@sha256:dc2e4794e75b861bf087e93037a2522fec398ffd8ac2b69591c4b0a50260f431 +ghcr.io/cozystack/cozystack/ubuntu-container-disk:v1.30-v1.5.4@sha256:c4a4fb57a7fc921eea1941f825b9494791c6c73e5bec596b1a296cfae538e17f diff --git a/packages/apps/kubernetes/images/ubuntu-container-disk-v1.31.tag b/packages/apps/kubernetes/images/ubuntu-container-disk-v1.31.tag index 31ef9a6ef6..dfab7d5390 100644 --- a/packages/apps/kubernetes/images/ubuntu-container-disk-v1.31.tag +++ b/packages/apps/kubernetes/images/ubuntu-container-disk-v1.31.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/ubuntu-container-disk:v1.31-v1.5.2@sha256:4e3479b4f469581d87574c72246995d1df6262be42e60022084aca658ead8755 +ghcr.io/cozystack/cozystack/ubuntu-container-disk:v1.31-v1.5.4@sha256:0b9349137056e05bd99ede79fc650e293aa296b65126fa7a916a299096606340 diff --git a/packages/apps/kubernetes/images/ubuntu-container-disk-v1.32.tag b/packages/apps/kubernetes/images/ubuntu-container-disk-v1.32.tag index e8ac0ea62d..658eec68a5 100644 --- a/packages/apps/kubernetes/images/ubuntu-container-disk-v1.32.tag +++ b/packages/apps/kubernetes/images/ubuntu-container-disk-v1.32.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/ubuntu-container-disk:v1.32-v1.5.2@sha256:79df2a0da2b1c13eee44f32793e4ca726bf4987935cecc34d204898d58cbbce1 +ghcr.io/cozystack/cozystack/ubuntu-container-disk:v1.32-v1.5.4@sha256:7042a8ad12a73c5a18002eae195a888cbda5b21251043c27faf8d11bec9cb3d2 diff --git a/packages/apps/kubernetes/images/ubuntu-container-disk-v1.33.tag b/packages/apps/kubernetes/images/ubuntu-container-disk-v1.33.tag index 4fefbb977c..ee04e54950 100644 --- a/packages/apps/kubernetes/images/ubuntu-container-disk-v1.33.tag +++ b/packages/apps/kubernetes/images/ubuntu-container-disk-v1.33.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/ubuntu-container-disk:v1.33-v1.5.2@sha256:200d41bfb8108d43e1e938df47175dd4dbaae23d48e6d1c20ae6c6432213eeee +ghcr.io/cozystack/cozystack/ubuntu-container-disk:v1.33-v1.5.4@sha256:7cb962b00258485ae576d0ce41209fb6209a41ac617cb1e89a1116210e9bb791 diff --git a/packages/apps/kubernetes/images/ubuntu-container-disk-v1.34.tag b/packages/apps/kubernetes/images/ubuntu-container-disk-v1.34.tag index ea724cd7f6..fa782e44dd 100644 --- a/packages/apps/kubernetes/images/ubuntu-container-disk-v1.34.tag +++ b/packages/apps/kubernetes/images/ubuntu-container-disk-v1.34.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/ubuntu-container-disk:v1.34-v1.5.2@sha256:21c0112abe14caba7a001b542d587ec1dab477e5f0fe65dba4383cac331c206c +ghcr.io/cozystack/cozystack/ubuntu-container-disk:v1.34-v1.5.4@sha256:3a084dfad0171d9779b56ac36331521088f3506b2bf00da9c115848df4e37272 diff --git a/packages/apps/kubernetes/images/ubuntu-container-disk-v1.35.tag b/packages/apps/kubernetes/images/ubuntu-container-disk-v1.35.tag index fabef3b996..c7dba3886d 100644 --- a/packages/apps/kubernetes/images/ubuntu-container-disk-v1.35.tag +++ b/packages/apps/kubernetes/images/ubuntu-container-disk-v1.35.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/ubuntu-container-disk:v1.35-v1.5.2@sha256:f90a024f2e2b4ef1b3ea653f5f47699218114e41bbd9b7c46e7c0ed00558aebb +ghcr.io/cozystack/cozystack/ubuntu-container-disk:v1.35-v1.5.4@sha256:cac4d5ed8a4178761309bb308f961c990f5c0521d7bf943a601deee498a07112 diff --git a/packages/apps/mariadb/images/mariadb-backup.tag b/packages/apps/mariadb/images/mariadb-backup.tag index ade215ad6e..cafcedfea9 100644 --- a/packages/apps/mariadb/images/mariadb-backup.tag +++ b/packages/apps/mariadb/images/mariadb-backup.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/mariadb-backup:v1.5.2@sha256:3cdaed3f19531586756902516feb099a5bece89c2c6685754d945486f0e99999 +ghcr.io/cozystack/cozystack/mariadb-backup:v1.5.4@sha256:0ddb3184c0da5a064401d10e48b1839f13ad5c2a82b8d29d6f0b2928322868a5 diff --git a/packages/core/installer/values.yaml b/packages/core/installer/values.yaml index f99b06af7a..467e1fb64f 100644 --- a/packages/core/installer/values.yaml +++ b/packages/core/installer/values.yaml @@ -7,9 +7,9 @@ bareNamespace: false cozystackOperator: # Deployment variant: talos, generic, hosted variant: talos - image: ghcr.io/cozystack/cozystack/cozystack-operator:v1.5.2@sha256:43fe115a1d55b80d1b56b8e9c589a8dc34a6d454111c91b3b161c7704ea23358 + image: ghcr.io/cozystack/cozystack/cozystack-operator:v1.5.4@sha256:cf69537ff3a0fd1a4fc81ea3d7022fb0da31fdff6f2e4113c73a33a0e0b65f06 platformSourceUrl: 'oci://ghcr.io/cozystack/cozystack/cozystack-packages' - platformSourceRef: 'digest=sha256:b2d3fc7a706b6e755d98385dcfc514945610166c4f5e61ba8cdc1a1acd8188e9' + platformSourceRef: 'digest=sha256:7d1f77973c35d4a37a37eb2c563317e22b86c6fed7d757fa4d136386812827bd' # When non-empty, overrides the operator's --helmrelease-interval flag # (operator default: 5m). E2E sets this to 30s; production should leave empty. helmReleaseInterval: "" diff --git a/packages/core/platform/values.yaml b/packages/core/platform/values.yaml index 7f7ad3be5e..e1f9d371cd 100644 --- a/packages/core/platform/values.yaml +++ b/packages/core/platform/values.yaml @@ -13,7 +13,7 @@ migrations: # targetVersion in-tree by design; run-migrations.sh refuses to advance past a # migration file missing from the image (exit 1), so a stale pin fails loudly # rather than silently skipping the SeaweedFS database hand-over. - image: ghcr.io/cozystack/cozystack/platform-migrations:v1.5.2@sha256:5d601b0709c578945f2e04d655b77d29008c94666a4350e67960c9f6bf9d2bde + image: ghcr.io/cozystack/cozystack/platform-migrations:v1.5.4@sha256:24510ca1f3b789d6cab6cca3810283db490930d78ed35d31a740b17f33106986 targetVersion: 46 # Bundle deployment configuration bundles: diff --git a/packages/core/testing/values.yaml b/packages/core/testing/values.yaml index dfc3ecf4a6..9fff71a2b2 100644 --- a/packages/core/testing/values.yaml +++ b/packages/core/testing/values.yaml @@ -1,2 +1,2 @@ e2e: - image: ghcr.io/cozystack/cozystack/e2e-sandbox:v1.5.2@sha256:e664363d5ce0b24431f7fd06dd27a4e397a0a33439247f922f5bf0d9474f759f + image: ghcr.io/cozystack/cozystack/e2e-sandbox:v1.5.4@sha256:41bd612ab75b490a3cdcdd5fca4808faab14a56f87e5ff8d4355bc969f8e13af diff --git a/packages/extra/bootbox/images/matchbox.tag b/packages/extra/bootbox/images/matchbox.tag index 00ba23f5f8..63a03d7ae5 100644 --- a/packages/extra/bootbox/images/matchbox.tag +++ b/packages/extra/bootbox/images/matchbox.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/matchbox:v1.5.2@sha256:8fd8c75c9bfb70ed1f9263166ddb562a3c4b4f3c954fb8556ac2b096e84d7de3 +ghcr.io/cozystack/cozystack/matchbox:v1.5.4@sha256:f8eb5bbaec74e107be3b164e87e031c93d08f964753d2e4d7d47b166dc13b6dc diff --git a/packages/extra/seaweedfs/images/objectstorage-sidecar.tag b/packages/extra/seaweedfs/images/objectstorage-sidecar.tag index 1bb9c23f5f..fe16a3cc01 100644 --- a/packages/extra/seaweedfs/images/objectstorage-sidecar.tag +++ b/packages/extra/seaweedfs/images/objectstorage-sidecar.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/objectstorage-sidecar:v1.5.2@sha256:fdc09634688d5579a329b48aabf6706cc55bcd9503b0331fa47764706f0d6f62 +ghcr.io/cozystack/cozystack/objectstorage-sidecar:v1.5.4@sha256:598e1f6aea3bd88a847d68b0602833df9cf32a0d4f99ddefbe1e73ff8e3aaafd diff --git a/packages/system/backup-controller/values.yaml b/packages/system/backup-controller/values.yaml index b41126fabb..ea0fbcf950 100644 --- a/packages/system/backup-controller/values.yaml +++ b/packages/system/backup-controller/values.yaml @@ -1,5 +1,5 @@ backupController: - image: "ghcr.io/cozystack/cozystack/backup-controller:v1.5.2@sha256:11d55b79cf136f9ec161410e2357dde7bc063246f905d5ac962ad8432af04962" + image: "ghcr.io/cozystack/cozystack/backup-controller:v1.5.4@sha256:55724072553d13228135628cbfc6fb7988694e7b477e9c38e547c1654800ef19" replicas: 2 debug: false metrics: diff --git a/packages/system/backupstrategy-controller/values.yaml b/packages/system/backupstrategy-controller/values.yaml index 80223adf32..dadf57c094 100644 --- a/packages/system/backupstrategy-controller/values.yaml +++ b/packages/system/backupstrategy-controller/values.yaml @@ -1,5 +1,5 @@ backupStrategyController: - image: "ghcr.io/cozystack/cozystack/backupstrategy-controller:v1.5.2@sha256:d965283f935e4b24b655bc76e37dcb3b731bec832bb5c2e3e407b0284c7526a3" + image: "ghcr.io/cozystack/cozystack/backupstrategy-controller:v1.5.4@sha256:df74d5bf4c29a8ea5bc2998c8f5b96bdb9a6678938f46b88b775497ec3eb62b8" # chBackupClientImage is the image rendered into the Altinity strategy # Pod: it drives clickhouse-backup's HTTP API via curl + jq. We reuse # platform-migrations (already pre-built and digested by release CI in diff --git a/packages/system/bucket/images/s3manager.tag b/packages/system/bucket/images/s3manager.tag index 918a9386e4..0ad27e5c43 100644 --- a/packages/system/bucket/images/s3manager.tag +++ b/packages/system/bucket/images/s3manager.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/s3manager:v1.5.2@sha256:8b36ab67904e34b2fb843b964c20d4822f3a6673f6745afb6a834efd02e017e2 +ghcr.io/cozystack/cozystack/s3manager:v1.5.4@sha256:10f58ec0324cc466b522f9141364d5137819b0530a01f74e25070e3aa63699e8 diff --git a/packages/system/cilium/values.yaml b/packages/system/cilium/values.yaml index 411e2a2e3f..5120707683 100644 --- a/packages/system/cilium/values.yaml +++ b/packages/system/cilium/values.yaml @@ -15,7 +15,7 @@ cilium: mode: "kubernetes" image: repository: ghcr.io/cozystack/cozystack/cilium - tag: v1.5.2 + tag: v1.5.4 digest: "sha256:94dc74c9301f0da5695c80e77b77ee0486ad0086b5be60a94136a2cafd410b73" envoy: enabled: true diff --git a/packages/system/cozystack-api/values.yaml b/packages/system/cozystack-api/values.yaml index e8409ac932..6399c85d1d 100644 --- a/packages/system/cozystack-api/values.yaml +++ b/packages/system/cozystack-api/values.yaml @@ -1,3 +1,3 @@ cozystackAPI: - image: ghcr.io/cozystack/cozystack/cozystack-api:v1.5.2@sha256:ef50b155e419cf8f0b4d57af9fa385ecbacd5c1248f13591ffcca75c2f9f2a07 + image: ghcr.io/cozystack/cozystack/cozystack-api:v1.5.4@sha256:febaefa1018d6407d58332f9804729fe080eb127caf85c40497c1a6591c8d399 replicas: 2 diff --git a/packages/system/cozystack-controller/values.yaml b/packages/system/cozystack-controller/values.yaml index edd418094f..f5db5800be 100644 --- a/packages/system/cozystack-controller/values.yaml +++ b/packages/system/cozystack-controller/values.yaml @@ -1,4 +1,4 @@ cozystackController: - image: ghcr.io/cozystack/cozystack/cozystack-controller:v1.5.2@sha256:464c9cda4dba5e83e17f26acf386cafaf3688be7c24e267692c97a3acb018a15 + image: ghcr.io/cozystack/cozystack/cozystack-controller:v1.5.4@sha256:239f6f84c06e5a1584fe6b90c7f25bbac096af866e45a2cc30774fbb284012bc debug: false disableTelemetry: false diff --git a/packages/system/dashboard/values.yaml b/packages/system/dashboard/values.yaml index 2feec3b17b..29f59f11b9 100644 --- a/packages/system/dashboard/values.yaml +++ b/packages/system/dashboard/values.yaml @@ -1,4 +1,4 @@ console: - image: ghcr.io/cozystack/cozystack/cozystack-ui:v1.5.2@sha256:c8f86e56952044999b9fa0aa1bbeb2b7e8fa4cfcd2f56ce3cff82e8276a87f68 + image: ghcr.io/cozystack/cozystack/cozystack-ui:v1.5.4@sha256:418dd6ddbfa4516d72278fd5579fd7e52fbe2da9be70fc8e6d0b3d2f29082685 tokenProxy: - image: ghcr.io/cozystack/cozystack/token-proxy:v1.5.2@sha256:0382e8a9014a842d2eddf1bac5b598f42a4ada7a9be673d9272f1610d47631c4 + image: ghcr.io/cozystack/cozystack/token-proxy:v1.5.4@sha256:470790ccf0b8ef871927440a91b76114aa2aa238fe04436f523d3e8be566b190 diff --git a/packages/system/flux-shard-operator/values.yaml b/packages/system/flux-shard-operator/values.yaml index 4ab207720b..7036760275 100644 --- a/packages/system/flux-shard-operator/values.yaml +++ b/packages/system/flux-shard-operator/values.yaml @@ -1,5 +1,5 @@ fluxShardOperator: - image: ghcr.io/cozystack/cozystack/flux-shard-operator:v1.5.2@sha256:a75b88c03e5d969b322d418957f30c92941ae2ec8e1b1852719fa7d9eb9aa2a5 + image: ghcr.io/cozystack/cozystack/flux-shard-operator:v1.5.4@sha256:a285e51bdedda916dcfc970b079211a161026816792053042833cd2a900a6b95 debug: false replicas: 2 ## Number of helm-controller shards to provision and distribute tenants diff --git a/packages/system/grafana-operator/images/grafana-dashboards.tag b/packages/system/grafana-operator/images/grafana-dashboards.tag index 78a5db48d2..fd24a9d4ee 100644 --- a/packages/system/grafana-operator/images/grafana-dashboards.tag +++ b/packages/system/grafana-operator/images/grafana-dashboards.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/grafana-dashboards:v1.5.2@sha256:ec97b9ee9aa591b5737da595ed7772b22714b9517f9a365d67e7977dd45dd77c +ghcr.io/cozystack/cozystack/grafana-dashboards:v1.5.4@sha256:a5e45336aaa84b94bc02201e6d0d0c85a5bc9f1b6605e425e99bce50e41e58dd diff --git a/packages/system/kamaji/values.yaml b/packages/system/kamaji/values.yaml index 97c536b381..9a35010048 100644 --- a/packages/system/kamaji/values.yaml +++ b/packages/system/kamaji/values.yaml @@ -3,7 +3,7 @@ kamaji: deploy: false image: pullPolicy: IfNotPresent - tag: v1.5.2@sha256:fcf5f4451dc935d141d12ef40a966cb6f7bdb46461431a64803e1f018b0c89de + tag: v1.5.4@sha256:fffbe9efc6b5fc79d2d2fddc4836af93e2f3975e97a100768decf79aa58592d7 repository: ghcr.io/cozystack/cozystack/kamaji resources: limits: @@ -36,4 +36,4 @@ kamaji: periodSeconds: 10 timeoutSeconds: 1 extraArgs: - - --migrate-image=ghcr.io/cozystack/cozystack/kamaji:v1.5.2@sha256:fcf5f4451dc935d141d12ef40a966cb6f7bdb46461431a64803e1f018b0c89de + - --migrate-image=ghcr.io/cozystack/cozystack/kamaji:v1.5.4@sha256:fffbe9efc6b5fc79d2d2fddc4836af93e2f3975e97a100768decf79aa58592d7 diff --git a/packages/system/kubeovn-plunger/values.yaml b/packages/system/kubeovn-plunger/values.yaml index dbdf45cbab..dc4c549f4f 100644 --- a/packages/system/kubeovn-plunger/values.yaml +++ b/packages/system/kubeovn-plunger/values.yaml @@ -1,4 +1,4 @@ portSecurity: true routes: "" -image: ghcr.io/cozystack/cozystack/kubeovn-plunger:v1.5.2@sha256:908b5903c014f6fa4348f57b1fced929a2ba823aadfbcda7ad9b76134bcf3fce +image: ghcr.io/cozystack/cozystack/kubeovn-plunger:v1.5.4@sha256:d821ac11fffc0e277b78491f1fd9ed8d5c6d17fc21d8048ff50eef40077783dc ovnCentralName: ovn-central diff --git a/packages/system/kubeovn-webhook/values.yaml b/packages/system/kubeovn-webhook/values.yaml index d55271c83b..024ffb2e24 100644 --- a/packages/system/kubeovn-webhook/values.yaml +++ b/packages/system/kubeovn-webhook/values.yaml @@ -1,3 +1,3 @@ portSecurity: true routes: "" -image: ghcr.io/cozystack/cozystack/kubeovn-webhook:v1.5.2@sha256:ca354b6c2ae6f648bc63798d19a1316fe37cb142246bf08ebb7ce27e6e8f21b2 +image: ghcr.io/cozystack/cozystack/kubeovn-webhook:v1.5.4@sha256:8fddf99339698520692c214ce2c4c73be40539ca7a7baf8c7f8ef77162395b66 diff --git a/packages/system/kubevirt-csi-node/values.yaml b/packages/system/kubevirt-csi-node/values.yaml index b65b551268..b3f2b94a1a 100644 --- a/packages/system/kubevirt-csi-node/values.yaml +++ b/packages/system/kubevirt-csi-node/values.yaml @@ -1,3 +1,3 @@ storageClass: replicated csiDriver: - image: ghcr.io/cozystack/cozystack/kubevirt-csi-driver:v1.5.2@sha256:808553aa64adf49c7c555a05d16f68ba5fb68fbd266e27eeff497e4d97d6dea1 + image: ghcr.io/cozystack/cozystack/kubevirt-csi-driver:v1.5.4@sha256:51b9a6c0b59dcb6dbffcbdeec613f1106a0f4bf8e5009ee41e64fea6dab21f8d diff --git a/packages/system/lineage-controller-webhook/values.yaml b/packages/system/lineage-controller-webhook/values.yaml index 6b1dadd08b..01ef3843a8 100644 --- a/packages/system/lineage-controller-webhook/values.yaml +++ b/packages/system/lineage-controller-webhook/values.yaml @@ -1,5 +1,5 @@ lineageControllerWebhook: - image: ghcr.io/cozystack/cozystack/lineage-controller-webhook:v1.5.2@sha256:5868789439d1384cf78d9fb2f20d5457060b0d5dfcc4f93f96a13ea27ed73b81 + image: ghcr.io/cozystack/cozystack/lineage-controller-webhook:v1.5.4@sha256:9dc9ee368b9339472caa58f8a0cb7abde6e9c0dbb17c366612ae982f1bf0a2f5 debug: false replicas: 2 # DEPRECATED. Injects KUBERNETES_SERVICE_HOST=status.hostIP and diff --git a/packages/system/linstor-gui/values.yaml b/packages/system/linstor-gui/values.yaml index fbcd8c806e..f0c303789b 100644 --- a/packages/system/linstor-gui/values.yaml +++ b/packages/system/linstor-gui/values.yaml @@ -3,7 +3,7 @@ ## @param image.tag LINSTOR GUI container image tag (digest recommended) image: repository: ghcr.io/cozystack/cozystack/linstor-gui - tag: v1.5.2@sha256:6e11829de86709f5e21636cdd1a1ada1239a7dcc1741f91e9d95d9f6ade603a2 + tag: v1.5.4@sha256:0d49354d49eac7f52698b2bed6604c0bec7f3a36730c6ce52b1a3690ca554eec ## @section Deployment ## @param replicas Number of linstor-gui replicas replicas: 1 diff --git a/packages/system/linstor/values.yaml b/packages/system/linstor/values.yaml index 105fb101c3..21962888f9 100644 --- a/packages/system/linstor/values.yaml +++ b/packages/system/linstor/values.yaml @@ -1,7 +1,7 @@ piraeusServer: image: repository: ghcr.io/cozystack/cozystack/piraeus-server - tag: v1.5.2@sha256:8a4038a01032a4bbf65ca8da767c52b95ca21e6cdef73460b4860953cc0eeb4f + tag: v1.5.4@sha256:7b49eced130fa7d96f668cc001d276d6eb00b9e9d9359068e5308dc41244a213 # Talos-specific workarounds (disable for generic Linux like Ubuntu/Debian) talos: enabled: true @@ -13,4 +13,4 @@ linstor: linstorCSI: image: repository: ghcr.io/cozystack/cozystack/linstor-csi - tag: v1.5.2@sha256:c7ff3f3a19f1079bd0fe67b26319fbc07ee318a826f207ee7ec92a6c7b8a4761 + tag: v1.5.4@sha256:7d0891fabe538db848d84ee9a21797ae1b9f2953c2627a5358a93e14db8cf964 diff --git a/packages/system/metallb/values.yaml b/packages/system/metallb/values.yaml index a44f269de3..e299305f45 100644 --- a/packages/system/metallb/values.yaml +++ b/packages/system/metallb/values.yaml @@ -37,11 +37,11 @@ metallb: controller: image: repository: ghcr.io/cozystack/cozystack/metallb-controller - tag: v1.5.2@sha256:b290031f40bfc072c0b329e07d7c17eca3301e4aad2e1988fc962645d92b5fc0 + tag: v1.5.4@sha256:9d8ba76cdb9c7c6221334ad05d706dee22b138b3e90c1fe8fc884925b7480c02 speaker: image: repository: ghcr.io/cozystack/cozystack/metallb-speaker - tag: v1.5.2@sha256:3aad3ac50e8da94342332b363099c04316d8eda0dcb8cdc3d89f1bcf5dc04e7d + tag: v1.5.4@sha256:87df3c82d0b6ea223b26fd5d6fbba6e940c13e56418dfc1cd90863d145795be9 # The vendored metallb chart's values.yaml leaves `frr-k8s.prometheus:` as # a YAML block containing only commented examples — it parses to null. # Helm-controller's deep-merge then writes that null over the frr-k8s diff --git a/packages/system/monitoring/images/grafana.tag b/packages/system/monitoring/images/grafana.tag index 48c341e4bf..2f63218655 100644 --- a/packages/system/monitoring/images/grafana.tag +++ b/packages/system/monitoring/images/grafana.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/grafana:v1.5.2@sha256:086f1502edf64d2aabcda6e6505b7ee4673b0d3a14cdb7ca9e90a1d91d6e78f4 +ghcr.io/cozystack/cozystack/grafana:v1.5.4@sha256:c9f94d455dfa045062e6ec48d22a5ac7f851ec9c3556636723d9a9fad0105b0b diff --git a/packages/system/multus/templates/multus-daemonset-thick.yml b/packages/system/multus/templates/multus-daemonset-thick.yml index e2bf51f364..d35d044e73 100644 --- a/packages/system/multus/templates/multus-daemonset-thick.yml +++ b/packages/system/multus/templates/multus-daemonset-thick.yml @@ -155,7 +155,7 @@ spec: serviceAccountName: multus containers: - name: kube-multus - image: ghcr.io/cozystack/cozystack/multus-cni:v1.5.2@sha256:679ca9637b685c886d6760cfcd881b2f199ed068abee88daceded142edebd0c1 + image: ghcr.io/cozystack/cozystack/multus-cni:v1.5.4@sha256:e1f57cda741e44d3dda78be3a2b36209a8f406e2b29319902c9f5e5bbd884b1d command: [ "/usr/src/multus-cni/bin/multus-daemon" ] resources: requests: @@ -201,7 +201,7 @@ spec: fieldPath: spec.nodeName initContainers: - name: install-multus-binary - image: ghcr.io/cozystack/cozystack/multus-cni:v1.5.2@sha256:679ca9637b685c886d6760cfcd881b2f199ed068abee88daceded142edebd0c1 + image: ghcr.io/cozystack/cozystack/multus-cni:v1.5.4@sha256:e1f57cda741e44d3dda78be3a2b36209a8f406e2b29319902c9f5e5bbd884b1d command: - "/usr/src/multus-cni/bin/install_multus" - "-d" diff --git a/packages/system/objectstorage-controller/values.yaml b/packages/system/objectstorage-controller/values.yaml index b3a238cc32..db1774b79f 100644 --- a/packages/system/objectstorage-controller/values.yaml +++ b/packages/system/objectstorage-controller/values.yaml @@ -1,3 +1,3 @@ objectstorage: controller: - image: "ghcr.io/cozystack/cozystack/objectstorage-controller:v1.5.2@sha256:e27da94691155fc3eb14a22cf23fba78e53c4ba524ca3042709efae3753c565b" + image: "ghcr.io/cozystack/cozystack/objectstorage-controller:v1.5.4@sha256:02626cbffe9c03e57b8ea63d16f0edeeddbdc0eeb69c30e830bb33749e0e847e" diff --git a/packages/system/seaweedfs/values.yaml b/packages/system/seaweedfs/values.yaml index 2da808d626..974bec6f73 100644 --- a/packages/system/seaweedfs/values.yaml +++ b/packages/system/seaweedfs/values.yaml @@ -224,7 +224,7 @@ seaweedfs: bucketClassName: "seaweedfs" region: "" sidecar: - image: "ghcr.io/cozystack/cozystack/objectstorage-sidecar:v1.5.2@sha256:fdc09634688d5579a329b48aabf6706cc55bcd9503b0331fa47764706f0d6f62" + image: "ghcr.io/cozystack/cozystack/objectstorage-sidecar:v1.5.4@sha256:598e1f6aea3bd88a847d68b0602833df9cf32a0d4f99ddefbe1e73ff8e3aaafd" certificates: commonName: "SeaweedFS CA" ipAddresses: []