diff --git a/.github/workflows/build-main.yaml b/.github/workflows/build-main.yaml index 927794676d..cfef7f3038 100644 --- a/.github/workflows/build-main.yaml +++ b/.github/workflows/build-main.yaml @@ -1,9 +1,23 @@ -name: Build cache (main) +name: Build cache (release-1.5) -# Warms the shared mode=max build cache so PR builds start hot. +# Warms the mode=max build cache for THIS branch so its PR builds start hot. +# +# On main this workflow fires on main pushes; on a release branch it has to fire +# on that branch's own pushes, because the cache main writes is main's content. +# Nothing had been warming release-1.5: main dropped ubuntu-container-disk from +# `make image` in 39cdc8866, six days before this branch was cut, so its six +# :*-buildcache refs were never written at all, and every other ref carries +# main's sources rather than this branch's. The result was a PR Build that ran +# cold end to end and timed out -- ~28min of it in ubuntu-container-disk alone +# before #3457 parallelised that target, and ~38min across all 28 packages after. +# +# The cache is written to its OWN namespace (CACHE_REGISTRY below), not the +# shared one. Writing the shared refs from here would replace main's layers with +# this branch's, so main's next PR build would miss on everything and blow its +# own 30-min timeout until the next main push re-warmed it. # # PR builds (pull-requests.yaml) read the cache but never write it -# (WRITE_CACHE unset -> 0). Only this serialized main build writes +# (WRITE_CACHE unset -> 0). Only this serialized branch build writes # CACHE_REGISTRY/:buildcache, so concurrent PR builds never race on the # cache manifest -- the 409 collisions PR #2711 fixed for image tags would # otherwise reappear on the cache ref. Cache is mode=max so all multistage @@ -19,23 +33,32 @@ name: Build cache (main) env: # Same per-CI registry as pull-requests.yaml; the cache is co-located here. REGISTRY: iad.ocir.io/idyksih5sir9/cozystack + # Branch-scoped cache namespace, kept in sync with the Build job in + # pull-requests.yaml. Both must point at the same place or PR builds read a + # cache nobody writes. + CACHE_REGISTRY: iad.ocir.io/idyksih5sir9/cozystack-cache-release-1.5 on: push: - branches: [main] + branches: [release-1.5] paths-ignore: - 'docs/**' -# Only the newest main commit's cache matters; cancel older in-flight warmers. +# Only the newest commit's cache matters; cancel older in-flight warmers. concurrency: - group: build-main-cache + group: build-release-1.5-cache cancel-in-progress: true jobs: warm-cache: name: Warm build cache - runs-on: [self-hosted] - timeout-minutes: 90 + # Was [self-hosted], which is the decommissioned cozy-runner-1 (#2937) -- this + # workflow could not have run on this branch even once. Same ephemeral shape + # the main warmer moved to; a cold serial `make build` here is CPU-bound. + runs-on: oracle-vm-24cpu-96gb-x86-64 + # The first run is fully cold and builds six ubuntu-container-disk images on + # top of the other 28 packages; later runs mostly re-export a warm cache. + timeout-minutes: 120 permissions: contents: read packages: write @@ -51,6 +74,16 @@ jobs: # .git/config where a later build step could read it. persist-credentials: false + # `make build` ends at packages/core/installer, whose image-packages target + # shells out to `flux push artifact`. The self-hosted runner this workflow + # used to target had flux baked in; the ephemeral shape does not, so without + # this the warmer builds all 28 packages and then fails on the last one. + # pull-requests.yaml installs it for the same reason. Idempotent. + - name: Set up build toolchain + run: | + command -v flux >/dev/null \ + || curl -fsSL https://fluxcd.io/install.sh | sudo bash + - name: Set up Docker config run: | if [ -d ~/.docker ]; then @@ -92,7 +125,7 @@ jobs: BUILDER: ${{ steps.buildx.outputs.name }} # WRITE_CACHE=1 turns on --cache-to (mode=max) for every image. WRITE_CACHE: '1' - # Publish a floating :main handle; do not move :latest (releases do). - IMAGE_TAG: main + # Floating handle for this branch; do not move :latest (releases do). + IMAGE_TAG: release-1.5 PUBLISH_VERSIONED: '0' PUBLISH_FLOATING: '0' diff --git a/.github/workflows/pr-labeler.yaml b/.github/workflows/pr-labeler.yaml index 63696fddfd..25c97005dc 100644 --- a/.github/workflows/pr-labeler.yaml +++ b/.github/workflows/pr-labeler.yaml @@ -103,6 +103,7 @@ jobs: // area/storage 'seaweedfs': 'area/storage', 'seaweedfs-cosi-driver': 'area/storage', + 'objectstorage-controller': 'area/storage', 'bucket': 'area/storage', 'linstor': 'area/storage', 'velero': 'area/storage', diff --git a/.github/workflows/pull-requests-release.yaml b/.github/workflows/pull-requests-release.yaml index 1d284227e5..13beba491a 100644 --- a/.github/workflows/pull-requests-release.yaml +++ b/.github/workflows/pull-requests-release.yaml @@ -14,7 +14,11 @@ concurrency: jobs: finalize: name: Finalize Release - runs-on: [self-hosted] + # Was [self-hosted], the decommissioned cozy-runner-1 (#2937), which nothing + # answers -- this job would sit in Queued and never publish the release. It + # is git plus GitHub API calls only, so the small ephemeral shape is ample; + # it is also the one both live branches use for this job. + runs-on: [oracle-vm-4cpu-16gb-x86-64] permissions: contents: write diff --git a/.github/workflows/pull-requests.yaml b/.github/workflows/pull-requests.yaml index e9664b015a..20c4236bf7 100644 --- a/.github/workflows/pull-requests.yaml +++ b/.github/workflows/pull-requests.yaml @@ -3,6 +3,11 @@ name: Pull Request env: # TODO: unhardcode this REGISTRY: iad.ocir.io/idyksih5sir9/cozystack + # Read the cache build-main.yaml writes for THIS branch, not the shared refs + # main writes -- those carry main's sources and mostly miss here. Must stay in + # sync with CACHE_REGISTRY in build-main.yaml. Builds only read it + # (WRITE_CACHE defaults to 0), so a miss is a cold build, never an error. + CACHE_REGISTRY: iad.ocir.io/idyksih5sir9/cozystack-cache-release-1.5 on: pull_request: types: [opened, synchronize, reopened] @@ -33,13 +38,24 @@ jobs: # buildkit; concurrent build jobs serialize on buildkit's single-writer bbolt # cache lock + exporter mutex and stall to the 30-min timeout (proven via a # live SIGUSR1 goroutine dump). One VM per job => one buildkit per job => that - # contention is severed by construction. `make build` is serial (one image at - # a time), so a small shape suffices -- 4cpu/16gb, not the 24cpu the e2e job - # uses -- and it leans on the warm mode=max cache (#2938) to stay under the - # 30-min wall. The `debug` label still routes to self-hosted for the - # breakpoint path. Phase 0.5 of #2937. - runs-on: ${{ contains(github.event.pull_request.labels.*.name, 'debug') && 'self-hosted' || 'oracle-vm-4cpu-16gb-x86-64' }} - timeout-minutes: 30 + # contention is severed by construction. Phase 0.5 of #2937. + # + # 24cpu/96gb rather than the 4cpu/16gb this used to run on: `make build` is + # still serial across images, but this branch is the only one that still + # builds ubuntu-container-disk, and it does so once per Kubernetes minor with + # no shared build cache to warm it (main dropped the image in 39cdc8866, six + # days before release-1.5 was cut, so nothing writes its :*-buildcache refs). + # Those six builds now run concurrently, and each drives its own libguestfs + # appliance -- ~9gb of RAM and a core apiece, which the small shape cannot + # give. The `debug` label still routes to self-hosted for the breakpoint path. + runs-on: ${{ contains(github.event.pull_request.labels.*.name, 'debug') && 'self-hosted' || 'oracle-vm-24cpu-96gb-x86-64' }} + # A warm cache puts this job around 15min. 30 was not enough for a cold one: + # the branch-scoped cache starts empty, and anything that invalidates it + # (a Dockerfile edit, a go.mod bump) drops a later build back to ~45min -- + # six container disks plus 28 packages built from scratch. Sized for that + # case rather than for the warm one, so a cold build reports its real + # failure instead of a timeout that says nothing about why. + timeout-minutes: 75 permissions: contents: read packages: write diff --git a/.github/workflows/tags.yaml b/.github/workflows/tags.yaml index 4faf63c136..ebe792eff0 100644 --- a/.github/workflows/tags.yaml +++ b/.github/workflows/tags.yaml @@ -15,7 +15,7 @@ concurrency: jobs: prepare-release: name: Prepare Release - runs-on: [self-hosted] + runs-on: oracle-vm-24cpu-96gb-x86-64 outputs: skip: ${{ steps.check_release.outputs.skip }} permissions: @@ -102,6 +102,14 @@ jobs: fetch-depth: 0 fetch-tags: true + # Ephemeral runners lack the flux CLI the installer's image-packages step + # shells out to; install if absent (idempotent). + - name: Set up build toolchain + if: steps.check_release.outputs.skip == 'false' + run: | + command -v flux >/dev/null \ + || curl -fsSL https://fluxcd.io/install.sh | sudo bash + - name: Login to GHCR if: steps.check_release.outputs.skip == 'false' uses: docker/login-action@v3 @@ -245,7 +253,7 @@ jobs: generate-changelog: name: Generate Changelog - runs-on: [self-hosted] + runs-on: ubuntu-latest needs: [prepare-release] permissions: contents: write @@ -450,7 +458,7 @@ jobs: update-website-docs: name: Update Website Docs - runs-on: [self-hosted] + runs-on: ubuntu-latest needs: [generate-changelog, prepare-release] # generate-changelog is non-blocking — run as long as prepare-release succeeded. # `always()` is needed so this job runs even if generate-changelog failed/skipped. diff --git a/api/apps/v1alpha1/kubernetes/types.go b/api/apps/v1alpha1/kubernetes/types.go index 57153057db..cdfe8d6b0c 100644 --- a/api/apps/v1alpha1/kubernetes/types.go +++ b/api/apps/v1alpha1/kubernetes/types.go @@ -255,8 +255,8 @@ type NodeGroup struct { // Minimum number of replicas. // +kubebuilder:default:=0 MinReplicas int `json:"minReplicas"` - // CPU and memory resources for each worker node. - Resources Resources `json:"resources"` + // Explicit CPU and memory for each worker node, as an alternative to `instanceType` sizing. Optional: when omitted, the node is sized by `instanceType`. When both `cpu` and `memory` are set, they take precedence and `instanceType` is ignored for that node group (the instancetype is omitted from the VM, since KubeVirt cannot override an instancetype's CPU/memory). Set both `cpu` and `memory` together or neither; setting only one is rejected at render time. + Resources Resources `json:"resources,omitempty"` // List of node roles. Roles []string `json:"roles,omitempty"` // StorageClass for worker node persistent disks. When empty, uses the management cluster default StorageClass (the one annotated storageclass.kubernetes.io/is-default-class: true). NOTE: deliberately not marked immutable — the field is optional and undefaulted, so a strict `self == oldSelf` rule would block any future attempt to set it on an existing node group. diff --git a/docs/operations/backup-classes.md b/docs/operations/backup-classes.md index f60cba0ad4..271012dde5 100644 --- a/docs/operations/backup-classes.md +++ b/docs/operations/backup-classes.md @@ -1,6 +1,6 @@ # Backup Classes -Cozystack ships a single platform-managed `BackupClass` named `cozy-default`. It is provisioned automatically when the `backupstrategy-controller` package is installed and references the system-managed S3 bucket `cozy-backups` in the `tenant-root` namespace. +Cozystack ships a single platform-managed `BackupClass` named `cozy-default`. It is provisioned automatically when the `backupstrategy-controller` package is installed and references the system-managed bucket provisioned through the `apps.cozystack.io/Bucket` CR `cozy-backups` in the `tenant-root` namespace (the real S3 bucket name is the COSI-assigned one from `BucketClaim.status.bucketName`). Tenants reference `cozy-default` from `BackupJob`, `Plan`, and `RestoreJob` resources — they do **not** supply S3 credentials, endpoints, or paths. The platform projects the system-managed credentials Secret into the tenant namespace per BackupJob (or, for long-lived references like Velero's `BackupStorageLocation`, into a fixed list of system namespaces on a periodic tick), and the default strategy templates encode `/` into every S3 path so two tenants with the same application name never collide. @@ -46,7 +46,7 @@ VM-driven (Velero) backups land in the same `cozy-backups` bucket under the `vel The strategy CR `cozy-default-foundationdb` is shipped, but it is **not** bound by `cozy-default` yet. Restore runs `fdbrestore` from inside the `cozy-foundationdb-operator` Deployment, which does not yet mount `cozy-backups-creds`. Until the operator deployment is updated to mount the projected Secret, FDB platform-default restore silently fails — admins who need it today should keep using a per-app `Bucket` plus a custom `BackupClass`, or wire the credentials file into the operator deployment themselves. -**Cleanup gotcha (zombie backup_agent).** Unlike CNPG/MariaDB/Altinity (one-shot operator-side Backup CRs), the FoundationDB driver creates a `foundationdb.org/FoundationDBBackup` CR that drives a **long-lived** `backup_agent` Deployment streaming continuously to S3. Deleting a Cozystack `Backup` (e.g. via retention sweeping) does NOT stop that Deployment — the agent keeps writing until the next BackupJob's `stopOtherFoundationDBBackups` call swaps it out, until an admin invokes `examples/backups/foundationdb/cleanup.sh`, or until the operator-side CR is deleted by hand. If a tenant deletes their last Cozystack Backup and never submits another BackupJob, the agent pods will continue running indefinitely and accumulate S3 PUTs. This is intentional today (the driver has no RBAC verb to stop the operator-side CR on Cozystack-Backup deletion) but admins should be aware of it. +**Cleanup gotcha (zombie backup_agent).** Unlike CNPG/MariaDB/Altinity (one-shot operator-side Backup CRs), the FoundationDB driver creates an `apps.foundationdb.org/FoundationDBBackup` CR that drives a **long-lived** `backup_agent` Deployment streaming continuously to S3. Deleting a Cozystack `Backup` (e.g. via retention sweeping) does NOT stop that Deployment — the agent keeps writing until the next BackupJob's `stopOtherFoundationDBBackups` call swaps it out, until an admin invokes `examples/backups/foundationdb/cleanup.sh`, or until the operator-side CR is deleted by hand. If a tenant deletes their last Cozystack Backup and never submits another BackupJob, the agent pods will continue running indefinitely and accumulate S3 PUTs. This is intentional today (the driver has no RBAC verb to stop the operator-side CR on Cozystack-Backup deletion) but admins should be aware of it. ## ClickHouse: opt-in to the system bucket @@ -76,7 +76,7 @@ kubectl -n tenant-root get secret bucket-cozy-backups-system-credentials kubectl -n cozy-velero get backupstoragelocation cozy-default ``` -The bucket lives in `tenant-root` and is provisioned through the `apps.cozystack.io/Bucket` CR. The system-managed credentials Secret never leaves that namespace. The backupstrategy-controller projects a copy under the name `cozy-backups-creds` into a tenant namespace right before each BackupJob runs, and refreshes the same Secret in `cozy-velero` (and any other namespace listed in `backupStorage.systemNamespaces`) on a 1-minute tick. The projected Secret carries multiple key formats so each driver finds what it needs in one place: +The bucket lives in `tenant-root` and is provisioned through the `apps.cozystack.io/Bucket` CR. The system-managed credentials Secret never leaves that namespace. The backupstrategy-controller projects a copy under the name `cozy-backups-creds` into a tenant namespace right before each BackupJob or RestoreJob runs, and refreshes the same Secret in `cozy-velero` (and any other namespace listed in `backupStorage.systemNamespaces`) on a 1-minute tick. The projected Secret carries multiple key formats so each driver finds what it needs in one place: | Key | Consumer | |-----------------------------------------------|-------------------------------------------| @@ -87,15 +87,15 @@ The bucket lives in `tenant-root` and is provisioned through the `apps.cozystack ### Bootstrap window -On a fresh-cluster install, the Velero `BackupStorageLocation` `cozy-default` is rendered before the credentials projector has had a chance to copy `cozy-backups-creds` into `cozy-velero`. The BSL reports `Unavailable` until the projector's first synchronous round completes (which happens immediately when the `backupstrategy-controller` Pod becomes Ready — typically tens of seconds after `helm install` returns, not minutes). Velero rejects new `Backup` AND `Restore` requests against `storageLocation: cozy-default` during that window. Plan VM backup automation accordingly, or wait for `kubectl -n cozy-velero get bsl cozy-default -o jsonpath='{.status.phase}' = Available` before submitting backups. +On a fresh-cluster install, the Velero `BackupStorageLocation` `cozy-default` is rendered before the credentials projector has had a chance to copy `cozy-backups-creds` into `cozy-velero`. The BSL reports `Unavailable` until the projector's first synchronous round completes (which runs as soon as the `backupstrategy-controller` acquires leadership — in practice moments after the Pod becomes Ready, typically tens of seconds after `helm install` returns, not minutes). Velero rejects new `Backup` AND `Restore` requests against `storageLocation: cozy-default` during that window. Plan VM backup automation accordingly, or wait for the BSL to become ready before submitting backups: `kubectl -n cozy-velero wait backupstoragelocation cozy-default --for=jsonpath='{.status.phase}'=Available --timeout=5m`. **Note on controller restarts.** The BSL flickers `Unavailable` on every `backupstrategy-controller` pod restart while the projector replays its first synchronous round. The window is short (single-digit seconds) but operators who alert on BSL availability should suppress alerts during the controller's `kube_pod_container_status_restarts_total{container=backupstrategy-controller}` events or use a longer evaluation window than the projector tick (60s). ### Cozy-default Bucket bootstrap -`cozy-default` ships an `apps.cozystack.io/Bucket cozy-backups` CR in `tenant-root`, which the bucket-application chart turns into a `BucketClaim`; the COSI driver then assigns the real S3 bucket name and writes it to the BucketClaim's `.status.bucketName`. The strategy templates and the Velero BSL all read that real bucket name (Helm `lookup` against the BucketClaim). On a fresh install the BucketClaim takes a short reconcile cycle to populate its status — until it does, the strategy templates render empty and only the `Bucket` CR + `BackupClass` are present in the cluster. Flux re-renders the HelmRelease on its standard interval (default 10 minutes), at which point the populated BucketClaim status causes the missing strategy templates to materialise. +`cozy-default` ships an `apps.cozystack.io/Bucket cozy-backups` CR in `tenant-root`, which the bucket-application chart turns into a `BucketClaim`; the COSI driver then assigns the real S3 bucket name and writes it to the BucketClaim's `.status.bucketName`. The strategy templates and the Velero BSL all read that real bucket name (Helm `lookup` against the BucketClaim). On a fresh install the BucketClaim takes a short reconcile cycle to populate its status — until it does, the strategy templates render empty and only the `Bucket` CR + `BackupClass` are present in the cluster. The HelmRelease re-reconciles on its interval (5 minutes by default — set by the cozystack operator's `helmrelease-interval` flag, not a Flux default), at which point the populated BucketClaim status causes the missing strategy templates to materialise. -If you need the BackupClass functional immediately (e.g. an e2e), trigger a Flux reconcile (`flux reconcile helmrelease backupstrategy-controller`) once you see `kubectl get bucketclaim -n tenant-root bucket-cozy-backups -o jsonpath='{.status.bucketName}'` non-empty. +If you need the BackupClass functional immediately (e.g. an e2e), trigger a Flux reconcile (`flux reconcile helmrelease backupstrategy-controller -n cozy-backup-controller`) once you see `kubectl get bucketclaim -n tenant-root bucket-cozy-backups -o jsonpath='{.status.bucketName}'` non-empty. ### Observability @@ -104,11 +104,11 @@ The credentials projector emits two Prometheus counters labelled by `namespace` - `cozystack_backup_credentials_projection_successes_total` - `cozystack_backup_credentials_projection_failures_total` -Alert on `rate(failures_total) > 0` or `absent_over_time(successes_total[10m])` to catch a stale BSL credential or a malformed source Secret without log scraping. +Alert on `rate(cozystack_backup_credentials_projection_failures_total[5m]) > 0` or `absent_over_time(cozystack_backup_credentials_projection_successes_total[10m])` to catch a stale BSL credential or a malformed source Secret without log scraping. ## Admin overrides for `cozy-default` -`cozy-default` is rendered by the `backupstrategy-controller` chart and owned by Flux's helm-controller. **Direct `kubectl edit backupclass cozy-default` is overwritten on the next helm reconcile** — the same applies to its companion `strategy.backups.cozystack.io/*` CRs (`cozy-default-cnpg`, `cozy-default-etcd`, `cozy-default-mariadb`, `cozy-default-altinity`, `cozy-default-foundationdb`, the two `cozy-default-velero-*`). The supported override path is the cozystack `Package` CR, which lets admins inject Helm values into platform components: +`cozy-default` is rendered by the `backupstrategy-controller` chart and owned by Flux's helm-controller. **Direct `kubectl edit backupclass cozy-default` is overwritten on the next helm reconcile** — the same applies to its companion `strategy.backups.cozystack.io/*` CRs (`cozy-default-cnpg`, `cozy-default-etcd`, `cozy-default-mariadb`, `cozy-default-altinity`, `cozy-default-foundationdb`, the two `cozy-default-velero-*`). The supported override path is the `backupStorage` block on the **`platform` component** of the `cozystack.cozystack-platform` Package CR: ```yaml apiVersion: cozystack.io/v1alpha1 @@ -117,7 +117,7 @@ metadata: name: cozystack.cozystack-platform spec: components: - backupstrategy-controller: + platform: values: backupStorage: provisionBucket: true # default; set false for external S3 @@ -130,10 +130,13 @@ spec: - cozy-velero ``` +The platform chart forwards this block into the child `Package cozystack.backupstrategy-controller` as `components.backupstrategy-controller.values.backupStorage` (`packages/core/platform/templates/bundles/system.yaml`), from where the cozystack operator merges it into the `backupstrategy-controller` HelmRelease over the chart defaults. Two paths that look plausible do **not** work: `spec.components.backupstrategy-controller` on the `cozystack.cozystack-platform` Package is silently ignored (the only component under that PackageSource is `platform`), and patching the child `Package cozystack.backupstrategy-controller` directly is reverted whenever the platform helm-reconcile re-renders it. + | Knob | Effect | |---|---| | `provisionBucket` | Toggle creation of the in-cluster `apps.cozystack.io/Bucket` CR. Set `false` for external S3 (see [Disabling the platform-managed bucket](#disabling-the-platform-managed-bucket)). | -| `bucketName` | K8s name of the Bucket CR + lookup key for the COSI BucketClaim. The actual S3 bucket name is the COSI-assigned UUID, surfaced through `BucketClaim.status.bucketName`. | +| `bucketName` | Two modes. With `provisionBucket: true` (default): K8s name of the Bucket CR + lookup key for the COSI BucketClaim — the actual S3 bucket name is the COSI-assigned UUID, surfaced through `BucketClaim.status.bucketName`. With `provisionBucket: false`: taken **verbatim as the real S3 bucket name** and baked into every strategy CR + the Velero BSL. | +| `namespace` | Namespace the Bucket CR (and its system-credentials Secret) lives in — `tenant-root` by default. Must be a tenant namespace (`tenant-*`): the Bucket chart's RBAC helper fails the Helm render for any other prefix. | | `bucketNameOverride` | Escape hatch for offline `helm template` renders — bypasses the live-cluster BucketClaim lookup. Leave empty in production. | | `endpoint` | S3 endpoint baked into every default strategy CR + the Velero BSL. Switching to `https://` silently enables TLS in the MariaDB strategy — ensure the CA bundle is reachable to the relevant operator/driver Pods before flipping it. | | `region` | Re-projected into `cozy-backups-creds` on the next reconcile. Pod-restart required for chart-emitted clients consuming the region via env (ClickHouse sidecar today). | @@ -158,7 +161,7 @@ The system-managed credentials Secret is the **only** way for in-cluster strateg ## Disabling the platform-managed bucket -If a deployment runs against an external S3 (no SeaweedFS), set `backupStorage.provisionBucket: false` in the `backupstrategy-controller` values and create the source credentials Secret in `tenant-root` manually (flat-key format: `accessKey` / `secretKey` / `endpoint` / `bucketName`; or the raw COSI `BucketInfo` JSON). Update `backupStorage.endpoint`, `backupStorage.region`, and (for VM backups) the chart's Velero BSL settings to point at the external S3. +If a deployment runs against an external S3 (no SeaweedFS), set `backupStorage.provisionBucket: false` via the `platform` component override described above and create the source credentials Secret in `tenant-root` manually (flat-key format: `accessKey` / `secretKey` / `endpoint` / `bucketName`; or the raw COSI `BucketInfo` JSON). In the same `backupStorage` block, update `endpoint`, `region`, **and `bucketName`**: with `provisionBucket: false` the strategies and the Velero BSL take `bucketName` verbatim as the real S3 bucket name (no COSI lookup), so it must name the actual bucket on the external S3 — the `bucketName` key inside the Secret alone is not enough. The Velero `BackupStorageLocation` picks the same values up automatically (the chart renders it from the same `backupStorage` block), so no separate BSL configuration is needed. Note that disabling the cluster-default BSL itself (the chart's `velero.bslEnabled` value) is **not** carried by the `backupStorage` override path — the platform Package forwards only the `backupStorage` block. ## Upgrade notes from chart-managed backups diff --git a/docs/operations/seaweedfs-431-rename-recovery.md b/docs/operations/seaweedfs-431-rename-recovery.md new file mode 100644 index 0000000000..cca31f2e87 --- /dev/null +++ b/docs/operations/seaweedfs-431-rename-recovery.md @@ -0,0 +1,386 @@ +# SeaweedFS 4.31 rename — audit & recovery + +This runbook covers clusters affected by the SeaweedFS chart-rename regression introduced when the vendored chart was bumped from `4.0.405` to `4.31.0` (Cozystack v1.5.0). Use it to classify each tenant, and to recover the ones that need an operator before they can be upgraded. + +**Scope — the default instance name is assumed throughout.** The supported way to run SeaweedFS is the tenant module: the tenant chart creates the instance under the fixed name `seaweedfs` (`packages/apps/tenant/templates/seaweedfs.yaml` hardcodes it; a tenant only enables or disables the module). Every shell selector below assumes that name. The API does not yet enforce it, so an instance created directly against `seaweedfses.apps.cozystack.io` under another name can exist — the audit script still classifies it (its release is `-system`), but **do not run the shell loops here against it: escalate instead**, because the name-based `grep seaweedfs` filters cannot see claims whose names the chart truncated (instance names of roughly 30+ characters), and the chart guard's reconstruction does not cover the zone/pool volume components of long-named instances. One accepted limit applies even to default-named instances: a zone or pool **key** of roughly 40+ characters pushes `seaweedfs-system-volume-` past the chart's truncation limit and similarly out of the guard's reconstruction — do not use keys that long. + +## Background + +Before 4.31 the chart named workloads after the chart (`seaweedfs-*`), ignoring the release name. 4.31 names them after the release, and the data-plane HelmRelease is `-system`, so every StatefulSet wanted to become `seaweedfs-system-*`. StatefulSet names are immutable, so the upgrade could not rename in place — Helm stood up a second, duplicate set beside the running one. Depending on cluster size the duplicate either deadlocks or splits: + +- **D-wedged** — with as many nodes as master replicas, the new masters cannot schedule (hard pod anti-affinity against the old masters). The new set stays `Pending`/`CrashLoopBackOff`, the old set keeps serving. Usually no data at risk — but "usually" is not something a Helm render can verify (a duplicate that served writes and later crashed or was scaled down is indistinguishable from one that never started), so the chart refuses these too rather than adopt on an assumption. +- **D-split** — with more nodes than masters, the new (empty) set comes up. Both sets carry identical pod labels, so the `seaweedfs-s3` Service load-balances across them, and both filers write to the **same `seaweedfs-db` Postgres** metadata store while pointing at different volume servers. This is a data-integrity incident, not just a duplicate: reads of existing objects through the new endpoint miss, new writes land on empty volumes, and the two master sets hand out volume IDs from independent sequences into one shared metadata table. + +The fix pins `fullnameOverride: seaweedfs` in `system/seaweedfs` values, so workloads are always named after the chart, exactly as they were before 4.31. Upgrading past the bump therefore **adopts the running set and its volumes in place**. + +Two states cannot be adopted that way, and the charts refuse to render for both rather than guess: + +- A tenant installed **fresh on 1.5.x**, whose data was written under the release-based names and lives on `data1-seaweedfs-system-volume-*` PVCs. Pinning the chart name there would rename the workloads *away* from that data, and Helm cannot move data between PVCs. Re-bind its volumes (Step 2) before upgrading. +- A tenant where **both** naming generations exist. One of them is an empty duplicate and one holds the data — but nothing durable in the object graph says which. Claim timestamps are not evidence: Step 2's own re-bind deletes and recreates claims, so a tenant interrupted mid-recovery has a brand-new claim holding real data. StatefulSets are recreated by the adoption hook. `readyReplicas: 0` does not prove a duplicate never served. Rendering would adopt the chart-named set, so a wrong guess strands or destroys data. Step 1 classifies these with signals a template does not have; once the empty generation is deleted, exactly one remains and the render proceeds on its own. + +The **enforcing** guard lives in `system/seaweedfs` (`templates/naming-guard.yaml`) — the `-system` HelmRelease pulls that chart straight from a platform-managed ExternalArtifact, so a platform upgrade re-renders it directly and nothing else stands between the upgrade and the tenant's workloads; `extra/seaweedfs` carries a sibling copy so the refusal is also visible on the SeaweedFS application itself. + +A tenant upgrading **1.4.x straight to 1.6 never renames**, so it only ever has one generation and is unaffected by any of this. Duplicates exist only on tenants that passed through 1.5.x. + +## Step 0 — `seaweedfs-db` ownership check (read-only, do this FIRST) + +Unrelated to the rename, but it lands on the same upgrade and it destroys data rather than duplicating it, so clear it before anything else. + +The v1.5.0 db split moved the CNPG `Cluster/seaweedfs-db` — the filer metadata store, i.e. the index for every object in the tenant's S3 — out of the `-system` release into its own `-db` release. Migration 43 performs the hand-over: it re-owns the Cluster to `-db` and stamps `helm.sh/resource-policy: keep` so the `-system` upgrade, whose chart no longer renders the Cluster, does not delete it as a removed resource. **Migration 43 shipped comparing the owning release name against the literal `seaweedfs-system`**, so it only ever fired for an instance named `seaweedfs`. `SeaweedFS` is a user-creatable kind: an instance named `foo` is owned by `foo-system`, was skipped, and had its Cluster pruned — CNPG takes the PVC with it. + +The prune is not a one-shot. Helm computes deletions by diffing the **last deployed** revision against the new manifest, so a tenant whose `-system` last succeeded on a pre-split revision recomputes the same deletion on *every* upgrade attempt — including attempts that fail for unrelated reasons and never become the new deployed revision. Such a tenant re-deletes the Cluster each time `-db` recreates it. + +Migration 43 is fixed to match the `-system` suffix, and migration 45 re-runs the hand-over for clusters that already ran the hardcoded version. Both are pre-upgrade hooks, so they land before `-system` re-renders. Audit anyway — a Cluster already deleted cannot be recovered by either: + +```sh +kubectl get cluster.postgresql.cnpg.io -A \ + -o custom-columns='NS:.metadata.namespace,NAME:.metadata.name,OWNER:.metadata.annotations.meta\.helm\.sh/release-name,KEEP:.metadata.annotations.helm\.sh/resource-policy' +``` + +Read the rows for `NAME=seaweedfs-db`: + +| OWNER | KEEP | Meaning | +|---|---|---| +| `-db` | `keep` | Handed over. Nothing to do. | +| `-db` | *(none)* | Installed fresh on ≥1.5.0. Safe: `-system` never rendered the Cluster, so it is not in that release's prune baseline. | +| `-system` | *(none)* | **At risk.** Migration 45 hands it over on the next upgrade. Do not reconcile `-system` before the migration runs. | +| *(no row at all)* | | **Already lost** — see below. | + +A tenant with a SeaweedFS instance but **no `seaweedfs-db` row** has already had its metadata deleted. Its S3 returns 500 and its objects are unreachable even though the volume PVCs still hold the bytes. Nothing in this runbook or in any migration can rebuild that index: the Cluster and its PVC are gone. Restore the `seaweedfs-db` Postgres from a backup, or treat the tenant's object storage as lost. Note that `-db` may report **Ready** while this is true — it rendered its Cluster successfully and a later `-system` prune removed it; Flux has not re-checked. Trust the `kubectl get cluster` output, not the HelmRelease status. + +## Step 1 — Audit the fleet (read-only) + +```sh +hack/seaweedfs-naming-audit.sh # whole cluster +hack/seaweedfs-naming-audit.sh tenant-foo # or named namespaces +``` + +**Read the exit code, not just the table.** The audit fails closed: any error it cannot interpret — a kubectl call that fails, a Helm release payload it cannot decode — makes it print `FATAL` and exit non-zero, and the table it printed up to that point is incomplete. A non-zero exit means you do not yet know the state of the fleet, so none of the steps below may be taken on the strength of it. Only a zero exit means the table is the whole answer; an empty table with exit 0 is a genuinely clean fleet. + +It mutates nothing. Earlier revisions of this runbook inlined the classification as a shell snippet here; it is a tested script now (`hack/seaweedfs-naming-audit.bats`), because it is what the chart's refusal hands you to and acting on it deletes PVCs. Two inline versions shipped wrong — one whose selector matched both generations at once and so inverted its own primary rule, one that could not see a long instance name at all — so it is not a snippet any more. + +It reports one class per SeaweedFS instance, matching exactly what the chart's guard decides: + +| CLASS | Meaning | Action | +|---|---|---| +| `L` | Only the chart-named generation. | None. The upgrade adopts it in place. | +| `S` | Only the release-named generation — installed fresh on 1.5.x, or a long instance name. | Step 2, before upgrading. | +| `MIXED` | Both generations. The chart **refuses**. | Below. | + +For `MIXED` it also names which generation is **original**, from two independent durable signals: the naming scheme revision 1 of the `-system` release was installed with, and each generation's **PersistentVolume** creation timestamps. It reads PV timestamps, never claim timestamps — Step 2 deletes each release-named claim and recreates it under the chart name against the same PV, so claim age is not durable and inverts for a tenant interrupted mid-re-bind. The direction rule is **relative, never a clock**: a generation is the candidate duplicate only when *every* one of its bound PVs is strictly newer than every bound PV of the other generation — the same precondition Step 2a enforces. A tenant interrupted mid-re-bind has both generations on original-vintage PVs, so their ranges **overlap** and the audit names no candidate: finish Step 2, do not run Step 2a. + +**Read the audit's own warning.** "Original" is not "the other one is empty", and the gap is exactly where it matters. A duplicate that never scheduled (safe to delete) and one that served writes and later crashed or was scaled down (holds unique objects, deleting destroys them) are **identical on every durable signal** — same revision-1 scheme, same `first_deployed` deltas. The audit narrows the question to one generation; it does not answer it. Before deleting anything, establish that the candidate is empty: + +Empty means: no volume files (`.dat`/`.idx`/`.vif`) in any data directory. Do **not** `kubectl exec` into the candidate's own pods to check — a wedged duplicate's pods never start, so a check that needs the pod running is unexecutable exactly for the class where it matters most. Mount the claims instead: + +```sh +ns= +# 1. Stop the candidate's workloads so its RWO claims can be mounted elsewhere. +# Reversible, and it is Step 3's first action anyway (Step 2a's for S-damaged). +# 2. A candidate claim still Pending has no PV and therefore no data — empty by +# construction; skip it. +# 3. Mount each BOUND candidate claim read-only in a scratch pod: +pvc= # e.g. data1-seaweedfs-system-volume-0; repeat per claim +kubectl -n "$ns" apply -f - < -- weed shell -c "volume.list" +``` + +If a candidate claim cannot be inspected, or the two views disagree, **stop and escalate**. Both generations holding real data is recoverable; deleting the wrong one is not. + +**D-wedged** — a duplicate that never scheduled — is `MIXED` too, and the chart refuses it like any other duplicate. On main it rendered through, because a duplicate reading `readyReplicas: 0` was taken as proof it never served. That is not proof: a duplicate that served writes and later crashed or was scaled down reads identically. Confirm emptiness as above, then remove it via Step 3 **before** upgrading. + +On a cluster with no spare nodes (nodes ≤ replicas) a wedged duplicate also blocks the adoption rollout even once the render passes: its pods are still *scheduled*, carry the same labels as the adopted set (including `app.kubernetes.io/instance`), and their hard pod anti-affinity keeps the adopted set's rolled pods from landing anywhere (observed as `seaweedfs-filer-1`/`seaweedfs-master-2` stuck Pending on `didn't match pod anti-affinity rules`, wedging the `-system` HelmRelease in upgrade/rollback loops). Step 3 removes it, which resolves that too. + +## Step 2 — `S` tenants: re-bind the volumes before upgrading + +The data is on `data1-seaweedfs-system-volume-N`; the fixed chart expects `data1-seaweedfs-volume-N`. Rather than copying objects, re-point the same PersistentVolume at a PVC with the new name. Nothing is written or moved; only the claim is renamed. Expect downtime for this tenant's S3. + +```sh +ns= + +# 1. Stop the operator from fighting the change. +app= # the SeaweedFS resource name; `seaweedfs` unless you renamed it +kubectl -n "$ns" patch helmrelease "${app}-system" --type merge -p '{"spec":{"suspend":true}}' +# Select the renamed workloads precisely. An S tenant has only the renamed set, +# but exclude the pre-4.31 chart-named seaweedfs-master/-filer/-volume[-] +# anyway so the same selector is reused in Step 3, where both sets coexist. +# seaweedfs-system-* and -system-seaweedfs-* are kept. +renamed_sts() { + kubectl -n "$ns" get sts -l app.kubernetes.io/name=seaweedfs -o name \ + | sed 's|statefulset.apps/||' | grep -vE '^seaweedfs-(master|filer|volume)($|-)' +} +for sts in $(renamed_sts); do + kubectl -n "$ns" scale sts "$sts" --replicas=0 +done +kubectl -n "$ns" scale deploy -l app.kubernetes.io/name=seaweedfs,app.kubernetes.io/component=s3 --replicas=0 + +# 2. For every volume PVC: protect its PV, then re-bind it under the new name. +for pvc in $(kubectl -n "$ns" get pvc -o name | sed 's|persistentvolumeclaim/||' \ + | grep -E '^data1-.*volume' | grep seaweedfs | grep -vE '^data1-seaweedfs-volume'); do + # data1--volume[-]-N -> data1-seaweedfs-volume[-]-N + new_pvc=$(echo "$pvc" | sed -E 's/^data1-.*-volume-/data1-seaweedfs-volume-/') + pv=$(kubectl -n "$ns" get pvc "$pvc" -o jsonpath='{.spec.volumeName}') + sc=$(kubectl -n "$ns" get pvc "$pvc" -o jsonpath='{.spec.storageClassName}') + size=$(kubectl -n "$ns" get pvc "$pvc" -o jsonpath='{.spec.resources.requests.storage}') + # Stash the PV's own reclaim policy ON THE PV before changing it, so a volume the + # cluster deliberately set to Retain does not silently come back as Delete -- and + # so the record survives this loop being interrupted, which a shell variable would + # not. Step 5 restores it from the annotation once the tenant is verified healthy. + reclaim=$(kubectl get pv "$pv" -o jsonpath='{.spec.persistentVolumeReclaimPolicy}') + if [ -z "$(kubectl get pv "$pv" -o jsonpath='{.metadata.annotations.cozystack\.io/original-reclaim-policy}')" ]; then + kubectl annotate pv "$pv" "cozystack.io/original-reclaim-policy=${reclaim}" || { echo "FAILED to record the original reclaim policy on $pv; aborting before any PVC is deleted" >&2; break; } + fi + + # Keep the PV (and the data) when the claim goes away. + kubectl patch pv "$pv" -p '{"spec":{"persistentVolumeReclaimPolicy":"Retain"}}' || { echo "FAILED to set Retain on $pv; aborting before deleting its PVC" >&2; break; } + kubectl -n "$ns" delete pvc "$pvc" + # A Released PV cannot be re-bound until its old claimRef is cleared. + kubectl patch pv "$pv" --type json -p '[{"op":"remove","path":"/spec/claimRef"}]' + + # The labels matter: they are what the StatefulSet's volumeClaimTemplate renders, + # and reconciliation does NOT retrofit claim-template labels onto an existing PVC. + # A claim recreated without them stays unlabelled for good, so any tooling that + # selects volume PVCs by app.kubernetes.io/name + instance -- including the + # post-delete volume reclaim 1.6 adds -- cannot see a recovered tenant's volumes, + # and they leak on a later app deletion. + kubectl -n "$ns" apply -f - <-system-seaweedfs-volume-*` for an instance running under another name, because 4.31's name helper appends the chart name when the release name does not contain it. + +## Step 2a — `S-damaged` tenants: remove the empty chart-named set first + +An `S` tenant that an unguarded 1.6.0 upgrade already reached has an extra problem: the upgrade **created** chart-named workloads (`seaweedfs-master/-filer/-volume`, an s3 Deployment) and **empty** `data1-seaweedfs-volume-*` PVCs beside the live renamed set, and deleted the renamed `-s3` Service (only `seaweedfs-s3` remains — its endpoints may still resolve to the renamed set's pods because both sets carry identical labels, which is luck, not design: if the chart-named pods ever become Ready, the Service splits reads across a live set and an empty one). + +> **STOP if you are resuming an interrupted Step 2.** This step deletes every `data1-seaweedfs-volume-*` claim. If Step 2 already re-bound some of them, those claims hold your data and are bound to the original PVs — deleting them is the data-loss path this step used to be reachable through by misclassification. Finish Step 2 instead. The check below refuses in that case, but read the Step 1 classification first and be sure. + +Verify the direction before touching anything. The chart-named generation must be the **newer** one, judged on the **PV** ages (claims are recreated by Step 2's re-bind, so their timestamps prove nothing). The precondition below refuses unless every chart-named claim is bound to a PV strictly newer than every release-named PV: + +```sh +#!/usr/bin/env bash +set -euo pipefail +ns= +app= + +# PRECONDITION. Every chart-named claim must be bound to a PV strictly newer than +# every release-named PV. A chart-named claim sitting on an OLD PV means Step 2 +# already re-bound it: it holds data, and this step would delete it. +pv_age() { kubectl get pv "$(kubectl -n "$ns" get pvc "$1" -o jsonpath='{.spec.volumeName}')" \ + -o jsonpath='{.metadata.creationTimestamp}'; } +newest_release_pv="" +for pvc in $(kubectl -n "$ns" get pvc -o name | sed 's|persistentvolumeclaim/||' \ + | grep -E '^data1-.*volume' | grep seaweedfs | grep -vE '^data1-seaweedfs-volume'); do + a=$(pv_age "$pvc"); [ "$a" \> "$newest_release_pv" ] && newest_release_pv="$a" +done +[ -n "$newest_release_pv" ] || { echo "REFUSING: no release-named volumes found; this is not an S-damaged tenant"; exit 1; } +for pvc in $(kubectl -n "$ns" get pvc -o name | sed 's|persistentvolumeclaim/||' | grep -E '^data1-seaweedfs-volume'); do + a=$(pv_age "$pvc") + if [ ! "$a" \> "$newest_release_pv" ]; then + echo "REFUSING: $pvc is bound to a PV created $a, NOT newer than the newest release-named PV ($newest_release_pv)." + echo "That claim is not an empty duplicate — Step 2 has most likely already re-bound it. Finish Step 2; do not run this step." + exit 1 + fi +done +echo "ok: every chart-named claim is bound to a strictly newer PV — safe to clear the duplicate" + +kubectl -n "$ns" patch helmrelease "${app}-system" --type merge -p '{"spec":{"suspend":true}}' +# The chart-named workloads were created by the aborted upgrade and never held +# data. Delete them so the re-bind can take over their names. +kubectl -n "$ns" delete sts seaweedfs-master seaweedfs-filer --ignore-not-found +kubectl -n "$ns" get sts -o name | sed 's|statefulset.apps/||' | grep -E '^seaweedfs-volume($|-)' \ + | xargs -r -I{} kubectl -n "$ns" delete sts {} +# The EMPTY chart-named claims block Step 2's re-bind (same names). +kubectl -n "$ns" get pvc -o name | sed 's|persistentvolumeclaim/||' | grep -E '^data1-seaweedfs-volume' \ + | xargs -r -I{} kubectl -n "$ns" delete pvc {} +``` + +Leave the HelmRelease suspended and continue with Step 2 (skip its suspend line; the renamed StatefulSets it scales down are the ones holding the data, exactly as in a plain `S` tenant). Do NOT scale down or delete anything named `*-system-*` before its volumes are re-bound. + +## Step 3 — `MIXED` tenants: remove the duplicate before upgrading + +The chart refuses while both generations exist, so the duplicate must be **gone before** the upgrade, not cleaned up after it. Do not skip to Step 4: it cannot run until this is done. + +Do **not** start by deleting PVCs. If the duplicate ever served, they may hold objects written through the split endpoint. + +1. **Take the duplicate out of service** so nothing else is written to it. Select it precisely — the generation Step 1 named as ORIGINAL is authoritative and must keep serving. For the common case (chart-named original, release-named duplicate): + + ```sh + ns= + kubectl -n "$ns" get sts -l app.kubernetes.io/name=seaweedfs -o name | sed 's|statefulset.apps/||' \ + | grep -vE '^seaweedfs-(master|filer|volume)($|-)' \ + | xargs -r -I{} kubectl -n "$ns" scale sts {} --replicas=0 + # The renamed s3 Deployment is the one that is NOT the chart-named `seaweedfs-s3`. + kubectl -n "$ns" get deploy -l app.kubernetes.io/name=seaweedfs,app.kubernetes.io/component=s3 -o name \ + | sed 's|deployment.apps/||' | grep -v '^seaweedfs-s3$' \ + | xargs -r -I{} kubectl -n "$ns" scale deploy {} --replicas=0 + ``` + + If Step 1 named the **release-named** generation as original, this tenant is `S-damaged`: the duplicate is the chart-named set. Use **Step 2a** instead, then Step 2 — the selectors are inverted there. + +2. **Confirm the authoritative set is serving**, and that the duplicate is out of the `seaweedfs-s3` endpoints: + + ```sh + kubectl -n "$ns" get endpoints seaweedfs-s3 -o yaml + ``` + +3. **If the duplicate ever served, escalate.** Step 1's emptiness check is what decides this. Recovering objects that exist only on a duplicate is not a procedure this runbook can give you: both master sets allocate volume IDs from independent sequences into one shared `seaweedfs-db`, so a fid written through the split endpoint can collide with a fid on the authoritative set, and there is no supported tool that reconciles two volume-ID spaces against one metadata store. Do not improvise it. Involve someone who can plan a per-object export, and treat the tenant as an incident. + +4. **Delete the duplicate's StatefulSets, Deployments and PVCs.** Only once (3) is settled, and only for a duplicate confirmed empty (or whose contents have been exported): + + ```sh + ns= + # StatefulSets and Deployments of the duplicate generation. + kubectl -n "$ns" get sts -l app.kubernetes.io/name=seaweedfs -o name | sed 's|statefulset.apps/||' \ + | grep -vE '^seaweedfs-(master|filer|volume)($|-)' \ + | xargs -r -I{} kubectl -n "$ns" delete sts {} + kubectl -n "$ns" get deploy -l app.kubernetes.io/name=seaweedfs -o name | sed 's|deployment.apps/||' \ + | grep -vE '^seaweedfs-(s3|objectstorage-provisioner)$' \ + | xargs -r -I{} kubectl -n "$ns" delete deploy {} + # Duplicate volume PVCs, BY NAME. Never by label: the live data PVCs carry the + # same app.kubernetes.io/instance=-system label and would match too. + kubectl -n "$ns" get pvc -o name | sed 's|persistentvolumeclaim/||' | grep -E '^data1-.*volume' \ + | grep seaweedfs | grep -vE '^data1-seaweedfs-volume' | xargs -r -I{} kubectl -n "$ns" delete pvc {} + ``` + + Never delete `data1-seaweedfs-volume-*` here — those are the live data PVCs of the authoritative set. (For an `S-damaged` tenant it is the other way round; that is Step 2a's job, and it has its own precondition check.) + +5. **Re-run the audit.** The tenant must now read `L` (or `S`, if you are on the Step 2 path), and the audit must exit zero — a `FATAL` here means the re-check never completed, not that the tenant is fine. Only then upgrade. + + ```sh + hack/seaweedfs-naming-audit.sh "$ns" + ``` + +## Step 4 — Upgrade, then clear the leftovers + +Every tenant must read `L` or `S` in the audit before you start: the chart refuses to render while both generations exist, so a `MIXED` tenant does not upgrade at all — Steps 2/2a/3 come first, not after. Once the fleet is clean, upgrade to a Cozystack version carrying the fix. On reconcile the `-system` release renders the chart-based names and adopts the running workloads and their volumes in place. + +A duplicate's StatefulSets and PVCs are removed in Step 3, before the upgrade. What can still be left behind afterwards are objects no release templated and nothing owns: cert-manager Secrets have no owner reference, and PVCs Helm never templated are never GC'd. Remove those once the tenant is verified healthy: + +```sh +ns= +# Duplicate volume PVCs, BY NAME. Never by label: the live data PVCs carry the +# same app.kubernetes.io/instance=seaweedfs-system label and would match too. +kubectl -n "$ns" get pvc -o name | sed 's|persistentvolumeclaim/||' | grep -E '^data1-.*volume' \ + | grep seaweedfs | grep -vE '^data1-seaweedfs-volume' | xargs -r -I{} kubectl -n "$ns" delete pvc {} +# Orphaned certificate/db secrets of the duplicate set. cert-manager names them +# --cert, so the renamed set's are everything matching the cert/db +# suffix EXCEPT the adopted chart-named seaweedfs--cert / seaweedfs-db-secret. +# This covers the default (seaweedfs-system-*) and any non-default (-system- +# seaweedfs-*) instance without hardcoding the name. +kubectl -n "$ns" get secret -o name | sed 's|secret/||' \ + | grep -E '(-cert|-db-secret)$' | grep seaweedfs \ + | grep -vE '^seaweedfs-(admin|ca|client|filer|master|volume|worker)-cert$|^seaweedfs-db-secret$' \ + | xargs -r -I{} kubectl -n "$ns" delete secret {} --ignore-not-found +``` + +Never delete `data1-seaweedfs-volume-*` — those are the live data PVCs of the adopted set. + +The same 4.31 bump also renamed the tenant's four **cluster-scoped** RBAC objects, and those leftovers are cluster-wide rather than namespaced. Pre-4.31 they were named after the per-namespace service account (`-seaweedfs-*`); 4.31 named them after the Helm release, which is identical in every tenant, so the fleet collided on one object. The fix puts them back on the service account name, which is where a 1.4.x tenant's objects already are — those are adopted in place and need no cleanup. A tenant that passed **through** 1.5.x, however, leaves behind whatever name that release used, and because more than one tenant claimed it, it may not be pruned by any release's manifest: + +```sh +# Stale shared/renamed cluster-scoped RBAC. The go-forward names all start with a +# tenant namespace; anything on the release-based names is a leftover. The fourth +# pre-4.31 object is matched separately: 4.31 renamed the master-rw BINDING from +# upstream's username-shaped system:serviceaccount::default to -rw-crb, so +# the old one carries neither suffix and no tenant prefix. +kubectl get clusterrole,clusterrolebinding \ + -o custom-columns='KIND:.kind,NAME:.metadata.name,OWNER:.metadata.annotations.meta\.helm\.sh/release-name,NS:.metadata.annotations.meta\.helm\.sh/release-namespace' \ + | grep -E 'objectstorage-provisioner|-rw-cr|^\S+\s+system:serviceaccount:.*:default' \ + | grep -vE '^\S+\s+tenant-' +``` + +A `system:serviceaccount:-seaweedfs:default` row is the pre-4.31 master-rw binding. It is superseded by `-seaweedfs-rw-crb` and is pruned automatically by the tenant's own upgrade (it is in that release's manifest), so it should not survive — if it does, the tenant has not upgraded yet. + +Each surviving row is inert once every tenant is upgraded and verified — no release renders those names any more. Confirm the tenant listed in `OWNER`/`NS` is healthy on the go-forward names first, then delete by name. Do not delete anything named `-seaweedfs-*`: those are live. + +**During** a rolling fleet upgrade there is a window, and it is worth expecting rather than debugging: Helm prunes by name without checking ownership, so the first tenant to reconcile onto the fixed chart deletes the shared `seaweedfs-objectstorage-provisioner` / `seaweedfs-rw-cr` objects that a not-yet-upgraded tenant is still bound through. Those tenants' COSI provisioners get 403s on bucket operations until they reconcile onto their own per-namespace RBAC. Existing buckets keep serving — S3 traffic does not go through the provisioner — so this is a provisioning outage, not a data-plane one, and it closes on its own as the remaining tenants reconcile. + +If a tenant's `seaweedfs-system` HelmRelease was suspended during triage, resume it so the fix can reconcile: + +```sh +kubectl -n patch helmrelease seaweedfs-system --type merge -p '{"spec":{"suspend":false}}' +kubectl -n annotate helmrelease seaweedfs-system reconcile.fluxcd.io/requestedAt="$(date +%s)" --overwrite +``` + +## Step 5 — Verify + +```sh +ns= +# Exactly one set, named after the chart, healthy. +kubectl -n "$ns" get sts -l app.kubernetes.io/name=seaweedfs +# All three HelmReleases Ready. +kubectl -n "$ns" get helmrelease seaweedfs seaweedfs-system seaweedfs-db +# S3 endpoints resolve only to the adopted set's ready pods. +kubectl -n "$ns" get endpoints seaweedfs-s3 +# Objects are readable through S3 (bucket list via any tenant bucket). +``` + +A recovered tenant has a single `seaweedfs-*` set, all three HelmReleases `Ready`, no `seaweedfs-system-*` StatefulSets, and no `data1-seaweedfs-system-volume-*` PVCs left behind. + +Only once that is true, restore the reclaim policy Step 2 stashed on each PV. Until this runs the volumes are `Retain`, which is deliberate: it is what makes an accidental claim deletion during recovery survivable. + +```sh +ns= +for pvc in $(kubectl -n "$ns" get pvc -o name | sed 's|persistentvolumeclaim/||' | grep -E '^data1-seaweedfs-volume'); do + pv=$(kubectl -n "$ns" get pvc "$pvc" -o jsonpath='{.spec.volumeName}') + orig=$(kubectl get pv "$pv" -o jsonpath='{.metadata.annotations.cozystack\.io/original-reclaim-policy}') + [ -n "$orig" ] || continue + kubectl patch pv "$pv" -p "{\"spec\":{\"persistentVolumeReclaimPolicy\":\"${orig}\"}}" && kubectl annotate pv "$pv" cozystack.io/original-reclaim-policy- +done +``` diff --git a/hack/e2e-apps/bucket.bats b/hack/e2e-apps/bucket.bats index e51053a601..9c885b4d4a 100644 --- a/hack/e2e-apps/bucket.bats +++ b/hack/e2e-apps/bucket.bats @@ -21,11 +21,24 @@ EOF # Wait for the bucket to be ready kubectl -n tenant-test wait hr bucket-${name} --timeout=5m --for=condition=ready timeout 60 sh -ec "until kubectl -n tenant-test get bucketclaims.objectstorage.k8s.io bucket-${name} >/dev/null 2>&1; do sleep 2; done" - kubectl -n tenant-test wait bucketclaims.objectstorage.k8s.io bucket-${name} --timeout=300s --for=jsonpath='{.status.bucketReady}' + # The COSI controller requeues the BucketClaim until the backend Bucket's + # readiness propagates, so this converges within tens of seconds. Assert + # bucketReady=true explicitly: a bare jsonpath match also matches the literal + # "false", so the unqualified form passed against an unready claim. The tight + # bound makes a propagation-race regression fail fast instead of hiding. + kubectl -n tenant-test wait bucketclaims.objectstorage.k8s.io bucket-${name} --timeout=120s --for=jsonpath='{.status.bucketReady}'=true || { + echo "=== BucketClaim did not converge to bucketReady=true ===" + kubectl -n tenant-test get bucketclaims.objectstorage.k8s.io bucket-${name} -o yaml 2>&1 || true + echo "=== backend Buckets (cluster-scoped) ===" + kubectl get buckets.objectstorage.k8s.io -o wide 2>&1 || true + echo "=== objectstorage-controller ===" + kubectl -n cozy-objectstorage-controller get pods 2>&1 || true + false + } timeout 60 sh -ec "until kubectl -n tenant-test get bucketaccesses.objectstorage.k8s.io bucket-${name}-admin >/dev/null 2>&1; do sleep 2; done" - kubectl -n tenant-test wait bucketaccesses.objectstorage.k8s.io bucket-${name}-admin --timeout=300s --for=jsonpath='{.status.accessGranted}' + kubectl -n tenant-test wait bucketaccesses.objectstorage.k8s.io bucket-${name}-admin --timeout=300s --for=jsonpath='{.status.accessGranted}'=true timeout 60 sh -ec "until kubectl -n tenant-test get bucketaccesses.objectstorage.k8s.io bucket-${name}-viewer >/dev/null 2>&1; do sleep 2; done" - kubectl -n tenant-test wait bucketaccesses.objectstorage.k8s.io bucket-${name}-viewer --timeout=300s --for=jsonpath='{.status.accessGranted}' + kubectl -n tenant-test wait bucketaccesses.objectstorage.k8s.io bucket-${name}-viewer --timeout=300s --for=jsonpath='{.status.accessGranted}'=true # Get admin (readwrite) credentials kubectl -n tenant-test get secret bucket-${name}-admin -ojsonpath='{.data.BucketInfo}' | base64 -d > bucket-admin-credentials.json diff --git a/hack/e2e-apps/harbor.bats b/hack/e2e-apps/harbor.bats index 0648b10373..8fe300939e 100644 --- a/hack/e2e-apps/harbor.bats +++ b/hack/e2e-apps/harbor.bats @@ -43,13 +43,22 @@ EOF timeout 60 sh -ec "until kubectl -n tenant-test get hr $release >/dev/null 2>&1; do sleep 2; done" kubectl -n tenant-test wait hr $release --timeout=5m --for=condition=ready - # Wait for COSI to provision bucket. The driver creates the Bucket and grants - # access quickly, but the central COSI controller's propagation of the Bucket's - # readiness back onto the namespaced BucketClaim can lag several minutes on a - # loaded runner, so allow the same 10m budget the dependent HelmRelease gets. + # Wait for COSI to provision the bucket. The driver creates the backend Bucket + # and grants access; the central COSI controller requeues the BucketClaim until + # the Bucket's readiness has propagated, so it converges within tens of seconds. + # A frozen claim (the propagation race this guards against) never converges, so + # the tight bound surfaces that regression instead of masking it. timeout 60 sh -ec "until kubectl -n tenant-test get bucketclaims.objectstorage.k8s.io $release-registry >/dev/null 2>&1; do sleep 2; done" kubectl -n tenant-test wait bucketclaims.objectstorage.k8s.io $release-registry \ - --timeout=600s --for=jsonpath='{.status.bucketReady}'=true + --timeout=120s --for=jsonpath='{.status.bucketReady}'=true || { + echo "=== BucketClaim did not converge to bucketReady=true ===" + kubectl -n tenant-test get bucketclaims.objectstorage.k8s.io $release-registry -o yaml 2>&1 || true + echo "=== backend Buckets (cluster-scoped) ===" + kubectl get buckets.objectstorage.k8s.io -o wide 2>&1 || true + echo "=== objectstorage-controller ===" + kubectl -n cozy-objectstorage-controller get pods 2>&1 || true + false + } timeout 60 sh -ec "until kubectl -n tenant-test get bucketaccesses.objectstorage.k8s.io $release-registry >/dev/null 2>&1; do sleep 2; done" kubectl -n tenant-test wait bucketaccesses.objectstorage.k8s.io $release-registry \ --timeout=60s --for=jsonpath='{.status.accessGranted}'=true diff --git a/hack/e2e-apps/mariadb.bats b/hack/e2e-apps/mariadb.bats index 0c1e3a5e86..94506ebb6c 100644 --- a/hack/e2e-apps/mariadb.bats +++ b/hack/e2e-apps/mariadb.bats @@ -40,7 +40,16 @@ EOF timeout 60 sh -ec "until kubectl -n tenant-test get hr mariadb-$name >/dev/null 2>&1; do sleep 2; done" kubectl -n tenant-test wait hr mariadb-$name --timeout=5m --for=condition=ready timeout 80 sh -ec "until kubectl -n tenant-test get svc mariadb-$name -o jsonpath='{.spec.ports[0].port}' | grep -q '3306'; do sleep 10; done" - timeout 80 sh -ec "until kubectl -n tenant-test get endpoints mariadb-$name -o jsonpath='{.subsets[*].addresses[*].ip}' | grep -q '[0-9]'; do sleep 10; done" + # The chart hands the operator a MariaDB CR and ships no built-in workload, so + # the HelmRelease above goes Ready as soon as helm applies the manifests — it + # never waits for a pod. This is the first gate that does: an endpoint address + # appears only once a replica has passed its startup and then its readiness + # probe. The startup probe grants each replica 310s of first-boot budget (see + # packages/apps/mariadb/templates/mariadb.yaml for why), so a ceiling under + # that fails runs the probe was still willing to wait for. One ready address + # satisfies this, so it has to clear one budget, not two, plus the operator + # reconcile, PVC bind and image pull that precede it. + timeout 600 sh -ec "until kubectl -n tenant-test get endpoints mariadb-$name -o jsonpath='{.subsets[*].addresses[*].ip}' | grep -q '[0-9]'; do sleep 10; done" timeout 60 sh -ec "until kubectl -n tenant-test get statefulset.apps/mariadb-$name >/dev/null 2>&1; do sleep 2; done" kubectl -n tenant-test wait statefulset.apps/mariadb-$name --timeout=110s --for=jsonpath='{.status.replicas}'=2 timeout 80 sh -ec "until kubectl -n tenant-test get svc mariadb-$name-metrics -o jsonpath='{.spec.ports[0].port}' | grep -q '9104'; do sleep 10; done" @@ -49,3 +58,66 @@ EOF kubectl -n tenant-test wait deployment.apps/mariadb-$name-metrics --timeout=90s --for=jsonpath='{.status.replicas}'=1 kubectl -n tenant-test delete mariadbs.apps.cozystack.io $name } + +@test "Create single-replica MariaDB" { + name='single' + kubectl -n tenant-test delete mariadbs.apps.cozystack.io $name --ignore-not-found --timeout=2m + kubectl apply -f- </dev/null 2>&1; do sleep 2; done" + # A single-replica MariaDB must be accepted by the operator's validating + # webhook: replication is guarded on replicas>1, so replicas=1 renders a CR + # the webhook admits. Dump the HR + rendered CR on failure so a webhook + # rejection (the regression this guards against) is legible in the log. + kubectl -n tenant-test wait hr mariadb-$name --timeout=5m --for=condition=ready \ + || { echo "HR mariadb-$name not ready — dumping state:"; \ + kubectl -n tenant-test describe hr mariadb-$name; \ + kubectl -n tenant-test get mariadbs.k8s.mariadb.com mariadb-$name -o yaml; false; } + # With replicas=1 the operator provisions the bare service (no + # -primary/-secondary): assert it exists and has an endpoint. + timeout 80 sh -ec "until kubectl -n tenant-test get svc mariadb-$name -o jsonpath='{.spec.ports[0].port}' | grep -q '3306'; do sleep 10; done" + # Same gate as the replicated case above — the HelmRelease does not wait for a + # pod, so this is where first boot is actually covered. One replica means one + # 310s startup budget, plus the operator reconcile, PVC bind and image pull + # ahead of it. + timeout 600 sh -ec "until kubectl -n tenant-test get endpoints mariadb-$name -o jsonpath='{.subsets[*].addresses[*].ip}' | grep -q '[0-9]'; do sleep 10; done" + timeout 60 sh -ec "until kubectl -n tenant-test get statefulset.apps/mariadb-$name >/dev/null 2>&1; do sleep 2; done" + kubectl -n tenant-test wait statefulset.apps/mariadb-$name --timeout=110s --for=jsonpath='{.status.replicas}'=1 + timeout 80 sh -ec "until kubectl -n tenant-test get svc mariadb-$name-metrics -o jsonpath='{.spec.ports[0].port}' | grep -q '9104'; do sleep 10; done" + timeout 40 sh -ec "until kubectl -n tenant-test get endpoints mariadb-$name-metrics -o jsonpath='{.subsets[*].addresses[*].ip}' | grep -q '[0-9]'; do sleep 10; done" + timeout 60 sh -ec "until kubectl -n tenant-test get deployment.apps/mariadb-$name-metrics >/dev/null 2>&1; do sleep 2; done" + kubectl -n tenant-test wait deployment.apps/mariadb-$name-metrics --timeout=90s --for=jsonpath='{.status.replicas}'=1 + kubectl -n tenant-test delete mariadbs.apps.cozystack.io $name --ignore-not-found +} diff --git a/hack/e2e-apps/postgres.bats b/hack/e2e-apps/postgres.bats index 44c0c5bd77..a4988c1da4 100644 --- a/hack/e2e-apps/postgres.bats +++ b/hack/e2e-apps/postgres.bats @@ -53,6 +53,25 @@ EOF # for some reason it takes longer for the read-only endpoint to be ready #timeout 120 sh -ec "until kubectl -n tenant-test get endpoints postgres-$name-ro -o jsonpath='{.subsets[*].addresses[*].ip}' | grep -q '[0-9]'; do sleep 10; done" timeout 120 sh -ec "until kubectl -n tenant-test get endpoints postgres-$name-rw -o jsonpath='{.subsets[*].addresses[*].ip}' | grep -q '[0-9]'; do sleep 10; done" + backup_name="$name-session-id-schema" + kubectl -n tenant-test delete backups.postgresql.cnpg.io "$backup_name" --ignore-not-found --timeout=2m + kubectl -n tenant-test apply -f - <-system +# release into a new -db release. The hand-over must, before -system +# next renders, re-own the Cluster to -db AND stamp +# helm.sh/resource-policy: keep — otherwise the -system upgrade prunes the +# Cluster as a removed resource, CNPG takes its PVC with it, and the tenant's +# filer metadata (all of its S3) is gone. +# +# Three properties are pinned here, each of which shipped broken: +# +# 1. INSTANCE NAME. Migration 43 compared the owner against the literal +# "seaweedfs-system". `SeaweedFS` is user-creatable, so an instance named +# `foo` is owned by `foo-system` and was silently skipped. Observed live: +# four default-named tenants carry release-name=seaweedfs-db + keep and their +# Clusters survived; the one tenant running `foo` has no Cluster at all. +# +# 2. OWNERSHIP IS NOT SAFETY. A Cluster already owned by -db can still +# need keep: where the hand-over was skipped, -system prunes it and +# -db RECREATES it under its own ownership with no keep, while +# -system's prune baseline still lists it. Live proof that the shape is +# real: tenant-l and tenant-root are -db-owned and their -system +# deployed revision still contains the Cluster — only keep saves them. +# +# 3. FAIL CLOSED. Migrations never re-run, so a swallowed error permanently +# leaves at-risk tenants exposed. `for ns in $(kubectl ...)` does not trip +# errexit: on failure the loop runs zero times and the script stamps the +# version anyway. Only "the resource type is not served" (no CNPG at all) and +# "gone between scan and read" may be treated as empty. +# +# These drive the real migration scripts end-to-end against a fake kubectl +# (hack/testdata/migration-seaweedfs-db/), mocking only the cluster boundary. +# +# SHELL. Production runs these under /bin/sh = busybox ash: the migrations image +# is FROM alpine, and run-migrations.sh execs `/migrations/` BY PATH, so the +# kernel honours the `#!/bin/sh` shebang. `set -euo pipefail` and +# errexit-in-function semantics differ from bash, and both are load-bearing here: +# the fail-closed returns from adopt_seaweedfs_db_clusters have to abort the +# script before it reaches its stamp. Asserting fail-closed in a shell that never +# runs it would be asserting nothing. +# +# So these tests do not invoke the migrations through the runner's shell at all: +# run_migration() executes them by path inside the image's own pinned base, read +# from the migrations Dockerfile. Same base image, same interpreter, same +# invocation form — production, rather than an approximation of it. This needs a +# working docker; there is deliberately no host-shell fallback, because every +# fallback available is a shell production never uses. +# +# What that replaced was `sh "$MIG_DIR/"`, which is neither production nor +# bash: on the GitHub ubuntu runner /bin/sh is dash, which has no `set -o +# pipefail` and aborts on line 1 with "set: Illegal option -o pipefail" before +# the script does any work, while on a developer box /bin/sh is often bash, which +# runs green and proves nothing about ash. +# +# cozytest.sh's awk parser recognizes only @test blocks and a bare `}` on its +# own line; there is no bats `run`/`$status`/`setup`. Assertions are direct +# shell tests that exit non-zero on failure. +# +# Run with: hack/cozytest.sh hack/migration-seaweedfs-db-adopt.bats +# ----------------------------------------------------------------------------- + +FAKEBIN="$PWD/hack/testdata/migration-seaweedfs-db" +MIG_DIR="$PWD/packages/core/platform/images/migrations/migrations" + +# The production base image, read out of the migrations Dockerfile rather than +# repeated here, so the interpreter under test cannot drift from the one the +# migrations actually ship on when that pin is bumped. +ALPINE=$(sed -n 's/^FROM \(alpine:[^ ]*\).*$/\1/p' \ + "$PWD/packages/core/platform/images/migrations/Dockerfile" | head -1) + +# run_migration -- run migrations/ the way run-migrations.sh does. +# +# By path, not `sh `: that is what makes the shebang, and therefore the +# interpreter, part of what is under test. The fake kubectl goes on PATH inside +# the container and $WORK is bind-mounted, so $FAKE_CMDLOG is the same file the +# assertions read back on the host. --network none because nothing here may +# reach a real cluster; --user keeps $WORK removable by the test afterwards. +# +# The explicit `return` is load-bearing: cozytest.sh's awk generator rewrites +# every bare `}` in column 0 into `return 0` + `}`, so a helper that falls off +# its own end returns 0 no matter what it ran, and every fail-closed assertion +# below would pass vacuously. Capture the status and return it by hand. +run_migration() { + _run_migration_rc=0 + docker run --rm --network none \ + --user "$(id -u):$(id -g)" \ + -v "$MIG_DIR:/migrations:ro" \ + -v "$FAKEBIN:/fakebin:ro" \ + -v "$WORK:/work" \ + -e PATH=/fakebin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ + -e FAKE_CMDLOG=/work/cmdlog \ + -e NAMESPACE="${NAMESPACE-}" \ + -e FAKE_CLUSTERS="${FAKE_CLUSTERS-}" \ + -e FAKE_LIST_FAIL="${FAKE_LIST_FAIL-}" \ + -e FAKE_GET_FAIL="${FAKE_GET_FAIL-}" \ + -e FAKE_ANNOTATE_FAIL="${FAKE_ANNOTATE_FAIL-}" \ + -e FAKE_KCTS="${FAKE_KCTS-}" \ + -e FAKE_KCS="${FAKE_KCS-}" \ + -e FAKE_KUBEADM_LIST_FAIL="${FAKE_KUBEADM_LIST_FAIL-}" \ + -e FAKE_KUBEADM_ANNOTATE_FAIL="${FAKE_KUBEADM_ANNOTATE_FAIL-}" \ + -e FAKE_KUBEADM_ANNOTATE_FAIL_NS="${FAKE_KUBEADM_ANNOTATE_FAIL_NS-}" \ + "$ALPINE" "/migrations/$1" || _run_migration_rc=$? + return "$_run_migration_rc" +} + +# prep resets env to a clean scenario. Tests set FAKE_* afterwards. +prep() { + # Fail here rather than at the first docker run, so the reason is legible. + docker info >/dev/null 2>&1 || { + echo "docker is required: these tests run the migrations inside $ALPINE," >&2 + echo "the base image of the migrations image, so that they exercise busybox" >&2 + echo "ash — the interpreter run-migrations.sh actually gives them." >&2 + return 1 + } + chmod +x "$FAKEBIN/kubectl" + WORK=$(mktemp -d) + export FAKE_CMDLOG="$WORK/cmdlog" + : > "$FAKE_CMDLOG" + export NAMESPACE=cozy-system + export FAKE_CLUSTERS="" + export FAKE_KCTS="" + export FAKE_KCS="" + unset FAKE_LIST_FAIL FAKE_GET_FAIL FAKE_ANNOTATE_FAIL || true + unset FAKE_KUBEADM_LIST_FAIL FAKE_KUBEADM_ANNOTATE_FAIL \ + FAKE_KUBEADM_ANNOTATE_FAIL_NS || true +} + +# --- 1. instance name ------------------------------------------------------- + +@test "hands over a default-named instance (seaweedfs-system -> seaweedfs-db)" { + prep + export FAKE_CLUSTERS="tenant-root seaweedfs-system -" + rc=0 + run_migration 43 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -eq 0 ] + grep -qF -- "ANNOTATE tenant-root release-name=seaweedfs-db resource-policy=keep" "$FAKE_CMDLOG" + # Migration 43 stamps 44 — asserting the number, not a bare "STAMP": a wrong + # version would loop run-migrations.sh forever. + grep -qF -- "STAMP 44" "$FAKE_CMDLOG" + rm -rf "$WORK" +} + +# THE original regression. Unfixed (owner compared against the literal +# "seaweedfs-system") this namespace is skipped entirely: no ANNOTATE line, and +# the Cluster is left with no keep for the foo-system upgrade to prune. +@test "hands over a NON-default instance name (foo-system -> foo-db)" { + prep + export FAKE_CLUSTERS="tenant-named foo-system -" + rc=0 + run_migration 43 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -eq 0 ] + grep -qF -- "ANNOTATE tenant-named release-name=foo-db resource-policy=keep" "$FAKE_CMDLOG" + rm -rf "$WORK" +} + +@test "migration 45 repairs a non-default instance the hardcoded 43 skipped, and stamps 46" { + prep + export FAKE_CLUSTERS="tenant-named foo-system -" + rc=0 + run_migration 45 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -eq 0 ] + grep -qF -- "ANNOTATE tenant-named release-name=foo-db resource-policy=keep" "$FAKE_CMDLOG" + grep -qF -- "STAMP 46" "$FAKE_CMDLOG" + rm -rf "$WORK" +} + +@test "handles a mixed fleet: every -system owner is handed over, in its own namespace" { + prep + # The shape of the upgrade stand: default-named tenants plus one `foo`. + export FAKE_CLUSTERS="tenant-root seaweedfs-system - +tenant-dsplit seaweedfs-system - +tenant-l seaweedfs-system - +tenant-named foo-system -" + rc=0 + run_migration 45 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -eq 0 ] + grep -qF -- "ANNOTATE tenant-root release-name=seaweedfs-db resource-policy=keep" "$FAKE_CMDLOG" + grep -qF -- "ANNOTATE tenant-dsplit release-name=seaweedfs-db resource-policy=keep" "$FAKE_CMDLOG" + grep -qF -- "ANNOTATE tenant-l release-name=seaweedfs-db resource-policy=keep" "$FAKE_CMDLOG" + grep -qF -- "ANNOTATE tenant-named release-name=foo-db resource-policy=keep" "$FAKE_CMDLOG" + [ "$(grep -c 'ANNOTATE' "$FAKE_CMDLOG")" -eq 4 ] + rm -rf "$WORK" +} + +# --- 2. ownership is not safety -------------------------------------------- + +# A -db-owned Cluster WITHOUT keep is exposed, not done: -system's +# prune baseline may still list the Cluster (live on the stand for tenant-l and +# tenant-root), in which case its next reconcile deletes it. Skipping on +# ownership alone — the shape the previous revision of this helper shipped — +# leaves the database to be pruned. +@test "protects a -db-owned Cluster that is still missing keep" { + prep + export FAKE_CLUSTERS="tenant-named foo-db -" + rc=0 + run_migration 45 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -eq 0 ] + grep -qF -- "ANNOTATE tenant-named release-name= resource-policy=keep" "$FAKE_CMDLOG" + rm -rf "$WORK" +} + +@test "protects a default-named -db-owned Cluster that is still missing keep" { + prep + export FAKE_CLUSTERS="tenant-fresh seaweedfs-db -" + rc=0 + run_migration 45 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -eq 0 ] + grep -qF -- "ANNOTATE tenant-fresh release-name= resource-policy=keep" "$FAKE_CMDLOG" + rm -rf "$WORK" +} + +@test "idempotent: a Cluster already owned by -db AND carrying keep is left alone" { + prep + export FAKE_CLUSTERS="tenant-root seaweedfs-db keep +tenant-named foo-db keep" + rc=0 + run_migration 45 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -eq 0 ] + [ "$(grep -c 'ANNOTATE' "$FAKE_CMDLOG")" -eq 0 ] + grep -qF -- "STAMP 46" "$FAKE_CMDLOG" + rm -rf "$WORK" +} + +@test "leaves a Cluster with no Helm owner annotation alone, but says so" { + prep + # Not Helm-managed. Guessing an owner would be worse than doing nothing, but + # an unowned SeaweedFS database must not pass silently. + export FAKE_CLUSTERS="tenant-manual - -" + rc=0 + run_migration 45 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -eq 0 ] + [ "$(grep -c 'ANNOTATE' "$FAKE_CMDLOG")" -eq 0 ] + grep -qF -- "carries no meta.helm.sh/release-name" "$WORK/out" + rm -rf "$WORK" +} + +@test "leaves a Cluster owned by an unrelated release alone" { + prep + export FAKE_CLUSTERS="tenant-x some-other-release -" + rc=0 + run_migration 45 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -eq 0 ] + [ "$(grep -c 'ANNOTATE' "$FAKE_CMDLOG")" -eq 0 ] + grep -qF -- "owned by unrelated release" "$WORK/out" + rm -rf "$WORK" +} + +@test "refuses a release literally named -system rather than annotating owner -db" { + prep + # "${current%-system}" would be empty, yielding release-name=-db, which no + # release will ever claim: the Cluster would be orphaned by the very step + # meant to protect it. + export FAKE_CLUSTERS="tenant-weird -system -" + rc=0 + run_migration 45 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -eq 0 ] + [ "$(grep -c 'ANNOTATE' "$FAKE_CMDLOG")" -eq 0 ] + grep -qF -- "no instance name" "$WORK/out" + rm -rf "$WORK" +} + +# --- 3. fail closed --------------------------------------------------------- + +@test "a failing fleet scan aborts the migration instead of stamping past it" { + prep + export FAKE_CLUSTERS="tenant-named foo-system -" + export FAKE_LIST_FAIL="Error from server (Timeout): the server was unable to return a response in the time allotted" + rc=0 + run_migration 45 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + # Must propagate: the Job retries rather than advancing the version. + [ "$rc" -ne 0 ] + grep -qF -- "refusing to stamp past an unverified fleet" "$WORK/out" + [ "$(grep -c 'STAMP' "$FAKE_CMDLOG")" -eq 0 ] + rm -rf "$WORK" +} + +@test "an unreadable owner annotation aborts rather than being read as not-Helm-managed" { + prep + export FAKE_CLUSTERS="tenant-named foo-system -" + export FAKE_GET_FAIL="Error from server (Forbidden): clusters.postgresql.cnpg.io \"seaweedfs-db\" is forbidden" + rc=0 + run_migration 45 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -ne 0 ] + grep -qF -- "cannot read the Helm owner" "$WORK/out" + [ "$(grep -c 'STAMP' "$FAKE_CMDLOG")" -eq 0 ] + rm -rf "$WORK" +} + +@test "a failed hand-over aborts rather than stamping a half-migrated fleet" { + prep + export FAKE_CLUSTERS="tenant-named foo-system -" + export FAKE_ANNOTATE_FAIL="Error from server (Conflict): the object has been modified" + rc=0 + run_migration 45 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -ne 0 ] + [ "$(grep -c 'STAMP' "$FAKE_CMDLOG")" -eq 0 ] + rm -rf "$WORK" +} + +# The fail-open that IS load-bearing: a cluster with no CNPG at all must not be +# blocked from upgrading. "The server doesn't have a resource type" is the only +# list failure allowed to mean "nothing to do". +@test "a cluster with no CNPG resource type stamps cleanly without annotating" { + prep + export FAKE_LIST_FAIL="error: the server doesn't have a resource type \"cluster\" in group \"postgresql.cnpg.io\"" + rc=0 + run_migration 45 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -eq 0 ] + grep -qF -- "resource type is not served" "$WORK/out" + [ "$(grep -c 'ANNOTATE' "$FAKE_CMDLOG")" -eq 0 ] + grep -qF -- "STAMP 46" "$FAKE_CMDLOG" + rm -rf "$WORK" +} + +@test "an empty fleet stamps without annotating" { + prep + export FAKE_CLUSTERS="" + rc=0 + run_migration 45 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -eq 0 ] + [ "$(grep -c 'ANNOTATE' "$FAKE_CMDLOG")" -eq 0 ] + grep -qF -- "STAMP 46" "$FAKE_CMDLOG" + rm -rf "$WORK" +} + +# --- 4. kubeadm bootstrap keep-pin ----------------------------------------- +# +# THE THREAT. 1.6 drops KubeadmConfigTemplate from the tenant `kubernetes` chart +# (workers move to TalosConfigTemplate), so its upgrade sees the resource in the +# previous release manifest and absent from the new one and deletes it — while the +# kubeadm-backed MachineSet is still around mid-rollover with its +# bootstrap.configRef pointing at it. 1.6 guards that with its own migration 45. +# A v1.5.4 cluster is stamped 46 and runs `seq 46 53`, so it never executes that +# slot; the pin has to already be on the objects, which is what this half of +# release-1.5's slot 45 does. +# +# So the property under test is the ANNOTATION LANDING ON THE OBJECT, not a +# function being callable. Every assertion below reads the PIN records the fake +# kubectl wrote, and the fake models the label selector rather than ignoring it: +# selecting on meta.helm.sh/release-name (an annotation, and therefore never a +# valid selector) returns no rows, so the classic silent-no-op shape of this bug +# fails these tests instead of passing them. +# +# PIN, not ANNOTATE, is the fake's verb for this half — the SeaweedFS assertions +# above count ANNOTATE lines, and a shared verb would couple the two halves' +# tests to each other. +# +# "DID NOT HAPPEN" IS ASSERTED AS `[ "$(grep -c ...)" -eq 0 ]`, NEVER `! grep -q`. +# POSIX and bash both exempt a !-negated pipeline from errexit — "the -e setting +# shall be ignored ... if the command's return value is being inverted with !" — +# so `! grep -q X file` cannot fail a cozytest test no matter what the file +# contains. It reads like an assertion and is a no-op. Measured, not assumed: with +# the two halves of slot 45 deliberately swapped, a `! grep -q 'PIN '` test stayed +# green while the cmdlog plainly contained the PIN line. The counting form puts the +# result in `[`, whose non-zero status does trip errexit. + +@test "pins an unannotated Helm-managed KubeadmConfigTemplate, and stamps 46" { + prep + export FAKE_KCTS="tenant-root kubernetes-md0 - Helm" + rc=0 + run_migration 45 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -eq 0 ] + grep -qF -- "PIN kubeadmconfigtemplate tenant-root/kubernetes-md0 resource-policy=keep" "$FAKE_CMDLOG" + grep -qF -- "STAMP 46" "$FAKE_CMDLOG" + rm -rf "$WORK" +} + +@test "pins every Helm-managed KubeadmConfigTemplate, each in its own namespace" { + prep + export FAKE_KCTS="tenant-root kubernetes-md0 - Helm +tenant-root kubernetes-gpu-md1 - Helm +tenant-a other-cluster-md0 - Helm" + rc=0 + run_migration 45 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -eq 0 ] + grep -qF -- "PIN kubeadmconfigtemplate tenant-root/kubernetes-md0 resource-policy=keep" "$FAKE_CMDLOG" + grep -qF -- "PIN kubeadmconfigtemplate tenant-root/kubernetes-gpu-md1 resource-policy=keep" "$FAKE_CMDLOG" + grep -qF -- "PIN kubeadmconfigtemplate tenant-a/other-cluster-md0 resource-policy=keep" "$FAKE_CMDLOG" + [ "$(grep -c 'PIN ' "$FAKE_CMDLOG")" -eq 3 ] + rm -rf "$WORK" +} + +# Re-running must not write. The pin reads helm.sh/resource-policy first and skips +# on "keep", so an already-pinned fleet produces no annotate call at all — an +# unconditional `kubectl annotate --overwrite` would still be correct on the +# cluster but would make "no-op" unobservable, and this is the assertion that +# keeps it observable. +@test "idempotent: an already-pinned KubeadmConfigTemplate is skipped without a write" { + prep + export FAKE_KCTS="tenant-root kubernetes-md0 keep Helm" + rc=0 + run_migration 45 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -eq 0 ] + [ "$(grep -c 'PIN ' "$FAKE_CMDLOG")" -eq 0 ] + grep -qF -- "already carries helm.sh/resource-policy=keep" "$WORK/out" + grep -qF -- "already-pinned=1" "$WORK/out" + grep -qF -- "STAMP 46" "$FAKE_CMDLOG" + rm -rf "$WORK" +} + +# A cluster with no tenant Kubernetes clusters at all. Zero matching objects is a +# clean run, not a failure: the CRDs are served (CAPI is installed platform-wide) +# and the selector simply matches nothing. +@test "zero Helm-managed kubeadm objects is success, not failure" { + prep + export FAKE_KCTS="" + export FAKE_KCS="" + rc=0 + run_migration 45 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -eq 0 ] + [ "$(grep -c 'PIN ' "$FAKE_CMDLOG")" -eq 0 ] + grep -qF -- "pinned=0 already-pinned=0 failures=0" "$WORK/out" + grep -qF -- "STAMP 46" "$FAKE_CMDLOG" + rm -rf "$WORK" +} + +# The KubeadmConfig children CAPI spawns from the template are owned by their +# Machine, not by Helm: no app.kubernetes.io/managed-by label, so Helm never +# prunes them and the pin must not touch them. Pinning one would leave keep on an +# object whose lifecycle belongs to CAPI. Verified against a live v1.5 stand, +# where the spawned KubeadmConfig carries no managed-by at all. +@test "leaves a CAPI-spawned KubeadmConfig alone: it is not Helm-managed" { + prep + export FAKE_KCS="tenant-root kubernetes-md0-lw46d-z2wqn - -" + rc=0 + run_migration 45 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -eq 0 ] + [ "$(grep -c 'PIN ' "$FAKE_CMDLOG")" -eq 0 ] + grep -qF -- "STAMP 46" "$FAKE_CMDLOG" + rm -rf "$WORK" +} + +# The fail-open that IS load-bearing, mirroring the CNPG case above: a cluster +# without the CAPI bootstrap provider has nothing to pin and must not be blocked +# from upgrading. +@test "a cluster with no kubeadm bootstrap provider stamps cleanly without pinning" { + prep + export FAKE_KUBEADM_LIST_FAIL="error: the server doesn't have a resource type \"kubeadmconfigtemplates\" in group \"bootstrap.cluster.x-k8s.io\"" + rc=0 + run_migration 45 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -eq 0 ] + grep -qF -- "is not served on this cluster" "$WORK/out" + [ "$(grep -c 'PIN ' "$FAKE_CMDLOG")" -eq 0 ] + grep -qF -- "STAMP 46" "$FAKE_CMDLOG" + rm -rf "$WORK" +} + +@test "a failing kubeadm fleet scan aborts instead of stamping past it" { + prep + export FAKE_KCTS="tenant-root kubernetes-md0 - Helm" + export FAKE_KUBEADM_LIST_FAIL="Error from server (Timeout): the server was unable to return a response in the time allotted" + rc=0 + run_migration 45 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + # Must propagate: the Job retries rather than advancing the version, because + # nothing later will pin what this pass could not see. + [ "$rc" -ne 0 ] + grep -qF -- "refusing to stamp past an unverified fleet" "$WORK/out" + [ "$(grep -c 'STAMP' "$FAKE_CMDLOG")" -eq 0 ] + rm -rf "$WORK" +} + +# The one per-object failure that is NOT a failure: an app deleted concurrently +# with this hook takes its KubeadmConfigTemplate away between the fleet scan and +# the annotate. Helm cannot prune what no longer exists, so nothing is at risk, and +# failing the pre-upgrade hook over it would block the platform upgrade on an +# object nobody needs. "not found" is accepted HERE and deliberately not for the +# fleet scan, where a list never answers NotFound and accepting it would let a real +# failure read as an empty fleet. +@test "an object that disappears between the scan and the pin is skipped, not fatal" { + prep + export FAKE_KCTS="tenant-doomed kubernetes-md0 - Helm" + export FAKE_KUBEADM_ANNOTATE_FAIL="Error from server (NotFound): kubeadmconfigtemplates.bootstrap.cluster.x-k8s.io \"kubernetes-md0\" not found" + rc=0 + run_migration 45 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -eq 0 ] + grep -qF -- "disappeared between the scan and the pin" "$WORK/out" + grep -qF -- "pinned=0 already-pinned=1 failures=0" "$WORK/out" + grep -qF -- "STAMP 46" "$FAKE_CMDLOG" + rm -rf "$WORK" +} + +@test "a failed pin aborts rather than stamping a half-pinned fleet" { + prep + export FAKE_KCTS="tenant-root kubernetes-md0 - Helm" + export FAKE_KUBEADM_ANNOTATE_FAIL="Error from server (Conflict): the object has been modified" + rc=0 + run_migration 45 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -ne 0 ] + grep -qF -- "could not be pinned" "$WORK/out" + [ "$(grep -c 'STAMP' "$FAKE_CMDLOG")" -eq 0 ] + rm -rf "$WORK" +} + +# Aggregation, and the reason it is worth having: one unpinnable object must not +# stop the others from being pinned, because the version is not stamped either way +# and the next attempt starts from the same place. The failing namespace is listed +# FIRST so a loop that aborted on the first failure would leave tenant-b unpinned +# and fail this test. +@test "a partial pin failure still pins the rest, then aborts without stamping" { + prep + export FAKE_KCTS="tenant-a broken-md0 - Helm +tenant-b healthy-md0 - Helm" + export FAKE_KUBEADM_ANNOTATE_FAIL="Error from server (Forbidden): kubeadmconfigtemplates.bootstrap.cluster.x-k8s.io is forbidden" + export FAKE_KUBEADM_ANNOTATE_FAIL_NS="tenant-a" + rc=0 + run_migration 45 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -ne 0 ] + grep -qF -- "PIN kubeadmconfigtemplate tenant-b/healthy-md0 resource-policy=keep" "$FAKE_CMDLOG" + [ "$(grep -cF -- "tenant-a/broken-md0 resource-policy=keep" "$FAKE_CMDLOG")" -eq 0 ] + grep -qF -- "pinned=1 already-pinned=0 failures=1" "$WORK/out" + [ "$(grep -c 'STAMP' "$FAKE_CMDLOG")" -eq 0 ] + rm -rf "$WORK" +} + +# --- 5. the two halves, in one slot ---------------------------------------- + +@test "runs both halves in one pass: SeaweedFS hand-over first, then the pin" { + prep + export FAKE_CLUSTERS="tenant-named foo-system -" + export FAKE_KCTS="tenant-root kubernetes-md0 - Helm" + rc=0 + run_migration 45 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -eq 0 ] + grep -qF -- "ANNOTATE tenant-named release-name=foo-db resource-policy=keep" "$FAKE_CMDLOG" + grep -qF -- "PIN kubeadmconfigtemplate tenant-root/kubernetes-md0 resource-policy=keep" "$FAKE_CMDLOG" + grep -qF -- "STAMP 46" "$FAKE_CMDLOG" + # Order is deliberate, not incidental: a missed hand-over loses a tenant's filer + # metadata, a missed pin gives a recoverable broken worker rollover. On a pass + # where only one half gets to run, it must be the irreversible one. + sw_line=$(grep -n 'ANNOTATE tenant-named' "$FAKE_CMDLOG" | head -1 | cut -d: -f1) + pin_line=$(grep -n 'PIN kubeadmconfigtemplate' "$FAKE_CMDLOG" | head -1 | cut -d: -f1) + [ "$sw_line" -lt "$pin_line" ] + rm -rf "$WORK" +} + +# The other side of that order: the SeaweedFS half failing must abort before the +# pin is attempted at all, and must not stamp. +@test "a SeaweedFS failure aborts before the pin half runs" { + prep + export FAKE_CLUSTERS="tenant-named foo-system -" + export FAKE_KCTS="tenant-root kubernetes-md0 - Helm" + export FAKE_LIST_FAIL="Error from server (Timeout): the server was unable to return a response in the time allotted" + rc=0 + run_migration 45 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -ne 0 ] + [ "$(grep -c 'PIN ' "$FAKE_CMDLOG")" -eq 0 ] + [ "$(grep -c 'STAMP' "$FAKE_CMDLOG")" -eq 0 ] + rm -rf "$WORK" +} + +# Migration 43 sources ONLY lib/seaweedfs-db-adopt.sh. The pin must not leak into +# it: a cluster running 43 is mid-upgrade to 44 and 1.6's slot 45 will still run +# for it, and the two libs must stay independently sourceable (neither may call +# the other's private helpers). +@test "migration 43 does not pin: the keep-pin belongs to slot 45 alone" { + prep + export FAKE_CLUSTERS="tenant-root seaweedfs-system -" + export FAKE_KCTS="tenant-root kubernetes-md0 - Helm" + rc=0 + run_migration 43 >"$WORK/out" 2>&1 || rc=$? + cat "$WORK/out"; cat "$FAKE_CMDLOG" + [ "$rc" -eq 0 ] + [ "$(grep -c 'PIN ' "$FAKE_CMDLOG")" -eq 0 ] + grep -qF -- "STAMP 44" "$FAKE_CMDLOG" + rm -rf "$WORK" +} diff --git a/hack/seaweedfs-guard-parity.bats b/hack/seaweedfs-guard-parity.bats new file mode 100644 index 0000000000..38c194e494 --- /dev/null +++ b/hack/seaweedfs-guard-parity.bats @@ -0,0 +1,144 @@ +#!/usr/bin/env bats +# ----------------------------------------------------------------------------- +# The SeaweedFS naming guard exists in two copies: +# +# packages/system/seaweedfs/templates/naming-guard.yaml (ENFORCING — the +# -system HelmRelease pulls this chart from a platform-managed +# ExternalArtifact, so a platform upgrade re-renders it directly and nothing +# else stands between the upgrade and the tenant's workloads) +# packages/extra/seaweedfs/templates/seaweedfs.yaml (sibling — warns on +# the SeaweedFS application itself, where the operator looks first) +# +# They cannot be shared: the two packages are separate charts and neither depends +# on cozy-lib, so there is no library either could include the classifier from. +# They were kept in sync by a comment saying "keep the two classifications in +# sync" — and this whole branch exists because the guard lived in the wrong +# chart. Silent drift between the copies is the same failure class, so pin it. +# +# Only the DETECTION block is compared: the two `fail` messages differ by design +# ("SeaweedFS release " vs "SeaweedFS ", matching each chart's voice) +# and the branch structure is asserted separately. It is the detection — which +# objects establish which generation — that must never diverge. +# +# The block starts at the reconstruction, not at the flag declarations: the line +# above it is deliberately different per chart (system/ IS the -system +# release; extra/ is and derives the child), and that is the ONLY sanctioned +# difference. Both are asserted separately below. +# +# Run with: hack/cozytest.sh hack/seaweedfs-guard-parity.bats +# ----------------------------------------------------------------------------- + +SYS="$PWD/packages/system/seaweedfs/templates/naming-guard.yaml" +EXTRA="$PWD/packages/extra/seaweedfs/templates/seaweedfs.yaml" + +# detection_block -- the generation-detection lines, from the first flag +# declaration through the two derived generation booleans. +detection_block() { + sed -n '/\$renamedVol := include/,/\$systemGen := or/p' "$1" +} + +# strip_tpl_comments -- the file with every {{/* ... */}} block removed, so +# an assertion can be made about the template LOGIC without the prose around it +# matching. Every comment opener in both files starts its own line, so dropping +# whole lines cannot take code with it. Written as a single awk program: cozytest.sh +# rewrites any bare `}` in column 0 into `return 0` + `}`, which would corrupt a +# multi-line awk body. +strip_tpl_comments() { + awk '/\{\{-? *\/\*/{c=1} !c{print} /\*\/ *-?\}\}/{c=0}' "$1" +} + +@test "both charts detect naming generations with byte-identical logic" { + a=$(mktemp); b=$(mktemp) + detection_block "$SYS" > "$a" + detection_block "$EXTRA" > "$b" + # Non-empty: a sed range that matched nothing would make this test vacuous. + [ -s "$a" ] + [ -s "$b" ] + diff -u "$a" "$b" + rm -f "$a" "$b" +} + +@test "each chart feeds the reconstruction the -system release name" { + # system/seaweedfs IS that release; extra/seaweedfs is and must derive it. + # Getting this wrong silently reconstructs the wrong prefix, so neither + # generation matches and the guard renders through. + grep -qF -- '$sysRelease := .Release.Name' "$SYS" + grep -qF -- '$sysRelease := printf "%s-system" .Release.Name' "$EXTRA" +} + +@test "both charts reconstruct the renamed prefix rather than prefix-matching alone" { + # An instance legitimately named `seaweedfs-volume` renders release-named objects + # (seaweedfs-volume-system-volume, data1-seaweedfs-volume-system-volume-0) that + # ALSO satisfy the chart-named prefixes. Release-named must be tested FIRST, + # against a reconstructed prefix, or live storage reads as legacy. + for f in "$SYS" "$EXTRA"; do + grep -qF -- '$renamedVol := include "seaweedfs.renamedVolumePrefix" $sysRelease' "$f" + # release-named branch precedes the chart-named fallback in both scans + pv=$(grep -n 'hasPrefix (printf "data1-%s" $renamedVol)' "$f" | cut -d: -f1) + lv=$(grep -n 'hasPrefix "data1-seaweedfs-volume"' "$f" | cut -d: -f1) + [ -n "$pv" ] && [ -n "$lv" ] && [ "$pv" -lt "$lv" ] + ps=$(grep -n 'hasPrefix $renamedVol .metadata.name' "$f" | cut -d: -f1) + ls=$(grep -n 'hasPrefix "seaweedfs-volume" .metadata.name' "$f" | cut -d: -f1) + [ -n "$ps" ] && [ -n "$ls" ] && [ "$ps" -lt "$ls" ] + done +} + +@test "both charts derive the generation flags from PVC and StatefulSet evidence" { + # The OR is load-bearing: a tenant whose PVCs are not provisioned yet is only + # visible through its label-matched StatefulSet. + for f in "$SYS" "$EXTRA"; do + grep -qF -- '$legacyGen := or $legacyPVC $legacySTS' "$f" + grep -qF -- '$systemGen := or $systemPVC $systemSTS' "$f" + done +} + +@test "both charts refuse when both generations are present" { + for f in "$SYS" "$EXTRA"; do + grep -qF -- 'if and $legacyGen $systemGen' "$f" + grep -qF -- 'has BOTH naming generations present' "$f" + done +} + +@test "both charts refuse a release-named-only tenant (class S)" { + for f in "$SYS" "$EXTRA"; do + grep -qF -- 'else if $systemGen' "$f" + grep -qF -- 'keeps its data on volumes named after the Helm release' "$f" + done +} + +@test "neither chart classifies on mutable claim timestamps or liveness" { + # The premise "PVCs are never recreated in place" is false: the runbook's own + # Step 2 re-bind deletes and recreates each claim, so a tenant interrupted + # part-way through reads as the exact inverse of the truth — and the step the + # old classification pointed at deletes the claim Step 2 just re-bound. + # readyReplicas is likewise only a snapshot, not proof a duplicate never + # served. Neither may come back as a discriminator. + # + # Counted rather than written as `! grep -qF ...`: POSIX and bash both exempt a + # !-negated pipeline from errexit — "the -e setting shall be ignored ... if the + # command's return value is being inverted with !" — so the negated form runs, + # returns 1, and the test carries on reporting success no matter what the file + # contains. These four assertions were the entire body of this test, so it + # asserted nothing at all. Putting the count inside `[` gives it a status errexit + # acts on. + # + # Matched against the TEMPLATE LOGIC ONLY, with {{/* */}} comment blocks stripped. + # All four names legitimately appear in the prose of both files — in the passages + # that explain why they were rejected as discriminators — so grepping the raw file + # would fail on the very documentation that records the decision this test exists + # to enforce. What must not come back is a live reference. + for f in "$SYS" "$EXTRA"; do + logic=$(strip_tpl_comments "$f") + [ "$(printf '%s\n' "$logic" | grep -cF -- 'creationTimestamp')" -eq 0 ] + [ "$(printf '%s\n' "$logic" | grep -cF -- 'readyReplicas')" -eq 0 ] + [ "$(printf '%s\n' "$logic" | grep -cF -- '$systemOldest')" -eq 0 ] + [ "$(printf '%s\n' "$logic" | grep -cF -- '$legacyOldest')" -eq 0 ] + done +} + +@test "both charts gate the guard behind the same cluster-view canary" { + for f in "$SYS" "$EXTRA"; do + grep -qF -- '$canary := lookup "v1" "Namespace" "" .Release.Namespace' "$f" + grep -qF -- 'refusing to upgrade blind' "$f" + done +} diff --git a/hack/seaweedfs-naming-audit.bats b/hack/seaweedfs-naming-audit.bats new file mode 100644 index 0000000000..eb605c5297 --- /dev/null +++ b/hack/seaweedfs-naming-audit.bats @@ -0,0 +1,605 @@ +#!/usr/bin/env bats +# ----------------------------------------------------------------------------- +# Unit tests for hack/seaweedfs-naming-audit.sh. +# +# The chart's naming guard refuses whenever both naming generations exist, so the +# audit IS the classifier — it is what an operator acts on, and acting on it +# deletes PVCs. Two earlier revisions of this classification shipped as an +# untested shell snippet inside the runbook, and both were wrong in ways that +# routed a live tenant into the step that strands its data: +# +# - a selector `^data1-(.*seaweedfs.*)-volume` also matched the CHART-named +# claims, so the release-named age range spanned both generations, every +# tenant read as "ranges overlap / mid-rebind", and a genuine S-damaged tenant +# was routed AWAY from Step 2a (which would have been correct) into Step 2, +# which deletes its data claim; +# - matching claims by name with `grep seaweedfs` cannot see a long instance +# name, whose claims the chart truncates past `seaweedfs`, so such a tenant +# read as "L — nothing to do" while the chart refused it. +# +# Moving the classifier into a file with tests is the point. These drive it +# against a fake kubectl, mocking only the cluster boundary. +# +# cozytest.sh's awk parser recognizes only @test blocks and a bare `}` on its own +# line; there is no bats `run`/`$status`/`setup`. +# +# Run with: hack/cozytest.sh hack/seaweedfs-naming-audit.bats +# ----------------------------------------------------------------------------- + +SEAWEEDFS_AUDIT_LIB=1 +export SEAWEEDFS_AUDIT_LIB +# shellcheck source=seaweedfs-naming-audit.sh +. "$PWD/hack/seaweedfs-naming-audit.sh" + +@test "reconstructs the renamed volume prefix for a default instance" { + [ "$(renamed_volume_prefix seaweedfs-system)" = "seaweedfs-system-volume" ] +} + +@test "reconstructs the renamed volume prefix for a non-default instance" { + # The release name does not contain the chart name, so 4.31 appends it. + [ "$(renamed_volume_prefix foo-system)" = "foo-system-seaweedfs-volume" ] +} + +@test "reconstructs the truncated prefix for a long instance name" { + # componentName cuts the fullname to 62-len("volume")=56 before appending, so + # `seaweedfs` falls off the tail — the case a `grep seaweedfs` name match cannot + # see, and the reason this is reconstructed rather than pattern-matched. + got=$(renamed_volume_prefix archive-of-quarterly-financial-statements-x1-system) + [ "$got" = "archive-of-quarterly-financial-statements-x1-system-seaw-volume" ] + # 56 chars of fullname + "-volume" + [ "${#got}" -eq 63 ] +} + +@test "reconstructs a distinct prefix for an instance named seaweedfs-volume" { + # The pathological case: this instance's RELEASE-named objects + # (seaweedfs-volume-system-volume, data1-seaweedfs-volume-system-volume-0) also + # satisfy the CHART-named prefixes. The reconstruction must return the + # release-named prefix so the release-named branch can be tested first. + got=$(renamed_volume_prefix seaweedfs-volume-system) + [ "$got" = "seaweedfs-volume-system-volume" ] + # It must NOT collide with the chart-named prefix. + [ "$got" != "seaweedfs-volume" ] +} + +@test "the reconstructed prefix never equals the chart-named prefix" { + # If it did, both generations would match one branch and the guard/audit could + # not separate them at all. + for r in seaweedfs-system foo-system seaweedfs-volume-system a-system; do + [ "$(renamed_volume_prefix "$r")" != "seaweedfs-volume" ] + done +} + +@test "clean duplicate: release-named PVs all strictly newer => legacy is original" { + # A tenant that passed through the 4.31 rename: legacy PVs at install time, + # duplicate PVs provisioned by the bad upgrade much later. + [ "$(classify_mixed_direction 1000 1010 5000 5020)" = "legacy-original" ] +} + +@test "S-damaged: chart-named PVs all strictly newer => release-named is original" { + # Installed fresh on 1.5.x (release-named PVs first); an unguarded 1.6 upgrade + # then created empty chart-named claims beside them. + [ "$(classify_mixed_direction 5000 5020 1000 1010)" = "renamed-original" ] +} + +@test "interrupted Step 2 re-bind: overlapping vintages => no candidate" { + # Step 2 re-binds release-named claims onto their ORIGINAL PVs under chart + # names, one claim at a time. Interrupted part-way, BOTH generations sit on + # original-vintage PVs, so the ranges interleave. The old absolute-window + # classifier fell through to a coin flip here and its advice deleted the + # un-re-bound claims; the relative rule must refuse instead. + [ "$(classify_mixed_direction 1000 1010 1005 1015)" = "overlap" ] +} + +@test "a tie is overlap, not a candidate" { + # Second-resolution timestamps: a duplicate provisioned within the same second + # as the newest original PV is not STRICTLY newer. Refusing is recoverable; + # naming the wrong candidate is not. + [ "$(classify_mixed_direction 1000 1010 1010 1020)" = "overlap" ] +} + +@test "direction needs no clock: vintages far from any anchor still classify" { + # The shipped StorageClasses are WaitForFirstConsumer, so PVs appear at + # pod-SCHEDULE time — on a cold cluster minutes after first_deployed. The rule + # must not care: only the two generations' ranges relative to EACH OTHER count. + # (The previous classifier anchored a 120s window on first_deployed and + # misclassified exactly this case.) + [ "$(classify_mixed_direction 100000 100600 200000 200600)" = "legacy-original" ] +} + +@test "the audit's reconstruction agrees with the chart helper it mirrors" { + # hack/seaweedfs-naming-audit.sh and + # packages/system/seaweedfs/templates/_naming.tpl reimplement the same two + # upstream helpers in two languages. If they drift, the audit classifies a + # tenant differently from the render that refuses it, and the operator is + # working from a different picture than the chart. Render the chart helper + # through helm and compare it to this script's output, release by release. + chart=$(mktemp -d) + printf 'apiVersion: v2\nname: probe\nversion: 0.0.0\n' > "$chart/Chart.yaml" + mkdir -p "$chart/templates" + cp packages/system/seaweedfs/templates/_naming.tpl "$chart/templates/" + cat > "$chart/templates/out.yaml" <<'EOF' +apiVersion: v1 +kind: ConfigMap +metadata: + name: probe +data: +{{- range $r := list "seaweedfs-system" "foo-system" "seaweedfs-volume-system" "archive-of-quarterly-financial-statements-x1-system" "a-system" }} + {{ $r }}: {{ include "seaweedfs.renamedVolumePrefix" $r }} +{{- end }} +EOF + helm template probe "$chart" 2>/dev/null | sed -n 's/^ \([a-z0-9-]*-system\): \(.*\)$/\1=\2/p' > "$chart/rendered" + [ -s "$chart/rendered" ] + [ "$(wc -l < "$chart/rendered")" -eq 5 ] + while IFS='=' read -r rel expected; do + [ -n "$rel" ] || continue + got=$(renamed_volume_prefix "$rel") + echo "chart: $rel -> $expected ; audit: $got" + [ "$got" = "$expected" ] + done < "$chart/rendered" + rm -rf "$chart" +} + +# ----------------------------------------------------------------------------- +# Fail-closed tests (issue #3431). +# +# The classification tests above mock nothing below the classifier. These drive +# the KUBECTL layer, where the fail-open bug lived: a kubectl call that failed +# used to return empty stdout, byte-identical to a genuinely clean fleet, so the +# audit printed an empty table and exited 0 -- the exact false "nothing to do" the +# operator is told to trust before deleting PVCs. They shim `kubectl` on PATH with +# a fake and assert the audit fails LOUDLY (non-zero exit + a FATAL naming the +# query) on any real error, while still treating a genuinely-absent object as +# clean. Two families: +# +# * enumerating LISTs (get ns / secret / pvc / sts) -- a failure is always fatal; +# * by-NAME GETs (a release secret in system_releases, a PVC/PV in pv_epoch) -- +# `--ignore-not-found` splits a real error (fatal) from a legitimate absence: +# an absent release revision is a clean skip; an absent PV degrades to the +# safe "cannot establish direction" note and must NOT become a deletion +# candidate (the range-narrowing flip Codex reproduced). +# +# The two golden tests guard the reverse: on a healthy cluster the output stays +# byte-for-byte what it was on origin/main, including the MIXED path that exercises +# every pv_epoch GET. +# +# The fake is a real executable on PATH, so it exercises the actual exit-status +# handling in run_kubectl and the propagation up through every caller -- a for +# loop over `$(...)` swallows the status, so this is where a regression would hide. + +# _release_blob [chart-name] -- the value kubectl returns for a Helm release +# secret's `.data.release`: base64(base64(gzip(json))). release_json base64-decodes +# twice and gunzips it, and system_releases reads the chart name from +# chart.metadata.name (as real Helm payloads carry it). Defaults to a cozy-seaweedfs +# release so the audit confirms the tenant; pass another chart name for a non- +# SeaweedFS release, or the literal EMPTY for a chartless {} payload. +# +# Three payload SHAPES beyond the default compact one, because the chart-name +# extraction is now fatal on a miss and every shape below is something a +# re-serializer or a user's values could produce: +# SPACED whitespace around the JSON punctuation (json.dumps' default) +# PRETTY indented and MULTI-LINE (jq . / yq -o=json), which a line-based +# matcher cannot read at all unless newlines are folded first +# DECOY a real cozy-seaweedfs chart PLUS a values subtree that spells +# chart.metadata.name with a different value. Helm marshals "config" +# (the values) AFTER "chart", so a last-match extraction returns the +# decoy and silently declares the tenant non-SeaweedFS. +_release_blob() { + _rb_name=${1:-cozy-seaweedfs} + case "$_rb_name" in + EMPTY) _rb_json='{}' ;; + SPACED) _rb_json='{"name": "seaweedfs-system", "chart": {"metadata": {"name": "cozy-seaweedfs", "version": "1.0.0"}}}' ;; + PRETTY) _rb_json='{ + "name": "seaweedfs-system", + "chart": { + "metadata": { + "name": "cozy-seaweedfs", + "version": "1.0.0" + } + } +}' ;; + DECOY) _rb_json='{"name":"seaweedfs-system","chart":{"metadata":{"name":"cozy-seaweedfs"}},"config":{"chart":{"metadata":{"name":"decoy-not-seaweedfs"}}}}' ;; + *) _rb_json='{"name":"seaweedfs-system","chart":{"metadata":{"name":"'"$_rb_name"'"}}}' ;; + esac + printf '%s' "$_rb_json" | gzip | base64 | base64 | tr -d '\n' +} + +# _write_fake_kubectl -- drop a fake kubectl +# into . +# one of ns | secretlist | secretget | secretget_absent | +# secret_missing_field | secret_bad_payload | pvclist | pvcget | +# sts | pvget | "" -- the single call to make behave badly. Real- +# error targets exit non-zero with a stderr message; secretget_absent +# mirrors an absent Secret (--ignore-not-found: exit 0, empty -o +# name); secret_missing_field / secret_bad_payload keep the Secret +# present but return an empty / undecodable .data.release payload. +# a PV name (pv-legacy|pv-legacy2|pv-renamed) to report as absent +# (exit 0, empty), exercising the "bound claim, PV gone" path. +# newline-separated `get pvc -o name` LIST output. +# It otherwise walks one confirmed seaweedfs-system tenant. The by-name PVC->PV and +# PV->timestamp maps are fixed here; timestamps are chosen so the two legacy PVs +# straddle the single renamed PV (true answer OVERLAP), which is what makes the +# range-narrowing flip observable. Kept POSIX and free of a column-0 `}` so +# cozytest.sh's awk converter passes the heredoc through untouched. +_write_fake_kubectl() { + # Grouped redirect (one open of the file). The closing brace is indented, so + # cozytest.sh's awk -- which only rewrites a `}` in column 0 -- leaves it and the + # heredoc alone. + { + printf '#!/bin/sh\n' + printf "FAIL='%s'\n" "$2" + printf "ABSENT_PV='%s'\n" "$3" + printf "BLOB='%s'\n" "$4" + printf "PVCS='%s'\n" "$5" + cat <<'FAKE' +verb=${1:-}; res=${2:-}; args="$*" +fail() { echo "fake kubectl: $1 (real error)" >&2; exit 1; } +# Anything this fake does not model must NOT look like a successful empty +# answer: that is the fail-open shape the audited script exists to reject, and +# it would let a new query added to the script pass these goldens unnoticed. +unmodelled() { echo "fake kubectl: unmodelled invocation: $args" >&2; exit 97; } +if [ "$verb $res" = "get ns" ]; then + [ "$FAIL" = ns ] && fail "get ns" + printf 'namespace/tenant-test\n'; exit 0 +fi +if [ "$verb $res" = "get secret" ]; then + case "$args" in + *sh.helm.release.v1*) + # release_json now asks two questions: existence (-o name) then payload + # (jsonpath .data.release). Mirror that split so absence, real error, and + # corrupt-payload are all reachable independently. + case "$args" in + *"-o name"*) + [ "$FAIL" = secretget ] && fail "release secret existence GET" + [ "$FAIL" = secretget_absent ] && exit 0 + printf 'secret/sh.helm.release.v1.seaweedfs-system.v1\n'; exit 0 ;; + *) + [ "$FAIL" = secret_missing_field ] && exit 0 + if [ "$FAIL" = secret_bad_payload ]; then printf '@@@not-base64@@@\n'; exit 0; fi + printf '%s\n' "$BLOB"; exit 0 ;; + esac ;; + *owner=helm*) printf '1\n'; exit 0 ;; + *) [ "$FAIL" = secretlist ] && fail "namespace secret LIST" + printf 'seaweedfs-system\n'; exit 0 ;; + esac +fi +if [ "$verb $res" = "get pvc" ]; then + case "$args" in + *"-o name"*) + [ "$FAIL" = pvclist ] && fail "pvc LIST" + [ -n "$PVCS" ] && printf '%s\n' "$PVCS" + exit 0 ;; + *data1-seaweedfs-system-volume-0*) [ "$FAIL" = pvcget ] && fail "pvc GET"; printf 'pv-renamed\n'; exit 0 ;; + *data1-seaweedfs-volume-1*) [ "$FAIL" = pvcget ] && fail "pvc GET"; printf 'pv-legacy2\n'; exit 0 ;; + *data1-seaweedfs-volume-0*) [ "$FAIL" = pvcget ] && fail "pvc GET"; printf 'pv-legacy\n'; exit 0 ;; + esac + unmodelled +fi +if [ "$verb $res" = "get sts" ]; then + [ "$FAIL" = sts ] && fail "sts LIST" + exit 0 +fi +if [ "$verb $res" = "get pv" ]; then + [ "$FAIL" = pvget ] && fail "get pv" + case "$args" in + *pv-legacy2*) [ "$ABSENT_PV" = pv-legacy2 ] && exit 0; printf '2099-01-01T00:00:00Z\n'; exit 0 ;; + *pv-legacy*) [ "$ABSENT_PV" = pv-legacy ] && exit 0; printf '2020-01-01T00:00:00Z\n'; exit 0 ;; + *pv-renamed*) [ "$ABSENT_PV" = pv-renamed ] && exit 0; printf '2020-06-01T00:00:00Z\n'; exit 0 ;; + esac + unmodelled +fi +unmodelled +FAKE + } > "$1/kubectl" + chmod +x "$1/kubectl" +} + +# _pvcs_mixed -- the `get pvc -o name` LIST for a MIXED tenant: two legacy claims +# and one release-named claim, whose PV vintages truly overlap. +_pvcs_mixed() { + printf '%s\n' \ + persistentvolumeclaim/data1-seaweedfs-volume-0 \ + persistentvolumeclaim/data1-seaweedfs-volume-1 \ + persistentvolumeclaim/data1-seaweedfs-system-volume-0 +} + +# _expected_L / _expected_mixed_overlap -- golden output built with the SAME printf +# contract the script uses, so a drift in row count, text, or padding fails the diff. +_expected_L() { + printf '%-24s %-14s %-8s %s\n' NAMESPACE RELEASE CLASS NOTE + printf '%-24s %-14s %-8s %s\n' --------- ------- ----- ---- + printf '%-24s %-14s %-8s %s\n' tenant-test seaweedfs-system L 'chart-named only; the upgrade adopts it, nothing to do' +} +_expected_mixed_overlap() { + printf '%-24s %-14s %-8s %s\n' NAMESPACE RELEASE CLASS NOTE + printf '%-24s %-14s %-8s %s\n' --------- ------- ----- ---- + printf '%-24s %-14s %-8s %s\n' tenant-test seaweedfs-system MIXED 'both generations; the chart REFUSES until one is removed' + printf '%-24s %-14s %-8s %s\n' '' '' '' 'PV vintages OVERLAP => no candidate. An interrupted Step 2 re-bind looks exactly like this (both generations on original PVs). Finish Step 2 if one is in progress; otherwise escalate. Do NOT run Step 2a.' + printf '%-24s %-14s %-8s %s\n' '' '' '' 'CANDIDATE ONLY: "original" does not mean the other set is EMPTY. A duplicate that' + printf '%-24s %-14s %-8s %s\n' '' '' '' 'served writes and later crashed looks identical here. Verify emptiness before deleting.' +} + +@test "fails closed when 'kubectl get ns' fails (whole-cluster mode)" { + # No namespace args -> main enumerates namespaces itself. If that LIST fails, + # the OLD code audited zero namespaces and still printed an empty clean table. + d=$(mktemp -d) + _write_fake_kubectl "$d" ns "" "$(_release_blob)" "" + rc=0 + out=$(PATH="$d:$PATH" SEAWEEDFS_AUDIT_LIB=0 sh "$PWD/hack/seaweedfs-naming-audit.sh" 2>&1) || rc=$? + rm -rf "$d" + echo "rc=$rc"; echo "$out" + [ "$rc" -ne 0 ] + printf '%s\n' "$out" | grep -q 'FATAL' + printf '%s\n' "$out" | grep -q 'list namespaces' +} + +@test "fails closed when the namespace-wide secret LIST fails" { + # system_releases enumerates Helm releases with a namespace-wide secret LIST -- + # the exact call that timed out in the field and reported a false clean. + d=$(mktemp -d) + _write_fake_kubectl "$d" secretlist "" "$(_release_blob)" "" + rc=0 + out=$(PATH="$d:$PATH" SEAWEEDFS_AUDIT_LIB=0 sh "$PWD/hack/seaweedfs-naming-audit.sh" tenant-test 2>&1) || rc=$? + rm -rf "$d" + echo "rc=$rc"; echo "$out" + [ "$rc" -ne 0 ] + printf '%s\n' "$out" | grep -q 'FATAL' + printf '%s\n' "$out" | grep -q 'list Helm release secrets' +} + +@test "fails closed when the by-name release-secret existence GET hits a real error" { + # Codex Finding 1: system_releases uses release_json to decide whether a release + # IS SeaweedFS. Both LISTs succeed, then a transient/forbidden by-name Secret GET + # used to leave `chart` empty -> the tenant was silently skipped -> exit 0, empty + # table. A real error here must now be fatal, not a false clean. + d=$(mktemp -d) + _write_fake_kubectl "$d" secretget "" "$(_release_blob)" "" + rc=0 + out=$(PATH="$d:$PATH" SEAWEEDFS_AUDIT_LIB=0 sh "$PWD/hack/seaweedfs-naming-audit.sh" tenant-test 2>&1) || rc=$? + rm -rf "$d" + echo "rc=$rc"; echo "$out" + [ "$rc" -ne 0 ] + printf '%s\n' "$out" | grep -q 'FATAL' + printf '%s\n' "$out" | grep -q 'Helm release secret' +} + +@test "an absent release-secret revision is a clean skip, not a failure" { + # The one legitimately-empty case for release_json: the revision secret does not + # exist (pruned by Helm history limit). The existence check (--ignore-not-found -o + # name) returns empty + exit 0, so the release is simply not confirmed as + # SeaweedFS. No error, no crash -- the audit completes and reports nothing here. + d=$(mktemp -d) + _write_fake_kubectl "$d" secretget_absent "" "$(_release_blob)" "" + rc=0 + out=$(PATH="$d:$PATH" SEAWEEDFS_AUDIT_LIB=0 sh "$PWD/hack/seaweedfs-naming-audit.sh" tenant-test 2>&1) || rc=$? + rm -rf "$d" + echo "rc=$rc"; echo "$out" + [ "$rc" -eq 0 ] + [ "$(printf '%s\n' "$out" | grep -c 'FATAL')" -eq 0 ] +} + +@test "fails closed when a present release secret has no .data.release payload" { + # Codex re-review round 2: the existence check passes (Secret EXISTS), but its + # .data.release field is empty/missing. That is CORRUPT state, not an absence -- + # and because the earlier fix used --ignore-not-found -o jsonpath, which returns + # empty+0 for BOTH, it used to read as a clean skip. It must be a loud stop. + d=$(mktemp -d) + _write_fake_kubectl "$d" secret_missing_field "" "$(_release_blob)" "" + rc=0 + out=$(PATH="$d:$PATH" SEAWEEDFS_AUDIT_LIB=0 sh "$PWD/hack/seaweedfs-naming-audit.sh" tenant-test 2>&1) || rc=$? + rm -rf "$d" + echo "rc=$rc"; echo "$out" + [ "$rc" -ne 0 ] + printf '%s\n' "$out" | grep -q 'FATAL' + printf '%s\n' "$out" | grep -q 'no .data.release payload' +} + +@test "fails closed when a present release secret has an undecodable payload" { + # Existence passes, .data.release is non-empty but is not base64(base64(gzip(...))), + # so every decode fails. The old `|| return 0` converted that decode failure into + # a clean 0 return -> silent skip. A corrupt payload must be fatal. + d=$(mktemp -d) + _write_fake_kubectl "$d" secret_bad_payload "" "$(_release_blob)" "" + rc=0 + out=$(PATH="$d:$PATH" SEAWEEDFS_AUDIT_LIB=0 sh "$PWD/hack/seaweedfs-naming-audit.sh" tenant-test 2>&1) || rc=$? + rm -rf "$d" + echo "rc=$rc"; echo "$out" + [ "$rc" -ne 0 ] + printf '%s\n' "$out" | grep -q 'FATAL' + printf '%s\n' "$out" | grep -q 'not decodable' +} + +@test "fails closed when a present release secret decodes to a chartless payload" { + # Codex round 3: the payload fetches and DECODES cleanly (valid JSON {}), so the + # decode guards all pass -- but it carries no chart name. system_releases then + # extracted chart='' and silently skipped the tenant -> exit 0, clean table. A + # decoded helm.sh/release.v1 release without a chart name is corrupt: fatal. + d=$(mktemp -d) + _write_fake_kubectl "$d" "" "" "$(_release_blob EMPTY)" "" + rc=0 + out=$(PATH="$d:$PATH" SEAWEEDFS_AUDIT_LIB=0 sh "$PWD/hack/seaweedfs-naming-audit.sh" tenant-test 2>&1) || rc=$? + rm -rf "$d" + echo "rc=$rc"; echo "$out" + [ "$rc" -ne 0 ] + printf '%s\n' "$out" | grep -q 'FATAL' + printf '%s\n' "$out" | grep -q 'could not read the chart name' + # The diagnostic must name the path it looked at, so a future Helm payload + # format change is diagnosable as such instead of reading as real corruption. + printf '%s\n' "$out" | grep -q '\.chart\.metadata\.name' +} + +@test "a present release secret for a non-SeaweedFS chart is a silent legitimate skip" { + # The counterpart the guard must NOT break: a real, well-formed release whose + # chart is simply not cozy-seaweedfs. It has a chart name (so it is not corrupt), + # but the wrong one, so it is filtered out exactly as before -- no row, no error. + d=$(mktemp -d) + _write_fake_kubectl "$d" "" "" "$(_release_blob cozy-postgres)" "" + rc=0 + out=$(PATH="$d:$PATH" SEAWEEDFS_AUDIT_LIB=0 sh "$PWD/hack/seaweedfs-naming-audit.sh" tenant-test 2>&1) || rc=$? + rm -rf "$d" + echo "rc=$rc"; echo "$out" + [ "$rc" -eq 0 ] + [ "$(printf '%s\n' "$out" | grep -c 'FATAL')" -eq 0 ] + # Not confirmed as SeaweedFS -> no classification row for the release. + [ "$(printf '%s\n' "$out" | grep -c 'seaweedfs-system')" -eq 0 ] +} + +@test "fails closed when 'kubectl get pvc' LIST fails" { + # The secret enumeration succeeds and confirms a seaweedfs-system tenant, so the + # audit reaches the per-namespace PVC LIST; a failure there must abort, not read + # as "this tenant has no claims". + d=$(mktemp -d) + _write_fake_kubectl "$d" pvclist "" "$(_release_blob)" "" + rc=0 + out=$(PATH="$d:$PATH" SEAWEEDFS_AUDIT_LIB=0 sh "$PWD/hack/seaweedfs-naming-audit.sh" tenant-test 2>&1) || rc=$? + rm -rf "$d" + echo "rc=$rc"; echo "$out" + [ "$rc" -ne 0 ] + printf '%s\n' "$out" | grep -q 'FATAL' + printf '%s\n' "$out" | grep -q 'list PVCs' +} + +@test "fails closed when 'kubectl get sts' LIST fails" { + # PVC LIST succeeds (empty), so the audit reaches the StatefulSet LIST; a + # failure there must abort rather than read as "no StatefulSets". + d=$(mktemp -d) + _write_fake_kubectl "$d" sts "" "$(_release_blob)" "" + rc=0 + out=$(PATH="$d:$PATH" SEAWEEDFS_AUDIT_LIB=0 sh "$PWD/hack/seaweedfs-naming-audit.sh" tenant-test 2>&1) || rc=$? + rm -rf "$d" + echo "rc=$rc"; echo "$out" + [ "$rc" -ne 0 ] + printf '%s\n' "$out" | grep -q 'FATAL' + printf '%s\n' "$out" | grep -q 'StatefulSets' +} + +@test "fails closed when the by-name PV GET hits a real error" { + # Codex Finding 2, error half: pv_epoch reads each bound PV's age by name. A real + # error (RBAC/timeout) must abort, not silently drop that PV from the range. + d=$(mktemp -d) + _write_fake_kubectl "$d" pvget "" "$(_release_blob)" "$(_pvcs_mixed)" + rc=0 + out=$(PATH="$d:$PATH" SEAWEEDFS_AUDIT_LIB=0 sh "$PWD/hack/seaweedfs-naming-audit.sh" tenant-test 2>&1) || rc=$? + rm -rf "$d" + echo "rc=$rc"; echo "$out" + [ "$rc" -ne 0 ] + printf '%s\n' "$out" | grep -q 'FATAL' + printf '%s\n' "$out" | grep -q "get PV '" +} + +@test "an absent PV degrades to the safe fallback, never a wrong candidate" { + # Codex Finding 2, absence half. True vintages: legacy PVs 2020 + 2099 straddle + # the single renamed PV 2020-06 => OVERLAP. Report the newest legacy PV (2099) as + # ABSENT: the observed legacy range collapses to {2020} and, unguarded, "every + # release-named PV is strictly newer" would fire -- naming the release-named set a + # deletion candidate on incomplete evidence. The generation must instead read as + # incomplete and fall to "cannot establish direction". + d=$(mktemp -d) + _write_fake_kubectl "$d" "" pv-legacy2 "$(_release_blob)" "$(_pvcs_mixed)" + rc=0 + out=$(PATH="$d:$PATH" SEAWEEDFS_AUDIT_LIB=0 sh "$PWD/hack/seaweedfs-naming-audit.sh" tenant-test 2>&1) || rc=$? + rm -rf "$d" + echo "rc=$rc"; echo "$out" + [ "$rc" -eq 0 ] + [ "$(printf '%s\n' "$out" | grep -c 'FATAL')" -eq 0 ] + printf '%s\n' "$out" | grep -q 'direction cannot be established from PV ages' + # The whole point: incomplete evidence must NOT be reported as a deletion candidate. + [ "$(printf '%s\n' "$out" | grep -c 'candidate duplicate')" -eq 0 ] +} + +@test "success path (L) is byte-identical to the expected fixture" { + # A single chart-named claim, no release-named one, no StatefulSets => L, adopt in + # place. Full-output compare, so an extra row / warning / duplicate line fails. + d=$(mktemp -d) + _write_fake_kubectl "$d" "" "" "$(_release_blob)" "persistentvolumeclaim/data1-seaweedfs-volume-0" + rc=0 + out=$(PATH="$d:$PATH" SEAWEEDFS_AUDIT_LIB=0 sh "$PWD/hack/seaweedfs-naming-audit.sh" tenant-test 2>&1) || rc=$? + printf '%s\n' "$out" > "$d/got" + _expected_L > "$d/want" + echo "rc=$rc"; echo "--- got ---"; cat "$d/got"; echo "--- want ---"; cat "$d/want" + [ "$rc" -eq 0 ] + diff "$d/want" "$d/got" + rm -rf "$d" +} + +@test "success path (MIXED/overlap) is byte-identical and exercises pv_epoch" { + # Both generations present with all PVs readable => the MIXED path runs every + # pv_epoch GET and lands on OVERLAP. Full-output compare against the golden. + d=$(mktemp -d) + _write_fake_kubectl "$d" "" "" "$(_release_blob)" "$(_pvcs_mixed)" + rc=0 + out=$(PATH="$d:$PATH" SEAWEEDFS_AUDIT_LIB=0 sh "$PWD/hack/seaweedfs-naming-audit.sh" tenant-test 2>&1) || rc=$? + printf '%s\n' "$out" > "$d/got" + _expected_mixed_overlap > "$d/want" + echo "rc=$rc"; echo "--- got ---"; cat "$d/got"; echo "--- want ---"; cat "$d/want" + [ "$rc" -eq 0 ] + diff "$d/want" "$d/got" + rm -rf "$d" +} + +@test "a whitespace-spaced Helm payload still classifies the tenant" { + # The chart-name read is FATAL on a miss, so any payload shape a re-serializer + # can produce must parse. Byte-compare against the same golden as the compact + # payload: the shape must make no difference to the report at all. + d=$(mktemp -d) + _write_fake_kubectl "$d" "" "" "$(_release_blob SPACED)" "persistentvolumeclaim/data1-seaweedfs-volume-0" + rc=0 + out=$(PATH="$d:$PATH" SEAWEEDFS_AUDIT_LIB=0 sh "$PWD/hack/seaweedfs-naming-audit.sh" tenant-test 2>&1) || rc=$? + printf '%s\n' "$out" > "$d/got" + _expected_L > "$d/want" + echo "rc=$rc"; echo "--- got ---"; cat "$d/got" + [ "$rc" -eq 0 ] + diff "$d/want" "$d/got" + rm -rf "$d" +} + +@test "a pretty-printed multi-line Helm payload still classifies the tenant" { + # Line-based sed/grep cannot see across newlines at all, so this shape is the + # one that fails hardest without the newline fold -- and jq/yq re-serialization + # is exactly how a payload would arrive indented. + d=$(mktemp -d) + _write_fake_kubectl "$d" "" "" "$(_release_blob PRETTY)" "persistentvolumeclaim/data1-seaweedfs-volume-0" + rc=0 + out=$(PATH="$d:$PATH" SEAWEEDFS_AUDIT_LIB=0 sh "$PWD/hack/seaweedfs-naming-audit.sh" tenant-test 2>&1) || rc=$? + printf '%s\n' "$out" > "$d/got" + _expected_L > "$d/want" + echo "rc=$rc"; echo "--- got ---"; cat "$d/got" + [ "$rc" -eq 0 ] + diff "$d/want" "$d/got" + rm -rf "$d" +} + +@test "a values subtree spelling chart.metadata.name cannot shadow the real chart" { + # Helm marshals "config" (the user's values) AFTER "chart", so a LAST-match + # extraction returns the decoy, the release reads as non-SeaweedFS, and the + # tenant vanishes from the report with exit 0 -- a false clean, the failure this + # whole script exists to prevent. First-match extraction is what stops it. + d=$(mktemp -d) + _write_fake_kubectl "$d" "" "" "$(_release_blob DECOY)" "persistentvolumeclaim/data1-seaweedfs-volume-0" + rc=0 + out=$(PATH="$d:$PATH" SEAWEEDFS_AUDIT_LIB=0 sh "$PWD/hack/seaweedfs-naming-audit.sh" tenant-test 2>&1) || rc=$? + printf '%s\n' "$out" > "$d/got" + _expected_L > "$d/want" + echo "rc=$rc"; echo "--- got ---"; cat "$d/got" + [ "$rc" -eq 0 ] + # The decoy name must never reach the report. + [ "$(printf '%s\n' "$out" | grep -c 'decoy-not-seaweedfs')" -eq 0 ] + diff "$d/want" "$d/got" + rm -rf "$d" +} + +@test "fails closed when the by-name PVC GET hits a real error" { + # pv_epoch resolves each claim's bound PV by name before reading the PV's age. + # A real error there (RBAC/timeout) must abort: silently dropping the claim + # shrinks the observed vintage range, which is what produces a wrong "candidate + # duplicate" verdict. The PV half of this pair was already covered; this is the + # PVC half, whose fake FAIL mode existed with no test behind it. + d=$(mktemp -d) + _write_fake_kubectl "$d" pvcget "" "$(_release_blob)" "$(_pvcs_mixed)" + rc=0 + out=$(PATH="$d:$PATH" SEAWEEDFS_AUDIT_LIB=0 sh "$PWD/hack/seaweedfs-naming-audit.sh" tenant-test 2>&1) || rc=$? + rm -rf "$d" + echo "rc=$rc"; echo "$out" + [ "$rc" -ne 0 ] + printf '%s\n' "$out" | grep -q 'FATAL' + printf '%s\n' "$out" | grep -q "get PVC '" +} diff --git a/hack/seaweedfs-naming-audit.sh b/hack/seaweedfs-naming-audit.sh new file mode 100755 index 0000000000..b66e91939e --- /dev/null +++ b/hack/seaweedfs-naming-audit.sh @@ -0,0 +1,406 @@ +#!/bin/sh +# SeaweedFS 4.31 rename — fleet audit (READ-ONLY). +# +# Classifies every SeaweedFS tenant into the three states the chart's naming guard +# (packages/system/seaweedfs/templates/naming-guard.yaml) decides between: +# +# L exactly the chart-named generation -> the upgrade adopts it. Nothing to do. +# S exactly the release-named generation -> re-bind first (runbook Step 2). +# MIXED both generations -> the chart REFUSES; an operator must remove the empty one. +# +# For MIXED it also reports which generation is ORIGINAL, from durable evidence: +# +# sh.helm.release.v1.-system.v1 — the manifest of revision 1 names either +# seaweedfs-master (born pre-4.31) or -master (born 4.31). Exact, but +# revision 1 may have been pruned by Helm's history limit. +# PersistentVolume creation timestamps — the original generation's PVs predate the +# duplicate's, which were provisioned later by the bad upgrade. Compared only +# against each other (see the relative rule below); info.first_deployed is +# printed as context but decides nothing. +# +# PV timestamps, not PVC ones: runbook Step 2 deletes each release-named claim and +# recreates it under the chart name against the SAME PV, so claim age is not durable +# and inverts for a tenant interrupted mid-re-bind. The PV survives and keeps its +# creationTimestamp. +# +# Direction is decided by a RELATIVE rule, never a clock window: a generation is +# the candidate duplicate only when EVERY one of its bound PVs is strictly newer +# than every bound PV of the other generation — the same precondition runbook +# Step 2a enforces before deleting anything. Overlapping vintages mean an +# interrupted Step 2 re-bind (both generations on original PVs) or interleaved +# provisioning, and the audit refuses to name a candidate. An absolute window +# measured from first_deployed was tried first and is unsound: the shipped +# StorageClasses are WaitForFirstConsumer, so PVs appear at pod-SCHEDULE time, +# and ordinary cold-cluster latency puts even the original generation's PVs +# minutes past first_deployed — which pushed a mid-rebind tenant into the branch +# whose advice deletes the un-re-bound claims. +# +# IMPORTANT — what this does NOT tell you. "Which generation is original" is not +# "the other one is empty". A duplicate that never scheduled (safe to delete) and one +# that served writes and later crashed (holds unique objects) are identical on every +# durable signal — same revision-1 scheme, same first_deployed deltas. Establishing +# emptiness needs the duplicate's volume files and the filer's volume list, which this +# script does not inspect. It narrows the question; it does not answer it. +# +# Usage: hack/seaweedfs-naming-audit.sh [namespace...] +# KUBECONFIG= hack/seaweedfs-naming-audit.sh +# Exit: 0 the audit RAN TO COMPLETION; the table is the whole answer +# (an empty table then means a genuinely clean fleet). +# non-zero a kubectl query FAILED, so the audit is INCOMPLETE and the +# table must NOT be trusted -- a message naming the failed query +# is on stderr. This is fail-CLOSED by design (see run_kubectl): +# the output gates a destructive runbook step, so an unreachable +# API aborts loudly rather than printing an empty table that +# reads identically to "nothing to do". +# Nothing is ever mutated on either path -- every kubectl call is read-only. +# +# POSIX sh, deliberately. hack/seaweedfs-naming-audit.bats sources this file, and +# cozytest.sh sources the converted test into its own /bin/sh — which is dash on +# the CI runner. A bash shebang would not save it: `.` runs the file's contents in +# the caller's shell and the shebang is just a comment. Keeping the script POSIX +# is what makes the tested shell and the executed shell the same one; anything +# bash-only here is untested in CI and unavailable at run time. `pipefail` in +# particular is not POSIX and dash 0.5.12 (Ubuntu) rejects it outright. +set -u + +# run_kubectl -- run ONE read-only kubectl query +# fail-CLOSED and print its stdout. This audit is the documented gate in front of +# a destructive runbook step (deleting a tenant's PVCs), so a query that FAILS +# must never be mistaken for a query that found nothing: an enumerating LIST +# returns empty stdout on a timeout or an RBAC denial, byte-identical to the same +# LIST succeeding on a genuinely clean fleet, and the old `2>/dev/null` on each +# call turned an unreachable API into a silent false "nothing to do". Every +# enumeration is routed through here so any failure is loud and terminal instead. +# +# On success prints kubectl's stdout verbatim -- which may legitimately be empty, +# because a LIST that matched nothing is honestly clean; keeping that case +# distinct from failure is the whole point. On a non-zero exit prints the failed +# query to stderr and returns that exit code; kubectl's own stderr stays on fd 2 +# and reaches the operator unfiltered. Callers MUST propagate the non-zero status +# (`|| return`/`|| exit`) -- a bare `for x in $(run_kubectl ...)` would swallow +# it, because a for-loop ignores the exit status of its word-list command, and an +# `exit` inside the `$(...)` subshell would only leave that subshell. +run_kubectl() { + _rk_what="$1"; shift + if _rk_out=$(kubectl "$@"); then + printf '%s' "$_rk_out" + return 0 + else + _rk_rc=$? + printf 'seaweedfs-naming-audit: FATAL: %s failed (kubectl %s -> exit %s). Refusing to report a clean fleet on an unreachable API.\n' \ + "$_rk_what" "$*" "$_rk_rc" >&2 + return "$_rk_rc" + fi +} + +# audit_fatal -- the counterpart to run_kubectl for corruption the API +# call itself did NOT report: the query succeeded, but what it returned is +# impossible for a healthy object (a helm.sh/release.v1 Secret with no decodable +# release payload). Same fail-CLOSED contract -- print a FATAL line naming the +# object to stderr and return non-zero; callers MUST propagate it. A silent skip +# here would be the identical false-clean this whole script guards against, just +# one field deeper than an unreachable API. +audit_fatal() { + printf 'seaweedfs-naming-audit: FATAL: %s Refusing to report a clean fleet on corrupt state.\n' "$1" >&2 + return 1 +} + +# renamed_volume_prefix -- reconstruct the name 4.31 gives the volume +# component of . Mirrors seaweedfs.fullname + seaweedfs.componentName +# (see packages/system/seaweedfs/templates/_naming.tpl, which the chart's guard +# uses for exactly the same purpose): the fullname gains -seaweedfs when the +# release name does not already contain it, is capped at 63, and componentName +# then cuts it to (62 - len("volume")) = 56 before appending -volume. +renamed_volume_prefix() { + release="$1" + case "$release" in + *seaweedfs*) full="$release" ;; + *) full="${release}-seaweedfs" ;; + esac + full=$(printf '%s' "$full" | cut -c1-63 | sed 's/-$//') + printf '%s-volume' "$(printf '%s' "$full" | cut -c1-56 | sed 's/-$//')" +} + +# release_json [rev] -- decode a Helm release secret's payload, or +# "" when that revision's secret does not exist. Helm stores base64(gzip(json)) in +# Secret.data.release, and Kubernetes base64s the data value again, hence the two +# decodes. +# +# This is a by-NAME GET, but it is NOT best-effort: system_releases decides whether +# a release IS SeaweedFS from the result, so a swallowed failure silently drops a +# real tenant -- the false-clean this script exists to prevent. EXISTENCE and +# EXTRACTION are therefore two separate questions, because `--ignore-not-found +# -o jsonpath` cannot tell them apart (it returns empty + exit 0 both for an absent +# Secret AND for a present Secret whose .data.release is missing): +# +# 1. Existence, via `--ignore-not-found -o name`. Empty + exit 0 = the revision +# was pruned by Helm's history limit -- a legitimate absence; return "" and +# let rev1_scheme / first_deployed treat it as "pruned". A real error (RBAC, +# timeout, apiserver down) stays non-zero and run_kubectl makes it fatal. +# 2. The Secret EXISTS, so its payload MUST decode. A helm.sh/release.v1 Secret +# with no .data.release, a payload that is not base64(base64(gzip(...))), or an +# empty JSON after decoding is CORRUPT state, not an absence -- and here that +# is safety-critical, so every such anomaly is audit_fatal, never a silent skip +# (contrast pv_epoch, where an empty field is a benign "no evidence"). +# +# The extraction GET drops --ignore-not-found deliberately: the Secret existed a +# moment ago, so a NotFound now is a mid-audit deletion race, and failing closed on +# it is correct. +release_json() { + _rj_secret="sh.helm.release.v1.$2.v${3:-1}" + _rj_exists=$(run_kubectl "check Helm release secret '$_rj_secret' in namespace '$1'" \ + get secret -n "$1" "$_rj_secret" --ignore-not-found -o name) || return $? + [ -n "$_rj_exists" ] || return 0 + _rj_field=$(run_kubectl "read .data.release of secret '$_rj_secret' in namespace '$1'" \ + get secret -n "$1" "$_rj_secret" -o jsonpath='{.data.release}') || return $? + [ -n "$_rj_field" ] || { audit_fatal "secret '$_rj_secret' in namespace '$1' exists but has no .data.release payload (corrupt Helm release)."; return 1; } + _rj_json=$(printf '%s' "$_rj_field" | base64 -d 2>/dev/null | base64 -d 2>/dev/null | gunzip 2>/dev/null) \ + || { audit_fatal "secret '$_rj_secret' in namespace '$1': .data.release is not decodable base64(base64(gzip(json))) (corrupt Helm release)."; return 1; } + [ -n "$_rj_json" ] || { audit_fatal "secret '$_rj_secret' in namespace '$1': release payload decoded to nothing (corrupt Helm release)."; return 1; } + printf '%s' "$_rj_json" +} + +# revisions -- retained revision numbers, oldest first. +revisions() { + _rev_out=$(run_kubectl "list Helm revision secrets for '$2' in namespace '$1'" \ + get secret -n "$1" -l "name=$2,owner=helm" \ + -o jsonpath='{range .items[*]}{.metadata.labels.version}{"\n"}{end}') || return $? + printf '%s\n' "$_rev_out" | sort -n +} + +# system_releases -- the SeaweedFS -system Helm releases in a namespace. +# Filtering on the `-system` suffix alone is not enough: every Cozystack app has a +# -system release (ingress-nginx-system, bucket-*-system, ...), and since the +# generation scan below matches PVCs by NAME across the whole namespace, an +# unrelated release in a namespace that happens to run SeaweedFS would be reported +# as a SeaweedFS tenant. Confirm the chart. +system_releases() { + _sr_secrets=$(run_kubectl "list Helm release secrets in namespace '$1'" \ + get secret -n "$1" \ + -o jsonpath='{range .items[?(@.type=="helm.sh/release.v1")]}{.metadata.labels.name}{"\n"}{end}') || return $? + for rel in $(printf '%s\n' "$_sr_secrets" | grep -E -- '-system$' | sort -u); do + _sr_revs=$(revisions "$1" "$rel") || return $? + for rev in $_sr_revs; do + _sr_json=$(release_json "$1" "$rel" "$rev") || return $? + # Empty "" here means the revision secret was pruned (release_json's absence + # path) -- a legitimate skip. A NON-empty payload, however, was fetched and + # decoded successfully, so it MUST name its chart: a helm.sh/release.v1 release + # always carries chart.metadata.name. A payload without one ({} , or any valid + # JSON lacking it) is corrupt/unexpected, and since "no chart name" is + # indistinguishable downstream from "not SeaweedFS", silently skipping it is + # the very false-clean this guard exists to stop -- so it is FATAL. A present + # but different chart name is a real, non-SeaweedFS release and stays a + # legitimate skip, filtered exactly as before by the cozy-seaweedfs test. + # + # Two properties make this match trustworthy, and both are load-bearing now + # that a miss is FATAL: + # + # * FIRST match, not last. Helm's Release marshals "chart" before + # "config" (the user's values), so a values subtree that happens to + # spell chart.metadata.name sits LATER in the payload -- and a greedy + # `sed 's/.*"chart"...'` would return that decoy instead of the real + # chart. A wrong name reads downstream as "not SeaweedFS" and drops a + # real release from the report: the exact silent false-clean this guard + # exists to stop. `grep -o | head -1` takes the leftmost match. + # * The chart -> metadata -> name key path stays ADJACENT. A looser "any + # 'name' after 'metadata'" matches chart.templates[].name, which Helm + # serializes immediately after metadata on EVERY healthy release, so it + # would return a template path for every tenant. + # + # Newlines are folded first so a pretty-printed payload parses at all (sed + # and grep are line-based), and whitespace around the punctuation is + # tolerated. Failing loudly on a payload this cannot read is the safe + # direction -- over-strictness stops the runbook, over-looseness lets it + # delete data -- so the message says what shape was expected. + if [ -n "$_sr_json" ]; then + _sr_chart=$(printf '%s' "$_sr_json" | tr '\n' ' ' \ + | grep -o '"chart"[[:space:]]*:[[:space:]]*{[[:space:]]*"metadata"[[:space:]]*:[[:space:]]*{[[:space:]]*"name"[[:space:]]*:[[:space:]]*"[^"]*"' \ + | head -1 \ + | sed -n 's/.*"\([^"]*\)"$/\1/p') + [ -n "$_sr_chart" ] || { audit_fatal "could not read the chart name of secret 'sh.helm.release.v1.$rel.v$rev' in namespace '$1' at .chart.metadata.name (expected 'name' as metadata's first key; a corrupt Helm release, or a payload format this parser does not handle)."; return 1; } + if [ "$_sr_chart" = cozy-seaweedfs ]; then printf '%s\n' "$rel"; fi + fi + break + done + done +} + +# first_deployed -- epoch seconds of the release's first install, +# from any retained revision (the field is identical on all of them). +first_deployed() { + _fd_revs=$(revisions "$1" "$2") || return $? + for rev in $_fd_revs; do + _fd_json=$(release_json "$1" "$2" "$rev") || return $? + # Same extraction discipline as the chart name above: fold newlines so a + # pretty-printed payload parses at all, tolerate whitespace around the + # punctuation, and take the FIRST match (.info precedes .config, so a values + # subtree cannot shadow the real timestamp). A miss here is not fatal -- the + # caller degrades to the documented safe fallback -- but a WRONG timestamp + # would silently change a vintage verdict, which is worse than none. + ts=$(printf '%s' "$_fd_json" | tr '\n' ' ' \ + | grep -o '"first_deployed"[[:space:]]*:[[:space:]]*"[^"]*"' \ + | head -1 \ + | sed -n 's/.*"\([^"]*\)"$/\1/p') + if [ -n "$ts" ]; then date -u -d "$(printf '%s' "$ts" | cut -c1-19)" +%s 2>/dev/null; return; fi + done +} + +# rev1_scheme -- "legacy" | "renamed" | "" (revision 1 pruned). +rev1_scheme() { + m=$(release_json "$1" "$2" 1) || return $? + [ -n "$m" ] || return 0 + if printf '%s' "$m" | grep -q 'name: seaweedfs-master'; then printf 'legacy' + elif printf '%s' "$m" | grep -qE "name: $2(-seaweedfs)?-master"; then printf 'renamed' + fi +} + +# pv_epoch -- creation epoch of the PV the claim is BOUND to, or "" when +# there is no usable PV age. Two by-NAME GETs. +# +# A real API error must abort (an epoch silently missing from a range can invert +# the strict-newer comparison and name the WRONG deletion candidate), so both GETs +# go through run_kubectl. But UNLIKE release_json, an EMPTY field here is NOT +# corruption and is NOT fatal -- because the consequence differs. When release_json +# returns empty for a present Secret the tenant is dropped from the report: a +# false-clean. When pv_epoch returns "" the claim merely contributes no epoch, +# which marks its whole generation INCOMPLETE (see audit_ns) and forces the safe +# "direction cannot be established -> classify by hand" branch. That degradation is +# conservative by construction: it never yields a false-clean and never names a +# candidate. So the two empty-field cases are deliberately kept benign: +# * PVC .spec.volumeName empty -- a Pending / unbound claim, a routine state; +# * PV .metadata.creationTimestamp empty -- shouldn't happen (the apiserver always +# stamps it), but if it ever did the only effect is one missing epoch, i.e. the +# same safe incomplete fallback, so a hard stop is not warranted here. +# `--ignore-not-found` keeps a genuinely-absent PVC/PV (NotFound) on that same +# benign path rather than turning it into a run_kubectl failure, and the final +# `|| return 0` keeps an unparseable timestamp there too; only a real run_kubectl +# error propagates. +pv_epoch() { + pv=$(run_kubectl "get PVC '$2' in namespace '$1'" \ + get pvc -n "$1" "$2" --ignore-not-found -o jsonpath='{.spec.volumeName}') || return $? + [ -n "$pv" ] || return 0 + t=$(run_kubectl "get PV '$pv' (bound by PVC '$2' in namespace '$1')" \ + get pv "$pv" --ignore-not-found -o jsonpath='{.metadata.creationTimestamp}') || return $? + [ -n "$t" ] || return 0 + date -u -d "$(printf '%s' "$t" | cut -c1-19)" +%s 2>/dev/null || return 0 +} + +# classify_mixed_direction -- direction of a MIXED +# tenant, from the bound-PV creation-epoch ranges of the chart-named (l*) and +# release-named (r*) generations. Prints one of: +# +# legacy-original every release-named PV strictly newer -> it is the candidate +# duplicate (runbook Step 3) +# renamed-original every chart-named PV strictly newer -> S-damaged, the +# chart-named set is the candidate duplicate (Step 2a, then 2) +# overlap vintages interleave or touch -> no candidate. An interrupted +# Step 2 re-bind puts BOTH generations on original-vintage PVs. +# +# Purely relative — no clock, no window, no first_deployed. Ties (second-resolution +# timestamps) count as overlap: refusing a candidate is recoverable, naming the +# wrong one is not. +classify_mixed_direction() { + if [ "$3" -gt "$2" ]; then printf 'legacy-original' + elif [ "$1" -gt "$4" ]; then printf 'renamed-original' + else printf 'overlap' + fi +} + +audit_ns() { + ns="$1" + _rels=$(system_releases "$ns") || return $? + for rel in $_rels; do + prefix=$(renamed_volume_prefix "$rel") + legacy_pvcs=""; renamed_pvcs="" + _pvcs=$(run_kubectl "list PVCs in namespace '$ns'" get pvc -n "$ns" -o name) || return $? + for pvc in $(printf '%s\n' "$_pvcs" | sed 's|persistentvolumeclaim/||'); do + case "$pvc" in + "data1-${prefix}"*) renamed_pvcs="$renamed_pvcs $pvc" ;; + data1-seaweedfs-volume*) legacy_pvcs="$legacy_pvcs $pvc" ;; + esac + done + legacy_sts=""; renamed_sts="" + _sts_list=$(run_kubectl "list SeaweedFS StatefulSets in namespace '$ns'" \ + get sts -n "$ns" -l app.kubernetes.io/name=seaweedfs -o name) || return $? + for sts in $(printf '%s\n' "$_sts_list" | sed 's|statefulset.apps/||'); do + case "$sts" in + "${prefix}"*) renamed_sts="$renamed_sts $sts" ;; + seaweedfs-volume*) legacy_sts="$legacy_sts $sts" ;; + esac + done + has_legacy=0; has_renamed=0 + [ -n "$legacy_pvcs$legacy_sts" ] && has_legacy=1 + [ -n "$renamed_pvcs$renamed_sts" ] && has_renamed=1 + [ "$has_legacy" = 0 ] && [ "$has_renamed" = 0 ] && continue + + if [ "$has_legacy" = 1 ] && [ "$has_renamed" = 0 ]; then + printf '%-24s %-14s %-8s %s\n' "$ns" "$rel" "L" "chart-named only; the upgrade adopts it, nothing to do" + continue + fi + if [ "$has_renamed" = 1 ] && [ "$has_legacy" = 0 ]; then + printf '%-24s %-14s %-8s %s\n' "$ns" "$rel" "S" "release-named only; re-bind the volumes (Step 2) BEFORE upgrading" + continue + fi + + # Both generations. Report which is ORIGINAL from durable evidence. + fd=$(first_deployed "$ns" "$rel") || return $? + scheme=$(rev1_scheme "$ns" "$rel") || return $? + printf '%-24s %-14s %-8s %s\n' "$ns" "$rel" "MIXED" "both generations; the chart REFUSES until one is removed" + if [ -n "$scheme" ]; then + printf '%-24s %-14s %-8s revision 1 was installed with the %s names => the %s generation is ORIGINAL\n' \ + "" "" "" "$scheme" "$scheme" + fi + # The direction rule is "EVERY PV of one generation strictly newer than every + # PV of the other". That holds only if we saw EVERY bound PV: a claim whose PV + # age we could not read (unbound, or PV gone) leaves the range unbounded on one + # side, and a silently-narrowed range can flip OVERLAP into a confident (wrong) + # candidate. So a generation with any unreadable claim is INCOMPLETE + # and forces the safe "cannot establish" branch -- distinct from a real API + # error, which pv_epoch has already turned into a hard failure above. + lmin=""; lmax=""; rmin=""; rmax=""; l_incomplete=0; r_incomplete=0 + for p in $legacy_pvcs; do + e=$(pv_epoch "$ns" "$p") || return $? + if [ -z "$e" ]; then l_incomplete=1; continue; fi + { [ -z "$lmin" ] || [ "$e" -lt "$lmin" ]; } && lmin=$e + { [ -z "$lmax" ] || [ "$e" -gt "$lmax" ]; } && lmax=$e + done + for p in $renamed_pvcs; do + e=$(pv_epoch "$ns" "$p") || return $? + if [ -z "$e" ]; then r_incomplete=1; continue; fi + { [ -z "$rmin" ] || [ "$e" -lt "$rmin" ]; } && rmin=$e + { [ -z "$rmax" ] || [ "$e" -gt "$rmax" ]; } && rmax=$e + done + if [ -n "$fd" ] && [ -n "$lmin" ] && [ -n "$rmin" ]; then + printf '%-24s %-14s %-8s oldest PV vs first_deployed: chart-named +%ss, release-named +%ss (context only, not the rule)\n' \ + "" "" "" "$((lmin - fd))" "$((rmin - fd))" + fi + if [ -n "$lmin" ] && [ -n "$rmin" ] && [ "$l_incomplete" = 0 ] && [ "$r_incomplete" = 0 ]; then + case $(classify_mixed_direction "$lmin" "$lmax" "$rmin" "$rmax") in + legacy-original) + printf '%-24s %-14s %-8s every release-named PV is strictly newer => chart-named is ORIGINAL, the release-named set is the candidate duplicate (Step 3)\n' "" "" "" ;; + renamed-original) + printf '%-24s %-14s %-8s every chart-named PV is strictly newer => release-named is ORIGINAL, the chart-named set is the candidate duplicate (Step 2a, then Step 2)\n' "" "" "" ;; + overlap) + printf '%-24s %-14s %-8s PV vintages OVERLAP => no candidate. An interrupted Step 2 re-bind looks exactly like this (both generations on original PVs). Finish Step 2 if one is in progress; otherwise escalate. Do NOT run Step 2a.\n' "" "" "" ;; + esac + else + printf '%-24s %-14s %-8s direction cannot be established from PV ages: a generation has no bound PVs (Pending/unbound claims, or StatefulSets only) or a bound PV age could not be read. Resolve those claims or classify by hand. Do NOT run Step 2a.\n' "" "" "" + fi + printf '%-24s %-14s %-8s CANDIDATE ONLY: "original" does not mean the other set is EMPTY. A duplicate that\n' "" "" "" + printf '%-24s %-14s %-8s served writes and later crashed looks identical here. Verify emptiness before deleting.\n' "" "" "" + done +} + +main() { + printf '%-24s %-14s %-8s %s\n' NAMESPACE RELEASE CLASS NOTE + printf '%-24s %-14s %-8s %s\n' --------- ------- ----- ---- + if [ "$#" -gt 0 ]; then + for ns in "$@"; do audit_ns "$ns" || exit $?; done + else + _ns_list=$(run_kubectl 'list namespaces (whole-cluster mode)' get ns -o name) || exit $? + for ns in $(printf '%s\n' "$_ns_list" | sed 's|namespace/||'); do audit_ns "$ns" || exit $?; done + fi +} + +# Sourced by hack/seaweedfs-naming-audit.bats to exercise the classifier directly. +[ "${SEAWEEDFS_AUDIT_LIB:-0}" = "1" ] || main "$@" diff --git a/hack/testdata/migration-seaweedfs-db/kubectl b/hack/testdata/migration-seaweedfs-db/kubectl new file mode 100755 index 0000000000..d005b52fcf --- /dev/null +++ b/hack/testdata/migration-seaweedfs-db/kubectl @@ -0,0 +1,205 @@ +#!/bin/sh +# Fake kubectl for hack/migration-seaweedfs-db-adopt.bats. It serves only the +# calls the two halves of migration 45 make — lib/seaweedfs-db-adopt.sh and +# lib/kubeadm-keep-pin.sh; behaviour is driven by FAKE_* env and every invocation +# is appended to $FAKE_CMDLOG so the test can assert on which objects were acted +# on and with which annotations. +# +# FAKE_CMDLOG file to append a one-line record of each call to +# FAKE_CLUSTERS newline list of " " — one +# per namespace holding a Cluster/seaweedfs-db. Use "-" for +# either annotation to model it being ABSENT (jsonpath prints +# nothing and exits 0). +# FAKE_LIST_FAIL non-empty => the -A fleet scan exits 1 with this as stderr +# FAKE_GET_FAIL non-empty => the per-namespace get exits 1 with this stderr +# FAKE_ANNOTATE_FAIL non-empty => `annotate` exits 1 (a failed hand-over) +# +# For the keep-pin half: +# +# FAKE_KCTS newline list of " " — one +# per KubeadmConfigTemplate. "-" models the annotation/label being +# ABSENT; managed-by is "Helm" for a chart-rendered object. +# FAKE_KCS the same, for KubeadmConfig. A CAPI-spawned child is modelled with +# managed-by "-": it is owned by its Machine, Helm never prunes it, +# and the pin must leave it alone. +# FAKE_KUBEADM_LIST_FAIL non-empty => the fleet scan for either kubeadm +# kind exits 1 with this as stderr +# FAKE_KUBEADM_ANNOTATE_FAIL non-empty => `annotate` on a kubeadm object +# exits 1 with this as stderr +# FAKE_KUBEADM_ANNOTATE_FAIL_NS restrict the above to one namespace, so a +# partial failure can be told apart from a total +# one — that is what proves the pin keeps going +# and reports every failure at the end rather +# than aborting on the first. +# +# The failure knobs exist because the point of the helper is that a kubectl error +# must ABORT the migration rather than let it stamp the version and never run +# again. A fake that can only succeed cannot test that. +set -u + +log() { [ -n "${FAKE_CMDLOG:-}" ] && printf '%s\n' "$*" >> "$FAKE_CMDLOG"; } +log "KUBECTL $*" + +args="$*" + +# arg_after KEY -- echoes the token following KEY in the argument vector. +arg_after() { k="$1"; shift; p=""; for a in "$@"; do [ "$p" = "$k" ] && { printf '%s' "$a"; return; }; p="$a"; done; } + +# field -- echo that annotation for the FAKE_CLUSTERS row of ns. +field() { + printf '%s\n' "${FAKE_CLUSTERS:-}" | while read -r n rel keep; do + if [ "$n" = "$1" ]; then + case "$2" in + rel) [ "${rel:--}" = "-" ] || printf '%s' "$rel" ;; + keep) [ "${keep:--}" = "-" ] || printf '%s' "$keep" ;; + esac + break + fi + done +} + +# kubeadm_rows -- echo the row list for whichever kubeadm kind names. +# "kubeadmconfigs" is not a substring of "kubeadmconfigtemplates", so the two +# patterns cannot both match; templates is tried first regardless. +kubeadm_rows() { + case "$1" in + *kubeadmconfigtemplates*) printf '%s\n' "${FAKE_KCTS:-}" ;; + *kubeadmconfigs*) printf '%s\n' "${FAKE_KCS:-}" ;; + esac +} + +# kubeadm_short -- singular kind name for the PIN log line. +kubeadm_short() { + case "$1" in + *kubeadmconfigtemplates*) printf 'kubeadmconfigtemplate' ;; + *kubeadmconfigs*) printf 'kubeadmconfig' ;; + esac +} + +case "$args" in + # Version stamp: kubectl apply --filename - + *"apply --filename -"*) + manifest=$(cat) + # Record the version actually stamped. A migration that stamped the wrong + # number would loop the runner forever, and asserting a bare "STAMP" would + # not notice. + v=$(printf '%s\n' "$manifest" | sed -n 's/^ version: "\{0,1\}\([0-9]\{1,\}\)"\{0,1\}$/\1/p' | head -1) + log "STAMP ${v:-unknown}" + exit 0 + ;; + + # Fleet scan: list namespaces holding a Cluster named seaweedfs-db. + *"get cluster.postgresql.cnpg.io -A"*) + if [ -n "${FAKE_LIST_FAIL:-}" ]; then + printf '%s\n' "$FAKE_LIST_FAIL" >&2 + exit 1 + fi + printf '%s\n' "${FAKE_CLUSTERS:-}" | while read -r ns rel keep; do + [ -n "$ns" ] && printf '%s\n' "$ns" + done + exit 0 + ;; + + # Per-namespace read of an annotation. + *"get cluster.postgresql.cnpg.io seaweedfs-db"*) + if [ -n "${FAKE_GET_FAIL:-}" ]; then + printf '%s\n' "$FAKE_GET_FAIL" >&2 + exit 1 + fi + ns=$(arg_after -n "$@") + case "$args" in + *"resource-policy"*) field "$ns" keep ;; + *) field "$ns" rel ;; + esac + exit 0 + ;; + + # The hand-over itself. + *"annotate cluster.postgresql.cnpg.io seaweedfs-db"*) + if [ -n "${FAKE_ANNOTATE_FAIL:-}" ]; then + printf '%s\n' "$FAKE_ANNOTATE_FAIL" >&2 + exit 1 + fi + ns=$(arg_after -n "$@") + rel="" + keep="" + for a in "$@"; do + case "$a" in + meta.helm.sh/release-name=*) rel="${a#meta.helm.sh/release-name=}" ;; + helm.sh/resource-policy=*) keep="${a#helm.sh/resource-policy=}" ;; + esac + done + log "ANNOTATE $ns release-name=${rel:-} resource-policy=${keep:-}" + exit 0 + ;; + + # --- kubeadm bootstrap objects: migration 45's keep-pin half -------------- + # Fleet scan for one kubeadm kind. + # + # THE LABEL SELECTOR IS MODELLED, NOT IGNORED. The pin selects on + # app.kubernetes.io/managed-by=Helm, which Helm injects as a LABEL into every + # resource it applies. meta.helm.sh/release-name is an ANNOTATION and can never + # be a selector: passing it matches nothing, the pin silently applies to nothing, + # and the run looks clean. A fake that answered every selector identically could + # not tell that apart from a working pin, so a row is returned only when the + # selector actually names its managed-by value. + *"get kubeadmconfig"*" --all-namespaces"*) + if [ -n "${FAKE_KUBEADM_LIST_FAIL:-}" ]; then + printf '%s\n' "$FAKE_KUBEADM_LIST_FAIL" >&2 + exit 1 + fi + sel=$(arg_after --selector "$@") + kubeadm_rows "$args" | while read -r ns name policy mgr; do + [ -n "${ns:-}" ] || continue + # NO --selector LISTS EVERYTHING, exactly as kubectl does. Modelling an + # absent selector as "matches nothing" would be backwards and would hide the + # over-pinning bug: dropping the selector would then read as a quiet no-op + # instead of as the pin stamping keep on Machine-owned objects whose + # lifecycle belongs to CAPI. + if [ -n "$sel" ]; then + [ "app.kubernetes.io/managed-by=${mgr:--}" = "$sel" ] || continue + fi + printf '%s/%s\n' "$ns" "$name" + done + exit 0 + ;; + + # Per-object read of helm.sh/resource-policy, which is what makes the pin skip + # an already-pinned object without a write. + *"get kubeadmconfig"*) + ns=$(arg_after --namespace "$@") + name=$(arg_after "$ns" "$@") + kubeadm_rows "$args" | while read -r n nm policy mgr; do + if [ "$n" = "$ns" ] && [ "$nm" = "$name" ]; then + [ "${policy:--}" = "-" ] || printf '%s' "$policy" + break + fi + done + exit 0 + ;; + + # The pin itself. Logged as PIN rather than ANNOTATE on purpose: the SeaweedFS + # assertions above count ANNOTATE lines, and reusing the verb would make them + # pass or fail for the other half's reasons. + *"annotate kubeadmconfig"*) + ns=$(arg_after --namespace "$@") + name=$(arg_after "$ns" "$@") + if [ -n "${FAKE_KUBEADM_ANNOTATE_FAIL:-}" ] && + { [ -z "${FAKE_KUBEADM_ANNOTATE_FAIL_NS:-}" ] || + [ "$ns" = "${FAKE_KUBEADM_ANNOTATE_FAIL_NS:-}" ]; }; then + printf '%s\n' "$FAKE_KUBEADM_ANNOTATE_FAIL" >&2 + exit 1 + fi + keep="" + for a in "$@"; do + case "$a" in + helm.sh/resource-policy=*) keep="${a#helm.sh/resource-policy=}" ;; + esac + done + log "PIN $(kubeadm_short "$args") $ns/$name resource-policy=${keep:-}" + exit 0 + ;; +esac + +log "UNHANDLED $args" +exit 0 diff --git a/internal/backupcontroller/cnpgstrategy_controller.go b/internal/backupcontroller/cnpgstrategy_controller.go index f4e3236fe1..a17737b54b 100644 --- a/internal/backupcontroller/cnpgstrategy_controller.go +++ b/internal/backupcontroller/cnpgstrategy_controller.go @@ -314,16 +314,62 @@ func cnpgBackupDeadlineExceeded(startedAt *metav1.Time) bool { // destructive step: // 1. The RestoreJob already records that we've purged // (restoreCondTargetPurged=True). Normal idempotent path. -// 2. The live Cluster already has spec.bootstrap.recovery populated. This -// only happens when an earlier reconcile purged successfully but the +// 2. The live Cluster is a freshly-recovered one that THIS restore's own +// purge + chart re-render produced (see cnpgClusterFreshlyRecovered). +// This only happens when an earlier reconcile purged successfully but the // status-condition write failed; the chart has since re-rendered the // Cluster with our restore-shaped values. Re-purging here would delete // the Cluster CNPG is actively bootstrapping from S3. -func cnpgPurgeNeeded(purgedCondition, liveClusterHasRecovery bool) bool { +// +// A live Cluster that carries spec.bootstrap.recovery but predates this +// RestoreJob is NOT a skip signal: it is leftover from an earlier, already- +// completed restore and still holds the old data. Skipping the purge on it +// (the previous behaviour, which keyed only on "has recovery bootstrap") made +// a repeat in-place restore a silent no-op - the job reported Succeeded while +// the PVC, disk, and data were never touched. +func cnpgPurgeNeeded(purgedCondition, liveClusterFreshlyRecovered bool) bool { if purgedCondition { return false } - return !liveClusterHasRecovery + return !liveClusterFreshlyRecovered +} + +// cnpgClusterFreshlyRecovered reports whether a live cnpg.io Cluster that +// carries spec.bootstrap.recovery was produced by THIS RestoreJob's own purge +// + chart re-render (its creationTimestamp is strictly after the job's +// StartedAt) rather than being left over from an earlier, already-completed +// restore. +// +// Only the former is safe to skip re-purging. A leftover recovery Cluster from +// a prior restore predates StartedAt and still holds the prior data, so it must +// be purged for the new restore to re-bootstrap from the backup. The fresh +// Cluster in the status-write-race case is always created after StartedAt (the +// job sets StartedAt on its first reconcile, long before it purges and the +// chart re-renders), so the timestamp comparison cleanly separates the two. +// +// The comparison is strict (created > started), so an exact tie resolves to +// "not fresh" and the caller purges. That matches the conservative default +// below: the only classification that must never be wrong is calling a stale +// leftover "fresh" (which reintroduces the silent no-op), and a fresh Cluster +// is always created well after StartedAt (see above), never exactly at it. +// +// An identity-based alternative was considered - recording the purged Cluster's +// UID on the TargetPurged condition and comparing UIDs on later reconciles - +// which is immune to clock skew. It was rejected because it leans on the same +// status write that the status-write-race path (the whole reason this skip +// exists) assumes can fail, so it cannot cover that case; the timestamp +// comparison needs no extra persisted state and the fresh-vs-stale gap +// (a full purge + re-render cycle) always dwarfs any plausible control-plane +// clock skew. +// +// Returns false when freshness cannot be determined (no recovery bootstrap, or +// a missing timestamp): the caller then proceeds to purge, which is the safe +// default for any pre-existing, non-fresh Cluster. +func cnpgClusterFreshlyRecovered(hasRecovery bool, clusterCreatedAt, restoreStartedAt *metav1.Time) bool { + if !hasRecovery || clusterCreatedAt == nil || restoreStartedAt == nil { + return false + } + return clusterCreatedAt.After(restoreStartedAt.Time) } // applyClusterBarmanObjectStore SSA-patches the live CNPG Cluster's @@ -614,12 +660,18 @@ func (r *RestoreJobReconciler) reconcileCNPGRestore(ctx context.Context, restore // check the live Cluster for bootstrap.recovery: if present, the chart // has already re-rendered after a previous purge, and we must NOT delete // it again. - hasRecovery, err := r.clusterHasRecoveryBootstrap(ctx, target.Namespace, clusterName) + hasRecovery, clusterCreatedAt, err := r.recoveryBootstrapClusterState(ctx, target.Namespace, clusterName) if err != nil { return ctrl.Result{}, err } purgedCondition := apimeta.IsStatusConditionTrue(restoreJob.Status.Conditions, restoreCondTargetPurged) - if cnpgPurgeNeeded(purgedCondition, hasRecovery) { + // Only skip the destructive purge for a recovery Cluster THIS restore just + // produced (created at/after StartedAt). A recovery Cluster left over from + // an earlier completed restore predates StartedAt and still holds the old + // data, so it must be purged - otherwise a repeat in-place restore silently + // no-ops. + freshlyRecovered := cnpgClusterFreshlyRecovered(hasRecovery, clusterCreatedAt, restoreJob.Status.StartedAt) + if cnpgPurgeNeeded(purgedCondition, freshlyRecovered) { // Gate the destructive flow on the source cluster having shipped // the backup's required WALs to object storage. archive_command runs // on the source primary; once we delete the Cluster + PVCs, any @@ -1126,11 +1178,15 @@ func (r *RestoreJobReconciler) cnpgClusterHealthy(ctx context.Context, namespace return cluster.Status.Phase == cnpgClusterHealthyPhase, nil } -// clusterHasRecoveryBootstrap returns true when the live cnpg.io Cluster's -// spec.bootstrap.recovery is populated - the signal that the chart has -// re-rendered with our restore-shaped values and the operator is using the -// recovery bootstrap path. Treats a missing Cluster as "not yet" rather -// than an error so the caller can keep polling while HelmRelease catches up. +// recoveryBootstrapClusterState fetches the live cnpg.io Cluster and reports +// whether its spec.bootstrap.recovery is populated - the signal that the chart +// has re-rendered with our restore-shaped values and the operator is using the +// recovery bootstrap path - together with the Cluster's creationTimestamp. The +// caller pairs the timestamp with the RestoreJob's StartedAt (via +// cnpgClusterFreshlyRecovered) to tell a Cluster this restore just produced +// apart from one left over by an earlier completed restore. Treats a missing +// Cluster as "not yet" rather than an error so the caller can keep polling +// while HelmRelease catches up. // // A Cluster carrying DeletionTimestamp is also treated as "not yet": that is // the in-flight purge case, where r.Delete has fired but cnpg.io's @@ -1143,18 +1199,22 @@ func (r *RestoreJobReconciler) cnpgClusterHealthy(ctx context.Context, namespace // drops the change and the cluster ends up with the original initdb spec. // Holding here forces the caller to requeue until the old CR is fully GC'd // and the chart re-creates a fresh one. -func (r *RestoreJobReconciler) clusterHasRecoveryBootstrap(ctx context.Context, namespace, clusterName string) (bool, error) { +func (r *RestoreJobReconciler) recoveryBootstrapClusterState(ctx context.Context, namespace, clusterName string) (hasRecovery bool, createdAt *metav1.Time, err error) { cluster := &cnpgtypes.Cluster{} if err := r.Get(ctx, types.NamespacedName{Namespace: namespace, Name: clusterName}, cluster); err != nil { if apierrors.IsNotFound(err) { - return false, nil + return false, nil, nil } - return false, err + return false, nil, err } if !cluster.DeletionTimestamp.IsZero() { - return false, nil + return false, nil, nil + } + if cluster.Spec.Bootstrap == nil || cluster.Spec.Bootstrap.Recovery == nil { + return false, nil, nil } - return cluster.Spec.Bootstrap != nil && cluster.Spec.Bootstrap.Recovery != nil, nil + created := cluster.CreationTimestamp + return true, &created, nil } // --------------------------------------------------------------------------- diff --git a/internal/backupcontroller/cnpgstrategy_controller_test.go b/internal/backupcontroller/cnpgstrategy_controller_test.go index f8ee1c7b93..3504e72780 100644 --- a/internal/backupcontroller/cnpgstrategy_controller_test.go +++ b/internal/backupcontroller/cnpgstrategy_controller_test.go @@ -549,43 +549,122 @@ func TestBuildPostgresAppRestorePatch_NoSecretRefIsSkipped(t *testing.T) { // freshly-recovered Cluster on a status-update failure. The controller used // to rely solely on a Status condition: if the post-purge Status().Update // raced or failed, the next reconcile would re-purge the just-restored -// Cluster, destroying recovery progress. Cross-checking the live Cluster's -// bootstrap.recovery makes the second purge a no-op. +// Cluster, destroying recovery progress. Cross-checking that the live +// Cluster is a *freshly-recovered* one (bootstrap.recovery + created after +// the job started) makes that second purge a no-op - while still purging a +// stale recovery Cluster left over from an earlier completed restore, so a +// repeat in-place restore is not a silent no-op. func TestCNPGPurgeNeeded(t *testing.T) { cases := []struct { - name string - purgedCondition bool - hasRecovery bool - want bool + name string + purgedCondition bool + liveClusterFreshlyRecovered bool + want bool }{ { - name: "fresh restore, old cluster still in place: purge", - purgedCondition: false, - hasRecovery: false, - want: true, + name: "fresh restore, old cluster still in place: purge", + purgedCondition: false, + liveClusterFreshlyRecovered: false, + want: true, }, { - name: "purge already recorded: skip", - purgedCondition: true, - hasRecovery: false, - want: false, + name: "purge already recorded: skip", + purgedCondition: true, + liveClusterFreshlyRecovered: false, + want: false, }, { - name: "live cluster already recovered (status write raced): skip - the bug fix", - purgedCondition: false, - hasRecovery: true, - want: false, + name: "live cluster freshly recovered by this restore (status write raced): skip", + purgedCondition: false, + liveClusterFreshlyRecovered: true, + want: false, }, { - name: "both true: skip (post-purge steady state)", - purgedCondition: true, - hasRecovery: true, - want: false, + name: "stale recovery cluster from a previous completed restore: purge (repeat-restore fix)", + purgedCondition: false, + liveClusterFreshlyRecovered: false, + want: true, + }, + { + name: "both true: skip (post-purge steady state)", + purgedCondition: true, + liveClusterFreshlyRecovered: true, + want: false, }, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - got := cnpgPurgeNeeded(tc.purgedCondition, tc.hasRecovery) + got := cnpgPurgeNeeded(tc.purgedCondition, tc.liveClusterFreshlyRecovered) + if got != tc.want { + t.Errorf("got %v want %v", got, tc.want) + } + }) + } +} + +// TestCNPGClusterFreshlyRecovered locks in the signal that separates a +// recovery Cluster THIS restore just produced from one left over by an +// earlier completed restore. The repeat-in-place-restore bug was exactly a +// stale recovery Cluster (created before StartedAt) being mistaken for a +// freshly-recovered one and skipping the purge. +func TestCNPGClusterFreshlyRecovered(t *testing.T) { + started := metav1.NewTime(time.Now()) + after := metav1.NewTime(started.Add(time.Minute)) + before := metav1.NewTime(started.Add(-time.Hour)) + + cases := []struct { + name string + hasRecovery bool + createdAt *metav1.Time + startedAt *metav1.Time + want bool + }{ + { + name: "no recovery bootstrap: not fresh", + hasRecovery: false, + createdAt: &after, + startedAt: &started, + want: false, + }, + { + name: "recovery cluster created after start (our own purge re-render): fresh", + hasRecovery: true, + createdAt: &after, + startedAt: &started, + want: true, + }, + { + name: "recovery cluster created before start (leftover from previous restore): not fresh", + hasRecovery: true, + createdAt: &before, + startedAt: &started, + want: false, + }, + { + name: "recovery cluster created exactly at start: not fresh (conservative tie -> purge)", + hasRecovery: true, + createdAt: &started, + startedAt: &started, + want: false, + }, + { + name: "missing cluster creation timestamp: not fresh", + hasRecovery: true, + createdAt: nil, + startedAt: &started, + want: false, + }, + { + name: "missing job start timestamp: not fresh", + hasRecovery: true, + createdAt: &after, + startedAt: nil, + want: false, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := cnpgClusterFreshlyRecovered(tc.hasRecovery, tc.createdAt, tc.startedAt) if got != tc.want { t.Errorf("got %v want %v", got, tc.want) } @@ -898,7 +977,7 @@ func TestApplyClusterBarmanObjectStore_NotFoundOnMissingCluster(t *testing.T) { } } -// TestClusterHasRecoveryBootstrap_TerminatingCluster locks in the +// TestRecoveryBootstrapClusterState_TerminatingCluster locks in the // DeletionTimestamp guard. Without it, a Cluster CR mid-deletion (after // purgeExistingCluster fired r.Delete but cnpg.io's finalizers haven't // drained yet) would get treated as "still has bootstrap" by the @@ -907,7 +986,7 @@ func TestApplyClusterBarmanObjectStore_NotFoundOnMissingCluster(t *testing.T) { // might SSA-merge the chart's bootstrap.recovery onto the terminating // CR and cnpg-operator's bootstrap-immutability check would drop the // change, leaving the cluster on the original initdb spec. -func TestClusterHasRecoveryBootstrap_TerminatingCluster(t *testing.T) { +func TestRecoveryBootstrapClusterState_TerminatingCluster(t *testing.T) { now := metav1.Now() t.Run("terminating cluster reports not-yet-recovered", func(t *testing.T) { cluster := &cnpgtypes.Cluster{ @@ -925,18 +1004,22 @@ func TestClusterHasRecoveryBootstrap_TerminatingCluster(t *testing.T) { } c := newCNPGStrategyTestClient(t, cluster) r := &RestoreJobReconciler{Client: c} - got, err := r.clusterHasRecoveryBootstrap(context.Background(), "tenant", "postgres-app") + got, createdAt, err := r.recoveryBootstrapClusterState(context.Background(), "tenant", "postgres-app") if err != nil { t.Fatalf("unexpected error: %v", err) } if got { t.Fatalf("expected false for terminating cluster, got true") } + if createdAt != nil { + t.Fatalf("expected nil createdAt for terminating cluster, got %v", createdAt) + } }) - t.Run("live recovery cluster reports recovered", func(t *testing.T) { + t.Run("live recovery cluster reports recovered with creation timestamp", func(t *testing.T) { + created := metav1.NewTime(time.Now().Add(-time.Hour)) cluster := &cnpgtypes.Cluster{ - ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "postgres-app"}, + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant", Name: "postgres-app", CreationTimestamp: created}, Spec: cnpgtypes.ClusterSpec{ Bootstrap: &cnpgtypes.BootstrapConfiguration{ Recovery: &cnpgtypes.RecoverySource{Source: "pg-src"}, @@ -945,25 +1028,31 @@ func TestClusterHasRecoveryBootstrap_TerminatingCluster(t *testing.T) { } c := newCNPGStrategyTestClient(t, cluster) r := &RestoreJobReconciler{Client: c} - got, err := r.clusterHasRecoveryBootstrap(context.Background(), "tenant", "postgres-app") + got, createdAt, err := r.recoveryBootstrapClusterState(context.Background(), "tenant", "postgres-app") if err != nil { t.Fatalf("unexpected error: %v", err) } if !got { t.Fatalf("expected true for live recovery cluster, got false") } + if createdAt == nil { + t.Fatalf("expected non-nil createdAt for live recovery cluster") + } }) t.Run("missing cluster reports not-yet", func(t *testing.T) { c := newCNPGStrategyTestClient(t) r := &RestoreJobReconciler{Client: c} - got, err := r.clusterHasRecoveryBootstrap(context.Background(), "tenant", "postgres-app") + got, createdAt, err := r.recoveryBootstrapClusterState(context.Background(), "tenant", "postgres-app") if err != nil { t.Fatalf("unexpected error: %v", err) } if got { t.Fatalf("expected false for missing cluster, got true") } + if createdAt != nil { + t.Fatalf("expected nil createdAt for missing cluster, got %v", createdAt) + } }) } @@ -1509,6 +1598,149 @@ func TestReconcileCNPGRestore_MissingStrategyFailsClosedAfterDeadline(t *testing } } +// TestReconcileCNPGRestore_RepeatInPlacePurgesStaleRecoveryCluster is the +// reconcile-level regression test for #3311. The pure-function tests +// (TestCNPGPurgeNeeded / TestCNPGClusterFreshlyRecovered) lock in the truth +// table, but the bug lived at the call site: a repeat in-place restore fed the +// purge decision the wrong signal and skipped the destructive purge, so the +// job reported Succeeded against untouched data. This drives the real +// reconcileCNPGRestore path with a fake client and asserts the destructive +// purge actually fires for a stale leftover recovery Cluster - and, in the +// mirror case, that a Cluster this restore just re-created is NOT re-purged +// (the status-write-race protection the guard was originally built for). +func TestReconcileCNPGRestore_RepeatInPlacePurgesStaleRecoveryCluster(t *testing.T) { + const ( + ns = "tenant" + appName = "app" + clusterName = "postgres-app" + cnpgBkName = "cnpgbk" + ) + apiGroup := backupsv1alpha1.DefaultApplicationAPIGroup + strategyGroup := strategyv1alpha1.GroupVersion.Group + ctx := context.Background() + + // startedAt anchors the discriminator: a stale leftover Cluster predates + // it, a freshly-recovered one postdates it. + startedAt := metav1.NewTime(time.Now()) + + mkBackupArtifact := func(t *testing.T) *backupsv1alpha1.Backup { + t.Helper() + snap, err := marshalCNPGBackupSnapshot(newPostgresApp(appName, ns), nil) + if err != nil { + t.Fatalf("marshal snapshot: %v", err) + } + return &backupsv1alpha1.Backup{ + ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: "bk"}, + Spec: backupsv1alpha1.BackupSpec{ + ApplicationRef: corev1.TypedLocalObjectReference{APIGroup: &apiGroup, Kind: postgresAppKind, Name: appName}, + StrategyRef: corev1.TypedLocalObjectReference{APIGroup: &strategyGroup, Kind: strategyv1alpha1.CNPGStrategyKind, Name: "cnpg-strategy"}, + DriverMetadata: map[string]string{ + cnpgServerNameKey: appName, + cnpgDestinationPathKey: "s3://bucket/" + appName + "/", + cnpgBackupNameKey: cnpgBkName, + }, + }, + Status: backupsv1alpha1.BackupStatus{UnderlyingResources: snap}, + } + } + strategy := &strategyv1alpha1.CNPG{ + ObjectMeta: metav1.ObjectMeta{Name: "cnpg-strategy"}, + Spec: strategyv1alpha1.CNPGSpec{ + Template: strategyv1alpha1.CNPGTemplate{ + BarmanObjectStore: strategyv1alpha1.BarmanObjectStoreTemplate{DestinationPath: "s3://bucket/"}, + }, + }, + } + // A completed cnpg.io/Backup with endWal set clears the WAL-archive gate + // so the reconcile can reach the purge step. + cnpgBackup := &cnpgtypes.Backup{ + ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: cnpgBkName}, + Spec: cnpgtypes.BackupSpec{Cluster: cnpgtypes.ClusterReference{Name: clusterName}}, + Status: cnpgtypes.BackupStatus{Phase: cnpgBackupPhaseComplete, EndWal: "000000010000000000000003"}, + } + mkRecoveryCluster := func(created metav1.Time) *cnpgtypes.Cluster { + return &cnpgtypes.Cluster{ + ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: clusterName, CreationTimestamp: created}, + Spec: cnpgtypes.ClusterSpec{ + Bootstrap: &cnpgtypes.BootstrapConfiguration{ + Recovery: &cnpgtypes.RecoverySource{Source: appName}, + }, + }, + } + } + mkClusterPVC := func() *corev1.PersistentVolumeClaim { + return &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: clusterName + "-1", Labels: map[string]string{cnpgClusterLabel: clusterName}}, + } + } + mkRestoreJob := func() *backupsv1alpha1.RestoreJob { + sa := startedAt + return &backupsv1alpha1.RestoreJob{ + ObjectMeta: metav1.ObjectMeta{Namespace: ns, Name: "rj"}, + Spec: backupsv1alpha1.RestoreJobSpec{BackupRef: corev1.LocalObjectReference{Name: "bk"}}, + Status: backupsv1alpha1.RestoreJobStatus{StartedAt: &sa, Phase: backupsv1alpha1.RestoreJobPhaseRunning}, + } + } + + t.Run("stale leftover recovery cluster from a prior restore is purged", func(t *testing.T) { + backup := mkBackupArtifact(t) + // creationTimestamp an hour before StartedAt: leftover from a prior restore. + stale := metav1.NewTime(startedAt.Add(-time.Hour)) + c := newCNPGStrategyTestClient(t, backup, mkRestoreJob(), strategy, cnpgBackup, + newPostgresApp(appName, ns), mkRecoveryCluster(stale), mkClusterPVC()) + r := &RestoreJobReconciler{Client: c, Interface: dynamicfake.NewSimpleDynamicClient(testCNPGScheme(t))} + + rj := &backupsv1alpha1.RestoreJob{} + if err := c.Get(ctx, client.ObjectKey{Namespace: ns, Name: "rj"}, rj); err != nil { + t.Fatalf("get seeded RestoreJob: %v", err) + } + if _, err := r.reconcileCNPGRestore(ctx, rj, backup); err != nil { + t.Fatalf("reconcileCNPGRestore: %v", err) + } + + // The stale Cluster (and its labelled PVC) must have been purged. + err := c.Get(ctx, client.ObjectKey{Namespace: ns, Name: clusterName}, &cnpgtypes.Cluster{}) + if !apierrors.IsNotFound(err) { + t.Fatalf("expected stale Cluster to be purged (NotFound), got err=%v", err) + } + pvcs := &corev1.PersistentVolumeClaimList{} + if err := c.List(ctx, pvcs, client.InNamespace(ns), client.MatchingLabels{cnpgClusterLabel: clusterName}); err != nil { + t.Fatalf("list PVCs: %v", err) + } + if len(pvcs.Items) != 0 { + t.Fatalf("expected cluster PVCs purged, still have %d", len(pvcs.Items)) + } + }) + + t.Run("freshly-recovered cluster from this restore is not re-purged", func(t *testing.T) { + backup := mkBackupArtifact(t) + // creationTimestamp a minute after StartedAt: this restore's own re-render. + fresh := metav1.NewTime(startedAt.Add(time.Minute)) + c := newCNPGStrategyTestClient(t, backup, mkRestoreJob(), strategy, cnpgBackup, + newPostgresApp(appName, ns), mkRecoveryCluster(fresh)) + r := &RestoreJobReconciler{Client: c, Interface: dynamicfake.NewSimpleDynamicClient(testCNPGScheme(t))} + + rj := &backupsv1alpha1.RestoreJob{} + if err := c.Get(ctx, client.ObjectKey{Namespace: ns, Name: "rj"}, rj); err != nil { + t.Fatalf("get seeded RestoreJob: %v", err) + } + if _, err := r.reconcileCNPGRestore(ctx, rj, backup); err != nil { + t.Fatalf("reconcileCNPGRestore: %v", err) + } + + // The freshly-recovered Cluster must survive - re-purging it would + // destroy the recovery this restore just started (the status-write-race + // protection the guard was built for). + got := &cnpgtypes.Cluster{} + if err := c.Get(ctx, client.ObjectKey{Namespace: ns, Name: clusterName}, got); err != nil { + t.Fatalf("expected freshly-recovered Cluster to survive, got err=%v", err) + } + if !got.DeletionTimestamp.IsZero() { + t.Fatalf("freshly-recovered Cluster must not be marked for deletion") + } + }) +} + // newCNPGStrategyTestClient returns a fake client.Client wired up with // the schemes the CNPG-strategy reconciler needs. func newCNPGStrategyTestClient(t *testing.T, objs ...client.Object) client.Client { diff --git a/internal/fluxshardoperator/provisioner.go b/internal/fluxshardoperator/provisioner.go index a0acb9400c..1a0066d59d 100644 --- a/internal/fluxshardoperator/provisioner.go +++ b/internal/fluxshardoperator/provisioner.go @@ -263,7 +263,15 @@ func mergeResourceList(dst *corev1.ResourceList, overrides corev1.ResourceList) // - the required podAntiAffinity cloned from flux-aio (which keeps its own // replicas off one node) is dropped, since it targets // app.kubernetes.io/name=flux and would otherwise leave every shard -// Pending on a single-node cluster. +// Pending on a single-node cluster; +// - the corporate-proxy env inherited from flux-aio is dropped, in both the +// upper- and lower-case spellings (HTTP_PROXY/HTTPS_PROXY/NO_PROXY): a +// standalone shard needs no external egress, and behind an unreachable +// proxy a stalled startup call leaves the manager never serving /healthz, +// so the liveness probe crashloops the pod; +// - a startupProbe is derived from the liveness handler (generous failure +// budget, liveness handler and TimeoutSeconds inherited) so a slow but +// progressing start is not killed by the short inherited liveness window. func BuildShardDeployment(flux *appsv1.Deployment, idx int, cfg *Config) (*appsv1.Deployment, error) { var src *corev1.Container for i := range flux.Spec.Template.Spec.Containers { @@ -316,6 +324,24 @@ func BuildShardDeployment(flux *appsv1.Deployment, idx int, cfg *Config) (*appsv // is not hostNetwork, so it must fall back to the in-cluster defaults. case "KUBERNETES_SERVICE_HOST", "KUBERNETES_SERVICE_PORT": continue + // Corporate-proxy env inherited from flux-aio. A standalone shard needs + // no external egress: source-controller does artifact fetching and + // cosign/TUF verification, and every hop the shard makes is in-cluster + // (artifacts advertise as flux.$(RUNTIME_NAMESPACE).svc, guest-cluster + // apiservers are reached over .svc kubeconfigs). NO_PROXY=.svc covers + // those, but not the management apiserver: the KUBERNETES_SERVICE_HOST + // case above makes the shard fall back to the kubelet-injected + // ClusterIP, a bare IP that no .svc suffix matches, so that startup + // call is the one that stalls through an unreachable proxy and leaves + // the manager never serving /healthz until the liveness probe + // crashloops the pod. Dropping the proxy env (and the then-pointless + // NO_PROXY) keeps every hop direct. A HelmRelease targeting a remote + // cluster via spec.kubeConfig reachable only through the proxy is the + // one theoretical exception; cozystack guest-cluster apiservers are + // in-cluster, so this does not apply here. + case "HTTP_PROXY", "HTTPS_PROXY", "NO_PROXY", + "http_proxy", "https_proxy", "no_proxy": + continue } env = append(env, e) } @@ -327,6 +353,25 @@ func BuildShardDeployment(flux *appsv1.Deployment, idx int, cfg *Config) (*appsv mergeResourceList(&hc.Resources.Requests, cfg.ShardResources.Requests) mergeResourceList(&hc.Resources.Limits, cfg.ShardResources.Limits) + // Guard startup with a startupProbe derived from the liveness handler. + // Without it the inherited ~30s liveness window kills a controller that is + // still syncing caches (a slow start on a large cluster, or a transient + // dependency), turning any slow start into an unrecoverable crashloop. The + // generous startup budget defers liveness until the manager is serving, + // then liveness still catches a wedged running pod. Only the budget fields + // are normalised; the liveness handler and its TimeoutSeconds are inherited, + // so a controller whose /healthz is slow under load keeps the same tolerance + // at startup as at runtime (overriding TimeoutSeconds down could make the + // startup probe stricter than liveness and recreate the crashloop). + if hc.LivenessProbe != nil && hc.StartupProbe == nil { + sp := hc.LivenessProbe.DeepCopy() + sp.InitialDelaySeconds = 0 + sp.PeriodSeconds = 10 + sp.SuccessThreshold = 1 + sp.FailureThreshold = 30 + hc.StartupProbe = sp + } + mounted := map[string]bool{} for _, m := range hc.VolumeMounts { mounted[m.Name] = true diff --git a/internal/fluxshardoperator/provisioner_test.go b/internal/fluxshardoperator/provisioner_test.go index 678f93833c..96ad34ff00 100644 --- a/internal/fluxshardoperator/provisioner_test.go +++ b/internal/fluxshardoperator/provisioner_test.go @@ -8,6 +8,7 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" ) // fluxAIODeployment models the relevant shape of the flux-aio "flux" @@ -92,8 +93,31 @@ func fluxAIODeployment() *appsv1.Deployment { FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.namespace"}, }}, {Name: "TUF_ROOT", Value: "/tmp/.sigstore"}, + // Corporate-proxy env present on flux-aio in + // proxied installs (operator/patch level, not the + // installer, which injects only KUBERNETES_SERVICE_*); + // harmless there, hangs a standalone shard at startup. + {Name: "HTTP_PROXY", Value: "http://proxy.example:3128"}, + {Name: "HTTPS_PROXY", Value: "http://proxy.example:3128"}, + {Name: "NO_PROXY", Value: ".svc"}, + {Name: "http_proxy", Value: "http://proxy.example:3128"}, + {Name: "https_proxy", Value: "http://proxy.example:3128"}, + {Name: "no_proxy", Value: ".svc"}, }, VolumeMounts: []corev1.VolumeMount{{Name: "tmp", MountPath: "/tmp"}}, + // flux-aio ships helm-controller with a bare httpGet + // liveness probe: no timing fields at all, so the + // kube-apiserver defaults (~30s window) apply. The + // startupProbe must inherit this handler and its unset + // TimeoutSeconds and only normalise the startup budget. + LivenessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/healthz", + Port: intstr.FromString("healthz-hc"), + }, + }, + }, }, {Name: "notification-controller", Image: "ghcr.io/fluxcd/notification-controller:v1.8.0"}, }, @@ -114,7 +138,18 @@ func TestBuildShardDeployment(t *testing.T) { }, } - dep, err := BuildShardDeployment(fluxAIODeployment(), 2, cfg) + flux := fluxAIODeployment() + // Capture the source liveness probe verbatim so the startupProbe assertions + // can check inheritance against what flux-aio actually ships, rather than + // against a hardcoded value baked into the test. + var srcLiveness *corev1.Probe + for i := range flux.Spec.Template.Spec.Containers { + if flux.Spec.Template.Spec.Containers[i].Name == "helm-controller" { + srcLiveness = flux.Spec.Template.Spec.Containers[i].LivenessProbe.DeepCopy() + } + } + + dep, err := BuildShardDeployment(flux, 2, cfg) if err != nil { t.Fatal(err) } @@ -188,6 +223,11 @@ func TestBuildShardDeployment(t *testing.T) { // A non-hostNetwork pod dialing the node-local KubePrism endpoint // crashloops on "dial tcp [::1]:7445: connect: connection refused". t.Fatalf("node-local apiserver endpoint env leaked through: %s", e.Name) + case "HTTP_PROXY", "HTTPS_PROXY", "NO_PROXY", + "http_proxy", "https_proxy", "no_proxy": + // A standalone shard behind an unreachable proxy hangs at startup + // and crashloops before it ever serves /healthz. + t.Fatalf("corporate-proxy env leaked through: %s", e.Name) } } envNames := make([]string, 0, len(hc.Env)) @@ -201,6 +241,42 @@ func TestBuildShardDeployment(t *testing.T) { if hc.Resources.Limits.Memory().String() != "1Gi" { t.Fatalf("resources override not applied: %v", hc.Resources) } + + if hc.StartupProbe == nil { + t.Fatal("startupProbe must be added so a slow start is not liveness-killed into a crashloop") + } + // Handler inherited from liveness verbatim. + if hc.StartupProbe.HTTPGet == nil || hc.StartupProbe.HTTPGet.Path != "/healthz" { + t.Fatalf("startupProbe must reuse the liveness /healthz handler: %+v", hc.StartupProbe) + } + // Assert the complete normalised startup budget, not just FailureThreshold: + // a threshold check alone still passes if PeriodSeconds later regresses to 1, + // which would silently restore a short crashloop window. + if hc.StartupProbe.InitialDelaySeconds != 0 || + hc.StartupProbe.PeriodSeconds != 10 || + hc.StartupProbe.SuccessThreshold != 1 || + hc.StartupProbe.FailureThreshold != 30 { + t.Fatalf("startupProbe budget not normalised to the expected contract "+ + "(delay=0 period=10 success=1 failure=30): %+v", hc.StartupProbe) + } + // TimeoutSeconds is inherited from the source liveness probe, never forced. + // flux-aio ships a bare probe (TimeoutSeconds 0), so a stray + // sp.TimeoutSeconds = N would diverge from the source and fail here. + if hc.StartupProbe.TimeoutSeconds != srcLiveness.TimeoutSeconds { + t.Fatalf("startupProbe must inherit the liveness TimeoutSeconds (%d), got %d", + srcLiveness.TimeoutSeconds, hc.StartupProbe.TimeoutSeconds) + } + // The startupProbe must be a DeepCopy of the liveness probe, never an alias. + // If it aliased, normalising the startup FailureThreshold to 30 would also + // stamp 30 onto liveness, producing exactly the never-failing liveness probe + // this change exists to avoid. + if hc.LivenessProbe == hc.StartupProbe { + t.Fatal("startupProbe must be a DeepCopy of liveness, not an alias sharing its backing probe") + } + if hc.LivenessProbe.FailureThreshold != srcLiveness.FailureThreshold { + t.Fatalf("liveness FailureThreshold was clobbered by the startup budget "+ + "(alias regression): source=%d live=%d", srcLiveness.FailureThreshold, hc.LivenessProbe.FailureThreshold) + } } func TestBuildShardDeploymentInheritsResourcesWhenUnset(t *testing.T) { diff --git a/packages/apps/clickhouse/images/altinity-clickhouse-backup.tag b/packages/apps/clickhouse/images/altinity-clickhouse-backup.tag index 238996cb2b..dd5d71c9eb 100644 --- a/packages/apps/clickhouse/images/altinity-clickhouse-backup.tag +++ b/packages/apps/clickhouse/images/altinity-clickhouse-backup.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/altinity-clickhouse-backup:v1.5.0@sha256:65d15779d062680ddb69210c1dfaf8920ea4be2beb069d028bcb5934870a4d72 +ghcr.io/cozystack/cozystack/altinity-clickhouse-backup:v1.5.4@sha256:65d15779d062680ddb69210c1dfaf8920ea4be2beb069d028bcb5934870a4d72 diff --git a/packages/apps/clickhouse/images/clickhouse-backup.tag b/packages/apps/clickhouse/images/clickhouse-backup.tag index 1a935dd581..f4927f42bb 100644 --- a/packages/apps/clickhouse/images/clickhouse-backup.tag +++ b/packages/apps/clickhouse/images/clickhouse-backup.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/clickhouse-backup:v1.5.0@sha256:8efef62d59fb82d39544823f81a797c2b9ac130a50241fee48e701161ce2670d +ghcr.io/cozystack/cozystack/clickhouse-backup:v1.5.4@sha256:1714da5dc14b58866d1e21bed8934d385bfe4c9873c8840b3b4be4a1abfe5f5d diff --git a/packages/apps/harbor/templates/harbor.yaml b/packages/apps/harbor/templates/harbor.yaml index 72bfa61a4c..8cb8b38c08 100644 --- a/packages/apps/harbor/templates/harbor.yaml +++ b/packages/apps/harbor/templates/harbor.yaml @@ -23,6 +23,46 @@ {{- end }} {{- end }} +{{- /* + storageClassName on a PVC is immutable, and a StatefulSet's volumeClaimTemplate + cannot be patched at all. The upstream chart renders both the jobservice PVC and + the trivy VCT without a storageClassName unless one is passed, so on first install + the default-StorageClass admission stamps the cluster default onto the live object. + To avoid an immutable-field rejection on upgrade we keep the rendered value equal + to the live one: preserve an existing object's storageClassName, and only fall back + to the configured storageClass for a fresh install. See issue #2368. + + A live object pinned to an explicit "" (no dynamic provisioning) must round-trip + too: upstream maps the "-" sentinel back to storageClassName: "", so forward "-" + rather than letting "" collapse to an omitted field. An object carrying no + storageClassName key at all keeps the empty fallback so the field stays omitted, + matching the live object. +*/}} +{{- $jobservicePvc := lookup "v1" "PersistentVolumeClaim" .Release.Namespace (printf "%s-jobservice" .Release.Name) }} +{{- $jobserviceStorageClass := .Values.storageClass }} +{{- if $jobservicePvc }} + {{- $jobserviceSpec := $jobservicePvc.spec | default dict }} + {{- if hasKey $jobserviceSpec "storageClassName" }} + {{- $jobserviceStorageClass = $jobserviceSpec.storageClassName | default "-" }} + {{- else }} + {{- $jobserviceStorageClass = "" }} + {{- end }} +{{- end }} + +{{- /* The trivy StatefulSet's VCT is named "data", so its PVCs are + data--trivy-0..N; the class is identical across ordinals, so the + first ordinal is representative. */}} +{{- $trivyPvc := lookup "v1" "PersistentVolumeClaim" .Release.Namespace (printf "data-%s-trivy-0" .Release.Name) }} +{{- $trivyStorageClass := .Values.storageClass }} +{{- if $trivyPvc }} + {{- $trivySpec := $trivyPvc.spec | default dict }} + {{- if hasKey $trivySpec "storageClassName" }} + {{- $trivyStorageClass = $trivySpec.storageClassName | default "-" }} + {{- else }} + {{- $trivyStorageClass = "" }} + {{- end }} +{{- end }} + apiVersion: v1 kind: Secret metadata: @@ -108,13 +148,20 @@ spec: bucket: {{ .Release.Name }}-registry secure: false v4auth: true - {{- if .Values.trivy.enabled }} + {{- if or $jobserviceStorageClass .Values.trivy.enabled }} persistentVolumeClaim: + {{- with $jobserviceStorageClass }} + jobservice: + jobLog: + storageClass: {{ . | quote }} + {{- end }} + {{- if .Values.trivy.enabled }} trivy: size: {{ .Values.trivy.size }} - {{- with .Values.storageClass }} - storageClass: {{ . }} + {{- with $trivyStorageClass }} + storageClass: {{ . | quote }} {{- end }} + {{- end }} {{- end }} portal: resources: {{- include "cozy-lib.resources.defaultingSanitize" (list "t1.nano" (dict) $) | nindent 10 }} diff --git a/packages/apps/harbor/tests/jobservice_storageclass_test.yaml b/packages/apps/harbor/tests/jobservice_storageclass_test.yaml new file mode 100644 index 0000000000..e1aac99339 --- /dev/null +++ b/packages/apps/harbor/tests/jobservice_storageclass_test.yaml @@ -0,0 +1,64 @@ +suite: jobservice storageClass propagation + +release: + name: harbor + namespace: tenant-test + +# Regression test for issue #2368: the jobservice PVC must receive the +# configured storageClass so it is created with an explicit storageClassName +# instead of relying on the default-StorageClass admission. Because PVC +# storageClassName is immutable, the template preserves an existing PVC's +# class when one is found; under `helm template`/unittest the cluster lookup +# returns empty, so these cases exercise the fresh-install fallback path. +# +# The rendered HelmRelease (documentIndex 1) carries the values that the +# cozy-harbor chart forwards to the upstream harbor subchart. +# helm-unittest renders every template in the chart, so the platform-injected +# globals consumed by the ingress/httproute/bucket templates must be supplied +# even though only templates/harbor.yaml is asserted on. +set: + _namespace: + host: example.org + ingress: nginx + gateway: "" + seaweedfs: seaweedfs + _cluster: + solver: http01 + +tests: + - it: forwards storageClass to jobservice jobLog on a fresh install + template: templates/harbor.yaml + documentIndex: 1 + set: + storageClass: local-3rep + asserts: + - equal: + path: spec.values.harbor.persistence.persistentVolumeClaim.jobservice.jobLog.storageClass + value: local-3rep + + - it: still forwards storageClass to trivy alongside jobservice + template: templates/harbor.yaml + documentIndex: 1 + set: + storageClass: local-3rep + asserts: + - equal: + path: spec.values.harbor.persistence.persistentVolumeClaim.trivy.storageClass + value: local-3rep + + - it: omits the jobservice block when storageClass is unset + template: templates/harbor.yaml + documentIndex: 1 + asserts: + - notExists: + path: spec.values.harbor.persistence.persistentVolumeClaim.jobservice + + - it: emits no persistentVolumeClaim block when storageClass is unset and trivy is disabled + template: templates/harbor.yaml + documentIndex: 1 + set: + trivy: + enabled: false + asserts: + - notExists: + path: spec.values.harbor.persistence.persistentVolumeClaim diff --git a/packages/apps/http-cache/images/nginx-cache.tag b/packages/apps/http-cache/images/nginx-cache.tag index 2439140c8a..245159e46f 100644 --- a/packages/apps/http-cache/images/nginx-cache.tag +++ b/packages/apps/http-cache/images/nginx-cache.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/nginx-cache:v1.5.0@sha256:6468799a55dacc6026198db15fa0c8add516cc481e89c47d107427b0f2ea1626 +ghcr.io/cozystack/cozystack/nginx-cache:v1.5.4@sha256:1ec93d5ce6f30f3bfd21a24236162883ac6d77afce7a03d234bd893ce7f4ab86 diff --git a/packages/apps/kafka/templates/kafka.yaml b/packages/apps/kafka/templates/kafka.yaml index 49876bacf7..b2a65d24f8 100644 --- a/packages/apps/kafka/templates/kafka.yaml +++ b/packages/apps/kafka/templates/kafka.yaml @@ -97,8 +97,25 @@ spec: name: {{ .Release.Name }}-metrics key: kafka-metrics-config.yml entityOperator: - topicOperator: {} - userOperator: {} + # Tenant namespaces ship a LimitRange that defaults containers to 128Mi. + # The topic-operator and user-operator are JVM processes and OOMKill at + # that default, leaving the entity-operator in CrashLoopBackOff so + # KafkaTopics/KafkaUsers never reconcile. Set explicit resources so they + # don't inherit the namespace default. + topicOperator: + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + memory: 512Mi + userOperator: + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + memory: 512Mi template: # `pod` accepts metadata + a flat set of pod-spec fields # (enableServiceLinks, securityContext, affinity, …) directly. diff --git a/packages/apps/kafka/tests/entityoperator_template_test.yaml b/packages/apps/kafka/tests/entityoperator_template_test.yaml index 95b7921387..b179400130 100644 --- a/packages/apps/kafka/tests/entityoperator_template_test.yaml +++ b/packages/apps/kafka/tests/entityoperator_template_test.yaml @@ -20,3 +20,15 @@ tests: # the phantom template.spec key must not reappear - notExists: path: spec.entityOperator.template.spec + + - it: topic-operator and user-operator set explicit resources (avoid 128Mi LimitRange OOM) + release: + name: test-kafka + namespace: tenant-test + asserts: + - equal: + path: spec.entityOperator.topicOperator.resources.limits.memory + value: 512Mi + - equal: + path: spec.entityOperator.userOperator.resources.limits.memory + value: 512Mi diff --git a/packages/apps/kubernetes/Makefile b/packages/apps/kubernetes/Makefile index c2f44309b5..186e564296 100644 --- a/packages/apps/kubernetes/Makefile +++ b/packages/apps/kubernetes/Makefile @@ -20,19 +20,30 @@ update: image: image-ubuntu-container-disk image-kubevirt-cloud-provider image-kubevirt-csi-driver image-cluster-autoscaler +# One container disk per Kubernetes minor. Each build spends ~4-5min inside a +# libguestfs appliance installing that minor's kubelet/kubeadm into its own copy +# of the cloud image, and the versions share no per-version work, so build them +# concurrently: the serial loop this replaced spent ~28min of the Build job's +# 30min budget on this target alone. The sub-make carries its own -j so the root +# `make build` stays serial, and buildkit dedups the shared guestfish and +# cloud-image stages across the concurrent solves. --output-sync=target groups +# each version's log under its target name -- six interleaved buildx progress +# streams are unreadable otherwise. image-ubuntu-container-disk: - $(foreach ver,$(KUBERNETES_VERSIONS), \ - docker buildx build images/ubuntu-container-disk \ - --build-arg KUBERNETES_VERSION=$(ver) \ - --tag $(REGISTRY)/ubuntu-container-disk:$(ver)-$(IMAGE_TAG) \ - $(if $(filter 1,$(PUBLISH_VERSIONED)),--tag $(REGISTRY)/ubuntu-container-disk:$(ver)) \ - $(call cache-args,ubuntu-container-disk,$(ver)-buildcache) \ - --metadata-file images/ubuntu-container-disk-$(ver).json \ - $(BUILDX_ARGS) && \ - echo "$(REGISTRY)/ubuntu-container-disk:$(ver)-$(IMAGE_TAG)@$$(yq e '."containerimage.digest"' images/ubuntu-container-disk-$(ver).json -o json -r)" \ - > images/ubuntu-container-disk-$(ver).tag && \ - rm -f images/ubuntu-container-disk-$(ver).json; \ - ) + $(MAKE) -j$(words $(KUBERNETES_VERSIONS)) --output-sync=target \ + $(addprefix image-ubuntu-container-disk-,$(KUBERNETES_VERSIONS)) + +image-ubuntu-container-disk-%: + docker buildx build images/ubuntu-container-disk \ + --build-arg KUBERNETES_VERSION=$* \ + --tag $(REGISTRY)/ubuntu-container-disk:$*-$(IMAGE_TAG) \ + $(if $(filter 1,$(PUBLISH_VERSIONED)),--tag $(REGISTRY)/ubuntu-container-disk:$*) \ + $(call cache-args,ubuntu-container-disk,$*-buildcache) \ + --metadata-file images/ubuntu-container-disk-$*.json \ + $(BUILDX_ARGS) + echo "$(REGISTRY)/ubuntu-container-disk:$*-$(IMAGE_TAG)@$$(yq e '."containerimage.digest"' images/ubuntu-container-disk-$*.json -o json -r)" \ + > images/ubuntu-container-disk-$*.tag + rm -f images/ubuntu-container-disk-$*.json image-kubevirt-cloud-provider: docker buildx build images/kubevirt-cloud-provider \ diff --git a/packages/apps/kubernetes/README.md b/packages/apps/kubernetes/README.md index 1912419f02..52417f8635 100644 --- a/packages/apps/kubernetes/README.md +++ b/packages/apps/kubernetes/README.md @@ -100,29 +100,29 @@ See the reference for components utilized in this service: ### Application-specific Parameters -| Name | Description | Type | Value | -| ----------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------- | ----------- | -| `nodeGroups` | Worker nodes configuration map. | `map[string]object` | `{...}` | -| `nodeGroups[name].minReplicas` | Minimum number of replicas. | `int` | `0` | -| `nodeGroups[name].maxReplicas` | Maximum number of replicas. | `int` | `10` | -| `nodeGroups[name].instanceType` | Virtual machine instance type. | `string` | `u1.medium` | -| `nodeGroups[name].diskSize` | Persistent disk size for kubelet and containerd data. | `quantity` | `20Gi` | -| `nodeGroups[name].storageClass` | StorageClass for worker node persistent disks. When empty, uses the management cluster default StorageClass (the one annotated storageclass.kubernetes.io/is-default-class: true). NOTE: deliberately not marked immutable — the field is optional and undefaulted, so a strict `self == oldSelf` rule would block any future attempt to set it on an existing node group. | `string` | `""` | -| `nodeGroups[name].roles` | List of node roles. | `[]string` | `[]` | -| `nodeGroups[name].resources` | CPU and memory resources for each worker node. | `object` | `{}` | -| `nodeGroups[name].resources.cpu` | CPU available. | `quantity` | `""` | -| `nodeGroups[name].resources.memory` | Memory (RAM) available. | `quantity` | `""` | -| `nodeGroups[name].gpus` | List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM). | `[]object` | `[]` | -| `nodeGroups[name].gpus[i].name` | Name of GPU, such as "nvidia.com/AD102GL_L40S". | `string` | `""` | -| `nodeGroups[name].kubelet` | Kubelet resource reservations for this node group. | `object` | `{}` | -| `nodeGroups[name].kubelet.systemReservedMemory` | Memory reserved for host OS. Auto-computed from instanceType if empty. | `string` | `""` | -| `nodeGroups[name].kubelet.kubeReservedMemory` | Memory reserved for kubelet and container runtime. Auto-computed from instanceType if empty. | `string` | `""` | -| `nodeGroups[name].kubelet.systemReservedCpu` | CPU reserved for host OS. Auto-computed from instanceType if empty. | `string` | `""` | -| `nodeGroups[name].kubelet.kubeReservedCpu` | CPU reserved for kubelet and container runtime. Auto-computed from instanceType if empty. | `string` | `""` | -| `nodeGroups[name].kubelet.evictionHardMemory` | Hard eviction threshold for memory (absolute like 200Mi or percentage like 7%). | `string` | `7%` | -| `nodeGroups[name].kubelet.evictionSoftMemory` | Soft eviction threshold for memory (absolute like 1Gi or percentage like 10%). | `string` | `10%` | -| `version` | Kubernetes major.minor version to deploy | `string` | `v1.35` | -| `host` | External hostname for Kubernetes cluster. Defaults to `.` if empty. | `string` | `""` | +| Name | Description | Type | Value | +| ----------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------- | ----------- | +| `nodeGroups` | Worker nodes configuration map. | `map[string]object` | `{...}` | +| `nodeGroups[name].minReplicas` | Minimum number of replicas. | `int` | `0` | +| `nodeGroups[name].maxReplicas` | Maximum number of replicas. | `int` | `10` | +| `nodeGroups[name].instanceType` | Virtual machine instance type. | `string` | `u1.medium` | +| `nodeGroups[name].diskSize` | Persistent disk size for kubelet and containerd data. | `quantity` | `20Gi` | +| `nodeGroups[name].storageClass` | StorageClass for worker node persistent disks. When empty, uses the management cluster default StorageClass (the one annotated storageclass.kubernetes.io/is-default-class: true). NOTE: deliberately not marked immutable — the field is optional and undefaulted, so a strict `self == oldSelf` rule would block any future attempt to set it on an existing node group. | `string` | `""` | +| `nodeGroups[name].roles` | List of node roles. | `[]string` | `[]` | +| `nodeGroups[name].resources` | Explicit CPU and memory for each worker node, as an alternative to `instanceType` sizing. Optional: when omitted, the node is sized by `instanceType`. When both `cpu` and `memory` are set, they take precedence and `instanceType` is ignored for that node group (the instancetype is omitted from the VM, since KubeVirt cannot override an instancetype's CPU/memory). Set both `cpu` and `memory` together or neither; setting only one is rejected at render time. | `object` | `{}` | +| `nodeGroups[name].resources.cpu` | CPU available. | `quantity` | `""` | +| `nodeGroups[name].resources.memory` | Memory (RAM) available. | `quantity` | `""` | +| `nodeGroups[name].gpus` | List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM). | `[]object` | `[]` | +| `nodeGroups[name].gpus[i].name` | Name of GPU, such as "nvidia.com/AD102GL_L40S". | `string` | `""` | +| `nodeGroups[name].kubelet` | Kubelet resource reservations for this node group. | `object` | `{}` | +| `nodeGroups[name].kubelet.systemReservedMemory` | Memory reserved for host OS. Auto-computed from instanceType if empty. | `string` | `""` | +| `nodeGroups[name].kubelet.kubeReservedMemory` | Memory reserved for kubelet and container runtime. Auto-computed from instanceType if empty. | `string` | `""` | +| `nodeGroups[name].kubelet.systemReservedCpu` | CPU reserved for host OS. Auto-computed from instanceType if empty. | `string` | `""` | +| `nodeGroups[name].kubelet.kubeReservedCpu` | CPU reserved for kubelet and container runtime. Auto-computed from instanceType if empty. | `string` | `""` | +| `nodeGroups[name].kubelet.evictionHardMemory` | Hard eviction threshold for memory (absolute like 200Mi or percentage like 7%). | `string` | `7%` | +| `nodeGroups[name].kubelet.evictionSoftMemory` | Soft eviction threshold for memory (absolute like 1Gi or percentage like 10%). | `string` | `10%` | +| `version` | Kubernetes major.minor version to deploy | `string` | `v1.35` | +| `host` | External hostname for Kubernetes cluster. Defaults to `.` if empty. | `string` | `""` | ### Cluster Addons diff --git a/packages/apps/kubernetes/images/cluster-autoscaler.tag b/packages/apps/kubernetes/images/cluster-autoscaler.tag index c7c07c5705..aaca7d5fd6 100644 --- a/packages/apps/kubernetes/images/cluster-autoscaler.tag +++ b/packages/apps/kubernetes/images/cluster-autoscaler.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/cluster-autoscaler:v1.5.0@sha256:1767a51cf78d6700cb0b39ea1562b1443421afa416cb6d85e724225499859616 +ghcr.io/cozystack/cozystack/cluster-autoscaler:v1.5.4@sha256:b4ac67d2f1ba9557a5e7d0d53ab4daf5efd2e78e441cf7cb5d99e8d54cd2f176 diff --git a/packages/apps/kubernetes/images/kubevirt-cloud-provider.tag b/packages/apps/kubernetes/images/kubevirt-cloud-provider.tag index b0c55db51d..9c217e54b8 100644 --- a/packages/apps/kubernetes/images/kubevirt-cloud-provider.tag +++ b/packages/apps/kubernetes/images/kubevirt-cloud-provider.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/kubevirt-cloud-provider:v1.5.0@sha256:c7224442a074a6a5e7af7ddec14be6cabf8ae9dd2569bfa38794a0dce9188ef9 +ghcr.io/cozystack/cozystack/kubevirt-cloud-provider:v1.5.4@sha256:88f04b08cda502fdcdd90f2bdc429a8300eb8580a202ad26bbeca51bb9bd6431 diff --git a/packages/apps/kubernetes/images/kubevirt-csi-driver.tag b/packages/apps/kubernetes/images/kubevirt-csi-driver.tag index 3aa37f76bd..361582ee7c 100644 --- a/packages/apps/kubernetes/images/kubevirt-csi-driver.tag +++ b/packages/apps/kubernetes/images/kubevirt-csi-driver.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/kubevirt-csi-driver:v1.5.0@sha256:dad088fc04cc54af6d5f487ca9296669331a5345f143ed64cd9027b1bb20b5de +ghcr.io/cozystack/cozystack/kubevirt-csi-driver:v1.5.4@sha256:51b9a6c0b59dcb6dbffcbdeec613f1106a0f4bf8e5009ee41e64fea6dab21f8d diff --git a/packages/apps/kubernetes/images/ubuntu-container-disk-v1.30.tag b/packages/apps/kubernetes/images/ubuntu-container-disk-v1.30.tag index f47b874a15..af83f18b57 100644 --- a/packages/apps/kubernetes/images/ubuntu-container-disk-v1.30.tag +++ b/packages/apps/kubernetes/images/ubuntu-container-disk-v1.30.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/ubuntu-container-disk:v1.30-v1.5.0@sha256:98b80b2d18cb1384c96b95e1c8b869618f1a6e21962dcc1ab2dc874e5dff71dd +ghcr.io/cozystack/cozystack/ubuntu-container-disk:v1.30-v1.5.4@sha256:c4a4fb57a7fc921eea1941f825b9494791c6c73e5bec596b1a296cfae538e17f diff --git a/packages/apps/kubernetes/images/ubuntu-container-disk-v1.31.tag b/packages/apps/kubernetes/images/ubuntu-container-disk-v1.31.tag index a6fae51679..dfab7d5390 100644 --- a/packages/apps/kubernetes/images/ubuntu-container-disk-v1.31.tag +++ b/packages/apps/kubernetes/images/ubuntu-container-disk-v1.31.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/ubuntu-container-disk:v1.31-v1.5.0@sha256:dd0094f5b43e00538ca236110f89792cb5aafa234626518f3bdae353b0060f70 +ghcr.io/cozystack/cozystack/ubuntu-container-disk:v1.31-v1.5.4@sha256:0b9349137056e05bd99ede79fc650e293aa296b65126fa7a916a299096606340 diff --git a/packages/apps/kubernetes/images/ubuntu-container-disk-v1.32.tag b/packages/apps/kubernetes/images/ubuntu-container-disk-v1.32.tag index 01e417d554..658eec68a5 100644 --- a/packages/apps/kubernetes/images/ubuntu-container-disk-v1.32.tag +++ b/packages/apps/kubernetes/images/ubuntu-container-disk-v1.32.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/ubuntu-container-disk:v1.32-v1.5.0@sha256:99f400892d464afc66289c7ac7f9db3e50ef664376dba262fd181252498622a9 +ghcr.io/cozystack/cozystack/ubuntu-container-disk:v1.32-v1.5.4@sha256:7042a8ad12a73c5a18002eae195a888cbda5b21251043c27faf8d11bec9cb3d2 diff --git a/packages/apps/kubernetes/images/ubuntu-container-disk-v1.33.tag b/packages/apps/kubernetes/images/ubuntu-container-disk-v1.33.tag index f4603e64fa..ee04e54950 100644 --- a/packages/apps/kubernetes/images/ubuntu-container-disk-v1.33.tag +++ b/packages/apps/kubernetes/images/ubuntu-container-disk-v1.33.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/ubuntu-container-disk:v1.33-v1.5.0@sha256:57ca7239396346ff55293588f8f3b10c11a8e8bd0a9f0198c05343582db6c184 +ghcr.io/cozystack/cozystack/ubuntu-container-disk:v1.33-v1.5.4@sha256:7cb962b00258485ae576d0ce41209fb6209a41ac617cb1e89a1116210e9bb791 diff --git a/packages/apps/kubernetes/images/ubuntu-container-disk-v1.34.tag b/packages/apps/kubernetes/images/ubuntu-container-disk-v1.34.tag index 23761242e6..fa782e44dd 100644 --- a/packages/apps/kubernetes/images/ubuntu-container-disk-v1.34.tag +++ b/packages/apps/kubernetes/images/ubuntu-container-disk-v1.34.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/ubuntu-container-disk:v1.34-v1.5.0@sha256:1f0af945d8c42af3bef9e17149e1530e3afc4d95ff79434dc81ba0fe729d7574 +ghcr.io/cozystack/cozystack/ubuntu-container-disk:v1.34-v1.5.4@sha256:3a084dfad0171d9779b56ac36331521088f3506b2bf00da9c115848df4e37272 diff --git a/packages/apps/kubernetes/images/ubuntu-container-disk-v1.35.tag b/packages/apps/kubernetes/images/ubuntu-container-disk-v1.35.tag index 325da2bdea..c7dba3886d 100644 --- a/packages/apps/kubernetes/images/ubuntu-container-disk-v1.35.tag +++ b/packages/apps/kubernetes/images/ubuntu-container-disk-v1.35.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/ubuntu-container-disk:v1.35-v1.5.0@sha256:aaf3993143e6f1f60a5e758dcb8e134a2c56c0bef3b44763dd097a5134b12cf2 +ghcr.io/cozystack/cozystack/ubuntu-container-disk:v1.35-v1.5.4@sha256:cac4d5ed8a4178761309bb308f961c990f5c0521d7bf943a601deee498a07112 diff --git a/packages/apps/kubernetes/templates/cluster.yaml b/packages/apps/kubernetes/templates/cluster.yaml index 0cc8d17554..cd6884edf0 100644 --- a/packages/apps/kubernetes/templates/cluster.yaml +++ b/packages/apps/kubernetes/templates/cluster.yaml @@ -25,10 +25,16 @@ spec: node-role.kubernetes.io/{{ . }}: "" {{- end }} spec: - {{- with .group.instanceType }} + {{- /* Explicit resources take precedence over instanceType. When a node + group is sized by resources (both cpu and memory set), the + instancetype is omitted from the VM: KubeVirt rejects a + VirtualMachine that references an instancetype and also overrides + either domain.cpu or domain.memory. */}} + {{- $sizedByResources := and .group.resources .group.resources.cpu .group.resources.memory }} + {{- if and .group.instanceType (not $sizedByResources) }} instancetype: kind: VirtualMachineClusterInstancetype - name: {{ . }} + name: {{ .group.instanceType }} {{- end }} runStrategy: Always dataVolumeTemplates: @@ -238,9 +244,24 @@ metadata: name: {{ .Release.Name }} namespace: {{ .Release.Namespace }} {{- range $groupName, $group := .Values.nodeGroups }} +{{- /* instanceType and explicit resources are alternative sizing sources, and + explicit resources win. KubeVirt rejects a VirtualMachine that references + an instancetype while also overriding either domain.cpu or domain.memory, + so when a group is sized by resources (both cpu and memory set) the + instancetype is omitted from the VM (see the VM spec above) and the lookup + below is skipped — a stale instancetype name cannot fail an otherwise + valid, resource-sized group. Explicit resources are all-or-nothing: a + partial spec (only one of cpu/memory) would emit one domain override + alongside the instancetype, which KubeVirt rejects, so it fails here with + a clear render-time error instead of a late admission rejection. */}} +{{- $hasCpu := and $group.resources $group.resources.cpu }} +{{- $hasMem := and $group.resources $group.resources.memory }} +{{- if and $group.instanceType (or (and $hasCpu (not $hasMem)) (and $hasMem (not $hasCpu))) }} +{{- fail (printf "nodeGroup %s: set both resources.cpu and resources.memory, or neither" $groupName) }} +{{- end }} {{/* Resolve instanceType once per group — reused for kubelet reservations and capacity annotations. */}} {{- $instanceType := dict }} -{{- if $group.instanceType }} +{{- if and $group.instanceType (not (and $hasCpu $hasMem)) }} {{- $instanceType = (lookup "instancetype.kubevirt.io/v1beta1" "VirtualMachineClusterInstancetype" "" $group.instanceType) }} {{- if not $instanceType }} {{- fail (printf "nodeGroup %s: specified instanceType %q not found in cluster" $groupName $group.instanceType) }} @@ -411,6 +432,33 @@ kind: KubeadmConfigTemplate metadata: name: {{ $.Release.Name }}-{{ $groupName }} namespace: {{ $.Release.Namespace }} + annotations: + {{- /* 1.6 drops this object from the chart entirely — workers move to + TalosConfigTemplate. On that upgrade Helm sees it in the previous + release manifest and absent from the new one and deletes it, while the + kubeadm-backed MachineSet is still mid-rollover with its + bootstrap.configRef pointing here: controller-manager floods with + reconcile errors and workers can hang pending with nothing to bootstrap + from. Being born with keep means the object is already protected no + matter which migration path a cluster takes to 1.6 — platform migration + 45 pins the templates that already exist, and this covers every one + created after that migration has run. + + Safe on the uninstall path, which is the only thing keep changes here: + CAPI stamps an ownerReference to the Cluster on this object (verified on + a live v1.5 stand, alongside MachineDeployment and + KubevirtMachineTemplate), and the Cluster itself is Helm-managed with no + keep. So `helm uninstall` deletes the Cluster and Kubernetes garbage + collection reclaims this template through that ownerReference. keep only + suppresses Helm's own delete; it has no bearing on owner-driven GC. + + The name is deterministic — -, not content-hashed + like KubevirtMachineTemplate above — so keep cannot accumulate a copy + per upgrade. The one residue is a removed nodeGroup, whose template + Helm no longer prunes; it is inert (nothing references it), it is + re-adopted in place if that nodeGroup comes back, and GC reclaims it + with the Cluster. */}} + helm.sh/resource-policy: keep spec: template: spec: diff --git a/packages/apps/kubernetes/templates/helmreleases/victoria-metrics-operator.yaml b/packages/apps/kubernetes/templates/helmreleases/victoria-metrics-operator.yaml index d6c068ca65..476e470d04 100644 --- a/packages/apps/kubernetes/templates/helmreleases/victoria-metrics-operator.yaml +++ b/packages/apps/kubernetes/templates/helmreleases/victoria-metrics-operator.yaml @@ -31,6 +31,8 @@ spec: dependsOn: - name: {{ .Release.Name }}-cilium namespace: {{ .Release.Namespace }} + {{- if .Values.addons.certManager.enabled }} - name: {{ .Release.Name }}-cert-manager-crds namespace: {{ .Release.Namespace }} + {{- end }} {{- end }} diff --git a/packages/apps/kubernetes/tests/cluster_test.yaml b/packages/apps/kubernetes/tests/cluster_test.yaml index 73ffa5d480..5ebff7cbd8 100644 --- a/packages/apps/kubernetes/tests/cluster_test.yaml +++ b/packages/apps/kubernetes/tests/cluster_test.yaml @@ -388,6 +388,105 @@ tests: - failedTemplate: errorMessage: 'nodeGroup "md0": ephemeralStorage is no longer supported and should have been automatically migrated to diskSize by platform migration 41. If you see this error after upgrading, the migration did not run — check the cozystack-migration-hook Job logs in cozy-system.' + ############################################### + # KubeadmConfigTemplate — Helm prune guard # + ############################################### + + # THE THREAT. 1.6 drops KubeadmConfigTemplate from this chart entirely — + # workers move to TalosConfigTemplate. On that upgrade Helm sees the object in + # the previous release manifest and absent from the new one and deletes it, + # while the kubeadm-backed MachineSet is still mid-rollover with its + # bootstrap.configRef pointing at it: controller-manager floods with reconcile + # errors and workers can hang pending with nothing to bootstrap from. + # + # Platform migration 45 pins the templates that already exist when a cluster + # upgrades. This annotation covers the ones created AFTERWARDS — a new tenant + # Kubernetes cluster, or a nodeGroup added on 1.5.4 — which that migration has + # already run past and which 1.6's own slot 45 will skip, because a 1.5.4 + # cluster is stamped 46 and runs `seq 46 53`. + # + # Safe on uninstall, the only path keep changes: CAPI stamps an ownerReference + # to the Cluster on this object, and the Cluster is Helm-managed with no keep, + # so `helm uninstall` deletes the Cluster and garbage collection reclaims this + # template through that reference. keep suppresses only Helm's own delete. + + - it: is born with helm.sh/resource-policy=keep so the 1.6 upgrade cannot prune it + release: + name: test-k8s + namespace: tenant-test + asserts: + - isKind: + of: KubeadmConfigTemplate + documentIndex: 4 + - equal: + path: metadata.name + value: test-k8s-md0 + documentIndex: 4 + - equal: + path: metadata.annotations["helm.sh/resource-policy"] + value: keep + documentIndex: 4 + + # The annotation must sit inside the per-nodeGroup range, not on one template by + # accident. With two groups every KubeadmConfigTemplate has to carry it, or the + # groups added later are exactly the ones left prunable. + - it: pins the KubeadmConfigTemplate of every node group, not just the first + release: + name: test-k8s + namespace: tenant-test + set: + _namespace: + etcd: etcd + ingress: nginx + host: example.com + _cluster: + cluster-domain: cozy.local + nodeGroups: + md0: + minReplicas: 0 + maxReplicas: 10 + instanceType: "" + diskSize: 20Gi + roles: + - worker + resources: {} + md1: + minReplicas: 1 + maxReplicas: 5 + instanceType: "" + diskSize: 40Gi + roles: + - worker + resources: + memory: "8Gi" + cpu: 4 + asserts: + # md0 at documentIndex 4; md1 at documentIndex 9, since each nodeGroup emits + # 5 documents (KubeadmConfigTemplate, KubevirtMachineTemplate, + # MachineDeployment, MachineHealthCheck, WorkloadMonitor). + - isKind: + of: KubeadmConfigTemplate + documentIndex: 4 + - equal: + path: metadata.name + value: test-k8s-md0 + documentIndex: 4 + - equal: + path: metadata.annotations["helm.sh/resource-policy"] + value: keep + documentIndex: 4 + - isKind: + of: KubeadmConfigTemplate + documentIndex: 9 + - equal: + path: metadata.name + value: test-k8s-md1 + documentIndex: 9 + - equal: + path: metadata.annotations["helm.sh/resource-policy"] + value: keep + documentIndex: 9 + ############################################### # KubeadmConfigTemplate — files block # ############################################### @@ -493,3 +592,143 @@ tests: path: spec.template.spec.virtualMachineTemplate.spec.template.metadata.labels["apps.cozystack.io/application.name"] value: foo documentIndex: 5 + + ############################################### + # resources is optional when instanceType set # + ############################################### + + # A node group may be sized either by instanceType or by explicit resources. + # When instanceType is the sizing source, resources may be omitted entirely. + # helm-unittest enforces values.schema.json, so the render below only succeeds + # if the schema does NOT require resources on each node group. + # + # instanceType is set to "" here only to bypass the live-cluster lookup() that + # validates a real VirtualMachineClusterInstancetype; the schema acceptance of + # an omitted resources key is independent of the instanceType value. + # + # The "worker" group sorts after "md0". On this release line each node group + # renders a KubeadmConfigTemplate before its KubevirtMachineTemplate, so the + # per-group document stride is 5 and worker's KubevirtMachineTemplate is + # document index 10 (md0 → 5, worker → 10). + + - it: accepts a node group that omits resources and renders no explicit cpu/memory + release: + name: test-k8s + namespace: tenant-test + set: + nodeGroups: + worker: + minReplicas: 1 + maxReplicas: 3 + instanceType: "" + diskSize: "20Gi" + roles: + - worker + asserts: + - isKind: + of: KubevirtMachineTemplate + documentIndex: 10 + - equal: + path: spec.template.spec.virtualMachineTemplate.spec.template.metadata.labels["cluster.x-k8s.io/deployment-name"] + value: test-k8s-worker + documentIndex: 10 + - notExists: + path: spec.template.spec.virtualMachineTemplate.spec.template.spec.domain.cpu + documentIndex: 10 + - notExists: + path: spec.template.spec.virtualMachineTemplate.spec.template.spec.domain.memory + documentIndex: 10 + + # Complementary case: when explicit resources ARE provided (md0 in common.yaml + # has cpu: 2 / memory: 4Gi), they take effect on the VM domain. Confirms the fix + # only relaxes the schema requirement and does not drop honoured resources. + - it: still renders explicit cpu and memory when resources are provided + release: + name: test-k8s + namespace: tenant-test + asserts: + - isKind: + of: KubevirtMachineTemplate + documentIndex: 5 + - equal: + path: spec.template.spec.virtualMachineTemplate.spec.template.spec.domain.cpu.cores + value: 2 + documentIndex: 5 + - equal: + path: spec.template.spec.virtualMachineTemplate.spec.template.spec.domain.memory.guest + value: 4Gi + documentIndex: 5 + + # instanceType and explicit resources are alternative sizing sources, and + # explicit resources take precedence. When both are set the VM is sized by the + # explicit resources and the instancetype is omitted, so KubeVirt does not + # reject a VM that both references an instancetype and overrides domain.cpu/memory. + # (md0 in common.yaml already carries cpu: 2 / memory: 4Gi; this only adds an + # instanceType on top to exercise the both-set case.) + - it: prefers explicit resources over instanceType when both are set + release: + name: test-k8s + namespace: tenant-test + set: + nodeGroups.md0.instanceType: "u1.medium" + asserts: + - isKind: + of: KubevirtMachineTemplate + documentIndex: 5 + - equal: + path: spec.template.spec.virtualMachineTemplate.spec.template.spec.domain.cpu.cores + value: 2 + documentIndex: 5 + - equal: + path: spec.template.spec.virtualMachineTemplate.spec.template.spec.domain.memory.guest + value: 4Gi + documentIndex: 5 + - notExists: + path: spec.template.spec.virtualMachineTemplate.spec.instancetype + documentIndex: 5 + + # Explicit resources are all-or-nothing when an instanceType is present. A + # partial spec (only cpu, or only memory) on a group that also carries a + # non-empty instanceType would emit one domain override alongside the + # instancetype, which KubeVirt rejects. Fail fast with a clear render-time + # error instead of shipping a VM the admission webhook rejects at apply time. + # (Fresh group names are used so the partial spec is real — a `set` on md0 + # would deep-merge with common.yaml's md0, which already carries both cpu and + # memory.) + - it: fails when only resources.cpu is set without resources.memory + release: + name: test-k8s + namespace: tenant-test + set: + nodeGroups: + mdcpuonly: + minReplicas: 0 + maxReplicas: 10 + instanceType: "u1.medium" + diskSize: 20Gi + roles: + - worker + resources: + cpu: 2 + asserts: + - failedTemplate: + errorMessage: "nodeGroup mdcpuonly: set both resources.cpu and resources.memory, or neither" + + - it: fails when only resources.memory is set without resources.cpu + release: + name: test-k8s + namespace: tenant-test + set: + nodeGroups: + mdmemonly: + minReplicas: 0 + maxReplicas: 10 + instanceType: "u1.medium" + diskSize: 20Gi + roles: + - worker + resources: + memory: 4Gi + asserts: + - failedTemplate: + errorMessage: "nodeGroup mdmemonly: set both resources.cpu and resources.memory, or neither" diff --git a/packages/apps/kubernetes/tests/victoria-metrics-operator_test.yaml b/packages/apps/kubernetes/tests/victoria-metrics-operator_test.yaml new file mode 100644 index 0000000000..d62a995efc --- /dev/null +++ b/packages/apps/kubernetes/tests/victoria-metrics-operator_test.yaml @@ -0,0 +1,77 @@ +suite: victoria-metrics-operator.yaml cert-manager-crds dependency gating + +templates: + - templates/helmreleases/victoria-metrics-operator.yaml + +values: + - values/common.yaml + +# The vmop HelmRelease is gated on monitoringAgents.enabled, but the +# cert-manager-crds HelmRelease it can depend on is only created when +# certManager.enabled is true. The dependsOn entry must therefore be gated on +# the same condition, otherwise the valid combination +# monitoringAgents.enabled=true + certManager.enabled=false leaves vmop blocked +# on a non-existent dependency. + +tests: + - it: renders the vmop HelmRelease when monitoringAgents is enabled + release: + name: kubernetes-test + namespace: tenant-test + set: + addons.monitoringAgents.enabled: true + asserts: + - hasDocuments: + count: 1 + - isKind: + of: HelmRelease + - equal: + path: metadata.name + value: kubernetes-test-cozy-victoria-metrics-operator + + - it: always depends on cilium + release: + name: kubernetes-test + namespace: tenant-test + set: + addons.monitoringAgents.enabled: true + asserts: + - contains: + path: spec.dependsOn + content: + name: kubernetes-test-cilium + namespace: tenant-test + + - it: does NOT depend on cert-manager-crds when certManager is disabled + release: + name: kubernetes-test + namespace: tenant-test + set: + addons.monitoringAgents.enabled: true + addons.certManager.enabled: false + asserts: + - notContains: + path: spec.dependsOn + content: + name: kubernetes-test-cert-manager-crds + namespace: tenant-test + - lengthEqual: + path: spec.dependsOn + count: 1 + + - it: depends on cert-manager-crds when certManager is enabled + release: + name: kubernetes-test + namespace: tenant-test + set: + addons.monitoringAgents.enabled: true + addons.certManager.enabled: true + asserts: + - contains: + path: spec.dependsOn + content: + name: kubernetes-test-cert-manager-crds + namespace: tenant-test + - lengthEqual: + path: spec.dependsOn + count: 2 diff --git a/packages/apps/kubernetes/values.schema.json b/packages/apps/kubernetes/values.schema.json index e1492b787d..db71b285b5 100644 --- a/packages/apps/kubernetes/values.schema.json +++ b/packages/apps/kubernetes/values.schema.json @@ -40,8 +40,7 @@ "diskSize", "instanceType", "maxReplicas", - "minReplicas", - "resources" + "minReplicas" ], "properties": { "diskSize": { @@ -128,7 +127,7 @@ "default": 0 }, "resources": { - "description": "CPU and memory resources for each worker node.", + "description": "Explicit CPU and memory for each worker node, as an alternative to `instanceType` sizing. Optional: when omitted, the node is sized by `instanceType`. When both `cpu` and `memory` are set, they take precedence and `instanceType` is ignored for that node group (the instancetype is omitted from the VM, since KubeVirt cannot override an instancetype's CPU/memory). Set both `cpu` and `memory` together or neither; setting only one is rejected at render time.", "type": "object", "properties": { "cpu": { diff --git a/packages/apps/kubernetes/values.yaml b/packages/apps/kubernetes/values.yaml index 9c1405b550..d8aab8cd1f 100644 --- a/packages/apps/kubernetes/values.yaml +++ b/packages/apps/kubernetes/values.yaml @@ -85,7 +85,7 @@ storageClass: replicated ## @field {string} [storageClass] - StorageClass for worker node persistent disks. When empty, uses the management cluster default StorageClass (the one annotated storageclass.kubernetes.io/is-default-class: true). NOTE: deliberately not marked immutable — the field is optional and undefaulted, so a strict `self == oldSelf` rule would block any future attempt to set it on an existing node group. ## @x-cozystack-options {source: storageclass} ## @field {[]string} roles - List of node roles. -## @field {Resources} resources - CPU and memory resources for each worker node. +## @field {Resources} [resources] - Explicit CPU and memory for each worker node, as an alternative to `instanceType` sizing. Optional: when omitted, the node is sized by `instanceType`. When both `cpu` and `memory` are set, they take precedence and `instanceType` is ignored for that node group (the instancetype is omitted from the VM, since KubeVirt cannot override an instancetype's CPU/memory). Set both `cpu` and `memory` together or neither; setting only one is rejected at render time. ## @field {[]GPU} gpus - List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM). ## @field {Kubelet} [kubelet] - Kubelet resource reservations for this node group. diff --git a/packages/apps/mariadb/Makefile b/packages/apps/mariadb/Makefile index eb12cf44a2..58eb927441 100644 --- a/packages/apps/mariadb/Makefile +++ b/packages/apps/mariadb/Makefile @@ -11,6 +11,10 @@ update: hack/update-versions.sh make generate +.PHONY: test +test: + helm unittest . + image: docker buildx build images/mariadb-backup \ $(call image-tags,mariadb-backup,$(MARIADB_BACKUP_TAG)) \ diff --git a/packages/apps/mariadb/images/mariadb-backup.tag b/packages/apps/mariadb/images/mariadb-backup.tag index 3559372269..cafcedfea9 100644 --- a/packages/apps/mariadb/images/mariadb-backup.tag +++ b/packages/apps/mariadb/images/mariadb-backup.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/mariadb-backup:v1.5.0@sha256:3cdaed3f19531586756902516feb099a5bece89c2c6685754d945486f0e99999 +ghcr.io/cozystack/cozystack/mariadb-backup:v1.5.4@sha256:0ddb3184c0da5a064401d10e48b1839f13ad5c2a82b8d29d6f0b2928322868a5 diff --git a/packages/apps/mariadb/templates/backup-cronjob.yaml b/packages/apps/mariadb/templates/backup-cronjob.yaml index ddb237cddd..9316e37a94 100644 --- a/packages/apps/mariadb/templates/backup-cronjob.yaml +++ b/packages/apps/mariadb/templates/backup-cronjob.yaml @@ -41,7 +41,9 @@ spec: name: {{ .Release.Name }} key: root-password - name: MYSQL_HOST - value: "{{ .Release.Name }}-{{ if eq (int .Values.replicas) 1 }}primary{{ else }}secondary{{ end }}" + # replicas>1 enables replication (offload dumps to a secondary); + # replicas=1 has no replication and only the bare service exists. + value: "{{ .Release.Name }}{{- if gt (int .Values.replicas) 1 }}-secondary{{- end }}" - name: AWS_ACCESS_KEY_ID valueFrom: secretKeyRef: diff --git a/packages/apps/mariadb/templates/dashboard-resourcemap.yaml b/packages/apps/mariadb/templates/dashboard-resourcemap.yaml index af89b39c03..a7c5731ea6 100644 --- a/packages/apps/mariadb/templates/dashboard-resourcemap.yaml +++ b/packages/apps/mariadb/templates/dashboard-resourcemap.yaml @@ -8,8 +8,14 @@ rules: resources: - services resourceNames: + # The bare service exists in both topologies; -primary/-secondary + # only when replication is on (replicas>1). Grant exactly what exists so the + # RBAC matches the services the ApplicationDefinition (mariadb-rd) lists. + - {{ .Release.Name }} + {{- if gt (int .Values.replicas) 1 }} - {{ .Release.Name }}-primary - {{ .Release.Name }}-secondary + {{- end }} verbs: ["get", "list", "watch"] - apiGroups: - "" diff --git a/packages/apps/mariadb/templates/mariadb.yaml b/packages/apps/mariadb/templates/mariadb.yaml index d3d5e6367b..d04f427407 100644 --- a/packages/apps/mariadb/templates/mariadb.yaml +++ b/packages/apps/mariadb/templates/mariadb.yaml @@ -12,6 +12,28 @@ spec: port: 3306 + # The operator leaves failureThreshold unset on the startup probe, so it + # defaults to 3. The kubelet only begins probing once initialDelaySeconds has + # elapsed and kills the container on the Nth consecutive failure, so the + # budget is initialDelay + (failureThreshold - 1) * period, here just 40s -- + # the same budget as the liveness probe the startup probe exists to defer. + # A fresh datadir bootstrap (mysql_install_db, then the entrypoint's + # temporary server applying the root password) does not reliably finish + # inside that budget under a constrained CPU limit or slow replicated + # storage. Overrunning it is not merely a slow start: the kubelet kills the + # container mid-bootstrap, leaving a datadir that already has the mysql + # system tables but never received the root grant. On restart the entrypoint + # keys DATABASE_ALREADY_EXISTS off that directory, skips initialization for + # good, and the server comes up with the install-time root account -- local + # only and passwordless -- instead of the credentials the operator holds. So + # every later probe fails with "Access denied for user 'root'" and the pod + # crash-loops permanently with no path back. A threshold of 30 lifts the + # budget to 310s. Widen only the startup budget; the liveness and readiness + # probes keep the operator defaults, so a server that hangs after a + # successful start is still caught as quickly as before. + startupProbe: + failureThreshold: 30 + replicas: {{ .Values.replicas }} replicasAllowEvenNumber: true affinity: @@ -29,11 +51,13 @@ spec: - {{ .Release.Name }} topologyKey: "kubernetes.io/hostname" + {{- if gt (int .Values.replicas) 1 }} replication: enabled: true #primary: # podIndex: 0 # automaticFailover: true + {{- end }} podMetadata: labels: @@ -63,6 +87,9 @@ spec: metadata: labels: app.kubernetes.io/instance: {{ $.Release.Name }} + {{- if and .Values.external (eq (int .Values.replicas) 1) }} + type: LoadBalancer + {{- end }} storage: size: {{ .Values.size }} resizeInUseVolumes: true @@ -71,7 +98,7 @@ spec: storageClassName: {{ . }} {{- end }} - {{- if .Values.external }} + {{- if and .Values.external (gt (int .Values.replicas) 1) }} primaryService: type: LoadBalancer {{- end }} diff --git a/packages/apps/mariadb/tests/startup_probe_test.yaml b/packages/apps/mariadb/tests/startup_probe_test.yaml new file mode 100644 index 0000000000..d6df7e0f41 --- /dev/null +++ b/packages/apps/mariadb/tests/startup_probe_test.yaml @@ -0,0 +1,52 @@ +suite: MariaDB startup probe budget + +# Regression cover for the widened startup budget. The rationale lives with +# the field itself, in templates/mariadb.yaml. + +templates: + - templates/mariadb.yaml + +tests: + - it: widens the startup probe budget for a single-replica instance + release: + name: mariadb-test + namespace: tenant-test + set: + replicas: 1 + asserts: + - equal: + path: spec.startupProbe.failureThreshold + value: 30 + + - it: widens the startup probe budget for a replicated instance + release: + name: mariadb-test + namespace: tenant-test + set: + replicas: 2 + asserts: + - equal: + path: spec.startupProbe.failureThreshold + value: 30 + + - it: leaves liveness and readiness probes to the operator defaults + release: + name: mariadb-test + namespace: tenant-test + asserts: + - notExists: + path: spec.livenessProbe + - notExists: + path: spec.readinessProbe + + - it: overrides only thresholds so the operator keeps its own probe handler + release: + name: mariadb-test + namespace: tenant-test + asserts: + - notExists: + path: spec.startupProbe.exec + - notExists: + path: spec.startupProbe.httpGet + - notExists: + path: spec.startupProbe.tcpSocket diff --git a/packages/apps/postgres/templates/db.yaml b/packages/apps/postgres/templates/db.yaml index 4e0b468194..c6a0e29411 100644 --- a/packages/apps/postgres/templates/db.yaml +++ b/packages/apps/postgres/templates/db.yaml @@ -189,6 +189,44 @@ spec: key: {{ $credSecretKey }} {{- end }} + {{/* + Every S3 path on this chart runs `barman-cloud-*` inside the instance + pod, and barman-cloud uploads through boto3: + - the chart-rendered spec.backup.barmanObjectStore above (legacy flow); + - the same field SSA-patched onto the live Cluster by the CNPG backup + driver in the platform useSystemBucket=true flow; + - externalClusters[].barmanObjectStore above, used by + bootstrap.recovery and by the RestoreJob flow (which sets + bootstrap.enabled on the Postgres app and lets this chart re-render). + + Since botocore ~1.36 (early 2025) the default RequestChecksumCalculation + is when_supported, so a flexible checksum (and the matching + x-amz-content-sha256 handling) rides on every PutObject. AWS S3 accepts + it, but non-AWS S3-compatible backends - Ceph RADOS Gateway, the + platform's own SeaweedFS system bucket, some MinIO / Cloudflare R2 + builds - reject it with "InvalidArgument: x-amz-content-sha256 must be + UNSIGNED-PAYLOAD, ...", which fails every backup and WAL-archive upload. + when_required computes a checksum only when the operation mandates one; + AWS S3 accepts that on a plain PutObject too, so it is a safe default + everywhere. + + spec.env lands in the instance pods, so the barman-cloud subprocess the + instance manager execs inherits it - one setting covers all of the paths + above. It stays Helm-owned: the backup driver's SSA patch carries only + spec.backup (applyClusterBarmanObjectStore in + internal/backupcontroller/cnpgstrategy_controller.go), so the two field + managers do not contend. + + Gated on backup/bootstrap being enabled rather than set unconditionally: + a Postgres release with no S3 configured never invokes barman-cloud, and + injecting an AWS_* variable into its instance pods would be noise. + */}} + {{- if or .Values.backup.enabled .Values.bootstrap.enabled }} + env: + - name: AWS_REQUEST_CHECKSUM_CALCULATION + value: when_required + {{- end }} + resources: {{- include "cozy-lib.resources.defaultingSanitize" (list .Values.resourcesPreset .Values.resources $) | nindent 4 }} imageName: ghcr.io/cloudnative-pg/postgresql:{{ include "postgres.versionMap" $ | trimPrefix "v" }} diff --git a/packages/apps/postgres/tests/backup_storage_test.yaml b/packages/apps/postgres/tests/backup_storage_test.yaml index 8adab98b1e..af5990319f 100644 --- a/packages/apps/postgres/tests/backup_storage_test.yaml +++ b/packages/apps/postgres/tests/backup_storage_test.yaml @@ -35,6 +35,18 @@ tests: documentSelector: path: kind value: Cluster + # The platform flow's barmanObjectStore is SSA-patched onto the live + # Cluster later, but it runs barman-cloud in these same instance pods, + # so the non-AWS S3 request-checksum pin has to be on the Cluster now. + - contains: + path: spec.env + content: + name: AWS_REQUEST_CHECKSUM_CALCULATION + value: when_required + template: templates/db.yaml + documentSelector: + path: kind + value: Cluster - hasDocuments: count: 0 template: templates/backup-secret.yaml @@ -60,6 +72,18 @@ tests: documentSelector: path: kind value: Cluster + # Non-AWS S3 gateways (Ceph RGW, the platform's own SeaweedFS) reject + # botocore's default flexible request checksum, so barman-cloud's boto3 + # is pinned to when_required via the instance pods' environment. + - contains: + path: spec.env + content: + name: AWS_REQUEST_CHECKSUM_CALCULATION + value: when_required + template: templates/db.yaml + documentSelector: + path: kind + value: Cluster - hasDocuments: count: 1 template: templates/backup-secret.yaml @@ -112,3 +136,50 @@ tests: asserts: - failedTemplate: errorMessage: "postgres: bootstrap.enabled=true requires S3 credentials: set backup.s3AccessKey+backup.s3SecretKey, backup.s3CredentialsSecret.name, or backup.useSystemBucket=true (with a backups.cozystack.io/RestoreJob)." + + - it: "bootstrap recovery with backup disabled: request-checksum pin still rendered" + # externalClusters[].barmanObjectStore drives recovery through the same + # barman-cloud client, so the pin must not hang off backup.enabled alone. + # This is also the RestoreJob flow: the CNPG driver patches the Postgres + # app's bootstrap.enabled and lets this chart re-render. + set: + backup: + enabled: false + useSystemBucket: false + s3AccessKey: AKIAEXAMPLE + s3SecretKey: s3cr3t + bootstrap: + enabled: true + oldName: old + asserts: + - exists: + path: spec.externalClusters + template: templates/db.yaml + documentSelector: + path: kind + value: Cluster + - contains: + path: spec.env + content: + name: AWS_REQUEST_CHECKSUM_CALCULATION + value: when_required + template: templates/db.yaml + documentSelector: + path: kind + value: Cluster + + - it: "no backup and no bootstrap: no AWS_* env injected into instance pods" + # A release with no S3 configured never invokes barman-cloud, so it gets + # no spec.env at all - the pin is scoped, not unconditional. + set: + backup: + enabled: false + bootstrap: + enabled: false + asserts: + - notExists: + path: spec.env + template: templates/db.yaml + documentSelector: + path: kind + value: Cluster diff --git a/packages/apps/tenant/templates/_helpers.tpl b/packages/apps/tenant/templates/_helpers.tpl index 6d687cb863..0f7d064d94 100644 --- a/packages/apps/tenant/templates/_helpers.tpl +++ b/packages/apps/tenant/templates/_helpers.tpl @@ -53,6 +53,38 @@ typing, so the "key absent" form is what distinguishes "unset" from explicit `false`. */}} +{{/* + tenant.ancestorTenantLabels emits the full set of + `tenant.cozystack.io/: ""` namespace labels for the tenant whose + namespace name is passed as the single argument (e.g. "tenant-ktj-htdev"). + + Every tenant descends from tenant-root, so that label is always emitted. + The remaining ancestors are encoded in the namespace name itself: tenant.name + constructs each child namespace as `-`, so every + progressive dash-prefix of the name is a real ancestor namespace + (tenant-ktj-htdev -> tenant-ktj, tenant-ktj-htdev). The last prefix is the + tenant's own name, so its self-label is included too. + + Deriving the chain from the name (rather than from a lookup of the parent + namespace's labels) is deterministic: it needs no cluster state, renders + identically offline, and converges on every reconcile regardless of the + order in which parent and child HelmReleases reconcile. It replaces an + earlier splitList over `.Release.Namespace` that only ever reached one level + up and therefore dropped tenant-root for tenants at depth >= 2 — breaking the + `-egress` CiliumClusterwideNetworkPolicy that grants an ancestor + reachability to its descendants via the `tenant.cozystack.io/` + namespace label. +*/}} +{{- define "tenant.ancestorTenantLabels" -}} +{{- $parts := splitList "-" . -}} +tenant.cozystack.io/tenant-root: "" +{{- range $i, $v := $parts }} +{{- if ne $i 0 }} +{{ printf "tenant.cozystack.io/%s: \"\"" (join "-" (slice $parts 0 (add $i 1))) }} +{{- end }} +{{- end }} +{{- end -}} + {{- define "tenant.gatewayEffective" -}} {{- if kindIs "invalid" .Values.gateway -}} false diff --git a/packages/apps/tenant/templates/etcd.yaml b/packages/apps/tenant/templates/etcd.yaml index 71e0b548d6..38da293aa1 100644 --- a/packages/apps/tenant/templates/etcd.yaml +++ b/packages/apps/tenant/templates/etcd.yaml @@ -13,6 +13,15 @@ metadata: apps.cozystack.io/application.group: apps.cozystack.io apps.cozystack.io/application.name: etcd spec: + # This release creates a VMPodScrape resource guarded by the + # victoria-metrics-operator admission webhook (failurePolicy: Fail). On a cold + # install the operator can be briefly unavailable on :9443, so gate on its + # HelmRelease becoming Ready to avoid an install failure → rollback. The system + # monitoring components gate on the operator the same way, via a variant-level + # dependsOn in packages/core/platform/sources/monitoring.yaml. + dependsOn: + - name: victoria-metrics-operator + namespace: cozy-victoria-metrics-operator chartRef: kind: ExternalArtifact name: cozystack-etcd-application-default-etcd diff --git a/packages/apps/tenant/templates/ingress.yaml b/packages/apps/tenant/templates/ingress.yaml index 229ee7c62d..b5ed2288f2 100644 --- a/packages/apps/tenant/templates/ingress.yaml +++ b/packages/apps/tenant/templates/ingress.yaml @@ -13,6 +13,15 @@ metadata: apps.cozystack.io/application.group: apps.cozystack.io apps.cozystack.io/application.name: ingress spec: + # This release creates VMPodScrape resources guarded by the + # victoria-metrics-operator admission webhook (failurePolicy: Fail). On a cold + # install the operator can be briefly unavailable on :9443, so gate on its + # HelmRelease becoming Ready to avoid an install failure → rollback. The system + # monitoring components gate on the operator the same way, via a variant-level + # dependsOn in packages/core/platform/sources/monitoring.yaml. + dependsOn: + - name: victoria-metrics-operator + namespace: cozy-victoria-metrics-operator chartRef: kind: ExternalArtifact name: cozystack-ingress-application-default-ingress diff --git a/packages/apps/tenant/templates/monitoring.yaml b/packages/apps/tenant/templates/monitoring.yaml index 29dbf95ab8..a66b11d45e 100644 --- a/packages/apps/tenant/templates/monitoring.yaml +++ b/packages/apps/tenant/templates/monitoring.yaml @@ -13,6 +13,17 @@ metadata: apps.cozystack.io/application.group: apps.cozystack.io apps.cozystack.io/application.name: monitoring spec: + # This release (via its monitoring-system child) creates VM* custom resources + # guarded by the victoria-metrics-operator admission webhook (failurePolicy: + # Fail). On a cold install the operator can be briefly unavailable on :9443, so + # gate on its HelmRelease becoming Ready to avoid an install failure → rollback. + # Gating the parent keeps the monitoring-system child (which emits the VM* CRs) + # from being created until the operator is Ready. The system monitoring + # components gate on the operator the same way, via a variant-level dependsOn in + # packages/core/platform/sources/monitoring.yaml. + dependsOn: + - name: victoria-metrics-operator + namespace: cozy-victoria-metrics-operator chartRef: kind: ExternalArtifact name: cozystack-monitoring-application-default-monitoring diff --git a/packages/apps/tenant/templates/namespace.yaml b/packages/apps/tenant/templates/namespace.yaml index 6e4ea1cb5c..92ceacab73 100644 --- a/packages/apps/tenant/templates/namespace.yaml +++ b/packages/apps/tenant/templates/namespace.yaml @@ -1,8 +1,14 @@ -{{/* Lookup for namespace uid (needed for ownerReferences) */}} +{{/* + Lookup the parent namespace, used below to stamp the child namespace's + ownerReference uid. During a real helm-controller reconcile the parent + namespace (.Release.Namespace) always exists, so the ownerReference is + always emitted. The lookup returns empty only when rendering offline + (helm template / helm unittest), where the ownerReference is simply omitted + so the rest of the template — notably the ancestor labels below, which are + derived purely from the namespace name and need no cluster state — still + renders and can be unit-tested. +*/}} {{- $existingNS := lookup "v1" "Namespace" "" .Release.Namespace }} -{{- if not $existingNS }} -{{- fail (printf "error lookup existing namespace: %s" .Release.Namespace) }} -{{- end }} {{- if ne (include "tenant.name" .) "tenant-root" }} {{/* Compute namespace values once for use in both Secret and labels */}} @@ -75,13 +81,7 @@ metadata: name: {{ $tenantName }} {{- if hasPrefix "tenant-" .Release.Namespace }} labels: - tenant.cozystack.io/{{ $tenantName }}: "" - {{- $parts := splitList "-" .Release.Namespace }} - {{- range $i, $v := $parts }} - {{- if ne $i 0 }} - tenant.cozystack.io/{{ join "-" (slice $parts 0 (add $i 1)) }}: "" - {{- end }} - {{- end }} + {{- include "tenant.ancestorTenantLabels" $tenantName | nindent 4 }} {{/* Labels for network policies */}} namespace.cozystack.io/etcd: {{ $etcd | quote }} namespace.cozystack.io/ingress: {{ $ingress | quote }} @@ -93,6 +93,7 @@ metadata: scheduler.cozystack.io/scheduling-class: {{ . | quote }} {{- end }} alpha.kubevirt.io/auto-memory-limits-ratio: "1.0" + {{- if $existingNS }} ownerReferences: - apiVersion: v1 blockOwnerDeletion: true @@ -101,6 +102,7 @@ metadata: name: {{ .Release.Namespace }} uid: {{ $existingNS.metadata.uid }} {{- end }} + {{- end }} --- apiVersion: v1 kind: Secret diff --git a/packages/apps/tenant/tests/namespace_ancestor_labels_test.yaml b/packages/apps/tenant/tests/namespace_ancestor_labels_test.yaml new file mode 100644 index 0000000000..773f7a86c9 --- /dev/null +++ b/packages/apps/tenant/tests/namespace_ancestor_labels_test.yaml @@ -0,0 +1,91 @@ +suite: tenant namespace ancestor labels +# Regression cover for the tenant.cozystack.io/ namespace labels +# emitted by namespace.yaml. These labels drive the -egress +# CiliumClusterwideNetworkPolicy (networkpolicy.yaml): an ancestor tenant is +# allowed to reach any namespace carrying tenant.cozystack.io/, so +# every tenant namespace MUST carry the label of each of its ancestors, +# tenant-root included, at every depth. +# +# The labels are derived from the namespace name by tenant.ancestorTenantLabels +# (see _helpers.tpl) with no cluster lookup, so helm-unittest can render them +# offline — the ownerReference lookup is the only cluster-coupled part of +# namespace.yaml and is skipped when the parent namespace lookup is empty. +templates: + - templates/namespace.yaml +set: + _cluster: + root-host: example.com +tests: + - it: depth-1 tenant (direct child of root) carries tenant-root and its own label + release: + name: tenant-ktj + namespace: tenant-root + asserts: + - documentIndex: 0 + isKind: + of: Namespace + - documentIndex: 0 + equal: + path: metadata.name + value: tenant-ktj + - documentIndex: 0 + equal: + path: metadata.labels["tenant.cozystack.io/tenant-root"] + value: "" + - documentIndex: 0 + equal: + path: metadata.labels["tenant.cozystack.io/tenant-ktj"] + value: "" + + - it: depth-2 tenant carries the full ancestor chain including tenant-root + release: + name: tenant-htdev + namespace: tenant-ktj + asserts: + - documentIndex: 0 + equal: + path: metadata.name + value: tenant-ktj-htdev + - documentIndex: 0 + equal: + path: metadata.labels["tenant.cozystack.io/tenant-root"] + value: "" + - documentIndex: 0 + equal: + path: metadata.labels["tenant.cozystack.io/tenant-ktj"] + value: "" + - documentIndex: 0 + equal: + path: metadata.labels["tenant.cozystack.io/tenant-ktj-htdev"] + value: "" + + - it: depth-3 tenant carries every ancestor from root down to itself + release: + name: tenant-foo + namespace: tenant-ktj-htdev + asserts: + - documentIndex: 0 + equal: + path: metadata.name + value: tenant-ktj-htdev-foo + - documentIndex: 0 + equal: + path: metadata.labels["tenant.cozystack.io/tenant-root"] + value: "" + - documentIndex: 0 + equal: + path: metadata.labels["tenant.cozystack.io/tenant-ktj"] + value: "" + - documentIndex: 0 + equal: + path: metadata.labels["tenant.cozystack.io/tenant-ktj-htdev"] + value: "" + - documentIndex: 0 + equal: + path: metadata.labels["tenant.cozystack.io/tenant-ktj-htdev-foo"] + value: "" + # A tenant must NOT claim descendants it does not have: the depth-3 + # namespace is not an ancestor of any deeper tenant, so no stray label. + - documentIndex: 0 + notExists: + path: metadata.labels["tenant.cozystack.io/tenant-ktj-htdev-foo-bar"] diff --git a/packages/apps/tenant/tests/vmo_webhook_dependson_test.yaml b/packages/apps/tenant/tests/vmo_webhook_dependson_test.yaml new file mode 100644 index 0000000000..bd2ac1ddec --- /dev/null +++ b/packages/apps/tenant/tests/vmo_webhook_dependson_test.yaml @@ -0,0 +1,80 @@ +suite: tenant HelmReleases gate on the victoria-metrics-operator + +# The victoria-metrics-operator admission webhook is configured with +# failurePolicy: Fail and rejects the VM* custom resources these tenant +# releases create (etcd/ingress: VMPodScrape; monitoring via its +# monitoring-system child: VMCluster/VMAlert/VLCluster/VMAlertmanager/ +# VMServiceScrape) whenever the operator pod is not yet serving on :9443. +# On a cold install the operator can be briefly unavailable, so every tenant +# release that creates VM* resources must depend on the operator HelmRelease +# becoming Ready — otherwise the release reconciles too early, the webhook +# rejects with connection refused, the install fails and rolls back. Pin the +# dependsOn so the gate cannot regress. + +# Restrict rendering to the gated HelmRelease templates so the chart's +# namespace.yaml `lookup` (which errors under helm-unittest) is not evaluated. +templates: + - templates/etcd.yaml + - templates/ingress.yaml + - templates/monitoring.yaml + - templates/seaweedfs.yaml + - templates/gateway.yaml + +release: + name: tenant-test + namespace: tenant-test + +set: + _cluster: + expose-ingress: tenant-root + expose-external-ips: "192.0.2.10" + gateway-enabled: "true" + etcd: true + ingress: true + monitoring: true + seaweedfs: true + gateway: true + host: "" + +tests: + - it: etcd HR depends on the victoria-metrics-operator + asserts: + - template: templates/etcd.yaml + equal: + path: spec.dependsOn + value: + - name: victoria-metrics-operator + namespace: cozy-victoria-metrics-operator + + - it: ingress HR depends on the victoria-metrics-operator + asserts: + - template: templates/ingress.yaml + equal: + path: spec.dependsOn + value: + - name: victoria-metrics-operator + namespace: cozy-victoria-metrics-operator + + - it: monitoring HR depends on the victoria-metrics-operator + asserts: + - template: templates/monitoring.yaml + equal: + path: spec.dependsOn + value: + - name: victoria-metrics-operator + namespace: cozy-victoria-metrics-operator + + # The gate is intentionally narrow: only the releases that create VM* custom + # resources carry it. Tenant releases that create none (seaweedfs, gateway) + # must not be needlessly blocked on the operator. + - it: seaweedfs HR carries no operator gate + asserts: + - template: templates/seaweedfs.yaml + notExists: + path: spec.dependsOn + + - it: gateway HR carries no operator gate + asserts: + - template: templates/gateway.yaml + notExists: + path: spec.dependsOn diff --git a/packages/apps/vpn/templates/secret.yaml b/packages/apps/vpn/templates/secret.yaml index 3ef4ac4b32..595bfd32ae 100644 --- a/packages/apps/vpn/templates/secret.yaml +++ b/packages/apps/vpn/templates/secret.yaml @@ -59,8 +59,6 @@ kind: Secret metadata: name: {{ .Release.Name }}-urls type: Opaque -foo: | - {{ toJson $passwords }} stringData: {{- range $user, $u := .Values.users }} "{{ $user }}": "ss://{{ regexReplaceAll "=" (replace "/" "_" (replace "+" "-" (printf "chacha20-ietf-poly1305:%s" (index $passwords $user) | b64enc))) "" }}@{{ $.Values.host | default (printf "%s.%s" $.Release.Name $host) }}:40000/?outline=1#{{ $.Release.Name }}" diff --git a/packages/core/installer/values.yaml b/packages/core/installer/values.yaml index c0aadf19ad..467e1fb64f 100644 --- a/packages/core/installer/values.yaml +++ b/packages/core/installer/values.yaml @@ -7,9 +7,9 @@ bareNamespace: false cozystackOperator: # Deployment variant: talos, generic, hosted variant: talos - image: ghcr.io/cozystack/cozystack/cozystack-operator:v1.5.0@sha256:05027a381a8a97276ec426dabb87e1b7df753d4d76925b68e2dd8819f7eb22df + image: ghcr.io/cozystack/cozystack/cozystack-operator:v1.5.4@sha256:cf69537ff3a0fd1a4fc81ea3d7022fb0da31fdff6f2e4113c73a33a0e0b65f06 platformSourceUrl: 'oci://ghcr.io/cozystack/cozystack/cozystack-packages' - platformSourceRef: 'digest=sha256:e2e7bc7664a9799332dc8d59ae312879ba3675929edfc0fd6ce005ec41cbeb7b' + platformSourceRef: 'digest=sha256:7d1f77973c35d4a37a37eb2c563317e22b86c6fed7d757fa4d136386812827bd' # When non-empty, overrides the operator's --helmrelease-interval flag # (operator default: 5m). E2E sets this to 30s; production should leave empty. helmReleaseInterval: "" diff --git a/packages/core/platform/images/migrations/migrations/43 b/packages/core/platform/images/migrations/migrations/43 index 0c421d1fce..a8d7f74a5f 100755 --- a/packages/core/platform/images/migrations/migrations/43 +++ b/packages/core/platform/images/migrations/migrations/43 @@ -1,31 +1,31 @@ #!/bin/sh # Migration 43 --> 44 -# Adopt existing Cluster/seaweedfs-db resources into the new seaweedfs-db +# Adopt existing Cluster/seaweedfs-db resources into the new -db # HelmRelease introduced in this release. # -# Pre-split, the CNPG Cluster lived inside the seaweedfs-system Helm release. -# Splitting moves it to a new release named seaweedfs-db. We rewrite the -# helm ownership annotations so Helm adopts the existing Cluster instead of -# erroring on the next reconcile, and stamp helm.sh/resource-policy: keep -# so the seaweedfs-system upgrade (which no longer renders the Cluster) does -# not delete it during the transition. +# Pre-split, the CNPG Cluster lived inside the -system Helm release. +# Splitting moves it to a new release named -db. We rewrite the helm +# ownership annotations so Helm adopts the existing Cluster instead of erroring +# on the next reconcile, and stamp helm.sh/resource-policy: keep so the +# -system upgrade (which no longer renders the Cluster) does not delete it +# during the transition. +# +# The hand-over itself lives in lib/seaweedfs-db-adopt.sh, shared with migration +# 45. This migration originally compared the owning release name against the +# literal "seaweedfs-system", which silently skipped every instance NOT named +# `seaweedfs`: SeaweedFS is a user-creatable kind, so an instance `foo` is owned +# by `foo-system`. Those tenants got no `keep`, and the -system upgrade +# pruned their Cluster — taking the filer metadata, and with it all of the +# tenant's S3. The shared helper matches the `-system` SUFFIX instead, covering +# every instance name; migration 45 repairs clusters that already ran the +# hardcoded version. set -euo pipefail -# Iterate every namespace that has a Cluster named "seaweedfs-db". -for ns in $(kubectl get cluster.postgresql.cnpg.io -A \ - -o jsonpath='{range .items[?(@.metadata.name=="seaweedfs-db")]}{.metadata.namespace}{"\n"}{end}'); do - current=$(kubectl get cluster.postgresql.cnpg.io seaweedfs-db -n "$ns" \ - -o jsonpath='{.metadata.annotations.meta\.helm\.sh/release-name}' 2>/dev/null || true) - if [ "$current" = "seaweedfs-system" ]; then - echo "Re-annotating Cluster/seaweedfs-db in $ns: seaweedfs-system -> seaweedfs-db" - kubectl annotate cluster.postgresql.cnpg.io seaweedfs-db -n "$ns" \ - meta.helm.sh/release-name=seaweedfs-db \ - helm.sh/resource-policy=keep \ - --overwrite - # release-namespace stays the same (tenant namespace, e.g. tenant-root) - fi -done +# shellcheck source=lib/seaweedfs-db-adopt.sh +. "$(dirname "$0")/lib/seaweedfs-db-adopt.sh" + +adopt_seaweedfs_db_clusters # Stamp version. # Mirror migration 42's labeled manifest. A label-less apply by the same field diff --git a/packages/core/platform/images/migrations/migrations/45 b/packages/core/platform/images/migrations/migrations/45 new file mode 100755 index 0000000000..38c11899e0 --- /dev/null +++ b/packages/core/platform/images/migrations/migrations/45 @@ -0,0 +1,91 @@ +#!/bin/sh +# Migration 45 --> 46 +# This slot does TWO independent jobs. Both are protective, both are idempotent, +# and neither may be reached by way of the other failing. +# +# 1. Repair the 1.5.0 db-split hand-over for SeaweedFS instances whose name is +# not the default `seaweedfs` (the original contents of this slot, below). +# 2. Pin the CAPI kubeadm bootstrap objects with helm.sh/resource-policy=keep, +# which is 1.6's OWN migration 45 folded in here — see +# lib/kubeadm-keep-pin.sh for why it has to be, and ORDER below for why it +# runs second. +# +# ORDER. The SeaweedFS repair runs first and the keep-pin second, because the two +# failure modes are not equally bad: a missed SeaweedFS hand-over loses a tenant's +# filer metadata, hence all of its S3, while a missed keep-pin gives a noisy +# broken tenant-Kubernetes worker rollover that is recoverable by hand. Both halves +# fail closed ahead of the version stamp at the bottom and the Job retries the +# whole slot, so a half-completed attempt is safe either way — but on an attempt +# where only one half gets to run, it should be the one whose failure is +# irreversible. Running the pin first would mean a pin failure stops the SeaweedFS +# repair from being attempted at all on that pass, which is strictly worse. +# +# --- 1. SeaweedFS db-split repair ------------------------------------------- +# +# Migration 43 performs the hand-over of Cluster/seaweedfs-db from the +# -system release to the -db release introduced by the split (PR +# #2601, v1.5.0). It matched the owning release name against the literal +# "seaweedfs-system", so it only ever fired for an instance named `seaweedfs`. +# `SeaweedFS` is a user-creatable kind: an instance named `foo` is owned by +# release `foo-system` and was skipped. Such a tenant kept +# meta.helm.sh/release-name: -system and never received +# helm.sh/resource-policy: keep, so the -system upgrade — whose post-split +# chart no longer renders the Cluster — deleted it as a removed resource. CNPG +# takes the PVC with the Cluster, so the filer metadata, and with it every object +# in that tenant's S3, is unreachable. +# +# Migration 43 is fixed in place, which covers clusters upgrading from before 43. +# Migrations never re-run, so any cluster already at version >= 44 ran the +# hardcoded version and is still exposed: this migration re-checks the fleet and +# performs the hand-over for any Cluster/seaweedfs-db still owned by a +# -system release. Running as a pre-upgrade hook, it lands before the +# platform applies the new artifacts and therefore before -system +# re-renders, closing the window on THIS upgrade. +# +# The prune is not a one-shot: Helm diffs the LAST DEPLOYED revision against the +# new manifest, so a tenant whose -system last succeeded on a pre-split +# revision recomputes the same deletion on EVERY upgrade attempt, including ones +# that fail for unrelated reasons. Such a tenant re-deletes the Cluster each time +# -db recreates it. Stamping `keep` breaks that loop even when the +# -system upgrade itself stays wedged. +# +# Idempotent (a Cluster already owned by -db is left alone), so it is a +# no-op on a correctly migrated cluster and on fresh installs. +# +# LIMIT — this migration cannot recover a Cluster that is ALREADY deleted. There +# is nothing left to re-annotate: the PVC went with it. Such a tenant needs its +# filer metadata restored from a backup; see +# docs/operations/seaweedfs-431-rename-recovery.md. + +set -euo pipefail + +# shellcheck source=lib/seaweedfs-db-adopt.sh +. "$(dirname "$0")/lib/seaweedfs-db-adopt.sh" +# shellcheck source=lib/kubeadm-keep-pin.sh +. "$(dirname "$0")/lib/kubeadm-keep-pin.sh" + +# Called bare, so a non-zero return trips errexit and aborts before the stamp +# below. Both helpers report their own reason on stderr first. +adopt_seaweedfs_db_clusters + +# --- 2. kubeadm bootstrap keep-pin ------------------------------------------ +pin_kubeadm_bootstrap_objects + +# Stamp version. +# Mirror migration 42's labeled manifest. A label-less apply by the same field +# manager (kubectl create | kubectl apply) would strip the +# platform.cozystack.io/no-delete label migration 42 added, dropping +# cozystack-version out of the cozystack-no-delete-guardrail +# ValidatingAdmissionPolicy. templates/cozystack-version.yaml only re-renders the +# labeled ConfigMap on first install, so the label must be carried inline here. +kubectl apply --filename - < +# True when a kubectl error means the resource type simply is not served, as +# opposed to the API being unreachable, forbidden, throttled, or not yet +# established: a cluster with no CAPI bootstrap provider has nothing to pin. +# Deliberately narrow — anything unrecognised is fatal — and deliberately NOT +# shared with lib/seaweedfs-db-adopt.sh, because migration 43 sources that lib +# alone and neither may depend on the other having been loaded. +_kkp_is_absent_err() { + grep -qiE "server doesn't have a resource type|server could not find the requested resource|could not find the requested resource|no matches for kind" "$1" +} + +# _kkp_is_gone_err +# True when a PER-OBJECT kubectl error means that one object is no longer there: +# an app being deleted concurrently with this hook can take its +# KubeadmConfigTemplate away between the fleet scan and the annotate. Nothing is at +# risk in that case — Helm cannot prune an object that does not exist — so it is a +# skip, not a failure, and must not fail a pre-upgrade hook and block the platform +# upgrade over an object nobody needs any more. +# +# Separate from _kkp_is_absent_err, and wider than it by exactly one phrase, +# because "not found" must NOT be accepted for the fleet SCAN: a list never +# answers NotFound, so accepting it there would let some genuine failure read as +# an empty fleet and leave every template un-pinned. +_kkp_is_gone_err() { + _kkp_is_absent_err "$1" || grep -qiE "not found" "$1" +} + +# _kkp_pin_one +# Stamp keep on one object, counting the outcome. Always returns 0: a failure is +# recorded in _kkp_failures and reported by the caller once the whole fleet has +# been walked, so one bad object cannot abort the loop before the rest are pinned. +_kkp_pin_one() { + _kkp_p_kind="$1" + _kkp_p_ns="$2" + _kkp_p_name="$3" + + # An unreadable current value is deliberately NOT fatal here: the annotate below + # is the operation that has to succeed and it IS checked, so a failed read costs + # only a redundant write. There is no fail-open hiding in this `|| true`. + _kkp_p_current=$(kubectl get "$_kkp_p_kind" --namespace "$_kkp_p_ns" "$_kkp_p_name" \ + --output 'jsonpath={.metadata.annotations.helm\.sh/resource-policy}' 2>/dev/null) \ + || _kkp_p_current="" + + if [ "$_kkp_p_current" = "keep" ]; then + echo "$_kkp_p_kind $_kkp_p_ns/$_kkp_p_name already carries $_KKP_KEEP_ANNOTATION — nothing to do" + _kkp_skipped=$((_kkp_skipped + 1)) + return 0 + fi + + if kubectl annotate "$_kkp_p_kind" --namespace "$_kkp_p_ns" "$_kkp_p_name" \ + "$_KKP_KEEP_ANNOTATION" --overwrite >/dev/null 2>"$_kkp_err"; then + echo "Pinned $_kkp_p_kind $_kkp_p_ns/$_kkp_p_name with $_KKP_KEEP_ANNOTATION" + _kkp_patched=$((_kkp_patched + 1)) + elif _kkp_is_gone_err "$_kkp_err"; then + echo "$_kkp_p_kind $_kkp_p_ns/$_kkp_p_name disappeared between the scan and the pin — skipping" + _kkp_skipped=$((_kkp_skipped + 1)) + else + echo "WARNING: failed to pin $_kkp_p_kind $_kkp_p_ns/$_kkp_p_name:" >&2 + cat "$_kkp_err" >&2 + _kkp_failures=$((_kkp_failures + 1)) + fi + return 0 +} + +# pin_kubeadm_bootstrap_objects +# Stamp helm.sh/resource-policy=keep on every Helm-managed kubeadm bootstrap +# object. Returns non-zero (aborting the migration before its version stamp) on +# any error that is not "this resource type is not served". +pin_kubeadm_bootstrap_objects() { + _kkp_err=$(mktemp) + _kkp_items=$(mktemp) + _kkp_patched=0 + _kkp_skipped=0 + _kkp_failures=0 + + for _kkp_kind in $_KKP_KINDS; do + echo "==> Pinning Helm-managed $_kkp_kind with $_KKP_KEEP_ANNOTATION" + + # Redirecting to a file inside `if !` keeps errexit from firing so the exit + # status can be inspected. `for x in $(kubectl ...)` would not trip errexit + # either: on failure it iterates zero times and lets the caller stamp the + # version regardless, which is the fail-open this guard exists for. + if ! kubectl get "$_kkp_kind" --all-namespaces \ + --selector "$_KKP_HELM_MANAGED_SELECTOR" \ + --output 'jsonpath={range .items[*]}{.metadata.namespace}{"/"}{.metadata.name}{"\n"}{end}' \ + >"$_kkp_items" 2>"$_kkp_err"; then + if _kkp_is_absent_err "$_kkp_err"; then + echo "$_kkp_kind is not served on this cluster — nothing to pin" + continue + fi + echo "FATAL: cannot list $_kkp_kind across namespaces; refusing to stamp past an unverified fleet:" >&2 + cat "$_kkp_err" >&2 + rm -f "$_kkp_err" "$_kkp_items" + return 1 + fi + + # Read from a file, not a pipe: `kubectl ... | while read` runs the loop body + # in a subshell, so the counters would be discarded on exit and a failed pin + # would be reported as a clean run. A namespace and an object name are both + # DNS labels and cannot contain "/", so the separator is unambiguous. + while IFS= read -r _kkp_item; do + [ -n "$_kkp_item" ] || continue + _kkp_ns="${_kkp_item%%/*}" + _kkp_name="${_kkp_item#*/}" + if [ -z "$_kkp_ns" ] || [ -z "$_kkp_name" ] || [ "$_kkp_ns" = "$_kkp_item" ]; then + echo "WARNING: cannot parse '$_kkp_item' as / for $_kkp_kind — refusing to guess" >&2 + _kkp_failures=$((_kkp_failures + 1)) + continue + fi + _kkp_pin_one "$_kkp_kind" "$_kkp_ns" "$_kkp_name" + done <"$_kkp_items" + done + + rm -f "$_kkp_err" "$_kkp_items" + + echo "==> kubeadm keep-pin summary: pinned=$_kkp_patched already-pinned=$_kkp_skipped failures=$_kkp_failures" + + if [ "$_kkp_failures" -gt 0 ]; then + echo "ERROR: $_kkp_failures kubeadm bootstrap object(s) could not be pinned; refusing to stamp the version so this migration retries on the next platform upgrade." >&2 + echo "ERROR: an un-pinned KubeadmConfigTemplate can be pruned by the 1.6 kubernetes chart upgrade, breaking the bootstrap.configRef of the kubeadm-backed MachineSet that is still rolling workers over to Talos." >&2 + return 1 + fi + + return 0 +} diff --git a/packages/core/platform/images/migrations/migrations/lib/seaweedfs-db-adopt.sh b/packages/core/platform/images/migrations/migrations/lib/seaweedfs-db-adopt.sh new file mode 100644 index 0000000000..d99264708c --- /dev/null +++ b/packages/core/platform/images/migrations/migrations/lib/seaweedfs-db-adopt.sh @@ -0,0 +1,179 @@ +# shellcheck shell=sh +# Shared helper for handing Cluster/seaweedfs-db over to the -db release. +# +# The 1.5.0 db split (PR #2601) moved the CNPG Cluster carrying SeaweedFS filer +# metadata out of the -system Helm release into a new -db release. +# The Cluster object itself did not move — only its ownership had to. Two things +# must be true BEFORE -system next renders, and they are independent: +# +# meta.helm.sh/release-name=-db so -db ADOPTS the existing +# Cluster instead of failing its install +# on Helm's ownership check; +# helm.sh/resource-policy=keep so the -system upgrade, whose new +# chart no longer renders the Cluster, +# does not GARBAGE-COLLECT it as a +# removed resource. +# +# Without `keep` the Cluster is deleted, CNPG takes its PVC with it, and the +# tenant's filer metadata — hence all of its S3 — is gone. Data loss, not outage. +# +# The window is not a one-shot. Helm prunes by diffing the LAST DEPLOYED revision +# against the new manifest, so as long as -system's last successful revision +# is a pre-split one that still contains the Cluster, EVERY subsequent upgrade +# attempt — including ones that fail for unrelated reasons and never become the +# new "deployed" revision — re-computes the same deletion. A tenant whose +# -system is wedged therefore re-deletes the Cluster on every retry, racing +# -db, which keeps recreating it. `keep` is what breaks that loop, and it +# must stay for as long as that pre-split revision remains the prune baseline +# (i.e. until -system upgrades successfully at least once). Clearing it +# again is deferred to the 1.7 batch migrations. +# +# Ownership is therefore NOT a proxy for safety. A Cluster can already be owned by +# -db and still need `keep`: where the hand-over was skipped, -system +# prunes the Cluster and -db simply RECREATES it under its own ownership +# with no keep — while -system's prune baseline still lists it, so the next +# reconcile deletes it again. Both shapes are live on the upgrade stand: +# tenant-l and tenant-root are -db-owned WITH keep and their -system +# deployed revision (rev 1) still contains the Cluster, so only keep saves them; +# tenant-fresh is -db-owned WITHOUT keep because it was installed after the +# split and its -system never rendered a Cluster. Telling those two apart +# needs the release's deployed manifest, which this script cannot read cheaply or +# reliably. The costs are asymmetric: stamping keep where it was not needed leaves +# an orphan on app delete (which the 1.6 extra/seaweedfs cleanup hook reclaims; +# on 1.5.x nothing does, so it has to be removed by hand); +# missing one loses the database. So keep is stamped on every Cluster owned by +# either side of the split. +# +# Migration 43 shipped this logic with the owning release name hardcoded to +# "seaweedfs-system", which is only correct for an instance named `seaweedfs`. +# `SeaweedFS` is a user-creatable kind, so an instance named e.g. `foo` is owned +# by `foo-system` and was silently skipped: no re-own, no keep, Cluster pruned. +# Matching on the `-system` SUFFIX instead covers every instance name, so this +# helper is sourced by both migration 43 (the original hand-over, for clusters +# that have not run it yet) and migration 45 (repair, for clusters that already +# ran the hardcoded version). +# +# Idempotent: a Cluster already owned by -db AND carrying keep is left +# alone, so re-running is a no-op and both migrations can safely fire on the same +# cluster. +# +# FAILS CLOSED. Migrations never re-run, so a transient error swallowed here would +# permanently leave at-risk tenants exposed with no later migration to catch them. +# Every kubectl failure is fatal EXCEPT the two that genuinely mean "nothing to +# do": the CNPG resource type not being served at all (a cluster without CNPG), +# and a Cluster disappearing between the scan and the read. A non-zero return +# aborts the migration before it stamps the version, so the Job retries. +# +# Sourced, not executed: +# . "$(dirname "$0")/lib/seaweedfs-db-adopt.sh" + +# _sdb_is_absent_err +# True when a kubectl error means the thing simply is not there, as opposed to the +# API being unreachable, forbidden, throttled, or not yet established. Kept +# deliberately narrow: anything unrecognised is treated as fatal. +_sdb_is_absent_err() { + grep -qiE "server doesn't have a resource type|server could not find the requested resource|could not find the requested resource|not found" "$1" +} + +# adopt_seaweedfs_db_clusters +# Re-own and/or protect every Cluster/seaweedfs-db the split left exposed. +# Returns non-zero (aborting the migration) on any error that is not "absent". +adopt_seaweedfs_db_clusters() { + _sdb_err=$(mktemp) + + # Fleet scan. Assigning inside `if !` keeps errexit from firing so the exit + # status can be inspected — `for ns in $(kubectl ...)` would silently iterate + # zero times on failure and let the caller stamp the version regardless, which + # is the whole bug this guard exists for. + if ! _sdb_namespaces=$(kubectl get cluster.postgresql.cnpg.io -A \ + -o jsonpath='{range .items[?(@.metadata.name=="seaweedfs-db")]}{.metadata.namespace}{"\n"}{end}' \ + 2>"$_sdb_err"); then + if _sdb_is_absent_err "$_sdb_err"; then + echo "CNPG Cluster resource type is not served on this cluster — no SeaweedFS databases to hand over" + rm -f "$_sdb_err" + return 0 + fi + echo "FATAL: cannot list Cluster/seaweedfs-db across namespaces; refusing to stamp past an unverified fleet:" >&2 + cat "$_sdb_err" >&2 + rm -f "$_sdb_err" + return 1 + fi + + for ns in $_sdb_namespaces; do + [ -n "$ns" ] || continue + + if ! _sdb_current=$(kubectl get cluster.postgresql.cnpg.io seaweedfs-db -n "$ns" \ + -o jsonpath='{.metadata.annotations.meta\.helm\.sh/release-name}' 2>"$_sdb_err"); then + if _sdb_is_absent_err "$_sdb_err"; then + echo "Cluster/seaweedfs-db in $ns disappeared between scan and read — skipping" + continue + fi + echo "FATAL: cannot read the Helm owner of Cluster/seaweedfs-db in $ns:" >&2 + cat "$_sdb_err" >&2 + rm -f "$_sdb_err" + return 1 + fi + + if ! _sdb_keep=$(kubectl get cluster.postgresql.cnpg.io seaweedfs-db -n "$ns" \ + -o jsonpath='{.metadata.annotations.helm\.sh/resource-policy}' 2>"$_sdb_err"); then + if _sdb_is_absent_err "$_sdb_err"; then + echo "Cluster/seaweedfs-db in $ns disappeared between scan and read — skipping" + continue + fi + echo "FATAL: cannot read the resource-policy of Cluster/seaweedfs-db in $ns:" >&2 + cat "$_sdb_err" >&2 + rm -f "$_sdb_err" + return 1 + fi + + case "$_sdb_current" in + # No Helm ownership annotation at all — distinct from "unreadable", which is + # fatal above. Guessing an owner would be worse than doing nothing, but a + # SeaweedFS database that nobody owns is worth saying out loud. + "") + echo "WARNING: Cluster/seaweedfs-db in $ns carries no meta.helm.sh/release-name — not Helm-managed, leaving it alone. If this tenant runs SeaweedFS, verify by hand that its database is not about to be pruned." >&2 + ;; + + # Owned by the data-plane release: this is the hand-over. The instance name + # is whatever precedes -system, so `foo-system` -> `foo-db` exactly as + # `seaweedfs-system` -> `seaweedfs-db`. + *-system) + _sdb_instance="${_sdb_current%-system}" + # A release literally named "-system" has no instance name; refusing is + # safer than annotating an owner of "-db" that no release will claim. + if [ -z "$_sdb_instance" ]; then + echo "WARNING: Cluster/seaweedfs-db in $ns is owned by a release named '$_sdb_current' with no instance name — skipping" >&2 + continue + fi + echo "Re-annotating Cluster/seaweedfs-db in $ns: $_sdb_current -> ${_sdb_instance}-db (+keep)" + kubectl annotate cluster.postgresql.cnpg.io seaweedfs-db -n "$ns" \ + meta.helm.sh/release-name="${_sdb_instance}-db" \ + helm.sh/resource-policy=keep \ + --overwrite + # release-namespace stays the same (the tenant namespace); both releases + # live there, so the ownership check only needed release-name rewritten. + ;; + + # Already owned by the db release. Ownership is correct, but that does NOT + # imply it is protected — see the note above. Stamp keep if it is missing. + *-db) + if [ "$_sdb_keep" = "keep" ]; then + echo "Cluster/seaweedfs-db in $ns already handed over to $_sdb_current and protected — nothing to do" + else + echo "Protecting Cluster/seaweedfs-db in $ns: owned by $_sdb_current but missing helm.sh/resource-policy=keep" + kubectl annotate cluster.postgresql.cnpg.io seaweedfs-db -n "$ns" \ + helm.sh/resource-policy=keep \ + --overwrite + fi + ;; + + # Owned by some unrelated release. Not ours to touch. + *) + echo "WARNING: Cluster/seaweedfs-db in $ns is owned by unrelated release '$_sdb_current' — leaving it alone" >&2 + ;; + esac + done + + rm -f "$_sdb_err" + return 0 +} diff --git a/packages/core/platform/sources/backupstrategy-controller.yaml b/packages/core/platform/sources/backupstrategy-controller.yaml index 2306c755f5..d28e599912 100644 --- a/packages/core/platform/sources/backupstrategy-controller.yaml +++ b/packages/core/platform/sources/backupstrategy-controller.yaml @@ -18,8 +18,16 @@ spec: # All three must be installed before the templates render, otherwise # the Helm release fails with "no matches for kind". Order is enforced # via dependsOn so a fresh-cluster bootstrap cannot race the CRDs. + # + # The cozy-backups Bucket is also created INTO the tenant-root namespace, + # which is provisioned by cozystack-basics (the static tenant-root release). + # Without the cozystack-basics edge the Bucket patch can race namespace + # creation on a fresh install and fail with `namespaces "tenant-root" not + # found`, which then races the install script's wait deadline. Depend on + # cozystack-basics so the target namespace exists first. dependsOn: - cozystack.networking + - cozystack.cozystack-basics - cozystack.backup-controller - cozystack.bucket-application - cozystack.objectstorage-controller diff --git a/packages/core/platform/sources/cozystack-basics.yaml b/packages/core/platform/sources/cozystack-basics.yaml index 8a5313ee66..0a71a24615 100644 --- a/packages/core/platform/sources/cozystack-basics.yaml +++ b/packages/core/platform/sources/cozystack-basics.yaml @@ -28,8 +28,11 @@ spec: # Drift detection is off on operator-generated HelmReleases, so a # capability gate in the templates would render the policy out at first # install and never add it back; ordering via dependsOn is the reliable - # fix. cozystack-basics is a dependency-graph leaf (nothing dependsOn it) - # and engine components consume none of its outputs, so this adds no cycle. + # fix. The cozystack-engine and gateway-api-crds packages it depends on + # consume none of cozystack-basics' outputs and do not depend back on it, + # so these edges add no cycle. (Downstream packages such as backupstrategy- + # controller dependsOn cozystack-basics, but those do not feed engine or + # gateway-api-crds, so they cannot close a loop here.) - cozystack.cozystack-engine - cozystack.gateway-api-crds components: diff --git a/packages/core/platform/templates/bundles/system.yaml b/packages/core/platform/templates/bundles/system.yaml index 84aff47ac7..faf1484dbe 100644 --- a/packages/core/platform/templates/bundles/system.yaml +++ b/packages/core/platform/templates/bundles/system.yaml @@ -195,7 +195,21 @@ {{include "cozystack.platform.package.default" (list "cozystack.monitoring-application" $) }} {{include "cozystack.platform.package.default" (list "cozystack.etcd-application" $) }} {{include "cozystack.platform.package.default" (list "cozystack.cozystack-basics" $) }} -{{include "cozystack.platform.package.default" (list "cozystack.backupstrategy-controller" $) }} +{{- /* + Forward admin backupStorage overrides (endpoint, region, systemSecretName, …) + into the emitted cozystack.backupstrategy-controller Package CR's + components.backupstrategy-controller.values, so the supported override path + on the cozystack-platform Package — spec.components.platform.values.backupStorage — + actually reaches the backupstrategy-controller HelmRelease (#3245). Same + pattern as the kubevirt / gpu-operator forwarding in bundles/iaas.yaml. The + components block is only rendered when the admin set something, so an unset + backupStorage keeps the emitted Package identical to the historical render. +*/ -}} +{{- $backupStrategyComponents := dict -}} +{{- if .Values.backupStorage -}} +{{- $_ := set $backupStrategyComponents "backupstrategy-controller" (dict "values" (dict "backupStorage" .Values.backupStorage)) -}} +{{- end -}} +{{include "cozystack.platform.package" (list "cozystack.backupstrategy-controller" "default" $ $backupStrategyComponents) }} {{include "cozystack.platform.package.default" (list "cozystack.backup-controller" $) }} {{- /* velero must be a default package: backupstrategy-controller (default) hard-depends on cozystack.velero, so an optional velero leaves the diff --git a/packages/core/platform/tests/bundles_backupstorage_wiring_test.yaml b/packages/core/platform/tests/bundles_backupstorage_wiring_test.yaml new file mode 100644 index 0000000000..3828bd0acf --- /dev/null +++ b/packages/core/platform/tests/bundles_backupstorage_wiring_test.yaml @@ -0,0 +1,91 @@ +suite: bundles.system backupStorage → backupstrategy-controller wiring +templates: + - templates/bundles/system.yaml +release: + name: cozystack + namespace: cozy-system +tests: + # Regression for #3245: the documented admin override for the cozy-default + # backup S3 coordinates rides on spec.components.platform.values.backupStorage + # of the cozystack-platform Package. The platform chart must forward that + # block into the emitted cozystack.backupstrategy-controller Package CR's + # components, otherwise the override silently never reaches the + # backupstrategy-controller HelmRelease. + - it: emits cozystack.backupstrategy-controller without a components block when backupStorage is unset + set: + bundles: + system: + enabled: true + variant: isp-full + documentSelector: + path: metadata.name + value: cozystack.backupstrategy-controller + asserts: + - equal: + path: kind + value: Package + - equal: + path: spec.variant + value: default + - notExists: + path: spec.components + + - it: forwards backupStorage overrides into the backupstrategy-controller component values + set: + bundles: + system: + enabled: true + variant: isp-full + backupStorage: + provisionBucket: false + endpoint: https://s3.example.com + region: eu-central-1 + systemSecretName: external-s3-credentials + documentSelector: + path: metadata.name + value: cozystack.backupstrategy-controller + asserts: + - equal: + path: spec.components.backupstrategy-controller.values.backupStorage.provisionBucket + value: false + - equal: + path: spec.components.backupstrategy-controller.values.backupStorage.endpoint + value: https://s3.example.com + - equal: + path: spec.components.backupstrategy-controller.values.backupStorage.region + value: eu-central-1 + - equal: + path: spec.components.backupstrategy-controller.values.backupStorage.systemSecretName + value: external-s3-credentials + + - it: forwards backupStorage overrides on the isp-full-generic variant too + set: + bundles: + system: + enabled: true + variant: isp-full-generic + backupStorage: + endpoint: https://s3.example.com + documentSelector: + path: metadata.name + value: cozystack.backupstrategy-controller + asserts: + - equal: + path: spec.components.backupstrategy-controller.values.backupStorage.endpoint + value: https://s3.example.com + + - it: forwards backupStorage overrides on the isp-hosted variant too + set: + bundles: + system: + enabled: true + variant: isp-hosted + backupStorage: + endpoint: https://s3.example.com + documentSelector: + path: metadata.name + value: cozystack.backupstrategy-controller + asserts: + - equal: + path: spec.components.backupstrategy-controller.values.backupStorage.endpoint + value: https://s3.example.com diff --git a/packages/core/platform/tests/sources_backupstrategy_dependson_test.yaml b/packages/core/platform/tests/sources_backupstrategy_dependson_test.yaml new file mode 100644 index 0000000000..e1234972bb --- /dev/null +++ b/packages/core/platform/tests/sources_backupstrategy_dependson_test.yaml @@ -0,0 +1,34 @@ +suite: backupstrategy-controller waits for the tenant-root namespace and its CRDs +# Regression guard for a fresh-install ordering race: backupstrategy-controller +# creates the cozy-backups Bucket INTO the tenant-root namespace, which is +# provisioned by cozystack-basics. Without the cozystack-basics edge the Bucket +# patch races namespace creation and fails with `namespaces "tenant-root" not +# found`, which then races the install script's wait deadline. The CRD edges +# (backup-controller / bucket-application / objectstorage-controller / velero) +# guard the `no matches for kind` race. All edges below must stay. +templates: + - templates/sources.yaml +release: + name: cozystack + namespace: cozy-system +tests: + - it: backupstrategy-controller dependsOn cozystack-basics and the CRD providers + documentSelector: + path: metadata.name + value: cozystack.backupstrategy-controller + asserts: + - contains: + path: spec.variants[0].dependsOn + content: cozystack.cozystack-basics + - contains: + path: spec.variants[0].dependsOn + content: cozystack.backup-controller + - contains: + path: spec.variants[0].dependsOn + content: cozystack.bucket-application + - contains: + path: spec.variants[0].dependsOn + content: cozystack.objectstorage-controller + - contains: + path: spec.variants[0].dependsOn + content: cozystack.velero diff --git a/packages/core/platform/values.yaml b/packages/core/platform/values.yaml index 6d1f095a8d..e1f9d371cd 100644 --- a/packages/core/platform/values.yaml +++ b/packages/core/platform/values.yaml @@ -5,8 +5,16 @@ sourceRef: path: / migrations: enabled: false - image: ghcr.io/cozystack/cozystack/platform-migrations:v1.5.0@sha256:8bf61f17e99eb4a3365394e3d97ac52a1d68cc84aa104894ecb09de25721dc1d - targetVersion: 45 + # The image digest is NOT hand-edited alongside targetVersion. The release + # build rebuilds the migrations image from this same source tree (which carries + # migrations/) and restamps this pin in the same commit via + # `make image-migrations` (yq --inplace) — so image and targetVersion are only + # consistent at a release tag, not mid-PR. The digest below therefore lags + # targetVersion in-tree by design; run-migrations.sh refuses to advance past a + # migration file missing from the image (exit 1), so a stale pin fails loudly + # rather than silently skipping the SeaweedFS database hand-over. + image: ghcr.io/cozystack/cozystack/platform-migrations:v1.5.4@sha256:24510ca1f3b789d6cab6cca3810283db490930d78ed35d31a740b17f33106986 + targetVersion: 46 # Bundle deployment configuration bundles: system: @@ -295,6 +303,18 @@ gateway: - cozy-monitoring - cozy-linstor-gui - default +# Backup storage overrides for the platform-managed cozy-default BackupClass. +# Everything set here is forwarded verbatim into the emitted +# cozystack.backupstrategy-controller Package CR as +# components.backupstrategy-controller.values.backupStorage, so it lands on the +# backupstrategy-controller HelmRelease and deep-merges over that chart's own +# backupStorage defaults (packages/system/backupstrategy-controller/values.yaml +# documents every knob: provisionBucket, bucketName, endpoint, region, +# forcePathStyle, systemSecretName, systemNamespaces, …). Admins set this via +# spec.components.platform.values.backupStorage on the +# cozystack.cozystack-platform Package — see docs/operations/backup-classes.md. +# Empty (the default) forwards nothing and keeps the child chart defaults. +backupStorage: {} # Pod scheduling configuration scheduling: globalAppTopologySpreadConstraints: "" diff --git a/packages/core/testing/values.yaml b/packages/core/testing/values.yaml index 7bfaf4fa97..9fff71a2b2 100644 --- a/packages/core/testing/values.yaml +++ b/packages/core/testing/values.yaml @@ -1,2 +1,2 @@ e2e: - image: ghcr.io/cozystack/cozystack/e2e-sandbox:v1.5.0@sha256:91b9a2985bcadea4e74cf9cd644a6d4c19442a7270fe14ac52e766b4ec0c93d8 + image: ghcr.io/cozystack/cozystack/e2e-sandbox:v1.5.4@sha256:41bd612ab75b490a3cdcdd5fca4808faab14a56f87e5ff8d4355bc969f8e13af diff --git a/packages/extra/bootbox/images/matchbox.tag b/packages/extra/bootbox/images/matchbox.tag index 2dd0ecebcc..63a03d7ae5 100644 --- a/packages/extra/bootbox/images/matchbox.tag +++ b/packages/extra/bootbox/images/matchbox.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/matchbox:v1.5.0@sha256:69e89917f3213484c77b1e1948309c340be871245ec5e5cf9850db4f88202e13 +ghcr.io/cozystack/cozystack/matchbox:v1.5.4@sha256:f8eb5bbaec74e107be3b164e87e031c93d08f964753d2e4d7d47b166dc13b6dc diff --git a/packages/extra/seaweedfs/Makefile b/packages/extra/seaweedfs/Makefile index fd0ad40fdd..799919b592 100644 --- a/packages/extra/seaweedfs/Makefile +++ b/packages/extra/seaweedfs/Makefile @@ -4,6 +4,22 @@ include ../../../hack/package.mk test: helm unittest . + $(MAKE) test-guard-fail-closed + +# Sibling of the same target in packages/system/seaweedfs: this chart's copy of +# the naming guard must also refuse an UPGRADE it cannot see the cluster for. +# helm-unittest 1.0.3 ignores release.isUpgrade, so tests/guard_fail_closed_test.yaml +# can only cover the render-through side and passes with or without the canary — +# the refusal needs a real renderer. +.PHONY: test-guard-fail-closed +test-guard-fail-closed: + @out=$$(helm template seaweedfs . -n tenant-root --is-upgrade \ + --set topology=Simple --set replicationFactor=2 \ + --set _namespace.host=example.org --set _namespace.ingress=tenant-root \ + --set _cluster.issuer-name=letsencrypt-prod --set _cluster.solver=http01 2>&1); \ + echo "$$out" | grep -q 'refusing to upgrade blind' \ + || { echo "FAIL: a blind upgrade rendered instead of refusing (naming-guard canary is not firing)"; echo "$$out" | head -5; exit 1; } + @echo "ok: naming guard refuses a blind upgrade" generate: cozyvalues-gen -v values.yaml -s values.schema.json -r README.md diff --git a/packages/extra/seaweedfs/images/objectstorage-sidecar.tag b/packages/extra/seaweedfs/images/objectstorage-sidecar.tag index 3648ad696e..fe16a3cc01 100644 --- a/packages/extra/seaweedfs/images/objectstorage-sidecar.tag +++ b/packages/extra/seaweedfs/images/objectstorage-sidecar.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/objectstorage-sidecar:v1.5.0@sha256:fdc09634688d5579a329b48aabf6706cc55bcd9503b0331fa47764706f0d6f62 +ghcr.io/cozystack/cozystack/objectstorage-sidecar:v1.5.4@sha256:598e1f6aea3bd88a847d68b0602833df9cf32a0d4f99ddefbe1e73ff8e3aaafd diff --git a/packages/extra/seaweedfs/templates/_naming.tpl b/packages/extra/seaweedfs/templates/_naming.tpl new file mode 100644 index 0000000000..4f8935db89 --- /dev/null +++ b/packages/extra/seaweedfs/templates/_naming.tpl @@ -0,0 +1,48 @@ +{{- /* seaweedfs.renamedVolumePrefix — reconstruct the name 4.31 gives the volume + component of the `-system` release, i.e. the RELEASE-NAMED generation the + fullnameOverride pin exists to avoid. + + Input: the `-system` release name, as a bare string. + {{ include "seaweedfs.renamedVolumePrefix" "foo-system" }} -> foo-system-seaweedfs-volume + + This replays two upstream helpers, so it must track them if the vendored chart + changes (hack/seaweedfs-guard-parity.bats pins the two copies of the guard + against each other; charts/seaweedfs/templates/shared/_helpers.tpl is the + source of truth for the rules below): + + seaweedfs.fullname — with no fullnameOverride, the release name, plus + `-` when the release name does not + already contain it; truncated to 63. + seaweedfs.componentName — truncates the fullname to (62 - len(suffix)) before + appending `-`, so for `volume` the fullname + is cut to 56. An instance name >= ~40 chars + therefore loses `seaweedfs` from the tail, which is + why a `contains "seaweedfs"` name filter cannot see + its claims and this reconstruction can. + + The guard deliberately does NOT call seaweedfs.fullname directly: that helper + reads .Values.fullnameOverride, which this chart PINS to `seaweedfs`, so it + returns the chart-named generation — the opposite of what is wanted here. + + ACCEPTED LIMIT — zone/pool volume components. MultiZone (and Simple-with- + pools) tenants get per-group components with suffix `volume-`, cut at + (62 - len(suffix)), i.e. SHORTER than the 56 this helper reconstructs. The + prefixes only diverge when the fullname exceeds that shorter cut: for the + supported instance (the tenant module hardcodes the name `seaweedfs`, + fullname 16 chars) that takes a zone/pool key of ~40+ characters, and for + the default `volume` group it can never happen (its suffix IS the 56 cut). + A guard prefix that diverges means a release-named zone component the guard + cannot see. Decided 2026-07-17 (1.6 triage): accepted and documented in + docs/operations/seaweedfs-431-rename-recovery.md (Scope) rather than + reconstructed per-key — instance names are fixed by the tenant module and + absurd keys are the only reachable trigger. Revisit if instance naming is + ever opened up or a key-length clamp lands in extra/seaweedfs. */}} +{{- define "seaweedfs.renamedVolumePrefix" -}} +{{- $release := . -}} +{{- $full := $release -}} +{{- if not (contains "seaweedfs" $release) -}} +{{- $full = printf "%s-seaweedfs" $release -}} +{{- end -}} +{{- $full = $full | trunc 63 | trimSuffix "-" -}} +{{- printf "%s-volume" ($full | trunc 56 | trimSuffix "-") -}} +{{- end -}} diff --git a/packages/extra/seaweedfs/templates/seaweedfs.yaml b/packages/extra/seaweedfs/templates/seaweedfs.yaml index 0ae2139770..3933234296 100644 --- a/packages/extra/seaweedfs/templates/seaweedfs.yaml +++ b/packages/extra/seaweedfs/templates/seaweedfs.yaml @@ -98,6 +98,99 @@ {{- $solver := (index .Values._cluster "solver") | default "http01" }} {{- $clusterIssuer := (index .Values._cluster "issuer-name") | default "letsencrypt-prod" }} {{- $wildcardSecret := (index .Values._cluster "wildcard-secret-name") | default "" }} +{{- /* Refuse to rename a running SeaweedFS away from its data. + + Before 4.31 the chart named workloads after the CHART, ignoring the release + name, so every instance ran as `seaweedfs-*` with its data on + `data1-seaweedfs-volume-*`. 4.31 names them after the release instead, so the + upgrade could not rename in place (StatefulSet names are immutable) and Helm + stood up a second, empty set while the data stayed put. system/seaweedfs pins + the chart-based name back, so those workloads and volumes are adopted in place. + + These states cannot be adopted and stop the render instead: + + - FRESH on 1.5.x: only the renamed volumes exist, so the data was written under + them. Adopting the chart-based name would rename the workloads AWAY from that + data and bring up an empty cluster beside it. Helm cannot move data between + PVCs, so the operator is sent to the PV re-bind runbook (Step 2). + - D-split: both sets exist AND the renamed volume servers are live, so they may + hold objects written through the split endpoint. Adopting the chart-based set + would strand those. The operator is sent to reconcile the split first (Step 3). + A duplicate that never served (D-wedged: zero ready replicas) is refused the + same way: readyReplicas is a snapshot, not proof it never served, so it cannot + be told apart from D-split and is never adopted blind (see the note below). + + The renamed volumes are matched by SHAPE rather than by reconstructing the + name: a volume PVC is always `data1--volume[-]-N`, and + `` is `seaweedfs` only for the adopted naming. Reconstructing it + instead would have to replay the chart's fullname helper AND its 56-character + component truncation, and would silently miss a tenant whose name is long or + does not contain the chart name. + + PVCs hold the data and outlive any workload, but carry no chart labels, so they + are matched on the name shape and must mention the chart to avoid tripping over + an unrelated `data1-*-volume-*` claim. StatefulSets do carry the labels, so they + are matched on those, which is also the ONLY signal for a tenant whose PVCs are + not provisioned yet and for one whose instance name is long enough that the + chart truncated `seaweedfs` off the PVC names. Either establishes a generation's + presence, so the two are OR-ed. + + Two generations present is UNDECIDABLE from inside a render and this guard does + not guess — see the long note in system/seaweedfs/templates/naming-guard.yaml. + In short: the release history DOES record birth order durably, but birth order + is the wrong question. It says which generation is original; adoption needs to + know whether the other one is EMPTY, and D-wedged and D-split are identical on + every durable signal. + + This copy of the guard warns on the SeaweedFS application itself, where the + operator looks first. The ENFORCING copy lives in system/seaweedfs + (templates/naming-guard.yaml): the -system HelmRelease pulls its chart + from a platform-managed ExternalArtifact, so a platform upgrade re-renders + THAT chart directly without ever re-rendering this one. Keep the two + classifications in sync — they are deliberately identical, and + hack/seaweedfs-guard-parity.bats fails the build if they drift. + + Fail-closed: lookup errors abort the render, but a client-side render returns + nothing — the release namespace is used as a canary, and an upgrade that + cannot see the cluster refuses instead of misclassifying. */}} +{{- $canary := lookup "v1" "Namespace" "" .Release.Namespace }} +{{- if and (not $canary) .Release.IsUpgrade }} +{{- fail (printf "SeaweedFS naming-migration guard for %s in namespace %s: cannot see the cluster (lookup returned nothing for the release namespace itself). This render decides whether workloads are renamed away from live data, so refusing to upgrade blind. Renders applied by helm-controller always see the cluster; if templating by hand, render server-side." .Release.Name .Release.Namespace) }} +{{- end }} +{{- if $canary }} +{{- /* extra/seaweedfs is the `` release; the renamed generation is named + after its CHILD `-system` release, so derive it. This one line is the only + difference from the copy in system/seaweedfs. */}} +{{- $sysRelease := printf "%s-system" .Release.Name }} +{{- $renamedVol := include "seaweedfs.renamedVolumePrefix" $sysRelease }} +{{- $legacyPVC := false }} +{{- $systemPVC := false }} +{{- $legacySTS := false }} +{{- $systemSTS := false }} +{{- range (lookup "v1" "PersistentVolumeClaim" .Release.Namespace "").items | default list }} +{{- if hasPrefix (printf "data1-%s" $renamedVol) .metadata.name }} +{{- $systemPVC = true }} +{{- else if hasPrefix "data1-seaweedfs-volume" .metadata.name }} +{{- $legacyPVC = true }} +{{- end }} +{{- end }} +{{- range (lookup "apps/v1" "StatefulSet" .Release.Namespace "").items | default list }} +{{- if eq (dig "app.kubernetes.io/name" "" (.metadata.labels | default dict)) "seaweedfs" }} +{{- if hasPrefix $renamedVol .metadata.name }} +{{- $systemSTS = true }} +{{- else if hasPrefix "seaweedfs-volume" .metadata.name }} +{{- $legacySTS = true }} +{{- end }} +{{- end }} +{{- end }} +{{- $legacyGen := or $legacyPVC $legacySTS }} +{{- $systemGen := or $systemPVC $systemSTS }} +{{- if and $legacyGen $systemGen }} +{{- fail (printf "SeaweedFS %s in namespace %s has BOTH naming generations present: the chart-named set (seaweedfs-volume / data1-seaweedfs-volume-*) AND the release-named set this chart no longer uses. One of them is an empty duplicate and one holds the data, and which is which cannot be established from inside a Helm render — claim timestamps are mutable (the Step 2 re-bind recreates claims), StatefulSets are recreated by the adoption hook, and a duplicate with zero ready replicas may still have served writes. Rendering would adopt the chart-named set, so guessing wrong strands or destroys the data. Refusing instead. Classify the tenant and delete the EMPTY generation so exactly one remains — this render then adopts the survivor with no further action. See docs/operations/seaweedfs-431-rename-recovery.md (Step 1 classifies; Step 2/2a/3 recover). If you are part-way through Step 2's re-bind, finish it." .Release.Name .Release.Namespace) }} +{{- else if $systemGen }} +{{- fail (printf "SeaweedFS %s in namespace %s keeps its data on volumes named after the Helm release — a tenant installed fresh on Cozystack 1.5.x (or one whose instance name is long enough that the chart truncated the volume PVC names, leaving only its StatefulSets to match on). Workloads are named seaweedfs-*, so rendering would rename them away from that data and bring up an empty cluster. Helm cannot move data between PVCs. Re-bind the existing PVs onto the data1-seaweedfs-volume-* PVC names before upgrading — see docs/operations/seaweedfs-431-rename-recovery.md (Step 2)." .Release.Name .Release.Namespace) }} +{{- end }} +{{- end }} apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: diff --git a/packages/extra/seaweedfs/tests/fullname_override_named_test.yaml b/packages/extra/seaweedfs/tests/fullname_override_named_test.yaml new file mode 100644 index 0000000000..4a6465c3ab --- /dev/null +++ b/packages/extra/seaweedfs/tests/fullname_override_named_test.yaml @@ -0,0 +1,190 @@ +suite: seaweedfs naming guard for a non-default instance name + +# SeaweedFS is a user-creatable kind, so an instance can run under a name other +# than `seaweedfs`. Two things follow, and the guard has to get both right: +# - Before 4.31 the chart ignored the release name, so such an instance ALSO +# stored its data on the chart-named data1-seaweedfs-volume-* PVCs. +# - On 4.31 the fullname helper appends the chart name when the release name +# does not contain it, so a fresh 1.5.x `archive` wrote its data to +# data1-archive-system-seaweedfs-volume-*, NOT data1-archive-system-volume-*. +# +# The guard only runs when it can see the cluster (the release namespace is its +# canary), so every provider registers v1/Namespace and includes the namespace. + +templates: + - templates/seaweedfs.yaml + +release: + name: archive + namespace: tenant-root + +set: &values + topology: Simple + replicationFactor: 2 + _namespace: + host: example.org + ingress: tenant-root + _cluster: + issuer-name: letsencrypt-prod + solver: http01 + +tests: + - it: adopts a pre-4.31 instance whose data is on the chart-named PVCs + set: *values + kubernetesProvider: + scheme: &scheme + "v1/Namespace": + gvr: + group: "" + version: "v1" + resource: "namespaces" + namespaced: false + "v1/PersistentVolumeClaim": + gvr: + group: "" + version: "v1" + resource: "persistentvolumeclaims" + namespaced: true + "apps/v1/StatefulSet": + gvr: + group: "apps" + version: "v1" + resource: "statefulsets" + namespaced: true + objects: + - &ns + kind: Namespace + apiVersion: v1 + metadata: + name: tenant-root + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: data1-seaweedfs-volume-0 + namespace: tenant-root + creationTimestamp: "2025-11-02T10:00:00Z" + - kind: StatefulSet + apiVersion: apps/v1 + metadata: + name: unrelated-app + namespace: tenant-root + documentSelector: + path: kind + value: HelmRelease + asserts: + - equal: + path: metadata.name + value: archive-system + + - it: refuses to render when this release's data is on its renamed volumes + set: *values + kubernetesProvider: + scheme: *scheme + objects: + - *ns + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: data1-archive-system-seaweedfs-volume-0 + namespace: tenant-root + creationTimestamp: "2026-06-20T07:01:59Z" + - kind: StatefulSet + apiVersion: apps/v1 + metadata: + name: unrelated-app + namespace: tenant-root + asserts: + - failedTemplate: + errorPattern: "SeaweedFS archive in namespace tenant-root keeps its data on volumes named after the Helm release" + + - it: refuses when both generations exist (was S-damaged) + set: *values + kubernetesProvider: + scheme: *scheme + objects: + - *ns + # A fresh-1.5.x `archive` that an unguarded upgrade already damaged: the + # renamed volumes are older (the data was born there) and the chart-named + # set is the newer, empty one. Must be S (Step 2a), never D-split. + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: data1-archive-system-seaweedfs-volume-0 + namespace: tenant-root + creationTimestamp: "2026-06-20T07:01:59Z" + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: data1-seaweedfs-volume-0 + namespace: tenant-root + creationTimestamp: "2026-06-20T08:28:58Z" + - kind: StatefulSet + apiVersion: apps/v1 + metadata: + name: archive-system-seaweedfs-volume + namespace: tenant-root + labels: + app.kubernetes.io/name: seaweedfs + status: + readyReplicas: "1" + asserts: + - failedTemplate: + errorPattern: "has BOTH naming generations present" + + - it: catches a long instance name whose renamed workloads the chart truncated + # Instance `archive-of-quarterly-financial-statements-x1` -> child release + # archive-of-quarterly-financial-statements-x1-system -> the fullname gains + # `-seaweedfs` (the release name does not contain it) and is 61 chars, so + # componentName cuts it to 56 before appending `-volume`: the chart name is gone + # from the tail and both the workload and its claims read + # ...-x1-system-seaw-volume. A `contains "seaweedfs"` name filter cannot see + # either of them; the reconstructed prefix matches both exactly. + release: + name: archive-of-quarterly-financial-statements-x1 + namespace: tenant-root + set: *values + kubernetesProvider: + scheme: *scheme + objects: + - *ns + - kind: StatefulSet + apiVersion: apps/v1 + metadata: + name: archive-of-quarterly-financial-statements-x1-system-seaw-volume + namespace: tenant-root + labels: + app.kubernetes.io/name: seaweedfs + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: data1-archive-of-quarterly-financial-statements-x1-system-seaw-volume-0 + namespace: tenant-root + asserts: + - failedTemplate: + errorPattern: "keeps its data on volumes named after the Helm release" + + - it: ignores an unrelated data1-*-volume-* PVC belonging to another app + set: *values + kubernetesProvider: + scheme: *scheme + objects: + - *ns + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: data1-clickhouse-volume-0 + namespace: tenant-root + - kind: StatefulSet + apiVersion: apps/v1 + metadata: + name: clickhouse-volume + namespace: tenant-root + labels: + app.kubernetes.io/name: clickhouse + documentSelector: + path: kind + value: HelmRelease + asserts: + - equal: + path: metadata.name + value: archive-system diff --git a/packages/extra/seaweedfs/tests/fullname_override_test.yaml b/packages/extra/seaweedfs/tests/fullname_override_test.yaml new file mode 100644 index 0000000000..590a9cf6f2 --- /dev/null +++ b/packages/extra/seaweedfs/tests/fullname_override_test.yaml @@ -0,0 +1,329 @@ +suite: seaweedfs legacy-naming guard + +# system/seaweedfs pins the chart-based name, so workloads keep their pre-4.31 +# names (`seaweedfs-master`, `data1-seaweedfs-volume-N`) and an upgrade past the +# 4.31 rename adopts the running set in place. +# +# Any tenant with BOTH naming generations present stops the render instead of +# adopting — which set is the empty duplicate cannot be told from inside a render, +# so the guard refuses and sends the operator to classify. The recovery path +# differs by case: a tenant an unguarded upgrade already damaged (the renamed +# volumes are OLDER — the data was born there and the chart-named set is the empty +# one → Step 2a + Step 2), and a D-split tenant (the legacy volumes are older AND +# the renamed volume servers are live → reconcile the split, Step 3). A duplicate +# that never served (D-wedged, zero ready replicas) is refused the same way: +# readyReplicas is a snapshot, not proof it never served, so it is indistinguishable +# from D-split and cannot be adopted blind. A tenant installed FRESH on 1.5.x (only +# the renamed volumes exist, no second generation) is refused separately → PV +# re-bind, Step 2. +# +# The guard only runs when it can see the cluster: the release namespace is its +# canary. Every test that exercises classification therefore registers +# v1/Namespace and provides the release namespace object; the fail-closed +# behaviour of the canary itself is covered in guard_fail_closed_test.yaml. +# +# Two helm-unittest quirks the mocks work around: +# - The template GETs the namespace and LISTs both PVCs and StatefulSets, so +# every mocked provider must register all three kinds — a fake client errors +# on an unregistered kind rather than returning empty. +# - status.readyReplicas is quoted ("1"/"0"): the fake client cannot deep-copy +# a bare Go int in a mocked object's status, and the template's `int` coerces +# the string back either way. + +templates: + - templates/seaweedfs.yaml + +release: + name: seaweedfs + namespace: tenant-root + +set: &values + topology: Simple + replicationFactor: 2 + _namespace: + host: example.org + ingress: tenant-root + _cluster: + issuer-name: letsencrypt-prod + solver: http01 + +tests: + - it: renders for a net-new install (no SeaweedFS data in the namespace) + set: *values + kubernetesProvider: + scheme: &scheme + "v1/Namespace": + gvr: + group: "" + version: "v1" + resource: "namespaces" + namespaced: false + "v1/PersistentVolumeClaim": + gvr: + group: "" + version: "v1" + resource: "persistentvolumeclaims" + namespaced: true + "apps/v1/StatefulSet": + gvr: + group: "apps" + version: "v1" + resource: "statefulsets" + namespaced: true + objects: + - &ns + kind: Namespace + apiVersion: v1 + metadata: + name: tenant-root + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: unrelated-app-data-0 + namespace: tenant-root + - kind: StatefulSet + apiVersion: apps/v1 + metadata: + name: unrelated-app + namespace: tenant-root + documentSelector: + path: kind + value: HelmRelease + asserts: + - equal: + path: metadata.name + value: seaweedfs-system + # The chart-based name is pinned in system/seaweedfs values, not re-pinned here. + - notExists: + path: spec.values.seaweedfs.fullnameOverride + + - it: renders for a cluster whose data is on the legacy seaweedfs-* PVCs + set: *values + kubernetesProvider: + scheme: *scheme + objects: + - *ns + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: data1-seaweedfs-volume-0 + namespace: tenant-root + creationTimestamp: "2025-11-02T10:00:00Z" + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: unrelated-app-data-0 + namespace: tenant-root + - kind: StatefulSet + apiVersion: apps/v1 + metadata: + name: unrelated-app + namespace: tenant-root + documentSelector: + path: kind + value: HelmRelease + asserts: + - equal: + path: metadata.name + value: seaweedfs-system + + # Was: "renders on a D-wedged cluster — adopts the legacy set". A duplicate + # reading zero ready replicas used to be treated as proof it never served, so + # this rendered through. readyReplicas is only a snapshot: a duplicate that + # crashed, was scaled down, or lost readiness AFTER serving writes looks + # identical, and adopting the legacy set would strand whatever it wrote. Both + # generations present is now refused regardless of liveness. + - it: refuses when both generations exist and the duplicate reads zero ready (was D-wedged, adopted) + set: *values + kubernetesProvider: + scheme: *scheme + objects: + - *ns + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: data1-seaweedfs-volume-0 + namespace: tenant-root + creationTimestamp: "2025-11-02T10:00:00Z" + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: data1-seaweedfs-system-volume-0 + namespace: tenant-root + creationTimestamp: "2026-06-20T12:00:00Z" + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: unrelated-app-data-0 + namespace: tenant-root + - kind: StatefulSet + apiVersion: apps/v1 + metadata: + name: seaweedfs-system-volume + namespace: tenant-root + labels: + app.kubernetes.io/name: seaweedfs + status: + readyReplicas: "0" + asserts: + - failedTemplate: + errorPattern: "has BOTH naming generations present" + + - it: refuses when both generations exist and the duplicate is live (was D-split) + set: *values + kubernetesProvider: + scheme: *scheme + objects: + - *ns + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: data1-seaweedfs-volume-0 + namespace: tenant-root + creationTimestamp: "2025-11-02T10:00:00Z" + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: data1-seaweedfs-system-volume-0 + namespace: tenant-root + creationTimestamp: "2026-06-20T12:00:00Z" + # The duplicate volume set is live: it may hold objects written through + # the split endpoint, so adoption would strand them. + - kind: StatefulSet + apiVersion: apps/v1 + metadata: + name: seaweedfs-system-volume + namespace: tenant-root + labels: + app.kubernetes.io/name: seaweedfs + status: + readyReplicas: "1" + asserts: + - failedTemplate: + errorPattern: "has BOTH naming generations present" + + - it: refuses when both generations exist and the renamed volumes are older (was S-damaged) + set: *values + kubernetesProvider: + scheme: *scheme + objects: + - *ns + # Mirrors a fresh-1.5.x tenant that an unguarded 1.6 upgrade already hit: + # the data was born on the renamed volumes (older), and the upgrade + # created the chart-named set (newer, EMPTY). The renamed set is live — + # which a presence-only guard misreads as D-split, sending the operator + # to quiesce the set that holds all the data. + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: data1-seaweedfs-system-volume-0 + namespace: tenant-root + creationTimestamp: "2026-06-20T07:01:59Z" + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: data1-seaweedfs-volume-0 + namespace: tenant-root + creationTimestamp: "2026-06-20T08:28:58Z" + - kind: StatefulSet + apiVersion: apps/v1 + metadata: + name: seaweedfs-system-volume + namespace: tenant-root + labels: + app.kubernetes.io/name: seaweedfs + status: + readyReplicas: "1" + - kind: StatefulSet + apiVersion: apps/v1 + metadata: + name: seaweedfs-volume + namespace: tenant-root + labels: + app.kubernetes.io/name: seaweedfs + status: + readyReplicas: "0" + asserts: + - failedTemplate: + errorPattern: "has BOTH naming generations present" + + - it: renders for a zoned legacy cluster (volume StatefulSets are per zone) + set: *values + kubernetesProvider: + scheme: *scheme + objects: + - *ns + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: data1-seaweedfs-volume-zone-a-0 + namespace: tenant-root + creationTimestamp: "2025-11-02T10:00:00Z" + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: unrelated-app-data-0 + namespace: tenant-root + - kind: StatefulSet + apiVersion: apps/v1 + metadata: + name: unrelated-app + namespace: tenant-root + documentSelector: + path: kind + value: HelmRelease + asserts: + - equal: + path: metadata.name + value: seaweedfs-system + + - it: refuses to render when the data lives only on seaweedfs-system-* PVCs + set: *values + kubernetesProvider: + scheme: *scheme + objects: + - *ns + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: data1-seaweedfs-system-volume-0 + namespace: tenant-root + creationTimestamp: "2026-06-20T07:01:59Z" + - kind: StatefulSet + apiVersion: apps/v1 + metadata: + name: unrelated-app + namespace: tenant-root + asserts: + - failedTemplate: + errorPattern: "keeps its data on volumes named after the Helm release.*Re-bind" + + - it: refuses to render on a seaweedfs-system-* cluster detected by its StatefulSet + set: *values + kubernetesProvider: + scheme: *scheme + objects: + - *ns + # No volume PVCs are visible at all (e.g. not provisioned yet, or a long + # name truncated the chart name out of them); the renamed StatefulSet + # alone is enough to refuse. + - kind: StatefulSet + apiVersion: apps/v1 + metadata: + name: seaweedfs-system-volume + namespace: tenant-root + labels: + app.kubernetes.io/name: seaweedfs + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: unrelated-app-data-0 + namespace: tenant-root + - kind: StatefulSet + apiVersion: apps/v1 + metadata: + name: unrelated-app + namespace: tenant-root + asserts: + - failedTemplate: + errorPattern: "keeps its data on volumes named after the Helm release" diff --git a/packages/extra/seaweedfs/tests/guard_fail_closed_test.yaml b/packages/extra/seaweedfs/tests/guard_fail_closed_test.yaml new file mode 100644 index 0000000000..3234d3e95e --- /dev/null +++ b/packages/extra/seaweedfs/tests/guard_fail_closed_test.yaml @@ -0,0 +1,42 @@ +suite: seaweedfs naming guard skips cleanly without a cluster view + +# The guard classifies a tenant by looking up its PVCs and StatefulSets. A +# failed API call already aborts the render (Helm propagates lookup errors), +# but a CLIENT-SIDE render (helm template, --dry-run=client) silently returns +# nothing — which, unguarded, would render the chart-based names over a class-S +# tenant. The release namespace is the canary: it always exists for a real +# install or upgrade, so when it cannot be seen, an UPGRADE refuses rather than +# classify blind, while an install without cluster access (CI lint, unittest +# with no provider — this test) skips the guard: it never touches live data. +# +# The refusal branch cannot be modelled here: helm-unittest 1.0.3 ignores +# release.isUpgrade (.Release.IsUpgrade renders false regardless). Verify it +# with a real renderer instead: +# helm template packages/extra/seaweedfs --is-upgrade ... 2>&1 \ +# | grep 'refusing to upgrade blind' + +templates: + - templates/seaweedfs.yaml + +release: + name: seaweedfs + namespace: tenant-root + +tests: + - it: still renders a client-side install (no cluster view, no data at risk) + set: + topology: Simple + replicationFactor: 2 + _namespace: + host: example.org + ingress: tenant-root + _cluster: + issuer-name: letsencrypt-prod + solver: http01 + documentSelector: + path: kind + value: HelmRelease + asserts: + - equal: + path: metadata.name + value: seaweedfs-system diff --git a/packages/system/backup-controller/values.yaml b/packages/system/backup-controller/values.yaml index b57ccc124d..ea0fbcf950 100644 --- a/packages/system/backup-controller/values.yaml +++ b/packages/system/backup-controller/values.yaml @@ -1,5 +1,5 @@ backupController: - image: "ghcr.io/cozystack/cozystack/backup-controller:v1.5.0@sha256:11d55b79cf136f9ec161410e2357dde7bc063246f905d5ac962ad8432af04962" + image: "ghcr.io/cozystack/cozystack/backup-controller:v1.5.4@sha256:55724072553d13228135628cbfc6fb7988694e7b477e9c38e547c1654800ef19" replicas: 2 debug: false metrics: diff --git a/packages/system/backupstrategy-controller/values.yaml b/packages/system/backupstrategy-controller/values.yaml index d36635507a..dadf57c094 100644 --- a/packages/system/backupstrategy-controller/values.yaml +++ b/packages/system/backupstrategy-controller/values.yaml @@ -1,5 +1,5 @@ backupStrategyController: - image: "ghcr.io/cozystack/cozystack/backupstrategy-controller:v1.5.0@sha256:d965283f935e4b24b655bc76e37dcb3b731bec832bb5c2e3e407b0284c7526a3" + image: "ghcr.io/cozystack/cozystack/backupstrategy-controller:v1.5.4@sha256:df74d5bf4c29a8ea5bc2998c8f5b96bdb9a6678938f46b88b775497ec3eb62b8" # chBackupClientImage is the image rendered into the Altinity strategy # Pod: it drives clickhouse-backup's HTTP API via curl + jq. We reuse # platform-migrations (already pre-built and digested by release CI in diff --git a/packages/system/bucket/images/s3manager.tag b/packages/system/bucket/images/s3manager.tag index 82cda22748..0ad27e5c43 100644 --- a/packages/system/bucket/images/s3manager.tag +++ b/packages/system/bucket/images/s3manager.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/s3manager:v1.5.0@sha256:8b36ab67904e34b2fb843b964c20d4822f3a6673f6745afb6a834efd02e017e2 +ghcr.io/cozystack/cozystack/s3manager:v1.5.4@sha256:10f58ec0324cc466b522f9141364d5137819b0530a01f74e25070e3aa63699e8 diff --git a/packages/system/cert-manager/Makefile b/packages/system/cert-manager/Makefile index c3ff574b72..a0885bf9aa 100644 --- a/packages/system/cert-manager/Makefile +++ b/packages/system/cert-manager/Makefile @@ -3,6 +3,9 @@ export NAMESPACE=cozy-$(NAME) include ../../../hack/package.mk +test: + helm unittest . + update: rm -rf charts helm repo add jetstack https://charts.jetstack.io diff --git a/packages/system/cert-manager/tests/cainjector_resources_test.yaml b/packages/system/cert-manager/tests/cainjector_resources_test.yaml new file mode 100644 index 0000000000..5dab84b0d8 --- /dev/null +++ b/packages/system/cert-manager/tests/cainjector_resources_test.yaml @@ -0,0 +1,30 @@ +suite: cert-manager cainjector resources + +# Pins the cainjector memory sizing that keeps CA injection working on a full +# cozystack install. cainjector loads all ValidatingWebhookConfigurations, +# APIServices and CRDs into its informer caches at startup; on a large cluster +# that working set exceeds the old 128Mi limit, so the single leader-elected +# cainjector Pod is OOMKilled during cache population and CrashLoops before it +# ever injects a caBundle. Every webhook then keeps an empty caBundle and +# admission calls fail with "x509: certificate signed by unknown authority" +# (failurePolicy: Fail). This guards the override in values.yaml against a +# blanket resource change or a `make update` silently lowering it again. + +templates: + - charts/cert-manager/templates/cainjector-deployment.yaml + +release: + name: cert-manager + namespace: cozy-cert-manager + +tests: + - it: sizes the cainjector memory limit above the OOM floor + asserts: + - isKind: + of: Deployment + - equal: + path: spec.template.spec.containers[0].resources.limits.memory + value: 512Mi + - equal: + path: spec.template.spec.containers[0].resources.requests.memory + value: 128Mi diff --git a/packages/system/cert-manager/tests/webhook_secure_port_test.yaml b/packages/system/cert-manager/tests/webhook_secure_port_test.yaml new file mode 100644 index 0000000000..6546b4bf53 --- /dev/null +++ b/packages/system/cert-manager/tests/webhook_secure_port_test.yaml @@ -0,0 +1,64 @@ +suite: cert-manager webhook secure port + +# Keeps the webhook off the kubelet's port. The values.yaml override this +# guards carries the reasoning; it is not repeated here. +# +# The first test is the regression guard: it fails if the override is dropped +# or if `make update` re-vendors a chart that ignores it, and it asserts the +# listener flag together with the advertised containerPort so the two cannot +# drift apart. The other two pin preconditions the fix depends on and already +# hold on their own — that the port lives in the Pod's network namespace +# rather than the node's, and that the Service reaches it by name, which is +# what lets a port change roll out without dropping an endpoint. +# +# An upstream rename of the key needs no test: the vendored values.schema.json +# sets additionalProperties: false, so the override would fail helm outright +# rather than be silently ignored. + +templates: + - charts/cert-manager/templates/webhook-deployment.yaml + - charts/cert-manager/templates/webhook-service.yaml + +release: + name: cert-manager + namespace: cozy-cert-manager + +tests: + - it: serves the webhook on 10260, not the kubelet's 10250 + template: charts/cert-manager/templates/webhook-deployment.yaml + asserts: + - isKind: + of: Deployment + - contains: + path: spec.template.spec.containers[0].args + content: --secure-port=10260 + - equal: + path: spec.template.spec.containers[0].ports[0] + value: + name: https + protocol: TCP + containerPort: 10260 + + - it: keeps the webhook off the host network, so the port lives in the Pod netns + template: charts/cert-manager/templates/webhook-deployment.yaml + asserts: + # The template emits the key only when the value is truthy, so absence + # is the only shape a disabled hostNetwork can take here. isNullOrEmpty + # would not do: it fails on a path that does not exist. + - notExists: + path: spec.template.spec.hostNetwork + + - it: resolves the Service to the webhook container by name, not by number + template: charts/cert-manager/templates/webhook-service.yaml + asserts: + - isKind: + of: Service + - equal: + path: spec.ports[0].name + value: https + - equal: + path: spec.ports[0].port + value: 443 + - equal: + path: spec.ports[0].targetPort + value: https diff --git a/packages/system/cert-manager/values.yaml b/packages/system/cert-manager/values.yaml index e69de29bb2..48e7884b18 100644 --- a/packages/system/cert-manager/values.yaml +++ b/packages/system/cert-manager/values.yaml @@ -0,0 +1,74 @@ +cert-manager: + resources: + limits: + cpu: 500m + memory: 512Mi + requests: + cpu: 50m + memory: 128Mi + webhook: + # Upstream defaults securePort to 10250, the port the kubelet serves on. + # That default is chosen for GKE private clusters, where apiservers may + # reach nodes only on 443 and 10250; it costs nothing there and is unsafe + # here. When a connection to the webhook Service resolves to a node IP + # rather than a Pod IP, it reaches the kubelet, which completes the TLS + # handshake with its node serving certificate, and the apiserver rejects + # every cert-manager admission call cluster-wide with "x509: certificate + # is valid for , not cert-manager-webhook.cozy-cert-manager.svc". + # The certificate is valid — it is simply the wrong server's — so the + # error reads as a cert-manager fault and hides the misroute that caused + # it. Serving on a port nothing else answers on turns that same misroute + # into a connection refusal, which names the real problem. This does not + # prevent the misroute; it removes the collision that makes one + # catastrophic and misattributed. + # + # What has to hold for 10260 to be an improvement is that nothing answers + # on it in the node's network namespace, since the misroute lands on a + # node IP. Nothing does, in either cluster this chart is installed into. + # On management nodes the only host-network workload of the kind that + # might is local-ccm, which links none of the cloud-provider framework + # that binds 10258 and 10260 by default, and binds no TCP port at all. + # No cloud-controller-manager runs on a tenant node either: the kubevirt + # one does build on that framework, but it is a pod-networked Deployment + # in the management cluster, so whatever it binds stays in a Pod's + # namespace. Anything host-network added later to either cluster has to + # keep clear of the port. + # + # Under webhook.hostNetwork the webhook itself would join that namespace, + # where the clusterwide policy in packages/system/cilium-networkpolicy + # denies 10250 from the world but has no entry for 10260. Enabling + # hostNetwork means revisiting that policy. + securePort: 10260 + resources: + limits: + cpu: 200m + memory: 128Mi + requests: + cpu: 10m + memory: 32Mi + cainjector: + # cainjector loads all ValidatingWebhookConfigurations, APIServices and CRDs + # into its informer caches at startup. On a full cozystack install that + # working set exceeds 128Mi, so cainjector is OOMKilled during cache + # population and CrashLoops before it ever injects a caBundle. Because it + # runs as a single leader-elected replica, that stalls CA injection for the + # whole cluster: webhook ValidatingWebhookConfigurations keep an empty + # caBundle and every admission call fails with + # "x509: certificate signed by unknown authority" (failurePolicy: Fail). + # Size the limit to the controller's 512Mi and lift the request off the + # OOM floor so injection completes deterministically. + resources: + limits: + cpu: 200m + memory: 512Mi + requests: + cpu: 10m + memory: 128Mi + startupapicheck: + resources: + limits: + cpu: 100m + memory: 64Mi + requests: + cpu: 10m + memory: 16Mi diff --git a/packages/system/cilium/values.yaml b/packages/system/cilium/values.yaml index 8a6ab3df66..5120707683 100644 --- a/packages/system/cilium/values.yaml +++ b/packages/system/cilium/values.yaml @@ -15,7 +15,7 @@ cilium: mode: "kubernetes" image: repository: ghcr.io/cozystack/cozystack/cilium - tag: v1.5.0 + tag: v1.5.4 digest: "sha256:94dc74c9301f0da5695c80e77b77ee0486ad0086b5be60a94136a2cafd410b73" envoy: enabled: true diff --git a/packages/system/cozystack-api/values.yaml b/packages/system/cozystack-api/values.yaml index db6102a35c..6399c85d1d 100644 --- a/packages/system/cozystack-api/values.yaml +++ b/packages/system/cozystack-api/values.yaml @@ -1,3 +1,3 @@ cozystackAPI: - image: ghcr.io/cozystack/cozystack/cozystack-api:v1.5.0@sha256:ef50b155e419cf8f0b4d57af9fa385ecbacd5c1248f13591ffcca75c2f9f2a07 + image: ghcr.io/cozystack/cozystack/cozystack-api:v1.5.4@sha256:febaefa1018d6407d58332f9804729fe080eb127caf85c40497c1a6591c8d399 replicas: 2 diff --git a/packages/system/cozystack-basics/templates/gateway-hostname-policy.yaml b/packages/system/cozystack-basics/templates/gateway-hostname-policy.yaml index 193ac9efab..f048a4fd10 100644 --- a/packages/system/cozystack-basics/templates/gateway-hostname-policy.yaml +++ b/packages/system/cozystack-basics/templates/gateway-hostname-policy.yaml @@ -1,3 +1,5 @@ +{{- /* Render only where the ValidatingAdmissionPolicy API is served (GA since Kubernetes 1.30; the management cluster requires 1.33+). Same guard as packages/core/platform/templates/deletion-protection.yaml. */}} +{{- if .Capabilities.APIVersions.Has "admissionregistration.k8s.io/v1/ValidatingAdmissionPolicy" }} --- apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingAdmissionPolicy @@ -154,3 +156,4 @@ metadata: spec: policyName: cozystack-namespace-host-label-policy validationActions: [Deny] +{{- end }} diff --git a/packages/system/cozystack-basics/templates/route-hostname-policy.yaml b/packages/system/cozystack-basics/templates/route-hostname-policy.yaml index cde87d2494..ec39b80942 100644 --- a/packages/system/cozystack-basics/templates/route-hostname-policy.yaml +++ b/packages/system/cozystack-basics/templates/route-hostname-policy.yaml @@ -36,6 +36,8 @@ finished labelling yet — in which case the caller should retry. */}} {{- $celValidator := `(namespaceObject == null || !has(namespaceObject.metadata.labels) || !("namespace.cozystack.io/host" in namespaceObject.metadata.labels)) ? false : (!has(object.spec.hostnames) || object.spec.hostnames.all(h, h == namespaceObject.metadata.labels["namespace.cozystack.io/host"] || h.endsWith("." + namespaceObject.metadata.labels["namespace.cozystack.io/host"])))` -}} +{{- /* Render only where the ValidatingAdmissionPolicy API is served (GA since Kubernetes 1.30; the management cluster requires 1.33+). Same guard as packages/core/platform/templates/deletion-protection.yaml. */}} +{{- if .Capabilities.APIVersions.Has "admissionregistration.k8s.io/v1/ValidatingAdmissionPolicy" }} --- apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingAdmissionPolicy @@ -104,3 +106,4 @@ metadata: spec: policyName: cozystack-route-hostname-policy-tls validationActions: [Deny] +{{- end }} diff --git a/packages/system/cozystack-basics/tests/gateway-hostname-policy_test.yaml b/packages/system/cozystack-basics/tests/gateway-hostname-policy_test.yaml index 50940dee43..80749c4813 100644 --- a/packages/system/cozystack-basics/tests/gateway-hostname-policy_test.yaml +++ b/packages/system/cozystack-basics/tests/gateway-hostname-policy_test.yaml @@ -19,6 +19,13 @@ release: # Both gates are dropped; inheritance now flows via the label # selector on Gateway.spec.listeners[].allowedRoutes. +# The template now gates on the ValidatingAdmissionPolicy API (GA since +# Kubernetes 1.30); helm-unittest's default capability set omits it, so every +# rendering test declares it, exactly as the platform deletion-protection suite. +capabilities: + apiVersions: + - admissionregistration.k8s.io/v1/ValidatingAdmissionPolicy + tests: - it: renders 3 VAPs + 3 Bindings (6 documents total) asserts: diff --git a/packages/system/cozystack-basics/tests/hostname-policies-capability-gate_test.yaml b/packages/system/cozystack-basics/tests/hostname-policies-capability-gate_test.yaml new file mode 100644 index 0000000000..7bd8e9424c --- /dev/null +++ b/packages/system/cozystack-basics/tests/hostname-policies-capability-gate_test.yaml @@ -0,0 +1,23 @@ +suite: cozystack-basics hostname VAP policies are gated on the ValidatingAdmissionPolicy API +templates: + - templates/route-hostname-policy.yaml + - templates/gateway-hostname-policy.yaml +release: + name: cozystack-basics + namespace: cozy-system +# Neither template reads a value that could independently suppress it, so the +# only reason either renders nothing here is the missing ValidatingAdmissionPolicy +# API declared below. helm-unittest's per-test capabilities merge rather than +# replace the suite's, so the absent case is a separate suite pinned at suite +# level (mirrors the platform notes gate suite). +capabilities: + apiVersions: [] +tests: + - it: neither hostname VAP policy renders when the API is unavailable + asserts: + - hasDocuments: + count: 0 + template: templates/route-hostname-policy.yaml + - hasDocuments: + count: 0 + template: templates/gateway-hostname-policy.yaml diff --git a/packages/system/cozystack-basics/tests/route-hostname-policy_test.yaml b/packages/system/cozystack-basics/tests/route-hostname-policy_test.yaml index 36f87c7a47..16ef82a67a 100644 --- a/packages/system/cozystack-basics/tests/route-hostname-policy_test.yaml +++ b/packages/system/cozystack-basics/tests/route-hostname-policy_test.yaml @@ -6,6 +6,13 @@ release: name: cozystack-basics namespace: cozy-system +# The template now gates on the ValidatingAdmissionPolicy API (GA since +# Kubernetes 1.30); helm-unittest's default capability set omits it, so every +# rendering test declares it, exactly as the platform deletion-protection suite. +capabilities: + apiVersions: + - admissionregistration.k8s.io/v1/ValidatingAdmissionPolicy + tests: - it: renders ValidatingAdmissionPolicy + Binding for HTTPRoute and TLSRoute (Layer 7) # Layer 7 (route-hostname VAP) is the only VAP rendered by THIS diff --git a/packages/system/cozystack-controller/values.yaml b/packages/system/cozystack-controller/values.yaml index 2d4adaa88c..f5db5800be 100644 --- a/packages/system/cozystack-controller/values.yaml +++ b/packages/system/cozystack-controller/values.yaml @@ -1,4 +1,4 @@ cozystackController: - image: ghcr.io/cozystack/cozystack/cozystack-controller:v1.5.0@sha256:d3be63579fcabfa802a61ff2f4c2f54d5d6d52c1cbcdff52a3f14d101ca85afc + image: ghcr.io/cozystack/cozystack/cozystack-controller:v1.5.4@sha256:239f6f84c06e5a1584fe6b90c7f25bbac096af866e45a2cc30774fbb284012bc debug: false disableTelemetry: false diff --git a/packages/system/dashboard/templates/gatekeeper.yaml b/packages/system/dashboard/templates/gatekeeper.yaml index 8d4b13f74b..3aa6c5bef8 100644 --- a/packages/system/dashboard/templates/gatekeeper.yaml +++ b/packages/system/dashboard/templates/gatekeeper.yaml @@ -68,6 +68,8 @@ spec: {{- end }} - --whitelist-domain=keycloak.{{ $host }} - --email-domain=* + - --api-route=^/api(/|$) + - --api-route=^/apis(/|$) - --pass-access-token=true - --pass-authorization-header=true - --cookie-refresh=3m diff --git a/packages/system/dashboard/templates/keycloakclient.yaml b/packages/system/dashboard/templates/keycloakclient.yaml index fa33fea4a8..cd455af7ff 100644 --- a/packages/system/dashboard/templates/keycloakclient.yaml +++ b/packages/system/dashboard/templates/keycloakclient.yaml @@ -64,6 +64,8 @@ spec: directAccess: true public: false webUrl: "https://dashboard.{{ $host }}" + webOrigins: + - "+" defaultClientScopes: - groups - kubernetes-client diff --git a/packages/system/dashboard/values.yaml b/packages/system/dashboard/values.yaml index c5ff6b4b30..29f59f11b9 100644 --- a/packages/system/dashboard/values.yaml +++ b/packages/system/dashboard/values.yaml @@ -1,4 +1,4 @@ console: - image: ghcr.io/cozystack/cozystack/cozystack-ui:v1.5.0@sha256:f0329ac5689447d3aec0e4d420196f42bda466a7789312ac5ce9b11405f1948d + image: ghcr.io/cozystack/cozystack/cozystack-ui:v1.5.4@sha256:418dd6ddbfa4516d72278fd5579fd7e52fbe2da9be70fc8e6d0b3d2f29082685 tokenProxy: - image: ghcr.io/cozystack/cozystack/token-proxy:v1.5.0@sha256:0382e8a9014a842d2eddf1bac5b598f42a4ada7a9be673d9272f1610d47631c4 + image: ghcr.io/cozystack/cozystack/token-proxy:v1.5.4@sha256:470790ccf0b8ef871927440a91b76114aa2aa238fe04436f523d3e8be566b190 diff --git a/packages/system/flux-shard-operator/README.md b/packages/system/flux-shard-operator/README.md index 660a6542b6..9f06cb9b6a 100644 --- a/packages/system/flux-shard-operator/README.md +++ b/packages/system/flux-shard-operator/README.md @@ -6,7 +6,7 @@ Spreads tenant HelmReleases across multiple helm-controller shards so a noisy te The operator has three parts, all served by one Deployment (leader-elected controllers, webhook on every replica): -1. **Shard runtime.** Reconciles `shardCount` helm-controller Deployments (`helm-controller-shard`, `--watch-label-selector=sharding.fluxcd.io/key=shard`) in the flux namespace, cloned from the flux-aio `flux` Deployment's helm-controller container and sanitised (no host networking, no localhost cross-container wiring). The helm-controller image and feature-gates are inherited from flux-aio automatically. Deployments beyond `shardCount` are pruned once they drain, and the legacy hand-rolled `flux-tenants` Deployment is retired once no HelmRelease carries `sharding.fluxcd.io/key=tenants`. +1. **Shard runtime.** Reconciles `shardCount` helm-controller Deployments (`helm-controller-shard`, `--watch-label-selector=sharding.fluxcd.io/key=shard`) in the flux namespace, cloned from the flux-aio `flux` Deployment's helm-controller container and sanitised (no host networking, no localhost cross-container wiring, no inherited corporate-proxy env, and a startupProbe guarding a slow start). The helm-controller image and feature-gates are inherited from flux-aio automatically. Deployments beyond `shardCount` are pruned once they drain, and the legacy hand-rolled `flux-tenants` Deployment is retired once no HelmRelease carries `sharding.fluxcd.io/key=tenants`. 2. **Placement controller.** Owns the tenant→shard assignment. The unit of placement is the tenant: all HelmReleases of one tenant (parent `tenant-` plus everything in namespace `tenant-`) carry the same shard label, so a noisy tenant's blast radius is bounded to its shard's co-residents. Tenants are distributed greedy least-loaded, weighted by HelmRelease count (N tenants over N shards land exactly 1 per shard). The assignment is recorded as the `internal.cozystack.io/flux-shard` label on the tenant namespace; HelmRelease labels remain the source of truth on restarts. Moves are paced and deleting tenants are never moved. Watches are metadata-only, so the controller does not decode the helm-controller status-patch firehose. diff --git a/packages/system/flux-shard-operator/values.yaml b/packages/system/flux-shard-operator/values.yaml index 18bddbbba3..7036760275 100644 --- a/packages/system/flux-shard-operator/values.yaml +++ b/packages/system/flux-shard-operator/values.yaml @@ -1,5 +1,5 @@ fluxShardOperator: - image: ghcr.io/cozystack/cozystack/flux-shard-operator:v1.5.0@sha256:22f99597b1d21a913cb7b81f2faa3cdb983f2121164d220955d4d6e120ef4cd3 + image: ghcr.io/cozystack/cozystack/flux-shard-operator:v1.5.4@sha256:a285e51bdedda916dcfc970b079211a161026816792053042833cd2a900a6b95 debug: false replicas: 2 ## Number of helm-controller shards to provision and distribute tenants diff --git a/packages/system/fluxcd/Makefile b/packages/system/fluxcd/Makefile index 9aa32fe9b1..c398a1b7e6 100644 --- a/packages/system/fluxcd/Makefile +++ b/packages/system/fluxcd/Makefile @@ -8,4 +8,5 @@ apply-locally: update: rm -rf charts - helm pull oci://ghcr.io/controlplaneio-fluxcd/charts/flux-instance --untar --untardir charts + helm pull oci://ghcr.io/controlplaneio-fluxcd/charts/flux-instance --version 0.50.0 --untar --untardir charts + patch --no-backup-if-mismatch -p1 < patches/guard-distribution-artifact.diff diff --git a/packages/system/fluxcd/charts/flux-instance/templates/instance.yaml b/packages/system/fluxcd/charts/flux-instance/templates/instance.yaml index f1f58309e3..5217c24d9c 100644 --- a/packages/system/fluxcd/charts/flux-instance/templates/instance.yaml +++ b/packages/system/fluxcd/charts/flux-instance/templates/instance.yaml @@ -16,7 +16,9 @@ spec: distribution: version: {{ .Values.instance.distribution.version | quote }} registry: {{ .Values.instance.distribution.registry }} + {{- if .Values.instance.distribution.artifact }} artifact: {{ .Values.instance.distribution.artifact }} + {{- end }} {{- if .Values.instance.distribution.artifactPullSecret }} artifactPullSecret: {{ .Values.instance.distribution.artifactPullSecret }} {{- end }} diff --git a/packages/system/fluxcd/patches/guard-distribution-artifact.diff b/packages/system/fluxcd/patches/guard-distribution-artifact.diff new file mode 100644 index 0000000000..57348afe5f --- /dev/null +++ b/packages/system/fluxcd/patches/guard-distribution-artifact.diff @@ -0,0 +1,22 @@ +# Upstream bug: flux-instance's instance.yaml renders spec.distribution.artifact +# unconditionally, unlike the sibling optional fields (artifactPullSecret, +# imagePullSecret, variant). The umbrella's empty default then renders as +# `artifact: null`, which the FluxInstance CRD rejects (type: string, +# pattern ^oci://.*$, and the field is optional). Guard it like its siblings. +# +# Chart source: https://github.com/controlplaneio-fluxcd/charts (flux-instance) +# Not yet reported upstream; drop this hunk once upstream guards the field. +diff --git a/charts/flux-instance/templates/instance.yaml b/charts/flux-instance/templates/instance.yaml +index f1f58309e..5217c24d9 100644 +--- a/charts/flux-instance/templates/instance.yaml ++++ b/charts/flux-instance/templates/instance.yaml +@@ -16,7 +16,9 @@ spec: + distribution: + version: {{ .Values.instance.distribution.version | quote }} + registry: {{ .Values.instance.distribution.registry }} ++ {{- if .Values.instance.distribution.artifact }} + artifact: {{ .Values.instance.distribution.artifact }} ++ {{- end }} + {{- if .Values.instance.distribution.artifactPullSecret }} + artifactPullSecret: {{ .Values.instance.distribution.artifactPullSecret }} + {{- end }} diff --git a/packages/system/grafana-operator/images/grafana-dashboards.tag b/packages/system/grafana-operator/images/grafana-dashboards.tag index 7a4976500d..fd24a9d4ee 100644 --- a/packages/system/grafana-operator/images/grafana-dashboards.tag +++ b/packages/system/grafana-operator/images/grafana-dashboards.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/grafana-dashboards:v1.5.0@sha256:f56dc47b970ebf4720c555468aabfb0f6bad4e15efa5bbd6644bf3e5ae9d843e +ghcr.io/cozystack/cozystack/grafana-dashboards:v1.5.4@sha256:a5e45336aaa84b94bc02201e6d0d0c85a5bc9f1b6605e425e99bce50e41e58dd diff --git a/packages/system/kamaji/images/kamaji/patches/fix-datastore-unused-deletion.diff b/packages/system/kamaji/images/kamaji/patches/fix-datastore-unused-deletion.diff new file mode 100644 index 0000000000..53073699b1 --- /dev/null +++ b/packages/system/kamaji/images/kamaji/patches/fix-datastore-unused-deletion.diff @@ -0,0 +1,12 @@ +diff --git a/controllers/datastore_controller.go b/controllers/datastore_controller.go +--- a/controllers/datastore_controller.go ++++ b/controllers/datastore_controller.go +@@ -151,7 +151,7 @@ func (r *DataStore) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { + return reconcile.Result{}, nil + } + +- if meta.IsStatusConditionFalse(ds.Status.Conditions, kamajiv1alpha1.DataStoreConditionAllowedDeletionType) { ++ if meta.IsStatusConditionFalse(ds.Status.Conditions, kamajiv1alpha1.DataStoreConditionAllowedDeletionType) || len(tcpList.Items) == 0 { + logger.Info("DataStore is not used by any TenantControlPlane object") + + meta.SetStatusCondition(&ds.Status.Conditions, metav1.Condition{ diff --git a/packages/system/kamaji/values.yaml b/packages/system/kamaji/values.yaml index 0acb625ef8..9a35010048 100644 --- a/packages/system/kamaji/values.yaml +++ b/packages/system/kamaji/values.yaml @@ -3,7 +3,7 @@ kamaji: deploy: false image: pullPolicy: IfNotPresent - tag: v1.5.0@sha256:130d5013a9b35be0013b96efefc656210cc30c6fe92a8c39c9d2ab8d47f791e0 + tag: v1.5.4@sha256:fffbe9efc6b5fc79d2d2fddc4836af93e2f3975e97a100768decf79aa58592d7 repository: ghcr.io/cozystack/cozystack/kamaji resources: limits: @@ -36,4 +36,4 @@ kamaji: periodSeconds: 10 timeoutSeconds: 1 extraArgs: - - --migrate-image=ghcr.io/cozystack/cozystack/kamaji:v1.5.0@sha256:130d5013a9b35be0013b96efefc656210cc30c6fe92a8c39c9d2ab8d47f791e0 + - --migrate-image=ghcr.io/cozystack/cozystack/kamaji:v1.5.4@sha256:fffbe9efc6b5fc79d2d2fddc4836af93e2f3975e97a100768decf79aa58592d7 diff --git a/packages/system/keycloak-configure/Makefile b/packages/system/keycloak-configure/Makefile index 4cd5bd2ed1..b0a357e692 100644 --- a/packages/system/keycloak-configure/Makefile +++ b/packages/system/keycloak-configure/Makefile @@ -3,3 +3,6 @@ export NAMESPACE=cozy-keycloak include ../../../hack/common-envs.mk include ../../../hack/package.mk + +test: + helm unittest . diff --git a/packages/system/keycloak-configure/templates/delete.yaml b/packages/system/keycloak-configure/templates/delete.yaml index 99abd6e67e..f499c77f6d 100644 --- a/packages/system/keycloak-configure/templates/delete.yaml +++ b/packages/system/keycloak-configure/templates/delete.yaml @@ -39,7 +39,7 @@ spec: done done - kubectl patch hr keycloak-configure -n cozy-system --type=merge -p '{"metadata":{"finalizers":[]}}' + kubectl patch hr {{ .Release.Name }} -n {{ .Release.Namespace }} --type=merge -p '{"metadata":{"finalizers":[]}}' --- diff --git a/packages/system/keycloak-configure/tests/delete_test.yaml b/packages/system/keycloak-configure/tests/delete_test.yaml new file mode 100644 index 0000000000..9e740d5b83 --- /dev/null +++ b/packages/system/keycloak-configure/tests/delete_test.yaml @@ -0,0 +1,58 @@ +suite: flux teardown job patches the HelmRelease in the release namespace + +# The pre-delete teardown Job clears the HelmRelease finalizers as its last +# step. Its RBAC (Role + RoleBinding, resourceNames: [.Release.Name]) is created +# in .Release.Namespace, so the kubectl patch MUST target that same namespace and +# release name. A hardcoded namespace that differs from the install namespace +# makes the ServiceAccount Forbidden to patch the HelmRelease, the Job retries +# forever, and the Helm release wedges in "uninstalling". +release: + name: keycloak-configure + namespace: cozy-keycloak +tests: + - it: teardown patches the HelmRelease using the release name and namespace + template: templates/delete.yaml + documentSelector: + path: kind + value: Job + set: + _cluster: + root-host: example.com + kube-root-ca: "" + asserts: + - matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: kubectl patch hr keycloak-configure -n cozy-keycloak + + - it: teardown never targets a namespace other than the release namespace + template: templates/delete.yaml + documentSelector: + path: kind + value: Job + set: + _cluster: + root-host: example.com + kube-root-ca: "" + asserts: + - notMatchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: patch hr \S+ -n cozy-system + + # Prove the patch is parameterized on .Release.Name / .Release.Namespace + # rather than matching the default install namespace by coincidence. + - it: teardown tracks a non-default release name and namespace + template: templates/delete.yaml + documentSelector: + path: kind + value: Job + release: + name: custom-release + namespace: custom-ns + set: + _cluster: + root-host: example.com + kube-root-ca: "" + asserts: + - matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: kubectl patch hr custom-release -n custom-ns diff --git a/packages/system/kubeovn-plunger/values.yaml b/packages/system/kubeovn-plunger/values.yaml index aa48658363..dc4c549f4f 100644 --- a/packages/system/kubeovn-plunger/values.yaml +++ b/packages/system/kubeovn-plunger/values.yaml @@ -1,4 +1,4 @@ portSecurity: true routes: "" -image: ghcr.io/cozystack/cozystack/kubeovn-plunger:v1.5.0@sha256:908b5903c014f6fa4348f57b1fced929a2ba823aadfbcda7ad9b76134bcf3fce +image: ghcr.io/cozystack/cozystack/kubeovn-plunger:v1.5.4@sha256:d821ac11fffc0e277b78491f1fd9ed8d5c6d17fc21d8048ff50eef40077783dc ovnCentralName: ovn-central diff --git a/packages/system/kubeovn-webhook/images/kubeovn-webhook/cert.go b/packages/system/kubeovn-webhook/images/kubeovn-webhook/cert.go new file mode 100644 index 0000000000..1fcbf573b4 --- /dev/null +++ b/packages/system/kubeovn-webhook/images/kubeovn-webhook/cert.go @@ -0,0 +1,45 @@ +package main + +import "crypto/tls" + +// newReloadingTLSConfig returns a tls.Config that reloads the certificate and +// key from disk on every TLS handshake, so a cert-manager renewal of the mounted +// Secret is picked up by the running process without a restart. +// +// Without this, the key pair is read exactly once at startup and cached for the +// lifetime of the process: after the certificate expires (roughly one year after +// install) the pod keeps presenting the expired certificate, the kube-apiserver's +// TLS call to the webhook fails, and because the MutatingWebhookConfiguration uses +// failurePolicy: Fail, every pod creation in tenant namespaces is rejected. +// +// tls.Config.GetCertificate is invoked on every handshake whenever Certificates is +// left unset, so re-reading the files there is enough; no watcher, mtime tracking +// or cache is needed: +// - The cert/key files are mounted from a plain Secret volume with no subPath. +// Kubernetes' atomic writer swaps the whole ..data directory via a single +// symlink flip, so a reader always sees a complete old or complete new +// generation of both files, never a torn mid-write mixture. A per-handshake +// LoadX509KeyPair therefore cannot observe a partial renewal. +// - The per-handshake cost (a few KB of file I/O plus a PEM/key parse) is +// negligible next to the asymmetric crypto the handshake already performs. +// - The webhook configures no mTLS (no ClientCAs), so GetCertificate's +// limitation of not refreshing client CA pools does not apply. +// +// If the mounted files genuinely become unreadable (Secret deleted, permissions +// broken: an operator error, not a normal renewal) the handshake fails loudly +// rather than silently serving a stale certificate. +func newReloadingTLSConfig(certFile, keyFile string) (*tls.Config, error) { + // Fail fast at startup if the initial key pair is missing or malformed. + if _, err := tls.LoadX509KeyPair(certFile, keyFile); err != nil { + return nil, err + } + return &tls.Config{ + GetCertificate: func(*tls.ClientHelloInfo) (*tls.Certificate, error) { + cert, err := tls.LoadX509KeyPair(certFile, keyFile) + if err != nil { + return nil, err + } + return &cert, nil + }, + }, nil +} diff --git a/packages/system/kubeovn-webhook/images/kubeovn-webhook/cert_test.go b/packages/system/kubeovn-webhook/images/kubeovn-webhook/cert_test.go new file mode 100644 index 0000000000..2d39fe5999 --- /dev/null +++ b/packages/system/kubeovn-webhook/images/kubeovn-webhook/cert_test.go @@ -0,0 +1,139 @@ +package main + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "math/big" + "os" + "path/filepath" + "testing" + "time" +) + +// genSelfSigned returns a PEM-encoded self-signed cert/key pair carrying the given +// serial number, so tests can tell two generations of the certificate apart. +func genSelfSigned(t *testing.T, serial int64) (certPEM, keyPEM []byte) { + t.Helper() + + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatalf("generate key: %v", err) + } + + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(serial), + Subject: pkix.Name{CommonName: "kube-ovn-webhook-test"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + DNSNames: []string{"localhost"}, + BasicConstraintsValid: true, + } + + der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) + if err != nil { + t.Fatalf("create certificate: %v", err) + } + keyDER, err := x509.MarshalPKCS8PrivateKey(key) + if err != nil { + t.Fatalf("marshal key: %v", err) + } + + certPEM = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}) + keyPEM = pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: keyDER}) + return certPEM, keyPEM +} + +// writeKeyPair writes the cert/key files and sets their mtime, mirroring how +// cert-manager replaces the mounted Secret on renewal. +func writeKeyPair(t *testing.T, certFile, keyFile string, certPEM, keyPEM []byte, mtime time.Time) { + t.Helper() + if err := os.WriteFile(certFile, certPEM, 0o600); err != nil { + t.Fatalf("write cert: %v", err) + } + if err := os.WriteFile(keyFile, keyPEM, 0o600); err != nil { + t.Fatalf("write key: %v", err) + } + setModTime(t, certFile, mtime) + setModTime(t, keyFile, mtime) +} + +func setModTime(t *testing.T, name string, mtime time.Time) { + t.Helper() + if err := os.Chtimes(name, mtime, mtime); err != nil { + t.Fatalf("chtimes %s: %v", name, err) + } +} + +// servedSerial completes a TLS handshake against addr and returns the serial number +// of the leaf certificate the server actually presented. +func servedSerial(t *testing.T, addr string) int64 { + t.Helper() + conn, err := tls.Dial("tcp", addr, &tls.Config{InsecureSkipVerify: true}) //nolint:gosec // test-only, inspecting the served cert + if err != nil { + t.Fatalf("dial: %v", err) + } + defer conn.Close() + certs := conn.ConnectionState().PeerCertificates + if len(certs) == 0 { + t.Fatalf("server presented no certificate") + } + return certs[0].SerialNumber.Int64() +} + +// TestReloadingTLSConfigServesRenewedCertificate is the core regression test: it fails +// against the old code that loads the key pair once into tls.Config.Certificates, and +// passes once the certificate is served via a reloading GetCertificate callback. +func TestReloadingTLSConfigServesRenewedCertificate(t *testing.T) { + dir := t.TempDir() + certFile := filepath.Join(dir, "tls.crt") + keyFile := filepath.Join(dir, "tls.key") + + certA, keyA := genSelfSigned(t, 1) + writeKeyPair(t, certFile, keyFile, certA, keyA, time.Now().Add(-2*time.Second)) + + tlsConfig, err := newReloadingTLSConfig(certFile, keyFile) + if err != nil { + t.Fatalf("newReloadingTLSConfig: %v", err) + } + + ln, err := tls.Listen("tcp", "127.0.0.1:0", tlsConfig) + if err != nil { + t.Fatalf("listen: %v", err) + } + defer ln.Close() + go func() { + for { + conn, err := ln.Accept() + if err != nil { + return + } + go func() { + if tc, ok := conn.(*tls.Conn); ok { + _ = tc.Handshake() + } + conn.Close() + }() + } + }() + + addr := ln.Addr().String() + + if got := servedSerial(t, addr); got != 1 { + t.Fatalf("before renewal: expected serial 1, got %d", got) + } + + // cert-manager renews the Secret: the mounted files are replaced in place. + certB, keyB := genSelfSigned(t, 2) + writeKeyPair(t, certFile, keyFile, certB, keyB, time.Now().Add(2*time.Second)) + + if got := servedSerial(t, addr); got != 2 { + t.Fatalf("after renewal: expected renewed serial 2, got %d (certificate was not reloaded)", got) + } +} diff --git a/packages/system/kubeovn-webhook/images/kubeovn-webhook/main.go b/packages/system/kubeovn-webhook/images/kubeovn-webhook/main.go index 70185961d7..7f1d8ec7ab 100644 --- a/packages/system/kubeovn-webhook/images/kubeovn-webhook/main.go +++ b/packages/system/kubeovn-webhook/images/kubeovn-webhook/main.go @@ -1,10 +1,10 @@ package main import ( - "crypto/tls" "flag" "log" "net/http" + "time" ) var ( @@ -28,17 +28,19 @@ func main() { mux := http.NewServeMux() mux.HandleFunc("/mutate-pods", HandleMutatePods) - tlsCert, err := tls.LoadX509KeyPair(tlsCertFile, tlsKeyFile) + tlsConfig, err := newReloadingTLSConfig(tlsCertFile, tlsKeyFile) if err != nil { log.Fatalf("Failed to load key pair: %v", err) } server := &http.Server{ - Addr: ":8443", - TLSConfig: &tls.Config{ - Certificates: []tls.Certificate{tlsCert}, - }, - Handler: mux, + Addr: ":8443", + TLSConfig: tlsConfig, + Handler: mux, + ReadHeaderTimeout: 10 * time.Second, + ReadTimeout: 30 * time.Second, + WriteTimeout: 30 * time.Second, + IdleTimeout: 60 * time.Second, } log.Printf("Starting webhook server on %s", server.Addr) diff --git a/packages/system/kubeovn-webhook/templates/certmanager.yaml b/packages/system/kubeovn-webhook/templates/certmanager.yaml index f8eee740e4..cb2770360a 100644 --- a/packages/system/kubeovn-webhook/templates/certmanager.yaml +++ b/packages/system/kubeovn-webhook/templates/certmanager.yaml @@ -38,7 +38,7 @@ metadata: spec: secretName: {{ include "namespace-annotation-webhook.fullname" . }}-tls duration: 8760h - renewBefore: 24h + renewBefore: 720h issuerRef: name: {{ include "namespace-annotation-webhook.fullname" . }}-ca-issuer commonName: {{ include "namespace-annotation-webhook.fullname" . }}-tls diff --git a/packages/system/kubeovn-webhook/values.yaml b/packages/system/kubeovn-webhook/values.yaml index 6f5fecb0f8..024ffb2e24 100644 --- a/packages/system/kubeovn-webhook/values.yaml +++ b/packages/system/kubeovn-webhook/values.yaml @@ -1,3 +1,3 @@ portSecurity: true routes: "" -image: ghcr.io/cozystack/cozystack/kubeovn-webhook:v1.5.0@sha256:ca354b6c2ae6f648bc63798d19a1316fe37cb142246bf08ebb7ce27e6e8f21b2 +image: ghcr.io/cozystack/cozystack/kubeovn-webhook:v1.5.4@sha256:8fddf99339698520692c214ce2c4c73be40539ca7a7baf8c7f8ef77162395b66 diff --git a/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml b/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml index 9ce1798390..44d325d3dd 100644 --- a/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml +++ b/packages/system/kubernetes-rd/cozyrds/kubernetes.yaml @@ -22,7 +22,7 @@ spec: singular: kubernetes plural: kuberneteses openAPISchema: |- - {"title":"Chart Values","type":"object","properties":{"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":"replicated","x-kubernetes-validations":[{"rule":"self == oldSelf","message":"storageClass is immutable"}],"x-cozystack-options":{"source":"storageclass"}},"nodeGroups":{"description":"Worker nodes configuration map.","type":"object","default":{"md0":{"diskSize":"20Gi","gpus":[],"instanceType":"u1.medium","kubelet":{},"maxReplicas":10,"minReplicas":0,"resources":{},"roles":["ingress-nginx"],"storageClass":""}},"additionalProperties":{"type":"object","required":["diskSize","instanceType","maxReplicas","minReplicas","resources"],"properties":{"diskSize":{"description":"Persistent disk size for kubelet and containerd data.","default":"20Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"gpus":{"description":"List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).","type":"array","items":{"type":"object","required":["name"],"properties":{"name":{"description":"Name of GPU, such as \"nvidia.com/AD102GL_L40S\".","type":"string","x-cozystack-options":{"source":"gpu"}}}}},"instanceType":{"description":"Virtual machine instance type.","type":"string","default":"u1.medium","x-cozystack-options":{"source":"instancetype"}},"kubelet":{"description":"Kubelet resource reservations for this node group.","type":"object","properties":{"evictionHardMemory":{"description":"Hard eviction threshold for memory (absolute like 200Mi or percentage like 7%).","type":"string","default":"7%"},"evictionSoftMemory":{"description":"Soft eviction threshold for memory (absolute like 1Gi or percentage like 10%).","type":"string","default":"10%"},"kubeReservedCpu":{"description":"CPU reserved for kubelet and container runtime. Auto-computed from instanceType if empty.","type":"string"},"kubeReservedMemory":{"description":"Memory reserved for kubelet and container runtime. Auto-computed from instanceType if empty.","type":"string"},"systemReservedCpu":{"description":"CPU reserved for host OS. Auto-computed from instanceType if empty.","type":"string"},"systemReservedMemory":{"description":"Memory reserved for host OS. Auto-computed from instanceType if empty.","type":"string"}}},"maxReplicas":{"description":"Maximum number of replicas.","type":"integer","default":10},"minReplicas":{"description":"Minimum number of replicas.","type":"integer","default":0},"resources":{"description":"CPU and memory resources for each worker node.","type":"object","properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"roles":{"description":"List of node roles.","type":"array","items":{"type":"string"}},"storageClass":{"description":"StorageClass for worker node persistent disks. When empty, uses the management cluster default StorageClass (the one annotated storageclass.kubernetes.io/is-default-class: true). NOTE: deliberately not marked immutable — the field is optional and undefaulted, so a strict `self == oldSelf` rule would block any future attempt to set it on an existing node group.","type":"string","x-cozystack-options":{"source":"storageclass"}}}}},"version":{"description":"Kubernetes major.minor version to deploy","type":"string","default":"v1.35","enum":["v1.35","v1.34","v1.33","v1.32","v1.31","v1.30"]},"host":{"description":"External hostname for Kubernetes cluster. Defaults to `.` if empty.","type":"string","default":""},"addons":{"description":"Cluster addons configuration.","type":"object","default":{},"required":["certManager","cilium","coredns","fluxcd","gatewayAPI","gpuOperator","hami","ingressNginx","monitoringAgents","ouroboros","velero","verticalPodAutoscaler"],"properties":{"certManager":{"description":"Cert-manager addon.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable cert-manager.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"cilium":{"description":"Cilium CNI plugin.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"coredns":{"description":"CoreDNS addon.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"fluxcd":{"description":"FluxCD GitOps operator.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable FluxCD.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"gatewayAPI":{"description":"Gateway API addon.","type":"object","default":{},"required":["enabled"],"properties":{"enabled":{"description":"Enable Gateway API.","type":"boolean","default":false}}},"gpuOperator":{"description":"NVIDIA GPU Operator.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable GPU Operator.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"hami":{"description":"HAMi GPU virtualization middleware.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable HAMi (requires GPU Operator).","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"ingressNginx":{"description":"Ingress-NGINX controller.","type":"object","default":{},"required":["enabled","exposeMethod","valuesOverride"],"properties":{"enabled":{"description":"Enable the controller (requires nodes labeled `ingress-nginx`).","type":"boolean","default":false},"exposeMethod":{"description":"Method to expose the controller. Allowed values: `Proxied`, `LoadBalancer`.","type":"string","default":"Proxied","enum":["Proxied","LoadBalancer"]},"hosts":{"description":"Domains routed to this tenant cluster when `exposeMethod` is `Proxied`.","type":"array","default":[],"items":{"type":"string"}},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"monitoringAgents":{"description":"Monitoring agents.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable monitoring agents.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"ouroboros":{"description":"Hairpin-NAT fix for ingress-nginx with PROXY-protocol.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable ouroboros. Requires addons.ingressNginx.enabled (chart-render fail otherwise). Only useful when PROXY-protocol is wired on the tenant ingress-nginx via valuesOverride.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides. Operator-key wins over cozystack defaults.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"velero":{"description":"Velero backup/restore addon.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable Velero.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"verticalPodAutoscaler":{"description":"Vertical Pod Autoscaler.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}}}},"controlPlane":{"description":"Kubernetes control-plane configuration.","type":"object","default":{},"required":["apiServer","controllerManager","konnectivity","replicas","scheduler"],"properties":{"apiServer":{"description":"API Server configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for API Server.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"c1.medium","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]}}},"controllerManager":{"description":"Controller Manager configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Controller Manager.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"t1.micro","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]}}},"konnectivity":{"description":"Konnectivity configuration.","type":"object","default":{},"required":["server"],"properties":{"server":{"description":"Konnectivity Server configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Konnectivity.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"t1.micro","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]}}}}},"replicas":{"description":"Number of control-plane replicas.","type":"integer","default":2},"scheduler":{"description":"Scheduler configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Scheduler.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"t1.micro","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]}}}}},"images":{"description":"Optional image overrides for air-gapped or rate-limited registries.","type":"object","default":{},"properties":{"waitForKubeconfig":{"description":"Image used by the wait-for-kubeconfig init container. Empty falls back to images/busybox.tag.","type":"string","default":""}}}}} + {"title":"Chart Values","type":"object","properties":{"storageClass":{"description":"StorageClass used to store the data.","type":"string","default":"replicated","x-kubernetes-validations":[{"rule":"self == oldSelf","message":"storageClass is immutable"}],"x-cozystack-options":{"source":"storageclass"}},"nodeGroups":{"description":"Worker nodes configuration map.","type":"object","default":{"md0":{"diskSize":"20Gi","gpus":[],"instanceType":"u1.medium","kubelet":{},"maxReplicas":10,"minReplicas":0,"resources":{},"roles":["ingress-nginx"],"storageClass":""}},"additionalProperties":{"type":"object","required":["diskSize","instanceType","maxReplicas","minReplicas"],"properties":{"diskSize":{"description":"Persistent disk size for kubelet and containerd data.","default":"20Gi","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"gpus":{"description":"List of GPUs to attach (NVIDIA driver requires at least 4 GiB RAM).","type":"array","items":{"type":"object","required":["name"],"properties":{"name":{"description":"Name of GPU, such as \"nvidia.com/AD102GL_L40S\".","type":"string","x-cozystack-options":{"source":"gpu"}}}}},"instanceType":{"description":"Virtual machine instance type.","type":"string","default":"u1.medium","x-cozystack-options":{"source":"instancetype"}},"kubelet":{"description":"Kubelet resource reservations for this node group.","type":"object","properties":{"evictionHardMemory":{"description":"Hard eviction threshold for memory (absolute like 200Mi or percentage like 7%).","type":"string","default":"7%"},"evictionSoftMemory":{"description":"Soft eviction threshold for memory (absolute like 1Gi or percentage like 10%).","type":"string","default":"10%"},"kubeReservedCpu":{"description":"CPU reserved for kubelet and container runtime. Auto-computed from instanceType if empty.","type":"string"},"kubeReservedMemory":{"description":"Memory reserved for kubelet and container runtime. Auto-computed from instanceType if empty.","type":"string"},"systemReservedCpu":{"description":"CPU reserved for host OS. Auto-computed from instanceType if empty.","type":"string"},"systemReservedMemory":{"description":"Memory reserved for host OS. Auto-computed from instanceType if empty.","type":"string"}}},"maxReplicas":{"description":"Maximum number of replicas.","type":"integer","default":10},"minReplicas":{"description":"Minimum number of replicas.","type":"integer","default":0},"resources":{"description":"Explicit CPU and memory for each worker node, as an alternative to `instanceType` sizing. Optional: when omitted, the node is sized by `instanceType`. When both `cpu` and `memory` are set, they take precedence and `instanceType` is ignored for that node group (the instancetype is omitted from the VM, since KubeVirt cannot override an instancetype's CPU/memory). Set both `cpu` and `memory` together or neither; setting only one is rejected at render time.","type":"object","properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"roles":{"description":"List of node roles.","type":"array","items":{"type":"string"}},"storageClass":{"description":"StorageClass for worker node persistent disks. When empty, uses the management cluster default StorageClass (the one annotated storageclass.kubernetes.io/is-default-class: true). NOTE: deliberately not marked immutable — the field is optional and undefaulted, so a strict `self == oldSelf` rule would block any future attempt to set it on an existing node group.","type":"string","x-cozystack-options":{"source":"storageclass"}}}}},"version":{"description":"Kubernetes major.minor version to deploy","type":"string","default":"v1.35","enum":["v1.35","v1.34","v1.33","v1.32","v1.31","v1.30"]},"host":{"description":"External hostname for Kubernetes cluster. Defaults to `.` if empty.","type":"string","default":""},"addons":{"description":"Cluster addons configuration.","type":"object","default":{},"required":["certManager","cilium","coredns","fluxcd","gatewayAPI","gpuOperator","hami","ingressNginx","monitoringAgents","ouroboros","velero","verticalPodAutoscaler"],"properties":{"certManager":{"description":"Cert-manager addon.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable cert-manager.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"cilium":{"description":"Cilium CNI plugin.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"coredns":{"description":"CoreDNS addon.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"fluxcd":{"description":"FluxCD GitOps operator.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable FluxCD.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"gatewayAPI":{"description":"Gateway API addon.","type":"object","default":{},"required":["enabled"],"properties":{"enabled":{"description":"Enable Gateway API.","type":"boolean","default":false}}},"gpuOperator":{"description":"NVIDIA GPU Operator.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable GPU Operator.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"hami":{"description":"HAMi GPU virtualization middleware.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable HAMi (requires GPU Operator).","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"ingressNginx":{"description":"Ingress-NGINX controller.","type":"object","default":{},"required":["enabled","exposeMethod","valuesOverride"],"properties":{"enabled":{"description":"Enable the controller (requires nodes labeled `ingress-nginx`).","type":"boolean","default":false},"exposeMethod":{"description":"Method to expose the controller. Allowed values: `Proxied`, `LoadBalancer`.","type":"string","default":"Proxied","enum":["Proxied","LoadBalancer"]},"hosts":{"description":"Domains routed to this tenant cluster when `exposeMethod` is `Proxied`.","type":"array","default":[],"items":{"type":"string"}},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"monitoringAgents":{"description":"Monitoring agents.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable monitoring agents.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"ouroboros":{"description":"Hairpin-NAT fix for ingress-nginx with PROXY-protocol.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable ouroboros. Requires addons.ingressNginx.enabled (chart-render fail otherwise). Only useful when PROXY-protocol is wired on the tenant ingress-nginx via valuesOverride.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides. Operator-key wins over cozystack defaults.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"velero":{"description":"Velero backup/restore addon.","type":"object","default":{},"required":["enabled","valuesOverride"],"properties":{"enabled":{"description":"Enable Velero.","type":"boolean","default":false},"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}},"verticalPodAutoscaler":{"description":"Vertical Pod Autoscaler.","type":"object","default":{},"required":["valuesOverride"],"properties":{"valuesOverride":{"description":"Custom Helm values overrides.","type":"object","default":{},"x-kubernetes-preserve-unknown-fields":true}}}}},"controlPlane":{"description":"Kubernetes control-plane configuration.","type":"object","default":{},"required":["apiServer","controllerManager","konnectivity","replicas","scheduler"],"properties":{"apiServer":{"description":"API Server configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for API Server.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"c1.medium","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]}}},"controllerManager":{"description":"Controller Manager configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Controller Manager.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"t1.micro","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]}}},"konnectivity":{"description":"Konnectivity configuration.","type":"object","default":{},"required":["server"],"properties":{"server":{"description":"Konnectivity Server configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Konnectivity.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"t1.micro","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]}}}}},"replicas":{"description":"Number of control-plane replicas.","type":"integer","default":2},"scheduler":{"description":"Scheduler configuration.","type":"object","default":{},"required":["resources","resourcesPreset"],"properties":{"resources":{"description":"CPU and memory resources for Scheduler.","type":"object","default":{},"properties":{"cpu":{"description":"CPU available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true},"memory":{"description":"Memory (RAM) available.","pattern":"^(\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\\+|-)?(([0-9]+(\\.[0-9]*)?)|(\\.[0-9]+))))?$","anyOf":[{"type":"integer"},{"type":"string"}],"x-kubernetes-int-or-string":true}}},"resourcesPreset":{"description":"Preset if `resources` omitted.","type":"string","default":"t1.micro","enum":["t1.nano","t1.micro","t1.small","t1.medium","t1.large","t1.xlarge","t1.2xlarge","t1.4xlarge","c1.nano","c1.micro","c1.small","c1.medium","c1.large","c1.xlarge","c1.2xlarge","c1.4xlarge","s1.nano","s1.micro","s1.small","s1.medium","s1.large","s1.xlarge","s1.2xlarge","s1.4xlarge","u1.nano","u1.micro","u1.small","u1.medium","u1.large","u1.xlarge","u1.2xlarge","u1.4xlarge","m1.nano","m1.micro","m1.small","m1.medium","m1.large","m1.xlarge","m1.2xlarge","m1.4xlarge","nano","micro","small","medium","large","xlarge","2xlarge"]}}}}},"images":{"description":"Optional image overrides for air-gapped or rate-limited registries.","type":"object","default":{},"properties":{"waitForKubeconfig":{"description":"Image used by the wait-for-kubeconfig init container. Empty falls back to images/busybox.tag.","type":"string","default":""}}}}} release: prefix: kubernetes- labels: diff --git a/packages/system/kubevirt-csi-node/values.yaml b/packages/system/kubevirt-csi-node/values.yaml index 16570bf2ab..b3f2b94a1a 100644 --- a/packages/system/kubevirt-csi-node/values.yaml +++ b/packages/system/kubevirt-csi-node/values.yaml @@ -1,3 +1,3 @@ storageClass: replicated csiDriver: - image: ghcr.io/cozystack/cozystack/kubevirt-csi-driver:v1.5.0@sha256:dad088fc04cc54af6d5f487ca9296669331a5345f143ed64cd9027b1bb20b5de + image: ghcr.io/cozystack/cozystack/kubevirt-csi-driver:v1.5.4@sha256:51b9a6c0b59dcb6dbffcbdeec613f1106a0f4bf8e5009ee41e64fea6dab21f8d diff --git a/packages/system/kubevirt-operator/Makefile b/packages/system/kubevirt-operator/Makefile index 9aeb6de1a1..3f3a22649a 100644 --- a/packages/system/kubevirt-operator/Makefile +++ b/packages/system/kubevirt-operator/Makefile @@ -6,8 +6,8 @@ include ../../../hack/package.mk update: rm -rf templates mkdir templates - # v1.7.0 blocked by https://github.com/kubevirt/kubevirt/issues/16386 - export RELEASE=v1.8.2 && \ + # Skipping v1.7.x: live-migration regression, https://github.com/kubevirt/kubevirt/issues/16386 + export RELEASE=v1.8.4 && \ wget https://github.com/kubevirt/kubevirt/releases/download/$${RELEASE}/kubevirt-operator.yaml -O templates/kubevirt-operator.yaml && \ sed -i 's/namespace: kubevirt/namespace: $(NAMESPACE)/g' templates/kubevirt-operator.yaml awk -i inplace -v RS="---" '!/kind: Namespace/{printf "%s", $$0 RS}' templates/kubevirt-operator.yaml diff --git a/packages/system/kubevirt-operator/templates/kubevirt-operator.yaml b/packages/system/kubevirt-operator/templates/kubevirt-operator.yaml index 8d73f98226..f6acbc37d5 100644 --- a/packages/system/kubevirt-operator/templates/kubevirt-operator.yaml +++ b/packages/system/kubevirt-operator/templates/kubevirt-operator.yaml @@ -8536,14 +8536,14 @@ spec: - virt-operator env: - name: VIRT_OPERATOR_IMAGE - value: quay.io/kubevirt/virt-operator:v1.8.2 + value: quay.io/kubevirt/virt-operator:v1.8.4 - name: WATCH_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.annotations['olm.targetNamespaces'] - name: KUBEVIRT_VERSION - value: v1.8.2 - image: quay.io/kubevirt/virt-operator:v1.8.2 + value: v1.8.4 + image: quay.io/kubevirt/virt-operator:v1.8.4 imagePullPolicy: IfNotPresent livenessProbe: httpGet: diff --git a/packages/system/lineage-controller-webhook/values.yaml b/packages/system/lineage-controller-webhook/values.yaml index 4b36c58946..01ef3843a8 100644 --- a/packages/system/lineage-controller-webhook/values.yaml +++ b/packages/system/lineage-controller-webhook/values.yaml @@ -1,5 +1,5 @@ lineageControllerWebhook: - image: ghcr.io/cozystack/cozystack/lineage-controller-webhook:v1.5.0@sha256:5868789439d1384cf78d9fb2f20d5457060b0d5dfcc4f93f96a13ea27ed73b81 + image: ghcr.io/cozystack/cozystack/lineage-controller-webhook:v1.5.4@sha256:9dc9ee368b9339472caa58f8a0cb7abde6e9c0dbb17c366612ae982f1bf0a2f5 debug: false replicas: 2 # DEPRECATED. Injects KUBERNETES_SERVICE_HOST=status.hostIP and diff --git a/packages/system/linstor-gui/values.yaml b/packages/system/linstor-gui/values.yaml index a5bd893108..f0c303789b 100644 --- a/packages/system/linstor-gui/values.yaml +++ b/packages/system/linstor-gui/values.yaml @@ -3,7 +3,7 @@ ## @param image.tag LINSTOR GUI container image tag (digest recommended) image: repository: ghcr.io/cozystack/cozystack/linstor-gui - tag: v1.5.0@sha256:6e11829de86709f5e21636cdd1a1ada1239a7dcc1741f91e9d95d9f6ade603a2 + tag: v1.5.4@sha256:0d49354d49eac7f52698b2bed6604c0bec7f3a36730c6ce52b1a3690ca554eec ## @section Deployment ## @param replicas Number of linstor-gui replicas replicas: 1 diff --git a/packages/system/linstor/values.yaml b/packages/system/linstor/values.yaml index 09cf14cdaf..21962888f9 100644 --- a/packages/system/linstor/values.yaml +++ b/packages/system/linstor/values.yaml @@ -1,7 +1,7 @@ piraeusServer: image: repository: ghcr.io/cozystack/cozystack/piraeus-server - tag: v1.5.0@sha256:1a00af3642c4814c3552d1964daf2831a430bc451c55e786e5eee278fa50e9e1 + tag: v1.5.4@sha256:7b49eced130fa7d96f668cc001d276d6eb00b9e9d9359068e5308dc41244a213 # Talos-specific workarounds (disable for generic Linux like Ubuntu/Debian) talos: enabled: true @@ -13,4 +13,4 @@ linstor: linstorCSI: image: repository: ghcr.io/cozystack/cozystack/linstor-csi - tag: v1.5.0@sha256:c60519d12aa8d078e4891d6e436c709ff53f2cc96457994390ca9424b6505903 + tag: v1.5.4@sha256:7d0891fabe538db848d84ee9a21797ae1b9f2953c2627a5358a93e14db8cf964 diff --git a/packages/system/mariadb-rd/cozyrds/mariadb.yaml b/packages/system/mariadb-rd/cozyrds/mariadb.yaml index ab1249b2fe..122814052a 100644 --- a/packages/system/mariadb-rd/cozyrds/mariadb.yaml +++ b/packages/system/mariadb-rd/cozyrds/mariadb.yaml @@ -35,5 +35,6 @@ spec: exclude: [] include: - resourceNames: + - mariadb-{{ .name }} - mariadb-{{ .name }}-primary - mariadb-{{ .name }}-secondary diff --git a/packages/system/metallb/values.yaml b/packages/system/metallb/values.yaml index c269dbe96f..e299305f45 100644 --- a/packages/system/metallb/values.yaml +++ b/packages/system/metallb/values.yaml @@ -37,11 +37,11 @@ metallb: controller: image: repository: ghcr.io/cozystack/cozystack/metallb-controller - tag: v1.5.0@sha256:b290031f40bfc072c0b329e07d7c17eca3301e4aad2e1988fc962645d92b5fc0 + tag: v1.5.4@sha256:9d8ba76cdb9c7c6221334ad05d706dee22b138b3e90c1fe8fc884925b7480c02 speaker: image: repository: ghcr.io/cozystack/cozystack/metallb-speaker - tag: v1.5.0@sha256:3aad3ac50e8da94342332b363099c04316d8eda0dcb8cdc3d89f1bcf5dc04e7d + tag: v1.5.4@sha256:87df3c82d0b6ea223b26fd5d6fbba6e940c13e56418dfc1cd90863d145795be9 # The vendored metallb chart's values.yaml leaves `frr-k8s.prometheus:` as # a YAML block containing only commented examples — it parses to null. # Helm-controller's deep-merge then writes that null over the frr-k8s diff --git a/packages/system/monitoring/images/grafana.tag b/packages/system/monitoring/images/grafana.tag index 2eb03cd00a..2f63218655 100644 --- a/packages/system/monitoring/images/grafana.tag +++ b/packages/system/monitoring/images/grafana.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/grafana:v1.5.0@sha256:086f1502edf64d2aabcda6e6505b7ee4673b0d3a14cdb7ca9e90a1d91d6e78f4 +ghcr.io/cozystack/cozystack/grafana:v1.5.4@sha256:c9f94d455dfa045062e6ec48d22a5ac7f851ec9c3556636723d9a9fad0105b0b diff --git a/packages/system/multus/templates/multus-daemonset-thick.yml b/packages/system/multus/templates/multus-daemonset-thick.yml index 6664eb2144..d35d044e73 100644 --- a/packages/system/multus/templates/multus-daemonset-thick.yml +++ b/packages/system/multus/templates/multus-daemonset-thick.yml @@ -155,7 +155,7 @@ spec: serviceAccountName: multus containers: - name: kube-multus - image: ghcr.io/cozystack/cozystack/multus-cni:v1.5.0@sha256:f41d0d45bb3a654a6f024a8e416884541550f4c70481f1579647c6dd434f04d8 + image: ghcr.io/cozystack/cozystack/multus-cni:v1.5.4@sha256:e1f57cda741e44d3dda78be3a2b36209a8f406e2b29319902c9f5e5bbd884b1d command: [ "/usr/src/multus-cni/bin/multus-daemon" ] resources: requests: @@ -201,7 +201,7 @@ spec: fieldPath: spec.nodeName initContainers: - name: install-multus-binary - image: ghcr.io/cozystack/cozystack/multus-cni:v1.5.0@sha256:f41d0d45bb3a654a6f024a8e416884541550f4c70481f1579647c6dd434f04d8 + image: ghcr.io/cozystack/cozystack/multus-cni:v1.5.4@sha256:e1f57cda741e44d3dda78be3a2b36209a8f406e2b29319902c9f5e5bbd884b1d command: - "/usr/src/multus-cni/bin/install_multus" - "-d" diff --git a/packages/system/objectstorage-controller/images/objectstorage/patches/92-bucketclaim-propagate-ready.diff b/packages/system/objectstorage-controller/images/objectstorage/patches/92-bucketclaim-propagate-ready.diff index e5fc92431d..ae245287db 100644 --- a/packages/system/objectstorage-controller/images/objectstorage/patches/92-bucketclaim-propagate-ready.diff +++ b/packages/system/objectstorage-controller/images/objectstorage/patches/92-bucketclaim-propagate-ready.diff @@ -1,11 +1,11 @@ diff --git a/controller/pkg/bucketclaim/bucketclaim.go b/controller/pkg/bucketclaim/bucketclaim.go -index 2b55c49..038cf28 100644 +index 2b55c49..9195195 100644 --- a/controller/pkg/bucketclaim/bucketclaim.go +++ b/controller/pkg/bucketclaim/bucketclaim.go @@ -213,8 +213,23 @@ func (b *BucketClaimListener) provisionBucketClaimOperation(ctx context.Context, return b.recordError(inputBucketClaim, v1.EventTypeWarning, v1alpha1.FailedCreateBucket, err) } - + + // The Bucket for this claim may already exist from an earlier reconcile, + // in which case the sidecar may have provisioned the backend and flipped + // the Bucket to ready in the meantime. The controller does not watch @@ -25,5 +25,166 @@ index 2b55c49..038cf28 100644 - bucketClaim.Status.BucketReady = false + bucketClaim.Status.BucketReady = createdBucket.Status.BucketReady } - + // Update status with retry logic for conflict errors +@@ -282,6 +297,22 @@ func (b *BucketClaimListener) provisionBucketClaimOperation(ctx context.Context, + return b.recordError(inputBucketClaim, v1.EventTypeWarning, v1alpha1.FailedCreateBucket, err) + } + ++ // The backend Bucket is provisioned asynchronously by the sidecar, which ++ // flips Bucket.status.bucketReady on an object this controller does not ++ // watch. The periodic informer resync cannot re-drive convergence either: ++ // the generic controller drops no-op resync deltas (reflect.DeepEqual in ++ // internal/runtime/controller.go) and nothing mutates the BucketClaim after ++ // this point. So while the Bucket is not ready, return an error to force a ++ // rate-limited requeue; each retry re-reads the live Bucket above until its ++ // readiness has been propagated onto the claim. Surface a Warning event so ++ // an operator can see why the claim is still pending (the returned error is ++ // logged at klog.V(3), suppressed at default verbosity); event aggregation ++ // folds the per-requeue repetition into a single counted entry. ++ if !statusBucketReady { ++ err := fmt.Errorf("backend Bucket %q for BucketClaim %q not ready yet; requeuing", bucketName, bucketClaim.ObjectMeta.Name) ++ return b.recordError(inputBucketClaim, v1.EventTypeWarning, v1alpha1.WaitingForBucket, err) ++ } ++ + klog.V(3).Infof("Finished creating Bucket %v", bucketName) + return nil + } +diff --git a/controller/pkg/bucketclaim/bucketclaim_test.go b/controller/pkg/bucketclaim/bucketclaim_test.go +index fea6f16..837f7e5 100644 +--- a/controller/pkg/bucketclaim/bucketclaim_test.go ++++ b/controller/pkg/bucketclaim/bucketclaim_test.go +@@ -3,6 +3,7 @@ package bucketclaim + import ( + "context" + "fmt" ++ "strings" + "sync" + "testing" + +@@ -379,6 +380,18 @@ func TestRetryOnConflictStatusUpdate(t *testing.T) { + t.Fatalf("Error occurred when creating BucketClaim: %v", err) + } + ++ // Pre-create the backend Bucket already flipped to ready so readiness ++ // propagation lands true and Add completes. This test exercises the ++ // status-update conflict-retry path, not the not-ready requeue path ++ // (the latter is covered by TestProvisionRequeuesUntilBucketReady). ++ readyBucket := &v1alpha1.Bucket{ ++ ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("bucket-%s", bucketClaim.UID)}, ++ Status: v1alpha1.BucketStatus{BucketReady: true}, ++ } ++ if _, err := client.ObjectstorageV1alpha1().Buckets().Create(ctx, readyBucket, metav1.CreateOptions{}); err != nil { ++ t.Fatalf("Error occurred when pre-creating ready Bucket: %v", err) ++ } ++ + // Cleanup + defer util.DeleteObjects(ctx, client, *bucketClaim, *bucketclass) + +@@ -436,8 +449,8 @@ func TestRetryOnConflictStatusUpdate(t *testing.T) { + t.Errorf("Expected BucketName %s, got %s", expectedBucketName, updatedClaim.Status.BucketName) + } + +- if updatedClaim.Status.BucketReady != false { +- t.Errorf("Expected BucketReady to be false, got %v", updatedClaim.Status.BucketReady) ++ if updatedClaim.Status.BucketReady != true { ++ t.Errorf("Expected BucketReady to be true, got %v", updatedClaim.Status.BucketReady) + } + + // Verify finalizer was added +@@ -445,3 +458,95 @@ func TestRetryOnConflictStatusUpdate(t *testing.T) { + t.Errorf("Expected finalizer to be added, but it was not found") + } + } ++ ++// TestProvisionRequeuesUntilBucketReady pins the readiness-propagation contract. ++// The controller does not watch Bucket objects and the generic controller drops ++// no-op resync deltas, so the BucketClaim must be driven to ready by the ++// provision path itself: Add returns an error (forcing a rate-limited requeue) ++// while the backend Bucket is not ready, and returns nil with BucketReady=true ++// once the Bucket's readiness has propagated. Without the requeue the claim ++// would stay BucketReady=false forever and BucketAccess would never be granted. ++func TestProvisionRequeuesUntilBucketReady(t *testing.T) { ++ newListener := func(client *fakebucketclientset.Clientset) *BucketClaimListener { ++ l := NewBucketClaimListener() ++ l.InitializeKubeClient(fakekubeclientset.NewSimpleClientset()) ++ l.InitializeBucketClient(client) ++ l.InitializeEventRecorder(record.NewFakeRecorder(3)) ++ return l ++ } ++ ++ t.Run("requeues while backend Bucket is not ready", func(t *testing.T) { ++ ctx, cancel := context.WithCancel(context.Background()) ++ defer cancel() ++ ++ client := fakebucketclientset.NewSimpleClientset() ++ if _, err := util.CreateBucketClass(ctx, client, &goldClass); err != nil { ++ t.Fatalf("Error occurred when creating BucketClass: %v", err) ++ } ++ bucketClaim, err := util.CreateBucketClaim(ctx, client, &bucketClaim1) ++ if err != nil { ++ t.Fatalf("Error occurred when creating BucketClaim: %v", err) ++ } ++ ++ // The controller creates the Bucket with BucketReady=false and nothing ++ // flips it here, so Add must report an error to force a requeue. ++ err = newListener(client).Add(ctx, bucketClaim) ++ if err == nil { ++ t.Fatal("Add should return an error to requeue while the backend Bucket is not ready") ++ } ++ // Pin the stable substring the production requeue error carries. A ++ // refactor that drops the requeue (returning nil or a different error) ++ // would silently regress convergence; this keeps the contract loud. ++ if !strings.Contains(err.Error(), "not ready yet") { ++ t.Fatalf("Expected requeue error to contain %q, got: %v", "not ready yet", err) ++ } ++ ++ updated, err := client.ObjectstorageV1alpha1().BucketClaims(bucketClaim.Namespace).Get(ctx, bucketClaim.Name, metav1.GetOptions{}) ++ if err != nil { ++ t.Fatalf("Error occurred when reading BucketClaim: %v", err) ++ } ++ if updated.Status.BucketReady { ++ t.Errorf("Expected BucketReady to stay false while the backend Bucket is not ready") ++ } ++ // Status (name + finalizer) is still persisted before the requeue. ++ if updated.Status.BucketName != fmt.Sprintf("bucket-%s", bucketClaim.UID) { ++ t.Errorf("Expected BucketName to be set even while not ready, got %q", updated.Status.BucketName) ++ } ++ }) ++ ++ t.Run("converges once the backend Bucket is ready", func(t *testing.T) { ++ ctx, cancel := context.WithCancel(context.Background()) ++ defer cancel() ++ ++ client := fakebucketclientset.NewSimpleClientset() ++ if _, err := util.CreateBucketClass(ctx, client, &goldClass); err != nil { ++ t.Fatalf("Error occurred when creating BucketClass: %v", err) ++ } ++ bucketClaim, err := util.CreateBucketClaim(ctx, client, &bucketClaim1) ++ if err != nil { ++ t.Fatalf("Error occurred when creating BucketClaim: %v", err) ++ } ++ ++ // Simulate the sidecar having already provisioned the backend and ++ // flipped the Bucket to ready before this reconcile. ++ readyBucket := &v1alpha1.Bucket{ ++ ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("bucket-%s", bucketClaim.UID)}, ++ Status: v1alpha1.BucketStatus{BucketReady: true}, ++ } ++ if _, err := client.ObjectstorageV1alpha1().Buckets().Create(ctx, readyBucket, metav1.CreateOptions{}); err != nil { ++ t.Fatalf("Error occurred when pre-creating ready Bucket: %v", err) ++ } ++ ++ if err := newListener(client).Add(ctx, bucketClaim); err != nil { ++ t.Fatalf("Add should succeed once the backend Bucket is ready: %v", err) ++ } ++ ++ updated, err := client.ObjectstorageV1alpha1().BucketClaims(bucketClaim.Namespace).Get(ctx, bucketClaim.Name, metav1.GetOptions{}) ++ if err != nil { ++ t.Fatalf("Error occurred when reading BucketClaim: %v", err) ++ } ++ if !updated.Status.BucketReady { ++ t.Errorf("Expected BucketReady to propagate true once the backend Bucket is ready") ++ } ++ }) ++} diff --git a/packages/system/objectstorage-controller/images/objectstorage/patches/93-shorten-leaderelection-lease.diff b/packages/system/objectstorage-controller/images/objectstorage/patches/93-shorten-leaderelection-lease.diff new file mode 100644 index 0000000000..1181f62e67 --- /dev/null +++ b/packages/system/objectstorage-controller/images/objectstorage/patches/93-shorten-leaderelection-lease.diff @@ -0,0 +1,23 @@ +diff --git a/internal/runtime/controller.go b/internal/runtime/controller.go +index 902116a..4cbe4c3 100644 +--- a/internal/runtime/controller.go ++++ b/internal/runtime/controller.go +@@ -171,10 +171,14 @@ func NewObjectStorageControllerWithClientset(identity string, leaderLockName str + threadiness: threads, + + ResyncPeriod: 30 * time.Second, +- // leader election +- LeaseDuration: 150 * time.Second, +- RenewDeadline: 120 * time.Second, +- RetryPeriod: 60 * time.Second, ++ // Leader election. The COSI controller and the per-driver sidecar both ++ // run single-replica, so leader election provides no HA — it only gates ++ // how fast a freshly recreated pod takes over. Use the standard ++ // Kubernetes leader-election timings instead of the upstream 150/120/60s ++ // so a recreated pod resumes reconciliation in ~15s rather than ~150s. ++ LeaseDuration: 15 * time.Second, ++ RenewDeadline: 10 * time.Second, ++ RetryPeriod: 2 * time.Second, + + opMap: &sync.Map{}, + }, nil diff --git a/packages/system/objectstorage-controller/values.yaml b/packages/system/objectstorage-controller/values.yaml index 5fc47baad0..db1774b79f 100644 --- a/packages/system/objectstorage-controller/values.yaml +++ b/packages/system/objectstorage-controller/values.yaml @@ -1,3 +1,3 @@ objectstorage: controller: - image: "ghcr.io/cozystack/cozystack/objectstorage-controller:v1.5.0@sha256:e27da94691155fc3eb14a22cf23fba78e53c4ba524ca3042709efae3753c565b" + image: "ghcr.io/cozystack/cozystack/objectstorage-controller:v1.5.4@sha256:02626cbffe9c03e57b8ea63d16f0edeeddbdc0eeb69c30e830bb33749e0e847e" diff --git a/packages/system/postgres-operator/Makefile b/packages/system/postgres-operator/Makefile index 5279f8a746..1674722b11 100644 --- a/packages/system/postgres-operator/Makefile +++ b/packages/system/postgres-operator/Makefile @@ -12,3 +12,4 @@ update: helm repo update cnpg helm pull cnpg/cloudnative-pg --untar --untardir charts --version 0.26.1 rm -rf charts/cloudnative-pg/charts + patch --no-backup-if-mismatch -p4 < patches/cloudnative-pg-1.27.3.patch diff --git a/packages/system/postgres-operator/charts/cloudnative-pg/Chart.yaml b/packages/system/postgres-operator/charts/cloudnative-pg/Chart.yaml index 31e6afd0e2..7772b44160 100644 --- a/packages/system/postgres-operator/charts/cloudnative-pg/Chart.yaml +++ b/packages/system/postgres-operator/charts/cloudnative-pg/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v2 -appVersion: 1.27.1 +appVersion: 1.27.3 dependencies: - alias: monitoring condition: monitoring.grafanaDashboard.create diff --git a/packages/system/postgres-operator/charts/cloudnative-pg/templates/crds/crds.yaml b/packages/system/postgres-operator/charts/cloudnative-pg/templates/crds/crds.yaml index ef934e1b86..ec0b0588be 100644 --- a/packages/system/postgres-operator/charts/cloudnative-pg/templates/crds/crds.yaml +++ b/packages/system/postgres-operator/charts/cloudnative-pg/templates/crds/crds.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.19.0 + controller-gen.kubebuilder.io/version: v0.20.0 helm.sh/resource-policy: keep name: backups.postgresql.cnpg.io spec: @@ -215,6 +215,11 @@ spec: - key - name type: object + useDefaultAzureCredentials: + description: |- + Use the default Azure authentication flow, which includes DefaultAzureCredential. + This allows authentication using environment variables and managed identities. + type: boolean type: object backupId: description: The ID of the Barman backup @@ -312,6 +317,13 @@ spec: podName: description: The pod name type: string + sessionID: + description: |- + The instance manager session ID. This is a unique identifier generated at instance manager + startup and changes on every restart (including container reboots). Used to detect if + the instance manager was restarted during long-running operations like backups, which + would terminate any running backup process. + type: string type: object majorVersion: description: |- @@ -454,7 +466,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.19.0 + controller-gen.kubebuilder.io/version: v0.20.0 helm.sh/resource-policy: keep name: clusterimagecatalogs.postgresql.cnpg.io spec: @@ -536,7 +548,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.19.0 + controller-gen.kubebuilder.io/version: v0.20.0 helm.sh/resource-policy: keep name: clusters.postgresql.cnpg.io spec: @@ -1554,9 +1566,10 @@ spec: operator: description: |- Operator represents a key's relationship to the value. - Valid operators are Exists and Equal. Defaults to Equal. + Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal. Exists is equivalent to wildcard for value, so that a pod can tolerate all taints of a particular category. + Lt and Gt perform numeric comparisons (requires feature gate TaintTolerationComparisonOperators). type: string tolerationSeconds: description: |- @@ -1649,6 +1662,11 @@ spec: - key - name type: object + useDefaultAzureCredentials: + description: |- + Use the default Azure authentication flow, which includes DefaultAzureCredential. + This allows authentication using environment variables and managed identities. + type: boolean type: object data: description: |- @@ -2146,6 +2164,7 @@ spec: options: description: |- The list of options that must be passed to initdb when creating the cluster. + Deprecated: This could lead to inconsistent configurations, please use the explicit provided parameters instead. If defined, explicit values will be ignored. @@ -2470,8 +2489,9 @@ spec: integer) type: string targetTime: - description: The target time as a timestamp in the RFC3339 - standard + description: |- + The target time as a timestamp in RFC3339 format or PostgreSQL timestamp format. + Timestamps without an explicit timezone are interpreted as UTC. type: string targetXID: description: The target transaction ID @@ -2990,7 +3010,7 @@ spec: resources: description: |- resources represents the minimum resources the volume should have. - If RecoverVolumeExpansionFailure feature is enabled users are allowed to specify resource requirements + Users are allowed to specify resource requirements that are lower than previous value but must still be higher than capacity recorded in the status field of the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources @@ -3194,6 +3214,11 @@ spec: - key - name type: object + useDefaultAzureCredentials: + description: |- + Use the default Azure authentication flow, which includes DefaultAzureCredential. + This allows authentication using environment variables and managed identities. + type: boolean type: object data: description: |- @@ -4651,7 +4676,7 @@ spec: name: description: The name of the extension, required minLength: 1 - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ + pattern: ^[a-z0-9]([-a-z0-9_]*[a-z0-9])?$ type: string required: - image @@ -5430,6 +5455,24 @@ spec: description: Kubelet's generated CSRs will be addressed to this signer. type: string + userAnnotations: + additionalProperties: + type: string + description: |- + userAnnotations allow pod authors to pass additional information to + the signer implementation. Kubernetes does not restrict or validate this + metadata in any way. + + These values are copied verbatim into the `spec.unverifiedUserAnnotations` field of + the PodCertificateRequest objects that Kubelet creates. + + Entries are subject to the same validation as object metadata annotations, + with the addition that all keys must be domain-prefixed. No restrictions + are placed on values, except an overall size limitation on the entire field. + + Signers should document the keys and values they support. Signers should + deny requests that contain keys they do not recognize. + type: object required: - keyType - signerName @@ -5878,7 +5921,7 @@ spec: resources: description: |- resources represents the minimum resources the volume should have. - If RecoverVolumeExpansionFailure feature is enabled users are allowed to specify resource requirements + Users are allowed to specify resource requirements that are lower than previous value but must still be higher than capacity recorded in the status field of the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources @@ -6134,7 +6177,7 @@ spec: resources: description: |- resources represents the minimum resources the volume should have. - If RecoverVolumeExpansionFailure feature is enabled users are allowed to specify resource requirements + Users are allowed to specify resource requirements that are lower than previous value but must still be higher than capacity recorded in the status field of the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources @@ -6542,7 +6585,7 @@ spec: resources: description: |- resources represents the minimum resources the volume should have. - If RecoverVolumeExpansionFailure feature is enabled users are allowed to specify resource requirements + Users are allowed to specify resource requirements that are lower than previous value but must still be higher than capacity recorded in the status field of the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources @@ -7258,7 +7301,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.19.0 + controller-gen.kubebuilder.io/version: v0.20.0 helm.sh/resource-policy: keep name: databases.postgresql.cnpg.io spec: @@ -7631,7 +7674,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.19.0 + controller-gen.kubebuilder.io/version: v0.20.0 helm.sh/resource-policy: keep name: failoverquorums.postgresql.cnpg.io spec: @@ -7709,7 +7752,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.19.0 + controller-gen.kubebuilder.io/version: v0.20.0 helm.sh/resource-policy: keep name: imagecatalogs.postgresql.cnpg.io spec: @@ -7790,7 +7833,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.19.0 + controller-gen.kubebuilder.io/version: v0.20.0 helm.sh/resource-policy: keep name: poolers.postgresql.cnpg.io spec: @@ -10375,7 +10418,9 @@ spec: type: integer type: object resizePolicy: - description: Resources resize policy for the container. + description: |- + Resources resize policy for the container. + This field cannot be set on ephemeral containers. items: description: ContainerResizePolicy represents resource resize policy for the container. @@ -13598,7 +13643,9 @@ spec: type: integer type: object resizePolicy: - description: Resources resize policy for the container. + description: |- + Resources resize policy for the container. + This field cannot be set on ephemeral containers. items: description: ContainerResizePolicy represents resource resize policy for the container. @@ -14385,8 +14432,8 @@ spec: will be made available to those containers which consume them by name. - This is an alpha field and requires enabling the - DynamicResourceAllocation feature gate. + This is a stable field but requires that the + DynamicResourceAllocation feature gate is enabled. This field is immutable. items: @@ -14845,9 +14892,10 @@ spec: operator: description: |- Operator represents a key's relationship to the value. - Valid operators are Exists and Equal. Defaults to Equal. + Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal. Exists is equivalent to wildcard for value, so that a pod can tolerate all taints of a particular category. + Lt and Gt perform numeric comparisons (requires feature gate TaintTolerationComparisonOperators). type: string tolerationSeconds: description: |- @@ -15641,7 +15689,7 @@ spec: resources: description: |- resources represents the minimum resources the volume should have. - If RecoverVolumeExpansionFailure feature is enabled users are allowed to specify resource requirements + Users are allowed to specify resource requirements that are lower than previous value but must still be higher than capacity recorded in the status field of the claim. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources @@ -16527,6 +16575,24 @@ spec: description: Kubelet's generated CSRs will be addressed to this signer. type: string + userAnnotations: + additionalProperties: + type: string + description: |- + userAnnotations allow pod authors to pass additional information to + the signer implementation. Kubernetes does not restrict or validate this + metadata in any way. + + These values are copied verbatim into the `spec.unverifiedUserAnnotations` field of + the PodCertificateRequest objects that Kubelet creates. + + Entries are subject to the same validation as object metadata annotations, + with the addition that all keys must be domain-prefixed. No restrictions + are placed on values, except an overall size limitation on the entire field. + + Signers should document the keys and values they support. Signers should + deny requests that contain keys they do not recognize. + type: object required: - keyType - signerName @@ -16952,6 +17018,42 @@ spec: x-kubernetes-list-map-keys: - name x-kubernetes-list-type: map + workloadRef: + description: |- + WorkloadRef provides a reference to the Workload object that this Pod belongs to. + This field is used by the scheduler to identify the PodGroup and apply the + correct group scheduling policies. The Workload object referenced + by this field may not exist at the time the Pod is created. + This field is immutable, but a Workload object with the same name + may be recreated with different policies. Doing this during pod scheduling + may result in the placement not conforming to the expected policies. + properties: + name: + description: |- + Name defines the name of the Workload object this Pod belongs to. + Workload must be in the same namespace as the Pod. + If it doesn't match any existing Workload, the Pod will remain unschedulable + until a Workload object is created and observed by the kube-scheduler. + It must be a DNS subdomain. + type: string + podGroup: + description: |- + PodGroup is the name of the PodGroup within the Workload that this Pod + belongs to. If it doesn't match any existing PodGroup within the Workload, + the Pod will remain unschedulable until the Workload object is recreated + and observed by the kube-scheduler. It must be a DNS label. + type: string + podGroupReplicaKey: + description: |- + PodGroupReplicaKey specifies the replica key of the PodGroup to which this + Pod belongs. It is used to distinguish pods belonging to different replicas + of the same pod group. The pod group policy is applied separately to each replica. + When set, it must be a DNS label. + type: string + required: + - name + - podGroup + type: object required: - containers type: object @@ -17043,7 +17145,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.19.0 + controller-gen.kubebuilder.io/version: v0.20.0 helm.sh/resource-policy: keep name: publications.postgresql.cnpg.io spec: @@ -17239,7 +17341,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.19.0 + controller-gen.kubebuilder.io/version: v0.20.0 helm.sh/resource-policy: keep name: scheduledbackups.postgresql.cnpg.io spec: @@ -17431,7 +17533,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.19.0 + controller-gen.kubebuilder.io/version: v0.20.0 helm.sh/resource-policy: keep name: subscriptions.postgresql.cnpg.io spec: diff --git a/packages/system/postgres-operator/patches/cloudnative-pg-1.27.3.patch b/packages/system/postgres-operator/patches/cloudnative-pg-1.27.3.patch new file mode 100644 index 0000000000..e3a9edab58 --- /dev/null +++ b/packages/system/postgres-operator/patches/cloudnative-pg-1.27.3.patch @@ -0,0 +1,369 @@ +--- a/packages/system/postgres-operator/charts/cloudnative-pg/Chart.yaml ++++ b/packages/system/postgres-operator/charts/cloudnative-pg/Chart.yaml +@@ -1,5 +1,5 @@ + apiVersion: v2 +-appVersion: 1.27.1 ++appVersion: 1.27.3 + dependencies: + - alias: monitoring + condition: monitoring.grafanaDashboard.create +--- a/packages/system/postgres-operator/charts/cloudnative-pg/templates/crds/crds.yaml ++++ b/packages/system/postgres-operator/charts/cloudnative-pg/templates/crds/crds.yaml +@@ -3,7 +3,7 @@ + kind: CustomResourceDefinition + metadata: + annotations: +- controller-gen.kubebuilder.io/version: v0.19.0 ++ controller-gen.kubebuilder.io/version: v0.20.0 + helm.sh/resource-policy: keep + name: backups.postgresql.cnpg.io + spec: +@@ -215,6 +215,11 @@ + - key + - name + type: object ++ useDefaultAzureCredentials: ++ description: |- ++ Use the default Azure authentication flow, which includes DefaultAzureCredential. ++ This allows authentication using environment variables and managed identities. ++ type: boolean + type: object + backupId: + description: The ID of the Barman backup +@@ -312,6 +317,13 @@ + podName: + description: The pod name + type: string ++ sessionID: ++ description: |- ++ The instance manager session ID. This is a unique identifier generated at instance manager ++ startup and changes on every restart (including container reboots). Used to detect if ++ the instance manager was restarted during long-running operations like backups, which ++ would terminate any running backup process. ++ type: string + type: object + majorVersion: + description: |- +@@ -454,7 +466,7 @@ + kind: CustomResourceDefinition + metadata: + annotations: +- controller-gen.kubebuilder.io/version: v0.19.0 ++ controller-gen.kubebuilder.io/version: v0.20.0 + helm.sh/resource-policy: keep + name: clusterimagecatalogs.postgresql.cnpg.io + spec: +@@ -536,7 +548,7 @@ + kind: CustomResourceDefinition + metadata: + annotations: +- controller-gen.kubebuilder.io/version: v0.19.0 ++ controller-gen.kubebuilder.io/version: v0.20.0 + helm.sh/resource-policy: keep + name: clusters.postgresql.cnpg.io + spec: +@@ -1554,9 +1566,10 @@ + operator: + description: |- + Operator represents a key's relationship to the value. +- Valid operators are Exists and Equal. Defaults to Equal. ++ Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. ++ Lt and Gt perform numeric comparisons (requires feature gate TaintTolerationComparisonOperators). + type: string + tolerationSeconds: + description: |- +@@ -1649,6 +1662,11 @@ + - key + - name + type: object ++ useDefaultAzureCredentials: ++ description: |- ++ Use the default Azure authentication flow, which includes DefaultAzureCredential. ++ This allows authentication using environment variables and managed identities. ++ type: boolean + type: object + data: + description: |- +@@ -2146,6 +2164,7 @@ + options: + description: |- + The list of options that must be passed to initdb when creating the cluster. ++ + Deprecated: This could lead to inconsistent configurations, + please use the explicit provided parameters instead. + If defined, explicit values will be ignored. +@@ -2470,8 +2489,9 @@ + integer) + type: string + targetTime: +- description: The target time as a timestamp in the RFC3339 +- standard ++ description: |- ++ The target time as a timestamp in RFC3339 format or PostgreSQL timestamp format. ++ Timestamps without an explicit timezone are interpreted as UTC. + type: string + targetXID: + description: The target transaction ID +@@ -2990,7 +3010,7 @@ + resources: + description: |- + resources represents the minimum resources the volume should have. +- If RecoverVolumeExpansionFailure feature is enabled users are allowed to specify resource requirements ++ Users are allowed to specify resource requirements + that are lower than previous value but must still be higher than capacity recorded in the + status field of the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources +@@ -3194,6 +3214,11 @@ + - key + - name + type: object ++ useDefaultAzureCredentials: ++ description: |- ++ Use the default Azure authentication flow, which includes DefaultAzureCredential. ++ This allows authentication using environment variables and managed identities. ++ type: boolean + type: object + data: + description: |- +@@ -4651,7 +4676,7 @@ + name: + description: The name of the extension, required + minLength: 1 +- pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ ++ pattern: ^[a-z0-9]([-a-z0-9_]*[a-z0-9])?$ + type: string + required: + - image +@@ -5430,6 +5455,24 @@ + description: Kubelet's generated CSRs will be addressed + to this signer. + type: string ++ userAnnotations: ++ additionalProperties: ++ type: string ++ description: |- ++ userAnnotations allow pod authors to pass additional information to ++ the signer implementation. Kubernetes does not restrict or validate this ++ metadata in any way. ++ ++ These values are copied verbatim into the `spec.unverifiedUserAnnotations` field of ++ the PodCertificateRequest objects that Kubelet creates. ++ ++ Entries are subject to the same validation as object metadata annotations, ++ with the addition that all keys must be domain-prefixed. No restrictions ++ are placed on values, except an overall size limitation on the entire field. ++ ++ Signers should document the keys and values they support. Signers should ++ deny requests that contain keys they do not recognize. ++ type: object + required: + - keyType + - signerName +@@ -5878,7 +5921,7 @@ + resources: + description: |- + resources represents the minimum resources the volume should have. +- If RecoverVolumeExpansionFailure feature is enabled users are allowed to specify resource requirements ++ Users are allowed to specify resource requirements + that are lower than previous value but must still be higher than capacity recorded in the + status field of the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources +@@ -6134,7 +6177,7 @@ + resources: + description: |- + resources represents the minimum resources the volume should have. +- If RecoverVolumeExpansionFailure feature is enabled users are allowed to specify resource requirements ++ Users are allowed to specify resource requirements + that are lower than previous value but must still be higher than capacity recorded in the + status field of the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources +@@ -6542,7 +6585,7 @@ + resources: + description: |- + resources represents the minimum resources the volume should have. +- If RecoverVolumeExpansionFailure feature is enabled users are allowed to specify resource requirements ++ Users are allowed to specify resource requirements + that are lower than previous value but must still be higher than capacity recorded in the + status field of the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources +@@ -7258,7 +7301,7 @@ + kind: CustomResourceDefinition + metadata: + annotations: +- controller-gen.kubebuilder.io/version: v0.19.0 ++ controller-gen.kubebuilder.io/version: v0.20.0 + helm.sh/resource-policy: keep + name: databases.postgresql.cnpg.io + spec: +@@ -7631,7 +7674,7 @@ + kind: CustomResourceDefinition + metadata: + annotations: +- controller-gen.kubebuilder.io/version: v0.19.0 ++ controller-gen.kubebuilder.io/version: v0.20.0 + helm.sh/resource-policy: keep + name: failoverquorums.postgresql.cnpg.io + spec: +@@ -7709,7 +7752,7 @@ + kind: CustomResourceDefinition + metadata: + annotations: +- controller-gen.kubebuilder.io/version: v0.19.0 ++ controller-gen.kubebuilder.io/version: v0.20.0 + helm.sh/resource-policy: keep + name: imagecatalogs.postgresql.cnpg.io + spec: +@@ -7790,7 +7833,7 @@ + kind: CustomResourceDefinition + metadata: + annotations: +- controller-gen.kubebuilder.io/version: v0.19.0 ++ controller-gen.kubebuilder.io/version: v0.20.0 + helm.sh/resource-policy: keep + name: poolers.postgresql.cnpg.io + spec: +@@ -10375,7 +10418,9 @@ + type: integer + type: object + resizePolicy: +- description: Resources resize policy for the container. ++ description: |- ++ Resources resize policy for the container. ++ This field cannot be set on ephemeral containers. + items: + description: ContainerResizePolicy represents resource + resize policy for the container. +@@ -13598,7 +13643,9 @@ + type: integer + type: object + resizePolicy: +- description: Resources resize policy for the container. ++ description: |- ++ Resources resize policy for the container. ++ This field cannot be set on ephemeral containers. + items: + description: ContainerResizePolicy represents resource + resize policy for the container. +@@ -14388,2 +14435,2 @@ +- This is an alpha field and requires enabling the +- DynamicResourceAllocation feature gate. ++ This is a stable field but requires that the ++ DynamicResourceAllocation feature gate is enabled. +@@ -14845,9 +14892,10 @@ + operator: + description: |- + Operator represents a key's relationship to the value. +- Valid operators are Exists and Equal. Defaults to Equal. ++ Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. ++ Lt and Gt perform numeric comparisons (requires feature gate TaintTolerationComparisonOperators). + type: string + tolerationSeconds: + description: |- +@@ -15641,7 +15689,7 @@ + resources: + description: |- + resources represents the minimum resources the volume should have. +- If RecoverVolumeExpansionFailure feature is enabled users are allowed to specify resource requirements ++ Users are allowed to specify resource requirements + that are lower than previous value but must still be higher than capacity recorded in the + status field of the claim. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#resources +@@ -16527,6 +16575,24 @@ + description: Kubelet's generated CSRs + will be addressed to this signer. + type: string ++ userAnnotations: ++ additionalProperties: ++ type: string ++ description: |- ++ userAnnotations allow pod authors to pass additional information to ++ the signer implementation. Kubernetes does not restrict or validate this ++ metadata in any way. ++ ++ These values are copied verbatim into the `spec.unverifiedUserAnnotations` field of ++ the PodCertificateRequest objects that Kubelet creates. ++ ++ Entries are subject to the same validation as object metadata annotations, ++ with the addition that all keys must be domain-prefixed. No restrictions ++ are placed on values, except an overall size limitation on the entire field. ++ ++ Signers should document the keys and values they support. Signers should ++ deny requests that contain keys they do not recognize. ++ type: object + required: + - keyType + - signerName +@@ -16952,6 +17018,42 @@ + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map ++ workloadRef: ++ description: |- ++ WorkloadRef provides a reference to the Workload object that this Pod belongs to. ++ This field is used by the scheduler to identify the PodGroup and apply the ++ correct group scheduling policies. The Workload object referenced ++ by this field may not exist at the time the Pod is created. ++ This field is immutable, but a Workload object with the same name ++ may be recreated with different policies. Doing this during pod scheduling ++ may result in the placement not conforming to the expected policies. ++ properties: ++ name: ++ description: |- ++ Name defines the name of the Workload object this Pod belongs to. ++ Workload must be in the same namespace as the Pod. ++ If it doesn't match any existing Workload, the Pod will remain unschedulable ++ until a Workload object is created and observed by the kube-scheduler. ++ It must be a DNS subdomain. ++ type: string ++ podGroup: ++ description: |- ++ PodGroup is the name of the PodGroup within the Workload that this Pod ++ belongs to. If it doesn't match any existing PodGroup within the Workload, ++ the Pod will remain unschedulable until the Workload object is recreated ++ and observed by the kube-scheduler. It must be a DNS label. ++ type: string ++ podGroupReplicaKey: ++ description: |- ++ PodGroupReplicaKey specifies the replica key of the PodGroup to which this ++ Pod belongs. It is used to distinguish pods belonging to different replicas ++ of the same pod group. The pod group policy is applied separately to each replica. ++ When set, it must be a DNS label. ++ type: string ++ required: ++ - name ++ - podGroup ++ type: object + required: + - containers + type: object +@@ -17043,7 +17145,7 @@ + kind: CustomResourceDefinition + metadata: + annotations: +- controller-gen.kubebuilder.io/version: v0.19.0 ++ controller-gen.kubebuilder.io/version: v0.20.0 + helm.sh/resource-policy: keep + name: publications.postgresql.cnpg.io + spec: +@@ -17239,7 +17341,7 @@ + kind: CustomResourceDefinition + metadata: + annotations: +- controller-gen.kubebuilder.io/version: v0.19.0 ++ controller-gen.kubebuilder.io/version: v0.20.0 + helm.sh/resource-policy: keep + name: scheduledbackups.postgresql.cnpg.io + spec: +@@ -17431,7 +17533,7 @@ + kind: CustomResourceDefinition + metadata: + annotations: +- controller-gen.kubebuilder.io/version: v0.19.0 ++ controller-gen.kubebuilder.io/version: v0.20.0 + helm.sh/resource-policy: keep + name: subscriptions.postgresql.cnpg.io + spec: diff --git a/packages/system/postgres-operator/tests/cnpg-version_test.yaml b/packages/system/postgres-operator/tests/cnpg-version_test.yaml new file mode 100644 index 0000000000..3e52874d5e --- /dev/null +++ b/packages/system/postgres-operator/tests/cnpg-version_test.yaml @@ -0,0 +1,34 @@ +suite: CNPG operator and CRDs stay aligned + +templates: + - charts/cloudnative-pg/templates/deployment.yaml + - charts/cloudnative-pg/templates/rbac.yaml + - charts/cloudnative-pg/templates/config.yaml + - charts/cloudnative-pg/templates/monitoring-configmap.yaml + - charts/cloudnative-pg/templates/crds/crds.yaml + +release: + name: postgres-operator + namespace: cozy-postgres-operator + +tests: + - it: runs the CNPG 1.27.3 operator image + template: charts/cloudnative-pg/templates/deployment.yaml + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: ghcr.io/cloudnative-pg/cloudnative-pg:1.27.3 + - equal: + path: spec.template.spec.containers[0].env[0].value + value: ghcr.io/cloudnative-pg/cloudnative-pg:1.27.3 + - equal: + path: metadata.labels["app.kubernetes.io/version"] + value: 1.27.3 + + - it: preserves the CNPG 1.27.3 backup session ID status + template: charts/cloudnative-pg/templates/crds/crds.yaml + asserts: + - documentIndex: 0 + equal: + path: spec.versions[0].schema.openAPIV3Schema.properties.status.properties.instanceID.properties.sessionID.type + value: string diff --git a/packages/system/seaweedfs/Makefile b/packages/system/seaweedfs/Makefile index cf43956f2d..ea2c973a18 100644 --- a/packages/system/seaweedfs/Makefile +++ b/packages/system/seaweedfs/Makefile @@ -5,6 +5,20 @@ include ../../../hack/package.mk test: helm unittest . + $(MAKE) test-guard-fail-closed + +# The naming guard refuses to render an UPGRADE that cannot see the cluster +# (templates/naming-guard.yaml): a client-side lookup silently returns nothing, +# which would classify a class-S tenant as net-new and rename its workloads away +# from its data. helm-unittest 1.0.3 ignores release.isUpgrade, so .Release.IsUpgrade +# always renders false there and the refusal branch gets NO unit coverage — the +# suite passes with or without the canary. Verify with a real renderer instead. +.PHONY: test-guard-fail-closed +test-guard-fail-closed: + @out=$$(helm template seaweedfs-system . -n tenant-root --is-upgrade 2>&1); \ + echo "$$out" | grep -q 'refusing to upgrade blind' \ + || { echo "FAIL: a blind upgrade rendered instead of refusing (naming-guard canary is not firing)"; echo "$$out" | head -5; exit 1; } + @echo "ok: naming guard refuses a blind upgrade" update: rm -rf charts @@ -18,5 +32,7 @@ update: patch --no-backup-if-mismatch -p4 < patches/cosi-provisioner-sa-name.patch patch --no-backup-if-mismatch -p4 < patches/cosi-bucket-class-lock-readonly.patch patch --no-backup-if-mismatch -p4 < patches/s3-service-name.patch + patch --no-backup-if-mismatch -p4 < patches/s3-service-name-consumers.patch + patch --no-backup-if-mismatch -p4 < patches/cluster-scoped-names-per-namespace.patch #patch --no-backup-if-mismatch -p4 < patches/retention-policy-delete.yaml diff --git a/packages/system/seaweedfs/charts/seaweedfs/templates/cosi/cosi-cluster-role.yaml b/packages/system/seaweedfs/charts/seaweedfs/templates/cosi/cosi-cluster-role.yaml index b5c1035757..a6e5b761ee 100644 --- a/packages/system/seaweedfs/charts/seaweedfs/templates/cosi/cosi-cluster-role.yaml +++ b/packages/system/seaweedfs/charts/seaweedfs/templates/cosi/cosi-cluster-role.yaml @@ -4,7 +4,7 @@ kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: - name: {{ include "seaweedfs.fullname" . }}-objectstorage-provisioner + name: {{ include "seaweedfs.serviceAccountName" . }}-objectstorage-provisioner labels: app.kubernetes.io/name: {{ template "seaweedfs.name" . }} helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} @@ -53,7 +53,7 @@ rules: kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: - name: {{ include "seaweedfs.fullname" . }}-objectstorage-provisioner + name: {{ include "seaweedfs.serviceAccountName" . }}-objectstorage-provisioner labels: app.kubernetes.io/name: {{ template "seaweedfs.name" . }} helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} @@ -65,6 +65,6 @@ subjects: namespace: {{ .Release.Namespace }} roleRef: kind: ClusterRole - name: {{ include "seaweedfs.fullname" . }}-objectstorage-provisioner + name: {{ include "seaweedfs.serviceAccountName" . }}-objectstorage-provisioner apiGroup: rbac.authorization.k8s.io {{- end }} diff --git a/packages/system/seaweedfs/charts/seaweedfs/templates/cosi/cosi-deployment.yaml b/packages/system/seaweedfs/charts/seaweedfs/templates/cosi/cosi-deployment.yaml index d485e06787..d581b085db 100644 --- a/packages/system/seaweedfs/charts/seaweedfs/templates/cosi/cosi-deployment.yaml +++ b/packages/system/seaweedfs/charts/seaweedfs/templates/cosi/cosi-deployment.yaml @@ -79,9 +79,9 @@ spec: {{- else if .Values.s3.ingress.enabled }} value: "{{ printf "https://%s" .Values.s3.ingress.host }}" {{- else if .Values.s3.enabled }} - value: "{{ printf "https://%s.%s.svc" (include "seaweedfs.componentName" (list . "s3")) .Release.Namespace }}" + value: "{{ printf "https://%s-s3.%s.svc:%v" (include "seaweedfs.name" .) .Release.Namespace .Values.s3.port }}" {{- else }} - value: "{{ printf "https://%s.%s.svc" (include "seaweedfs.componentName" (list . "filer")) .Release.Namespace }}" + value: "{{ printf "https://%s-s3.%s.svc:%v" (include "seaweedfs.name" .) .Release.Namespace .Values.filer.s3.port }}" {{- end }} {{- with .Values.cosi.region }} - name: REGION diff --git a/packages/system/seaweedfs/charts/seaweedfs/templates/s3/s3-iceberg-ingress.yaml b/packages/system/seaweedfs/charts/seaweedfs/templates/s3/s3-iceberg-ingress.yaml index 5a760d7e9e..68db538dcf 100644 --- a/packages/system/seaweedfs/charts/seaweedfs/templates/s3/s3-iceberg-ingress.yaml +++ b/packages/system/seaweedfs/charts/seaweedfs/templates/s3/s3-iceberg-ingress.yaml @@ -5,11 +5,11 @@ paths: backend: {{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion }} service: - name: {{ include "seaweedfs.componentName" (list . "s3") }} + name: {{ template "seaweedfs.name" . }}-s3 port: number: {{ .Values.s3.icebergPort }} {{- else }} - serviceName: {{ include "seaweedfs.componentName" (list . "s3") }} + serviceName: {{ template "seaweedfs.name" . }}-s3 servicePort: {{ .Values.s3.icebergPort }} {{- end }} {{- end -}} diff --git a/packages/system/seaweedfs/charts/seaweedfs/templates/s3/s3-ingress.yaml b/packages/system/seaweedfs/charts/seaweedfs/templates/s3/s3-ingress.yaml index bc4eee1642..c92063f8c4 100644 --- a/packages/system/seaweedfs/charts/seaweedfs/templates/s3/s3-ingress.yaml +++ b/packages/system/seaweedfs/charts/seaweedfs/templates/s3/s3-ingress.yaml @@ -2,7 +2,7 @@ {{- $s3Enabled := or .Values.s3.enabled (and .Values.filer.s3.enabled (not .Values.allInOne.enabled)) (and .Values.allInOne.enabled .Values.allInOne.s3.enabled) }} {{- if and $s3Enabled .Values.s3.ingress.enabled }} {{- /* Determine service name based on deployment mode */}} -{{- $serviceName := ternary (include "seaweedfs.componentName" (list . "all-in-one")) (include "seaweedfs.componentName" (list . "s3")) .Values.allInOne.enabled }} +{{- $serviceName := ternary (include "seaweedfs.componentName" (list . "all-in-one")) (printf "%s-s3" (include "seaweedfs.name" .)) .Values.allInOne.enabled }} {{- $s3Port := .Values.allInOne.s3.port | default .Values.s3.port }} {{- /* Build hosts list - support both legacy .host (string) and new .hosts (array) for backwards compatibility */}} {{- $hosts := list }} diff --git a/packages/system/seaweedfs/charts/seaweedfs/templates/shared/cluster-role.yaml b/packages/system/seaweedfs/charts/seaweedfs/templates/shared/cluster-role.yaml index 14d09f9993..e31c2d5f2f 100644 --- a/packages/system/seaweedfs/charts/seaweedfs/templates/shared/cluster-role.yaml +++ b/packages/system/seaweedfs/charts/seaweedfs/templates/shared/cluster-role.yaml @@ -5,7 +5,7 @@ kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: - name: {{ include "seaweedfs.fullname" . }}-rw-cr + name: {{ include "seaweedfs.serviceAccountName" . }}-rw-cr labels: app.kubernetes.io/name: {{ template "seaweedfs.name" . }} helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} @@ -19,7 +19,7 @@ rules: kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: - name: {{ include "seaweedfs.fullname" . }}-rw-crb + name: {{ include "seaweedfs.serviceAccountName" . }}-rw-crb labels: app.kubernetes.io/name: {{ template "seaweedfs.name" . }} helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} @@ -32,5 +32,5 @@ subjects: roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: {{ include "seaweedfs.fullname" . }}-rw-cr + name: {{ include "seaweedfs.serviceAccountName" . }}-rw-cr {{- end }} \ No newline at end of file diff --git a/packages/system/seaweedfs/patches/cluster-scoped-names-per-namespace.patch b/packages/system/seaweedfs/patches/cluster-scoped-names-per-namespace.patch new file mode 100644 index 0000000000..2091c6d5cf --- /dev/null +++ b/packages/system/seaweedfs/patches/cluster-scoped-names-per-namespace.patch @@ -0,0 +1,60 @@ +diff --git a/packages/system/seaweedfs/charts/seaweedfs/templates/cosi/cosi-cluster-role.yaml b/packages/system/seaweedfs/charts/seaweedfs/templates/cosi/cosi-cluster-role.yaml +index b5c103575..a6e5b761e 100644 +--- a/packages/system/seaweedfs/charts/seaweedfs/templates/cosi/cosi-cluster-role.yaml ++++ b/packages/system/seaweedfs/charts/seaweedfs/templates/cosi/cosi-cluster-role.yaml +@@ -4,7 +4,7 @@ + kind: ClusterRole + apiVersion: rbac.authorization.k8s.io/v1 + metadata: +- name: {{ include "seaweedfs.fullname" . }}-objectstorage-provisioner ++ name: {{ include "seaweedfs.serviceAccountName" . }}-objectstorage-provisioner + labels: + app.kubernetes.io/name: {{ template "seaweedfs.name" . }} + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} +@@ -53,7 +53,7 @@ rules: + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: +- name: {{ include "seaweedfs.fullname" . }}-objectstorage-provisioner ++ name: {{ include "seaweedfs.serviceAccountName" . }}-objectstorage-provisioner + labels: + app.kubernetes.io/name: {{ template "seaweedfs.name" . }} + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} +@@ -65,6 +65,6 @@ subjects: + namespace: {{ .Release.Namespace }} + roleRef: + kind: ClusterRole +- name: {{ include "seaweedfs.fullname" . }}-objectstorage-provisioner ++ name: {{ include "seaweedfs.serviceAccountName" . }}-objectstorage-provisioner + apiGroup: rbac.authorization.k8s.io + {{- end }} +diff --git a/packages/system/seaweedfs/charts/seaweedfs/templates/shared/cluster-role.yaml b/packages/system/seaweedfs/charts/seaweedfs/templates/shared/cluster-role.yaml +index 14d09f999..e31c2d5f2 100644 +--- a/packages/system/seaweedfs/charts/seaweedfs/templates/shared/cluster-role.yaml ++++ b/packages/system/seaweedfs/charts/seaweedfs/templates/shared/cluster-role.yaml +@@ -5,7 +5,7 @@ + kind: ClusterRole + apiVersion: rbac.authorization.k8s.io/v1 + metadata: +- name: {{ include "seaweedfs.fullname" . }}-rw-cr ++ name: {{ include "seaweedfs.serviceAccountName" . }}-rw-cr + labels: + app.kubernetes.io/name: {{ template "seaweedfs.name" . }} + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} +@@ -19,7 +19,7 @@ rules: + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: +- name: {{ include "seaweedfs.fullname" . }}-rw-crb ++ name: {{ include "seaweedfs.serviceAccountName" . }}-rw-crb + labels: + app.kubernetes.io/name: {{ template "seaweedfs.name" . }} + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} +@@ -32,5 +32,5 @@ subjects: + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole +- name: {{ include "seaweedfs.fullname" . }}-rw-cr ++ name: {{ include "seaweedfs.serviceAccountName" . }}-rw-cr + {{- end }} +\ No newline at end of file diff --git a/packages/system/seaweedfs/patches/s3-service-name-consumers.patch b/packages/system/seaweedfs/patches/s3-service-name-consumers.patch new file mode 100644 index 0000000000..73b09dc15f --- /dev/null +++ b/packages/system/seaweedfs/patches/s3-service-name-consumers.patch @@ -0,0 +1,47 @@ +diff --git a/packages/system/seaweedfs/charts/seaweedfs/templates/cosi/cosi-deployment.yaml b/packages/system/seaweedfs/charts/seaweedfs/templates/cosi/cosi-deployment.yaml +index d485e0678..d581b085d 100644 +--- a/packages/system/seaweedfs/charts/seaweedfs/templates/cosi/cosi-deployment.yaml ++++ b/packages/system/seaweedfs/charts/seaweedfs/templates/cosi/cosi-deployment.yaml +@@ -79,9 +79,9 @@ spec: + {{- else if .Values.s3.ingress.enabled }} + value: "{{ printf "https://%s" .Values.s3.ingress.host }}" + {{- else if .Values.s3.enabled }} +- value: "{{ printf "https://%s.%s.svc" (include "seaweedfs.componentName" (list . "s3")) .Release.Namespace }}" ++ value: "{{ printf "https://%s-s3.%s.svc:%v" (include "seaweedfs.name" .) .Release.Namespace .Values.s3.port }}" + {{- else }} +- value: "{{ printf "https://%s.%s.svc" (include "seaweedfs.componentName" (list . "filer")) .Release.Namespace }}" ++ value: "{{ printf "https://%s-s3.%s.svc:%v" (include "seaweedfs.name" .) .Release.Namespace .Values.filer.s3.port }}" + {{- end }} + {{- with .Values.cosi.region }} + - name: REGION +diff --git a/packages/system/seaweedfs/charts/seaweedfs/templates/s3/s3-iceberg-ingress.yaml b/packages/system/seaweedfs/charts/seaweedfs/templates/s3/s3-iceberg-ingress.yaml +index 5a760d7e9..68db538dc 100644 +--- a/packages/system/seaweedfs/charts/seaweedfs/templates/s3/s3-iceberg-ingress.yaml ++++ b/packages/system/seaweedfs/charts/seaweedfs/templates/s3/s3-iceberg-ingress.yaml +@@ -5,11 +5,11 @@ paths: + backend: + {{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion }} + service: +- name: {{ include "seaweedfs.componentName" (list . "s3") }} ++ name: {{ template "seaweedfs.name" . }}-s3 + port: + number: {{ .Values.s3.icebergPort }} + {{- else }} +- serviceName: {{ include "seaweedfs.componentName" (list . "s3") }} ++ serviceName: {{ template "seaweedfs.name" . }}-s3 + servicePort: {{ .Values.s3.icebergPort }} + {{- end }} + {{- end -}} +diff --git a/packages/system/seaweedfs/charts/seaweedfs/templates/s3/s3-ingress.yaml b/packages/system/seaweedfs/charts/seaweedfs/templates/s3/s3-ingress.yaml +index bc4eee164..c92063f8c 100644 +--- a/packages/system/seaweedfs/charts/seaweedfs/templates/s3/s3-ingress.yaml ++++ b/packages/system/seaweedfs/charts/seaweedfs/templates/s3/s3-ingress.yaml +@@ -2,7 +2,7 @@ + {{- $s3Enabled := or .Values.s3.enabled (and .Values.filer.s3.enabled (not .Values.allInOne.enabled)) (and .Values.allInOne.enabled .Values.allInOne.s3.enabled) }} + {{- if and $s3Enabled .Values.s3.ingress.enabled }} + {{- /* Determine service name based on deployment mode */}} +-{{- $serviceName := ternary (include "seaweedfs.componentName" (list . "all-in-one")) (include "seaweedfs.componentName" (list . "s3")) .Values.allInOne.enabled }} ++{{- $serviceName := ternary (include "seaweedfs.componentName" (list . "all-in-one")) (printf "%s-s3" (include "seaweedfs.name" .)) .Values.allInOne.enabled }} + {{- $s3Port := .Values.allInOne.s3.port | default .Values.s3.port }} + {{- /* Build hosts list - support both legacy .host (string) and new .hosts (array) for backwards compatibility */}} + {{- $hosts := list }} diff --git a/packages/system/seaweedfs/templates/_naming.tpl b/packages/system/seaweedfs/templates/_naming.tpl new file mode 100644 index 0000000000..4f8935db89 --- /dev/null +++ b/packages/system/seaweedfs/templates/_naming.tpl @@ -0,0 +1,48 @@ +{{- /* seaweedfs.renamedVolumePrefix — reconstruct the name 4.31 gives the volume + component of the `-system` release, i.e. the RELEASE-NAMED generation the + fullnameOverride pin exists to avoid. + + Input: the `-system` release name, as a bare string. + {{ include "seaweedfs.renamedVolumePrefix" "foo-system" }} -> foo-system-seaweedfs-volume + + This replays two upstream helpers, so it must track them if the vendored chart + changes (hack/seaweedfs-guard-parity.bats pins the two copies of the guard + against each other; charts/seaweedfs/templates/shared/_helpers.tpl is the + source of truth for the rules below): + + seaweedfs.fullname — with no fullnameOverride, the release name, plus + `-` when the release name does not + already contain it; truncated to 63. + seaweedfs.componentName — truncates the fullname to (62 - len(suffix)) before + appending `-`, so for `volume` the fullname + is cut to 56. An instance name >= ~40 chars + therefore loses `seaweedfs` from the tail, which is + why a `contains "seaweedfs"` name filter cannot see + its claims and this reconstruction can. + + The guard deliberately does NOT call seaweedfs.fullname directly: that helper + reads .Values.fullnameOverride, which this chart PINS to `seaweedfs`, so it + returns the chart-named generation — the opposite of what is wanted here. + + ACCEPTED LIMIT — zone/pool volume components. MultiZone (and Simple-with- + pools) tenants get per-group components with suffix `volume-`, cut at + (62 - len(suffix)), i.e. SHORTER than the 56 this helper reconstructs. The + prefixes only diverge when the fullname exceeds that shorter cut: for the + supported instance (the tenant module hardcodes the name `seaweedfs`, + fullname 16 chars) that takes a zone/pool key of ~40+ characters, and for + the default `volume` group it can never happen (its suffix IS the 56 cut). + A guard prefix that diverges means a release-named zone component the guard + cannot see. Decided 2026-07-17 (1.6 triage): accepted and documented in + docs/operations/seaweedfs-431-rename-recovery.md (Scope) rather than + reconstructed per-key — instance names are fixed by the tenant module and + absurd keys are the only reachable trigger. Revisit if instance naming is + ever opened up or a key-length clamp lands in extra/seaweedfs. */}} +{{- define "seaweedfs.renamedVolumePrefix" -}} +{{- $release := . -}} +{{- $full := $release -}} +{{- if not (contains "seaweedfs" $release) -}} +{{- $full = printf "%s-seaweedfs" $release -}} +{{- end -}} +{{- $full = $full | trunc 63 | trimSuffix "-" -}} +{{- printf "%s-volume" ($full | trunc 56 | trimSuffix "-") -}} +{{- end -}} diff --git a/packages/system/seaweedfs/templates/cluster-scoped-rbac-guard.yaml b/packages/system/seaweedfs/templates/cluster-scoped-rbac-guard.yaml new file mode 100644 index 0000000000..d14d8d289a --- /dev/null +++ b/packages/system/seaweedfs/templates/cluster-scoped-rbac-guard.yaml @@ -0,0 +1,69 @@ +{{- /* Cluster-scoped RBAC uniqueness guard. + + ClusterRoles and ClusterRoleBindings are CLUSTER-scoped: exactly one object can + carry a given name, cluster-wide. This chart is installed once per tenant, so + every cluster-scoped name it renders must be unique per install. + + That is in direct tension with the adoption pin. `seaweedfs.fullnameOverride` + is pinned to `seaweedfs` (see values.yaml and templates/naming-guard.yaml) so + the NAMESPACED workloads keep their pre-4.31 chart-based names and are adopted + in place across the bump — which means `seaweedfs.fullname` is, by + construction, the SAME string in every tenant. Naming a cluster-scoped object + after it makes every tenant fight over one object. + + Upstream 4.31 did exactly that: it switched the four cluster-scoped names from + global.serviceAccountName to seaweedfs.fullname. Pre-4.31 they were + `-{objectstorage-provisioner,rw-cr}`, and Cozystack + sets that value to `-seaweedfs` (extra/seaweedfs), so they were + unique per tenant. patches/cluster-scoped-names-per-namespace.patch puts them + back on the service account name — restoring the pre-4.31 names byte-for-byte + and, with them, per-tenant uniqueness. + + Uniqueness therefore rests on a VALUE, so assert it instead of trusting it: + the name must carry the release namespace. Without this, a tenant that stops + setting the value per namespace silently re-collides — no error, no ownership + conflict, just one ClusterRoleBinding whose subject flips to whichever tenant + reconciled last while every other tenant's COSI provisioner loses its RBAC. + That failure is invisible until a bucket operation fails, which is why the + collision survived two releases. + + Checked only when the render can see the cluster (same namespace canary as + naming-guard.yaml): system/seaweedfs's own values.yaml carries a static + PLACEHOLDER serviceAccountName that only extra/seaweedfs overrides per + release, so a client-side render (CI lint, helm unittest) legitimately sees + the placeholder and touches no live RBAC. + + The compat shim must run first. extra/seaweedfs sets the OLD flat key + global.serviceAccountName, and seaweedfs.compat is what folds it into the + canonical global.seaweedfs.serviceAccountName (the flat key wins when + present). Every subchart template includes the shim before reading, and this + guard has to read the same value the subchart will, or it compares against + this chart's placeholder default and refuses every real upgrade. Including + the shim rather than re-deriving the precedence here is deliberate: a second + copy of that rule would be free to drift from the one that actually names the + objects. It is idempotent and output-free, so calling it here is safe. */}} +{{- include "seaweedfs.compat" . -}} + +{{- $canary := lookup "v1" "Namespace" "" .Release.Namespace }} +{{- if $canary }} +{{- $sa := include "seaweedfs.serviceAccountName" . }} +{{- /* Equality, not a prefix or substring test. extra/seaweedfs sets exactly + `-seaweedfs`, so that is what to assert. Anything looser accepts a + value belonging to a DIFFERENT namespace: `contains` lets namespace `tenant-a` + through on `tenant-ab-seaweedfs`, and `hasPrefix "tenant-a-"` still lets it + through on `tenant-a-b-seaweedfs` — which is also the natural value for + namespace `tenant-a-b`, i.e. two namespaces would accept one string and + collide on the very object this guard exists to keep unique. + + This asserts the VALUE the names are built from, not the rendered names — it + cannot catch the chart being reverted to name them after the pinned fullname. + tests/cluster_scoped_names_test.yaml covers that axis, by asserting that two + tenants render two different names. + + (The comment marker must be exactly `{{- /*`: Go's lexer looks for `/*` at a + fixed offset past the trim marker, so `{{- /*` lexes as a command and fails + with `unexpected "/" in command`.) */}} +{{- if ne $sa (printf "%s-seaweedfs" .Release.Namespace) }} +{{- fail (printf "SeaweedFS release %s in namespace %s would render cluster-scoped RBAC named %q, which is not this namespace's own -seaweedfs and so is not unique per tenant. Every SeaweedFS instance pins the same fullname, so ClusterRole/ClusterRoleBinding names are taken from global.seaweedfs.serviceAccountName, which extra/seaweedfs must set per namespace. With a shared name only one tenant can own the object, and the ClusterRoleBinding's subject flips to whichever tenant reconciled last — silently revoking every other tenant's COSI provisioner." .Release.Name .Release.Namespace $sa) }} +{{- end }} +{{- end }} diff --git a/packages/system/seaweedfs/templates/naming-guard.yaml b/packages/system/seaweedfs/templates/naming-guard.yaml new file mode 100644 index 0000000000..a60541cb47 --- /dev/null +++ b/packages/system/seaweedfs/templates/naming-guard.yaml @@ -0,0 +1,154 @@ +{{- /* Naming-migration guard — the ENFORCING copy. + + Before 4.31 the vendored chart named workloads after the CHART, ignoring the + release name, so every instance ran as `seaweedfs-*` with its data on + `data1-seaweedfs-volume-*`. 4.31 names them after the release; values.yaml pins + `fullnameOverride: seaweedfs` so an upgrade past the bump adopts the running + set and its volumes in place. This template stops the render instead of letting + Helm rename workloads away from live data, or adopt the wrong one of two sets: + + - exactly the legacy generation (or nothing) — decidable, and adoption is + correct. Renders. + - exactly the release-named generation — a tenant installed fresh on 1.5.x, + whose data is on `data1-[-seaweedfs]-volume-*`. Rendering the pinned + names would rename the workloads AWAY from that data. Refuses → runbook Step 2. + - BOTH generations — undecidable from inside a render, so it refuses rather + than guess. See the classification note below. → runbook Step 1, then 2/2a/3. + + extra/seaweedfs carries the same classification for its own render, but that + render is NOT in the path of a platform upgrade: this chart reaches + helm-controller through an ExternalArtifact, so a platform bump upgrades the + `-system` release directly from the new artifact without ever + re-rendering extra/seaweedfs. The only render guaranteed to sit between a + platform upgrade and the tenant's workloads is THIS one. (v1.6 regression: + the guard lived only in extra/seaweedfs, so upgrading a fresh-1.5.x tenant + created an empty chart-named set beside its live data.) + + Fail-closed: `lookup` errors abort the render (Helm propagates API errors), + but a client-side render (`helm template`, --dry-run=client) silently returns + nothing. The release namespace itself is used as a canary: it always exists + for any real install/upgrade, so an empty canary means the render cannot see + the cluster — refuse on upgrade rather than misclassify, and skip the guard + for client-side installs (CI lint/unittest), which never touch live data. */}} + +{{- /* The classification below and the whole adoption scheme assume chart-based + workload names. Refuse to render at all if the pin is ever lifted. */}} +{{- if ne (dig "fullnameOverride" "" (.Values.seaweedfs | default dict)) "seaweedfs" }} +{{- fail (printf "system/seaweedfs must pin seaweedfs.fullnameOverride=seaweedfs (got %q): workload adoption across the 4.31 rename and the naming-migration guard are both built on chart-based names. Do not lift the pin without a data-migration plan." (dig "fullnameOverride" "" (.Values.seaweedfs | default dict))) }} +{{- end }} + +{{- $canary := lookup "v1" "Namespace" "" .Release.Namespace }} +{{- if and (not $canary) .Release.IsUpgrade }} +{{- fail (printf "SeaweedFS naming-migration guard for release %s in namespace %s: cannot see the cluster (lookup returned nothing for the release namespace itself). This render decides whether workloads are renamed away from live data, so refusing to upgrade blind. Renders applied by helm-controller always see the cluster; if templating by hand, render server-side." .Release.Name .Release.Namespace) }} +{{- end }} +{{- if $canary }} + +{{- /* Two naming GENERATIONS can exist in a namespace: + legacy — the pre-4.31 chart-based names (`seaweedfs-volume[-]`, + `data1-seaweedfs-volume[-]-N`), which this chart pins and renders; + system — the 4.31 release-based names (`[-seaweedfs]-volume`, + `data1-[-seaweedfs]-volume-N`). + + The release-named generation is matched by RECONSTRUCTING the name 4.31 would + give this release's volume component, and the chart-named one by the pinned + name — release-named checked FIRST, because the two prefixes can nest. + Overlapping prefixes alone are not safe: for an instance legitimately named + `seaweedfs-volume`, the release-named StatefulSet `seaweedfs-volume-system-volume` + and claim `data1-seaweedfs-volume-system-volume-0` BOTH satisfy the chart-named + prefixes, so a prefix-only guard reads live release-named storage as legacy and + renders onto the empty chart-named claims, stranding the real generation. + + Reconstructing also removes the blind spot a name match alone has: 4.31's + componentName truncates the fullname to 56 chars before appending `-volume`, so + an instance name >= ~40 chars loses `seaweedfs` from its claim names and a + `contains "seaweedfs"` filter cannot see them. The reconstructed prefix carries + the same truncation, so it matches those claims exactly. + + PVCs hold the data and outlive any workload but carry no chart labels, so they + are matched on name alone. StatefulSets DO carry the labels, and are the only + signal for a tenant whose PVCs are not provisioned yet. Either establishes a + generation's presence, so the two are OR-ed. */}} +{{- /* This chart IS the `-system` release, so its own release name is what + 4.31 named the renamed generation after. extra/seaweedfs is the `` release + and derives the child instead; that one line is the only difference between the + two copies of the block below. */}} +{{- $sysRelease := .Release.Name }} +{{- $renamedVol := include "seaweedfs.renamedVolumePrefix" $sysRelease }} +{{- $legacyPVC := false }} +{{- $systemPVC := false }} +{{- $legacySTS := false }} +{{- $systemSTS := false }} +{{- range (lookup "v1" "PersistentVolumeClaim" .Release.Namespace "").items | default list }} +{{- if hasPrefix (printf "data1-%s" $renamedVol) .metadata.name }} +{{- $systemPVC = true }} +{{- else if hasPrefix "data1-seaweedfs-volume" .metadata.name }} +{{- $legacyPVC = true }} +{{- end }} +{{- end }} +{{- range (lookup "apps/v1" "StatefulSet" .Release.Namespace "").items | default list }} +{{- if eq (dig "app.kubernetes.io/name" "" (.metadata.labels | default dict)) "seaweedfs" }} +{{- if hasPrefix $renamedVol .metadata.name }} +{{- $systemSTS = true }} +{{- else if hasPrefix "seaweedfs-volume" .metadata.name }} +{{- $legacySTS = true }} +{{- end }} +{{- end }} +{{- end }} +{{- $legacyGen := or $legacyPVC $legacySTS }} +{{- $systemGen := or $systemPVC $systemSTS }} + +{{- /* Exactly one generation is decidable; two is not. + + This guard used to try to tell the cases apart by comparing PVC creation + timestamps — older generation wins, because "PVCs are never recreated in + place". That premise is false, and the runbook itself is the counter-example: + Step 2's re-bind DELETES each release-named claim and re-creates it under the + chart name against the same PV. A tenant interrupted part-way through that + loop has a chart-named claim created SECONDS ago holding real data, beside a + release-named claim with the original timestamp also holding real data — the + exact inversion of the rule. The guard then reported S-damaged and sent the + operator to a step that deletes the claim Step 2 had just re-bound. + + A durable signal for BIRTH ORDER does exist, and it is worth naming so nobody + re-derives the broken one: the release history. `sh.helm.release.v1.- + system.v1` records the naming scheme the tenant was installed under (its + manifest names either `seaweedfs-master` or `-master`), and + `info.first_deployed` — present on every retained revision, so it survives + history pruning — anchors the PV creationTimestamps that Step 2's re-bind + preserves. On the upgrade stand that classifies every tenant correctly, + including reporting "no duplicate, mid-rebind" for the interrupted-Step-2 case, + because there BOTH generations sit at first_deployed. + + This render still cannot read it — the release is base64(gzip(json)) in a + Secret and Go templates have no gunzip — but that is NOT the reason the guard + refuses. Birth order is the wrong question. It says which generation is + ORIGINAL; adoption needs to know whether the OTHER one is EMPTY, and those come + apart exactly where it matters: + + D-wedged — duplicate never scheduled, holds nothing -> adopting is safe + D-split — duplicate served writes, holds unique objects -> adopting strands them + + Both are born pre-4.31, so the history signal is IDENTICAL for both (verified: + tenant-root and tenant-dsplit have the same rev-1 scheme and the same + first_deployed deltas — ~10s legacy vs ~35min release-named — yet one duplicate + served and the other did not). The only thing that separated them was + readyReplicas, which is a snapshot and not evidence: a duplicate that crashed, + was scaled down, or lost readiness after serving reads exactly like one that + never started. + + So this render does not guess. Two generations => refuse, and let an operator + establish emptiness with context the template does not have (the duplicate's + volume files, the filer's volume list). Once the empty generation is gone + exactly one remains, which IS decidable, and the render proceeds on its own. + + The cost is bounded: a duplicate only exists on a tenant that passed through + the broken 4.31 rename in 1.5.x, i.e. one that is already damaged and already + needs an operator. A 1.4.x tenant upgrading straight to 1.6 never renames, so + it only ever has the legacy generation and renders untouched. */}} +{{- if and $legacyGen $systemGen }} +{{- fail (printf "SeaweedFS release %s in namespace %s has BOTH naming generations present: the chart-named set (seaweedfs-volume / data1-seaweedfs-volume-*) AND the release-named set this chart no longer uses. One of them is an empty duplicate and one holds the data, and which is which cannot be established from inside a Helm render — claim timestamps are mutable (the Step 2 re-bind recreates claims), StatefulSets are recreated by the adoption hook, and a duplicate with zero ready replicas may still have served writes. Rendering would adopt the chart-named set, so guessing wrong strands or destroys the data. Refusing instead. Classify the tenant and delete the EMPTY generation so exactly one remains — this render then adopts the survivor with no further action. See docs/operations/seaweedfs-431-rename-recovery.md (Step 1 classifies; Step 2/2a/3 recover). If you are part-way through Step 2's re-bind, finish it." .Release.Name .Release.Namespace) }} +{{- else if $systemGen }} +{{- fail (printf "SeaweedFS release %s in namespace %s keeps its data on volumes named after the Helm release — a tenant installed fresh on Cozystack 1.5.x (or one whose instance name is long enough that the chart truncated the volume PVC names, leaving only its StatefulSets to match on). This chart pins the pre-4.31 chart-based names, so rendering would rename the workloads away from that data and stand up an empty cluster beside it. Helm cannot move data between PVCs. Re-bind the existing PVs onto the data1-seaweedfs-volume-* PVC names first — docs/operations/seaweedfs-431-rename-recovery.md (Step 2)." .Release.Name .Release.Namespace) }} +{{- end }} + +{{- end }} diff --git a/packages/system/seaweedfs/tests/cluster_scoped_names_test.yaml b/packages/system/seaweedfs/tests/cluster_scoped_names_test.yaml new file mode 100644 index 0000000000..8574ebc427 --- /dev/null +++ b/packages/system/seaweedfs/tests/cluster_scoped_names_test.yaml @@ -0,0 +1,138 @@ +suite: seaweedfs cluster-scoped RBAC names stay unique per tenant + +# ClusterRoles and ClusterRoleBindings are CLUSTER-scoped: exactly one object can +# carry a given name, cluster-wide. Every SeaweedFS instance pins the same +# `fullnameOverride: seaweedfs` (#3282 — that pin is what adopts the running +# NAMESPACED workloads across the 4.31 rename), so `seaweedfs.fullname` is +# identical in every tenant and must never name a cluster-scoped object. +# +# Upstream 4.31 switched these four names from global.serviceAccountName (which +# Cozystack sets per namespace) to seaweedfs.fullname, which is how five tenants +# ended up fighting over ONE ClusterRole. patches/cluster-scoped-names-per- +# namespace.patch puts them back on the service account name, restoring the +# pre-4.31 names byte-for-byte. +# +# These assertions fail on unpatched code: it renders `seaweedfs-*` for every +# namespace, so the two namespaces below produce the SAME name instead of two. + +templates: + - charts/seaweedfs/templates/cosi/cosi-cluster-role.yaml + - charts/seaweedfs/templates/shared/cluster-role.yaml + +release: + name: seaweedfs-system + namespace: tenant-root + +tests: + - it: names the COSI ClusterRole/Binding after the per-namespace service account + template: charts/seaweedfs/templates/cosi/cosi-cluster-role.yaml + set: + global.seaweedfs.serviceAccountName: tenant-root-seaweedfs + asserts: + # ClusterRole — pre-4.31 name, restored. + - equal: + path: metadata.name + value: tenant-root-seaweedfs-objectstorage-provisioner + documentIndex: 0 + # ClusterRoleBinding — same name, and its roleRef must follow the rename or + # the binding would point at another tenant's ClusterRole. + - equal: + path: metadata.name + value: tenant-root-seaweedfs-objectstorage-provisioner + documentIndex: 1 + - equal: + path: roleRef.name + value: tenant-root-seaweedfs-objectstorage-provisioner + documentIndex: 1 + # The subject was ALREADY per-namespace (cosi-provisioner-sa-name.patch); + # it is what the cluster-scoped names are now aligned with. + - equal: + path: subjects[0].name + value: tenant-root-seaweedfs-objectstorage-provisioner + documentIndex: 1 + - equal: + path: subjects[0].namespace + value: tenant-root + documentIndex: 1 + + - it: names the master-rw ClusterRole/Binding after the per-namespace service account + template: charts/seaweedfs/templates/shared/cluster-role.yaml + set: + global.seaweedfs.serviceAccountName: tenant-root-seaweedfs + asserts: + - equal: + path: metadata.name + value: tenant-root-seaweedfs-rw-cr + documentIndex: 0 + - equal: + path: metadata.name + value: tenant-root-seaweedfs-rw-crb + documentIndex: 1 + - equal: + path: roleRef.name + value: tenant-root-seaweedfs-rw-cr + documentIndex: 1 + - equal: + path: subjects[0].name + value: tenant-root-seaweedfs + documentIndex: 1 + + # THE regression test. A second tenant, running an instance under a DIFFERENT + # name (`foo` -> release foo-system), must not collide with tenant-root above. + # Unpatched, both render `seaweedfs-objectstorage-provisioner` / `seaweedfs-rw-cr` + # — observed live as one ClusterRole annotated to a single owning tenant while + # four others silently lost their COSI RBAC. + - it: renders a different cluster-scoped name for a different tenant (collision regression) + template: charts/seaweedfs/templates/cosi/cosi-cluster-role.yaml + release: + name: foo-system + namespace: tenant-named + set: + global.seaweedfs.serviceAccountName: tenant-named-seaweedfs + asserts: + - equal: + path: metadata.name + value: tenant-named-seaweedfs-objectstorage-provisioner + documentIndex: 0 + - equal: + path: metadata.name + value: tenant-named-seaweedfs-objectstorage-provisioner + documentIndex: 1 + - equal: + path: roleRef.name + value: tenant-named-seaweedfs-objectstorage-provisioner + documentIndex: 1 + + - it: renders a different master-rw name for a different tenant (collision regression) + template: charts/seaweedfs/templates/shared/cluster-role.yaml + release: + name: foo-system + namespace: tenant-named + set: + global.seaweedfs.serviceAccountName: tenant-named-seaweedfs + asserts: + - equal: + path: metadata.name + value: tenant-named-seaweedfs-rw-cr + documentIndex: 0 + - equal: + path: metadata.name + value: tenant-named-seaweedfs-rw-crb + documentIndex: 1 + + # The pin must NOT leak into cluster-scoped names, and must NOT be lifted from + # namespaced ones. Guards the two halves against each other. + - it: keeps the cluster-scoped name independent of the pinned fullname + template: charts/seaweedfs/templates/cosi/cosi-cluster-role.yaml + set: + global.seaweedfs.serviceAccountName: tenant-root-seaweedfs + seaweedfs.fullnameOverride: seaweedfs + asserts: + - notMatchRegex: + path: metadata.name + pattern: "^seaweedfs-" + documentIndex: 0 + - notMatchRegex: + path: metadata.name + pattern: "^seaweedfs-system-" + documentIndex: 0 diff --git a/packages/system/seaweedfs/tests/cluster_scoped_rbac_guard_test.yaml b/packages/system/seaweedfs/tests/cluster_scoped_rbac_guard_test.yaml new file mode 100644 index 0000000000..dd8aeb29a7 --- /dev/null +++ b/packages/system/seaweedfs/tests/cluster_scoped_rbac_guard_test.yaml @@ -0,0 +1,177 @@ +suite: seaweedfs cluster-scoped RBAC uniqueness guard + +# Companion to cluster_scoped_names_test.yaml: that suite pins the NAMES the +# chart renders, this one pins the guard that refuses when those names would not +# be unique per tenant. +# +# The guard reads global.seaweedfs.serviceAccountName and requires it to carry +# the release namespace, because the pinned fullname is identical in every tenant +# and so cannot be what distinguishes a cluster-scoped object. It runs only when +# the render can see the cluster (namespace canary, same as naming-guard.yaml) — +# system/seaweedfs's values.yaml default is a placeholder that only +# extra/seaweedfs overrides per release, so a client-side render must not trip. + +templates: + - templates/cluster-scoped-rbac-guard.yaml + +release: + name: seaweedfs-system + namespace: tenant-fresh + +tests: + - it: renders for a client-side install with no cluster view (CI lint, unittest) + asserts: + - hasDocuments: + count: 0 + + - it: renders when the cluster-scoped RBAC name carries the namespace + set: + global: + seaweedfs: + serviceAccountName: tenant-fresh-seaweedfs + kubernetesProvider: + scheme: &scheme + "v1/Namespace": + gvr: + group: "" + version: "v1" + resource: "namespaces" + namespaced: false + objects: + - &ns + kind: Namespace + apiVersion: v1 + metadata: + name: tenant-fresh + asserts: + - hasDocuments: + count: 0 + + # M2 REGRESSION — the compat shim. Production does NOT set the canonical + # global.seaweedfs.serviceAccountName that the other cases here use: + # extra/seaweedfs sets the OLD FLAT key global.serviceAccountName, which only + # `include "seaweedfs.compat"` folds into the canonical path (and the flat key + # wins when present). Without that include the guard reads this chart's + # placeholder default and refuses EVERY real upgrade on EVERY cluster — a bug + # that only `helm upgrade --dry-run=server` caught, because every mock set the + # canonical key. This case sets the key production actually sets. + - it: accepts the flat global.serviceAccountName that extra/seaweedfs actually sets + set: + global: + serviceAccountName: tenant-fresh-seaweedfs + kubernetesProvider: + scheme: *scheme + objects: + - *ns + asserts: + - hasDocuments: + count: 0 + + # The shim must not become a way to bypass the guard: a flat key that is not + # per-namespace is still refused. + - it: refuses a flat global.serviceAccountName that is not per-namespace + set: + global: + serviceAccountName: seaweedfs + kubernetesProvider: + scheme: *scheme + objects: + - *ns + asserts: + - failedTemplate: + errorPattern: "not unique per tenant" + + # A substring match accepts `tenant-ab-seaweedfs` for namespace `tenant-a`; a + # prefix match still accepts `tenant-a-b-seaweedfs`, which is ALSO the natural + # value for namespace `tenant-a-b` — so two namespaces would accept one string + # and collide on the object this guard exists to keep unique. Only equality with + # `-seaweedfs` — the value extra/seaweedfs actually sets — is safe. + - it: refuses a service account from a namespace that only shares a prefix + release: + name: seaweedfs-system + namespace: tenant-a + set: + global: + seaweedfs: + serviceAccountName: tenant-ab-seaweedfs + kubernetesProvider: + scheme: *scheme + objects: + - kind: Namespace + apiVersion: v1 + metadata: + name: tenant-a + asserts: + - failedTemplate: + errorPattern: "not unique per tenant" + + # The prefix variant: `tenant-a-b-seaweedfs` starts with `tenant-a-` but belongs + # to namespace tenant-a-b. hasPrefix passes this; equality does not. + - it: refuses a service account whose namespace merely extends this one + release: + name: seaweedfs-system + namespace: tenant-a + set: + global: + seaweedfs: + serviceAccountName: tenant-a-b-seaweedfs + kubernetesProvider: + scheme: *scheme + objects: + - kind: Namespace + apiVersion: v1 + metadata: + name: tenant-a + asserts: + - failedTemplate: + errorPattern: "not unique per tenant" + + # The regression this guard exists for: the bare chart default (or any value + # that is the same string in every tenant) collides cluster-wide. + - it: refuses when the RBAC name would be identical in every tenant + set: + global: + seaweedfs: + serviceAccountName: seaweedfs + kubernetesProvider: + scheme: *scheme + objects: + - *ns + asserts: + - failedTemplate: + errorPattern: "not unique per tenant" + + # The placeholder shipped in system/seaweedfs/values.yaml is namespace-shaped + # but belongs to no real namespace — it must not be mistaken for a valid value + # once the render can see the cluster. + - it: refuses when the value is the shipped placeholder, not this namespace + set: + global: + seaweedfs: + serviceAccountName: tenant-foo-seaweedfs + kubernetesProvider: + scheme: *scheme + objects: + - *ns + asserts: + - failedTemplate: + errorPattern: "not unique per tenant" + + - it: refuses for a non-default instance name whose SA is not per-namespace + release: + name: foo-system + namespace: tenant-named + set: + global: + seaweedfs: + serviceAccountName: seaweedfs + kubernetesProvider: + scheme: *scheme + objects: + - kind: Namespace + apiVersion: v1 + metadata: + name: tenant-named + asserts: + - failedTemplate: + errorPattern: "not unique per tenant" diff --git a/packages/system/seaweedfs/tests/cosi_image_test.yaml b/packages/system/seaweedfs/tests/cosi_image_test.yaml new file mode 100644 index 0000000000..7840d200de --- /dev/null +++ b/packages/system/seaweedfs/tests/cosi_image_test.yaml @@ -0,0 +1,38 @@ +suite: seaweedfs COSI driver image pin + +# Pins the COSI driver image to a release that enforces read-only bucket access. +# +# The vendored chart default is seaweedfs-cosi-driver v0.1.2 (see +# charts/seaweedfs/values.yaml), which hardcodes read-write S3 actions for every +# BucketAccess and ignores the accessPolicy parameter. The chart ships a +# -readonly BucketAccessClass with parameters.accessPolicy: readonly, so on +# v0.1.2 a credential meant to be read-only is silently granted read-write. +# The package values.yaml overrides cosi.image to v0.3.1, which honours +# accessPolicy: readonly. +# +# This test does not set the image itself; it relies on the package values.yaml +# override, so it goes red if that override is removed and the image falls back +# to the vulnerable v0.1.2 — turning a silent security regression into a build +# failure. Note: `make update` re-vendors charts/ and resets the vendored +# default to v0.1.2 but does not touch the package values.yaml, so the override +# (and this guard) must be preserved across any re-vendor. + +templates: + - charts/seaweedfs/templates/cosi/cosi-deployment.yaml + +release: + name: seaweedfs + namespace: cozy-seaweedfs + +tests: + - it: pins the cosi-driver container to v0.3.1 (read-only access enforcement) + template: charts/seaweedfs/templates/cosi/cosi-deployment.yaml + set: + seaweedfs.cosi.enabled: true + asserts: + - equal: + path: spec.template.spec.containers[0].name + value: seaweedfs-cosi-driver + - equal: + path: spec.template.spec.containers[0].image + value: ghcr.io/seaweedfs/seaweedfs-cosi-driver:v0.3.1 diff --git a/packages/system/seaweedfs/tests/fullname_override_test.yaml b/packages/system/seaweedfs/tests/fullname_override_test.yaml new file mode 100644 index 0000000000..545e949b55 --- /dev/null +++ b/packages/system/seaweedfs/tests/fullname_override_test.yaml @@ -0,0 +1,62 @@ +suite: seaweedfs workload naming is pinned to the chart name + +# Upstream chart 4.31 names workloads after the Helm release (`seaweedfs.fullname`) +# instead of after the chart (`seaweedfs.name`, pre-4.31). The shipped release is +# `seaweedfs-system`, so the bump silently renamed every StatefulSet/Deployment to +# `seaweedfs-system-*`. Those names are immutable, so an upgrade cannot rename in +# place — Helm brings up a second, duplicate set beside the running one while the +# data stays on the original `data1-seaweedfs-volume-*` PVCs. +# +# values.yaml pins `fullnameOverride: seaweedfs` so the rendered names stay +# chart-based regardless of the release name, matching the convention already used +# by other system packages (cozy-proxy, flux-operator, linstor-scheduler, ...). +# These tests pin that contract: names must NOT follow the release name. + +release: + name: seaweedfs-system + namespace: tenant-root + +tests: + - it: names the master StatefulSet after the chart, not the release + template: charts/seaweedfs/templates/master/master-statefulset.yaml + asserts: + - equal: + path: metadata.name + value: seaweedfs-master + - equal: + path: spec.serviceName + value: seaweedfs-master + + - it: names the volume StatefulSet after the chart, so its data PVCs keep the legacy names + template: charts/seaweedfs/templates/volume/volume-statefulset.yaml + asserts: + - equal: + path: metadata.name + value: seaweedfs-volume + # Together with the StatefulSet name this yields data1-seaweedfs-volume-N — + # the PVC names the running clusters already hold their data on. + - equal: + path: spec.volumeClaimTemplates[0].metadata.name + value: data1 + + - it: names the filer StatefulSet after the chart + template: charts/seaweedfs/templates/filer/filer-statefulset.yaml + asserts: + - equal: + path: metadata.name + value: seaweedfs-filer + + - it: keeps the selector labels release-scoped so adopted pods keep matching + template: charts/seaweedfs/templates/master/master-statefulset.yaml + asserts: + # A StatefulSet selector is immutable: in-place adoption only works because + # 4.31 left these labels identical to 4.05. + - equal: + path: spec.selector.matchLabels["app.kubernetes.io/name"] + value: seaweedfs + - equal: + path: spec.selector.matchLabels["app.kubernetes.io/instance"] + value: seaweedfs-system + - equal: + path: spec.selector.matchLabels["app.kubernetes.io/component"] + value: master diff --git a/packages/system/seaweedfs/tests/naming_guard_test.yaml b/packages/system/seaweedfs/tests/naming_guard_test.yaml new file mode 100644 index 0000000000..293028aca5 --- /dev/null +++ b/packages/system/seaweedfs/tests/naming_guard_test.yaml @@ -0,0 +1,568 @@ +suite: seaweedfs naming-migration guard (enforcing copy) + +# This chart is what a platform upgrade actually re-renders: the -system +# HelmRelease pulls it from a platform-managed ExternalArtifact, so bumping the +# platform upgrades the release directly — extra/seaweedfs (which carries a +# sibling copy of this guard for operator visibility) is NOT in that path. +# +# The guard decides between exactly three outcomes: +# legacy generation only (or nothing) -> render (adoption is correct) +# release-named generation only -> refuse, class S (runbook Step 2) +# BOTH generations -> refuse, undecidable (runbook Step 1) +# +# It used to tell the both-generations cases apart by comparing PVC +# creationTimestamps ("PVCs are never recreated in place, so the older generation +# is where the data was born"). That premise is false — the runbook's own Step 2 +# re-bind DELETES each release-named claim and recreates it under the chart name +# against the same PV — and acting on the wrong answer deletes live data. The +# guard no longer guesses; the tests below pin that. +# +# The guard reads the cluster through lookup with the release namespace as its +# canary, so classification tests register v1/Namespace + v1/PersistentVolumeClaim +# + apps/v1/StatefulSet and include the namespace object. readyReplicas is quoted +# ("1"/"0") because the fake client cannot deep-copy a bare Go int. + +templates: + - templates/naming-guard.yaml + +release: + name: seaweedfs-system + namespace: tenant-fresh + +tests: + - it: renders for a client-side install with no cluster view (CI lint, unittest) + asserts: + - hasDocuments: + count: 0 + + # NOTE: the fail-closed refusal (client-side UPGRADE with no cluster view) + # cannot be modelled here — helm-unittest 1.0.3 ignores release.isUpgrade, so + # .Release.IsUpgrade always renders false. The chart's `make test` target + # covers it with a real renderer (helm template --is-upgrade). + + - it: refuses to render if the fullnameOverride pin is ever lifted + set: + seaweedfs: + fullnameOverride: something-else + asserts: + - failedTemplate: + errorPattern: "must pin seaweedfs.fullnameOverride=seaweedfs" + + - it: renders for a net-new tenant (namespace visible, no SeaweedFS data) + kubernetesProvider: + scheme: &scheme + "v1/Namespace": + gvr: + group: "" + version: "v1" + resource: "namespaces" + namespaced: false + "v1/PersistentVolumeClaim": + gvr: + group: "" + version: "v1" + resource: "persistentvolumeclaims" + namespaced: true + "apps/v1/StatefulSet": + gvr: + group: "apps" + version: "v1" + resource: "statefulsets" + namespaced: true + objects: + - &ns + kind: Namespace + apiVersion: v1 + metadata: + name: tenant-fresh + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: unrelated-app-data-0 + namespace: tenant-fresh + - kind: StatefulSet + apiVersion: apps/v1 + metadata: + name: unrelated-app + namespace: tenant-fresh + asserts: + - hasDocuments: + count: 0 + + - it: renders for a legacy-only tenant (data on the chart-named PVCs, adopt in place) + kubernetesProvider: + scheme: *scheme + objects: + - *ns + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: data1-seaweedfs-volume-0 + namespace: tenant-fresh + - kind: StatefulSet + apiVersion: apps/v1 + metadata: + name: seaweedfs-volume + namespace: tenant-fresh + labels: + app.kubernetes.io/name: seaweedfs + asserts: + - hasDocuments: + count: 0 + + - it: refuses class S — data only on release-named volumes (fresh 1.5.x install) + kubernetesProvider: + scheme: *scheme + objects: + - *ns + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: data1-seaweedfs-system-volume-0 + namespace: tenant-fresh + - kind: StatefulSet + apiVersion: apps/v1 + metadata: + name: seaweedfs-system-volume + namespace: tenant-fresh + labels: + app.kubernetes.io/name: seaweedfs + asserts: + - failedTemplate: + errorPattern: "keeps its data on volumes named after the Helm release.*Step 2" + + - it: refuses class S for a non-default instance name (foo-system release) + release: + name: foo-system + namespace: tenant-fresh + kubernetesProvider: + scheme: *scheme + objects: + - *ns + # 4.31 appends the chart name when the release name does not contain it: + # a fresh 1.5.x `foo` wrote to data1-foo-system-seaweedfs-volume-*. + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: data1-foo-system-seaweedfs-volume-0 + namespace: tenant-fresh + # The fake client builds its list kinds from the objects present, so a + # suite that LISTs StatefulSets must include at least one or the lookup + # errors instead of returning empty. + - &unrelated_sts + kind: StatefulSet + apiVersion: apps/v1 + metadata: + name: unrelated-app + namespace: tenant-fresh + asserts: + - failedTemplate: + errorPattern: "keeps its data on volumes named after the Helm release" + + # --------------------------------------------------------------------------- + # Both generations present => refuse. Four shapes the old timestamp classifier + # routed FOUR different ways; all four get the same answer now. + # --------------------------------------------------------------------------- + + # Old code: legacy older + duplicate live -> "D-split" (Step 3). Now: refuse. + - it: refuses when both generations exist and the duplicate is live (was D-split) + kubernetesProvider: + scheme: *scheme + objects: + - *ns + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: data1-seaweedfs-volume-0 + namespace: tenant-fresh + creationTimestamp: "2025-11-02T10:00:00Z" + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: data1-seaweedfs-system-volume-0 + namespace: tenant-fresh + creationTimestamp: "2026-06-20T12:00:00Z" + - kind: StatefulSet + apiVersion: apps/v1 + metadata: + name: seaweedfs-system-volume + namespace: tenant-fresh + labels: + app.kubernetes.io/name: seaweedfs + status: + readyReplicas: "1" + asserts: + - failedTemplate: + errorPattern: "has BOTH naming generations present" + + # B5 REGRESSION. Old code: legacy older + zero ready replicas -> "D-wedged", + # which rendered THROUGH and adopted. readyReplicas is only a snapshot: a + # duplicate that crashed, was scaled down, or lost readiness AFTER serving + # writes reads identically to one that never served. Adoption would strand + # whatever it wrote. + - it: refuses when both generations exist and the duplicate reads zero ready (was D-wedged, adopted) + kubernetesProvider: + scheme: *scheme + objects: + - *ns + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: data1-seaweedfs-volume-0 + namespace: tenant-fresh + creationTimestamp: "2025-11-02T10:00:00Z" + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: data1-seaweedfs-system-volume-0 + namespace: tenant-fresh + creationTimestamp: "2026-06-20T12:00:00Z" + - kind: StatefulSet + apiVersion: apps/v1 + metadata: + name: seaweedfs-system-volume + namespace: tenant-fresh + labels: + app.kubernetes.io/name: seaweedfs + status: + readyReplicas: "0" + - kind: StatefulSet + apiVersion: apps/v1 + metadata: + name: seaweedfs-volume + namespace: tenant-fresh + labels: + app.kubernetes.io/name: seaweedfs + status: + readyReplicas: "1" + asserts: + - failedTemplate: + errorPattern: "has BOTH naming generations present" + + # Old code: release-named older -> "S-damaged" (Step 2a). Now: refuse. + - it: refuses when both generations exist and the release-named claims are older (was S-damaged) + kubernetesProvider: + scheme: *scheme + objects: + - *ns + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: data1-seaweedfs-system-volume-0 + namespace: tenant-fresh + creationTimestamp: "2026-06-20T07:01:59Z" + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: data1-seaweedfs-volume-0 + namespace: tenant-fresh + creationTimestamp: "2026-06-20T08:28:58Z" + - *unrelated_sts + asserts: + - failedTemplate: + errorPattern: "has BOTH naming generations present" + + # B1 REGRESSION — the data-loss path. A tenant interrupted part-way through the + # runbook's Step 2 re-bind: data1-seaweedfs-volume-0 was DELETED and RECREATED + # against its original PV seconds ago, so it is the NEWEST claim while holding + # real data; data1-seaweedfs-system-volume-1 has not been re-bound yet, so it + # keeps the ORIGINAL timestamp and also holds real data. The old timestamp rule + # read "release-named older" and reported S-damaged, sending the operator to + # Step 2a — which deletes data1-seaweedfs-volume-*, i.e. the claim Step 2 had + # just re-bound, with the PV's reclaim policy already restored to Delete. + - it: refuses a tenant interrupted mid Step-2 re-bind rather than calling it S-damaged + kubernetesProvider: + scheme: *scheme + objects: + - *ns + # Not yet re-bound: original claim, original timestamp, real data. + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: data1-seaweedfs-system-volume-1 + namespace: tenant-fresh + creationTimestamp: "2026-06-20T07:01:59Z" + # Already re-bound by Step 2: brand-new claim, ORIGINAL PV, real data. + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: data1-seaweedfs-volume-0 + namespace: tenant-fresh + creationTimestamp: "2026-07-17T09:30:00Z" + - *unrelated_sts + asserts: + - failedTemplate: + errorPattern: "has BOTH naming generations present" + + # --------------------------------------------------------------------------- + # B2 REGRESSION — long instance name. seaweedfs.componentName truncates the + # fullname to 56 chars before appending -volume, so an instance name >= ~40 + # chars drops `seaweedfs` off the PVC name and the name match cannot see it. + # The StatefulSet is label-matched and still can, which is why the generation + # flags OR the two signals instead of keying off the PVC evidence alone. + # --------------------------------------------------------------------------- + + - it: refuses both-generations for a long instance name whose PVC names were truncated + release: + name: archive-of-quarterly-financial-statements-x1-system + namespace: tenant-fresh + kubernetesProvider: + scheme: *scheme + objects: + - *ns + # The renamed volume PVC, truncated past the chart name: unmatchable. + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: data1-archive-of-quarterly-financial-statements-x1-system-seaw-volume-0 + namespace: tenant-fresh + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: data1-seaweedfs-volume-0 + namespace: tenant-fresh + # Only the label-matched StatefulSet reveals the renamed generation. Old + # code fell through to the D-split branch here, sending the operator to + # quiesce the set holding the data — the exact misclassification the + # parent commit exists to remove. + - kind: StatefulSet + apiVersion: apps/v1 + metadata: + name: archive-of-quarterly-financial-statements-x1-system-seaw-volume + namespace: tenant-fresh + labels: + app.kubernetes.io/name: seaweedfs + status: + readyReplicas: "1" + asserts: + - failedTemplate: + errorPattern: "has BOTH naming generations present" + + - it: refuses a truncated long instance name whose duplicate reads zero ready (rendered unguarded before) + release: + name: archive-of-quarterly-financial-statements-x1-system + namespace: tenant-fresh + kubernetesProvider: + scheme: *scheme + objects: + - *ns + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: data1-archive-of-quarterly-financial-statements-x1-system-seaw-volume-0 + namespace: tenant-fresh + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: data1-seaweedfs-volume-0 + namespace: tenant-fresh + # Old code: no branch fired at all here — it rendered through and adopted + # the empty chart-named set. Blocker fully open. + - kind: StatefulSet + apiVersion: apps/v1 + metadata: + name: archive-of-quarterly-financial-statements-x1-system-seaw-volume + namespace: tenant-fresh + labels: + app.kubernetes.io/name: seaweedfs + status: + readyReplicas: "0" + asserts: + - failedTemplate: + errorPattern: "has BOTH naming generations present" + + - it: refuses class S for a truncated long instance name with no legacy generation + release: + name: archive-of-quarterly-financial-statements-x1-system + namespace: tenant-fresh + kubernetesProvider: + scheme: *scheme + objects: + - *ns + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: data1-archive-of-quarterly-financial-statements-x1-system-seaw-volume-0 + namespace: tenant-fresh + - kind: StatefulSet + apiVersion: apps/v1 + metadata: + name: archive-of-quarterly-financial-statements-x1-system-seaw-volume + namespace: tenant-fresh + labels: + app.kubernetes.io/name: seaweedfs + asserts: + - failedTemplate: + errorPattern: "keeps its data on volumes named after the Helm release" + + # --------------------------------------------------------------------------- + # An instance legitimately named `seaweedfs-volume`. Its release is + # seaweedfs-volume-system, which CONTAINS "seaweedfs", so 4.31 names its volume + # workloads seaweedfs-volume-system-volume and its claims + # data1-seaweedfs-volume-system-volume-N. Both satisfy the chart-named prefixes + # ("seaweedfs-volume" / "data1-seaweedfs-volume"), so a prefix-only guard reads + # this tenant's LIVE release-named storage as legacy, sees no release-named + # generation, renders, and stands an empty cluster on data1-seaweedfs-volume-* + # while the real data is stranded. Reconstructing the renamed prefix and testing + # it FIRST is what separates them. + # --------------------------------------------------------------------------- + + - it: refuses class S for an instance named seaweedfs-volume (release-named storage is not legacy) + release: + name: seaweedfs-volume-system + namespace: tenant-fresh + kubernetesProvider: + scheme: *scheme + objects: + - *ns + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: data1-seaweedfs-volume-system-volume-0 + namespace: tenant-fresh + - kind: StatefulSet + apiVersion: apps/v1 + metadata: + name: seaweedfs-volume-system-volume + namespace: tenant-fresh + labels: + app.kubernetes.io/name: seaweedfs + asserts: + - failedTemplate: + errorPattern: "keeps its data on volumes named after the Helm release" + + - it: refuses both-generations for an instance named seaweedfs-volume + release: + name: seaweedfs-volume-system + namespace: tenant-fresh + kubernetesProvider: + scheme: *scheme + objects: + - *ns + # Live release-named storage... + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: data1-seaweedfs-volume-system-volume-0 + namespace: tenant-fresh + # ...beside a genuine chart-named duplicate. + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: data1-seaweedfs-volume-0 + namespace: tenant-fresh + - kind: StatefulSet + apiVersion: apps/v1 + metadata: + name: seaweedfs-volume-system-volume + namespace: tenant-fresh + labels: + app.kubernetes.io/name: seaweedfs + asserts: + - failedTemplate: + errorPattern: "has BOTH naming generations present" + + # --------------------------------------------------------------------------- + # MultiZone (and Simple-with-pools) tenants have NO plain `volume` component — + # extra/seaweedfs sets volume.enabled=false and renders one component per zone + # key, suffix `volume-`. For the supported instance (the tenant module + # hardcodes the name `seaweedfs`, fullname 16 chars) nothing is truncated, so + # the reconstructed prefix `seaweedfs-system-volume` must prefix-match every + # zone component. These pin that. The reconstruction does NOT cover zone + # components of long-named instances or ~40+ char zone keys — an accepted, + # documented limit (_naming.tpl, runbook Scope), deliberately not tested. + # --------------------------------------------------------------------------- + + - it: renders for a legacy-only MultiZone tenant (zone components adopt in place) + kubernetesProvider: + scheme: *scheme + objects: + - *ns + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: data1-seaweedfs-volume-md-0 + namespace: tenant-fresh + - kind: StatefulSet + apiVersion: apps/v1 + metadata: + name: seaweedfs-volume-md + namespace: tenant-fresh + labels: + app.kubernetes.io/name: seaweedfs + asserts: + - hasDocuments: + count: 0 + + - it: refuses class S for a MultiZone tenant with only zone components (fresh 1.5.x) + kubernetesProvider: + scheme: *scheme + objects: + - *ns + # No plain `volume` component exists on a MultiZone tenant — the zone + # objects are the ONLY release-named evidence, so if the prefix match + # missed them this tenant would render and be renamed away from its data. + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: data1-seaweedfs-system-volume-md-0 + namespace: tenant-fresh + - kind: StatefulSet + apiVersion: apps/v1 + metadata: + name: seaweedfs-system-volume-md + namespace: tenant-fresh + labels: + app.kubernetes.io/name: seaweedfs + asserts: + - failedTemplate: + errorPattern: "keeps its data on volumes named after the Helm release" + + - it: refuses both-generations for a MultiZone tenant + kubernetesProvider: + scheme: *scheme + objects: + - *ns + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: data1-seaweedfs-volume-md-0 + namespace: tenant-fresh + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: data1-seaweedfs-system-volume-md-0 + namespace: tenant-fresh + - kind: StatefulSet + apiVersion: apps/v1 + metadata: + name: seaweedfs-system-volume-md + namespace: tenant-fresh + labels: + app.kubernetes.io/name: seaweedfs + status: + readyReplicas: "0" + asserts: + - failedTemplate: + errorPattern: "has BOTH naming generations present" + + - it: ignores unrelated data1-*-volume-* claims of other apps + kubernetesProvider: + scheme: *scheme + objects: + - *ns + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: data1-clickhouse-volume-0 + namespace: tenant-fresh + - kind: StatefulSet + apiVersion: apps/v1 + metadata: + name: clickhouse-volume + namespace: tenant-fresh + labels: + app.kubernetes.io/name: clickhouse + asserts: + - hasDocuments: + count: 0 diff --git a/packages/system/seaweedfs/tests/s3_service_name_consumers_test.yaml b/packages/system/seaweedfs/tests/s3_service_name_consumers_test.yaml new file mode 100644 index 0000000000..b536b7c57d --- /dev/null +++ b/packages/system/seaweedfs/tests/s3_service_name_consumers_test.yaml @@ -0,0 +1,109 @@ +suite: seaweedfs s3 consumers resolve the renamed s3 service + +# Pins the regression where consumers of the S3 service 503 / fail to connect +# because they still resolve a service name that no longer exists. +# +# The s3-service-name.patch renamed the S3 Service to `-s3` (seaweedfs-s3), +# but several consumers still resolved it via componentName -> `-s3`: +# - the S3 ingress backend, +# - the iceberg ingress backend, +# - the COSI provisioner ENDPOINT (in-cluster, s3.ingress.enabled=false path). +# name and fullname only coincide when the release name equals the chart name; +# the shipped release is `seaweedfs-system`, so those consumers pointed at +# `seaweedfs-system-s3` while the real service is `seaweedfs-s3`. With no +# matching endpoints, ingress-nginx returns 503 and COSI cannot reach S3. +# +# values.yaml now pins fullnameOverride: seaweedfs, which makes name == fullname +# and would hide this bug by accident. So each test re-creates the original +# condition explicitly (fullnameOverride: seaweedfs-system => name != fullname) +# and keeps guarding the divergence itself, independent of what the shipped +# default happens to be. The all-in-one case documents the other ternary branch, +# which is unchanged and correct. + +templates: + - charts/seaweedfs/templates/s3/s3-service.yaml + - charts/seaweedfs/templates/s3/s3-ingress.yaml + - charts/seaweedfs/templates/s3/s3-iceberg-ingress.yaml + - charts/seaweedfs/templates/cosi/cosi-deployment.yaml + +release: + name: seaweedfs-system + namespace: cozy-seaweedfs + +tests: + - it: s3 service is named with the name helper, not componentName + template: charts/seaweedfs/templates/s3/s3-service.yaml + set: + seaweedfs.fullnameOverride: seaweedfs-system + seaweedfs.s3.enabled: true + asserts: + - equal: + path: metadata.name + value: seaweedfs-s3 + + - it: s3 ingress backend points at the s3 service, not a fullname-derived name + template: charts/seaweedfs/templates/s3/s3-ingress.yaml + set: + seaweedfs.fullnameOverride: seaweedfs-system + seaweedfs.s3.enabled: true + seaweedfs.s3.ingress.enabled: true + seaweedfs.s3.ingress.host: s3.example.com + asserts: + - equal: + path: spec.rules[0].http.paths[0].backend.service.name + value: seaweedfs-s3 + + - it: all-in-one ingress backend still resolves to the all-in-one service + template: charts/seaweedfs/templates/s3/s3-ingress.yaml + set: + seaweedfs.fullnameOverride: seaweedfs-system + seaweedfs.allInOne.enabled: true + seaweedfs.allInOne.s3.enabled: true + seaweedfs.s3.ingress.enabled: true + seaweedfs.s3.ingress.host: s3.example.com + asserts: + - equal: + path: spec.rules[0].http.paths[0].backend.service.name + value: seaweedfs-system-all-in-one + + - it: iceberg ingress backend points at the s3 service too + template: charts/seaweedfs/templates/s3/s3-iceberg-ingress.yaml + set: + seaweedfs.fullnameOverride: seaweedfs-system + seaweedfs.s3.enabled: true + seaweedfs.s3.icebergPort: 8334 + seaweedfs.s3.icebergIngress.enabled: true + seaweedfs.s3.icebergIngress.host: iceberg.example.com + asserts: + - equal: + path: spec.rules[0].http.paths[0].backend.service.name + value: seaweedfs-s3 + + - it: COSI in-cluster ENDPOINT resolves the s3 service when ingress is disabled + template: charts/seaweedfs/templates/cosi/cosi-deployment.yaml + set: + seaweedfs.fullnameOverride: seaweedfs-system + seaweedfs.cosi.enabled: true + seaweedfs.s3.enabled: true + seaweedfs.s3.ingress.enabled: false + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: ENDPOINT + value: https://seaweedfs-s3.cozy-seaweedfs.svc:8333 + + - it: COSI in-cluster ENDPOINT (filer S3 mode) targets the s3 service on the filer s3 port + template: charts/seaweedfs/templates/cosi/cosi-deployment.yaml + set: + seaweedfs.fullnameOverride: seaweedfs-system + seaweedfs.cosi.enabled: true + seaweedfs.s3.enabled: false + seaweedfs.filer.s3.enabled: true + seaweedfs.s3.ingress.enabled: false + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: ENDPOINT + value: https://seaweedfs-s3.cozy-seaweedfs.svc:8333 diff --git a/packages/system/seaweedfs/values.yaml b/packages/system/seaweedfs/values.yaml index c22027f63d..974bec6f73 100644 --- a/packages/system/seaweedfs/values.yaml +++ b/packages/system/seaweedfs/values.yaml @@ -11,6 +11,20 @@ global: monitoring: enabled: true seaweedfs: + # Decouple the workload names from the Helm release name. Upstream 4.31 started + # naming resources after the release, and the data-plane release is + # `-system`, so every StatefulSet was renamed to `-system-*`. Those + # names are immutable, so Helm could not rename in place — it stood up a second, + # empty set beside the running one while the data stayed on `data1-seaweedfs-volume-*`. + # Pinning the chart name restores the pre-4.31 names, so the running workloads + # and their volumes are adopted across the bump. extra/seaweedfs does NOT override + # this per release — it relies on this pin. The cases the pin cannot serve (a + # tenant installed fresh on 1.5.x, whose data is on the renamed volumes, and a + # live D-split duplicate) refuse to render in templates/naming-guard.yaml — the + # guard MUST live in this chart, because a platform upgrade re-renders this + # chart directly from its ExternalArtifact without re-rendering extra/seaweedfs. + # The guard also refuses to render at all if this pin is ever lifted. + fullnameOverride: seaweedfs master: # 30 GB per volume × 10 GiB PVC = 0 → rounded to 1 maxVolume per # volume server. With defaultReplication=001 (one replica on a @@ -57,6 +71,14 @@ seaweedfs: extraEnvironmentVars: WEED_LEVELDB2_ENABLED: "false" WEED_POSTGRES2_ENABLED: "true" + # Without explicit pool settings the Go sql pool keeps zero idle + # connections, so every filer metadata lookup opens a fresh PostgreSQL + # connection (TCP + TLS + SCRAM, ~300ms each), adding seconds of latency + # to every S3 operation. Keep max_open * filer.replicas below the + # database max_connections (CNPG default: 100). + WEED_POSTGRES2_CONNECTION_MAX_IDLE: "20" + WEED_POSTGRES2_CONNECTION_MAX_OPEN: "40" + WEED_POSTGRES2_CONNECTION_MAX_LIFETIME_SECONDS: "600" WEED_POSTGRES2_CREATETABLE: | CREATE TABLE IF NOT EXISTS "%s" ( dirhash BIGINT, @@ -191,13 +213,18 @@ seaweedfs: topologyKey: kubernetes.io/hostname cosi: enabled: true + # Override the vendored chart default (v0.1.2), which hardcodes read-write + # S3 actions for every BucketAccess and ignores the accessPolicy parameter. + # v0.3.x honours accessPolicy: readonly, so a -readonly BucketAccessClass + # actually issues read-only (Read,List) credentials instead of read-write. + image: "ghcr.io/seaweedfs/seaweedfs-cosi-driver:v0.3.1" podLabels: policy.cozystack.io/allow-to-apiserver: "true" driverName: "seaweedfs.objectstorage.k8s.io" bucketClassName: "seaweedfs" region: "" sidecar: - image: "ghcr.io/cozystack/cozystack/objectstorage-sidecar:v1.5.0@sha256:fdc09634688d5579a329b48aabf6706cc55bcd9503b0331fa47764706f0d6f62" + image: "ghcr.io/cozystack/cozystack/objectstorage-sidecar:v1.5.4@sha256:598e1f6aea3bd88a847d68b0602833df9cf32a0d4f99ddefbe1e73ff8e3aaafd" certificates: commonName: "SeaweedFS CA" ipAddresses: [] diff --git a/packages/system/velero/Makefile b/packages/system/velero/Makefile index 44eba95114..749efca032 100644 --- a/packages/system/velero/Makefile +++ b/packages/system/velero/Makefile @@ -3,9 +3,15 @@ export NAMESPACE=cozy-$(NAME) include ../../../hack/package.mk +test: + helm unittest . + update: rm -rf charts # Velero helm repo add tanzu https://vmware-tanzu.github.io/helm-charts helm repo update tanzu - helm pull tanzu/velero --untar --untardir charts + helm pull tanzu/velero --untar --untardir charts --version 11.0.0 + # Render a startupProbe from .Values.startupProbe (upstream chart omits it), + # so a slow server start under heavy parallel install does not trip liveness. + patch --no-backup-if-mismatch -p4 < patches/add-startup-probe.patch diff --git a/packages/system/velero/charts/velero/templates/deployment.yaml b/packages/system/velero/charts/velero/templates/deployment.yaml index d2a72639ad..a46d732006 100644 --- a/packages/system/velero/charts/velero/templates/deployment.yaml +++ b/packages/system/velero/charts/velero/templates/deployment.yaml @@ -199,6 +199,9 @@ spec: {{- toYaml . | nindent 12 }} {{- end }} {{- if .Values.metrics.enabled }} + {{- with .Values.startupProbe }} + startupProbe: {{- toYaml . | nindent 12 }} + {{- end }} {{- with .Values.livenessProbe }} livenessProbe: {{- toYaml . | nindent 12 }} {{- end }} diff --git a/packages/system/velero/patches/add-startup-probe.patch b/packages/system/velero/patches/add-startup-probe.patch new file mode 100644 index 0000000000..78c8945a8c --- /dev/null +++ b/packages/system/velero/patches/add-startup-probe.patch @@ -0,0 +1,14 @@ +diff --git a/packages/system/velero/charts/velero/templates/deployment.yaml b/packages/system/velero/charts/velero/templates/deployment.yaml +index d2a72639a..a46d73200 100644 +--- a/packages/system/velero/charts/velero/templates/deployment.yaml ++++ b/packages/system/velero/charts/velero/templates/deployment.yaml +@@ -199,6 +199,9 @@ spec: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- if .Values.metrics.enabled }} ++ {{- with .Values.startupProbe }} ++ startupProbe: {{- toYaml . | nindent 12 }} ++ {{- end }} + {{- with .Values.livenessProbe }} + livenessProbe: {{- toYaml . | nindent 12 }} + {{- end }} diff --git a/packages/system/velero/tests/velero_test.yaml b/packages/system/velero/tests/velero_test.yaml new file mode 100644 index 0000000000..8a03e1acc5 --- /dev/null +++ b/packages/system/velero/tests/velero_test.yaml @@ -0,0 +1,108 @@ +suite: cozy-velero chart rendering invariants +release: + name: velero + namespace: cozy-velero +tests: + - it: renders the velero server Deployment on the pinned upstream image + template: charts/velero/templates/deployment.yaml + documentSelector: + path: metadata.name + value: velero + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: velero/velero:v1.17.0 + + - it: pins the backup-plugin initContainers to velero-1.17-compatible releases + template: charts/velero/templates/deployment.yaml + documentSelector: + path: metadata.name + value: velero + asserts: + - equal: + path: spec.template.spec.initContainers[0].image + value: velero/velero-plugin-for-aws:v1.12.1 + - equal: + path: spec.template.spec.initContainers[1].image + value: quay.io/kubevirt/kubevirt-velero-plugin:v0.8.0 + + # Cozystack disables the upgrade-crds Job (upgradeCRDs: false) because CRDs + # ship via the chart's crds/ directory, making the Job redundant, and because + # on this chart (11.0.0) the Job runs a kubectl image whose tag defaults to the + # chart's Kubernetes version and need not match the cluster. Pin that + # invariant — it silently regressed once. + - it: does not emit the upgrade-crds Job when upgradeCRDs is disabled + template: charts/velero/templates/upgrade-crds/upgrade-crds.yaml + asserts: + - hasDocuments: + count: 0 + + # Velero binds :8085 (http-monitoring) late under a heavy parallel install, so + # the upstream liveness probe (initialDelaySeconds 10, ~130s total budget) would + # kill it mid-startup and crashloop it out of the install-gate window. A + # startupProbe (patched into the upstream chart, which omits one) holds liveness + # and readiness off until the server is up, granting 30 * 10s = 300s of startup + # grace while leaving steady-state liveness at the tight upstream default. Pin + # the startupProbe budget AND that liveness was NOT loosened, so neither + # silently regresses. + - it: gates startup with a startupProbe and keeps liveness tight + template: charts/velero/templates/deployment.yaml + documentSelector: + path: metadata.name + value: velero + asserts: + - equal: + path: spec.template.spec.containers[0].startupProbe.periodSeconds + value: 10 + - equal: + path: spec.template.spec.containers[0].startupProbe.failureThreshold + value: 30 + - equal: + path: spec.template.spec.containers[0].startupProbe.timeoutSeconds + value: 5 + - equal: + path: spec.template.spec.containers[0].startupProbe.successThreshold + value: 1 + - equal: + path: spec.template.spec.containers[0].startupProbe.httpGet.path + value: /metrics + - equal: + path: spec.template.spec.containers[0].startupProbe.httpGet.port + value: http-monitoring + # No initialDelaySeconds: the first startup check fires immediately, so the + # full 300s budget is spent probing. Pin its absence so a future change + # cannot silently re-introduce a delay that eats into the grace window. + - notExists: + path: spec.template.spec.containers[0].startupProbe.initialDelaySeconds + # Liveness stays at the tight upstream default — the startupProbe owns the + # grace, so loosening liveness would only weaken steady-state crash + # detection. Pin the full budget (initialDelay/period/failureThreshold) so a + # future change cannot silently re-delay or loosen liveness. + - equal: + path: spec.template.spec.containers[0].livenessProbe.initialDelaySeconds + value: 10 + - equal: + path: spec.template.spec.containers[0].livenessProbe.periodSeconds + value: 30 + - equal: + path: spec.template.spec.containers[0].livenessProbe.failureThreshold + value: 5 + + # All three probes live behind {{- if .Values.metrics.enabled }} — :8085 only + # exists when metrics are on. Pin that the startupProbe follows the same gate as + # liveness/readiness, so disabling metrics drops all probes together and never + # leaves a startupProbe pointed at an unbound port. + - it: omits all probes (including the startupProbe) when metrics are disabled + template: charts/velero/templates/deployment.yaml + documentSelector: + path: metadata.name + value: velero + set: + velero.metrics.enabled: false + asserts: + - notExists: + path: spec.template.spec.containers[0].startupProbe + - notExists: + path: spec.template.spec.containers[0].livenessProbe + - notExists: + path: spec.template.spec.containers[0].readinessProbe diff --git a/packages/system/velero/values.yaml b/packages/system/velero/values.yaml index 7c94b768c4..9bf2200aaa 100644 --- a/packages/system/velero/values.yaml +++ b/packages/system/velero/values.yaml @@ -30,3 +30,27 @@ velero: # Increase timeout for item operations to 24 hours to prevent timeouts # during backups of very large volumes. The Velero default is 4 hours. defaultItemOperationTimeout: 24h + + # Velero binds its metrics/health endpoint (:8085, http-monitoring) only after + # the server finishes loading plugins and connecting to the API server. Under a + # heavy parallel platform install that can take minutes, but the upstream + # liveness probe starts checking /metrics at initialDelaySeconds=10 and kills + # the container after failureThreshold=5 (~130s total) before :8085 is bound. + # The kubelet SIGTERMs velero (exitCode 0/Completed), it BackOff-restarts, and + # never escapes the loop inside the install-gate window, blocking the gate. + # + # Give the server an explicit startup budget. While the startupProbe is failing + # the kubelet holds off the liveness and readiness probes entirely, so steady- + # state liveness stays at the tight upstream default (10s / 30s / 5 failures) + # once startup completes. failureThreshold * periodSeconds = 30 * 10s = 300s of + # grace. The upstream chart does not render a startupProbe, so the velero + # package patches it in (patches/add-startup-probe.patch, re-applied on update). + startupProbe: + httpGet: + path: /metrics + port: http-monitoring + scheme: HTTP + periodSeconds: 10 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 30