From 568e1cdb25d89b3f6b012d6c06e7171f5462ec8c Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Wed, 22 Jul 2026 00:09:52 +0300 Subject: [PATCH 01/51] fix(cozystack-basics): gate the hostname VAP policies on the VAP API Render the route, gateway and ingress hostname ValidatingAdmissionPolicies only where the admissionregistration.k8s.io/v1/ValidatingAdmissionPolicy API is served, matching the deletion-protection guard, so a cluster without that API does not receive an unrenderable resource. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin (cherry picked from commit 267ea513adeee270e97f8d91d21b6242fc09f8f1) --- .../templates/gateway-hostname-policy.yaml | 3 ++ .../templates/ingress-hostname-policy.yaml | 3 +- .../templates/route-hostname-policy.yaml | 3 ++ .../tests/gateway-hostname-policy_test.yaml | 7 +++++ ...ostname-policies-capability-gate_test.yaml | 30 +++++++++++++++++++ .../tests/ingress-hostname-policy_test.yaml | 7 +++++ .../tests/route-hostname-policy_test.yaml | 7 +++++ 7 files changed, 59 insertions(+), 1 deletion(-) create mode 100644 packages/system/cozystack-basics/tests/hostname-policies-capability-gate_test.yaml diff --git a/packages/system/cozystack-basics/templates/gateway-hostname-policy.yaml b/packages/system/cozystack-basics/templates/gateway-hostname-policy.yaml index 193ac9efab..f048a4fd10 100644 --- a/packages/system/cozystack-basics/templates/gateway-hostname-policy.yaml +++ b/packages/system/cozystack-basics/templates/gateway-hostname-policy.yaml @@ -1,3 +1,5 @@ +{{- /* Render only where the ValidatingAdmissionPolicy API is served (GA since Kubernetes 1.30; the management cluster requires 1.33+). Same guard as packages/core/platform/templates/deletion-protection.yaml. */}} +{{- if .Capabilities.APIVersions.Has "admissionregistration.k8s.io/v1/ValidatingAdmissionPolicy" }} --- apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingAdmissionPolicy @@ -154,3 +156,4 @@ metadata: spec: policyName: cozystack-namespace-host-label-policy validationActions: [Deny] +{{- end }} diff --git a/packages/system/cozystack-basics/templates/ingress-hostname-policy.yaml b/packages/system/cozystack-basics/templates/ingress-hostname-policy.yaml index 828a49115d..aa4b15261f 100644 --- a/packages/system/cozystack-basics/templates/ingress-hostname-policy.yaml +++ b/packages/system/cozystack-basics/templates/ingress-hostname-policy.yaml @@ -57,7 +57,8 @@ labelling yet — in which case the caller should retry. */}} {{- $rootHost := index .Values._cluster "root-host" }} -{{- if $rootHost }} +{{- /* Also gate on the ValidatingAdmissionPolicy API (GA since Kubernetes 1.30; the management cluster requires 1.33+), so a cluster without it does not get an unrenderable resource. Same guard as packages/core/platform/templates/deletion-protection.yaml. */}} +{{- if and $rootHost (.Capabilities.APIVersions.Has "admissionregistration.k8s.io/v1/ValidatingAdmissionPolicy") }} {{- $A := `namespaceObject.metadata.labels["namespace.cozystack.io/host"]` }} {{- $celValidator := printf `(namespaceObject == null || !has(namespaceObject.metadata.labels) || !("namespace.cozystack.io/host" in namespaceObject.metadata.labels)) ? false : (!has(object.spec.defaultBackend) && (!has(object.spec.rules) || object.spec.rules.all(r, has(r.host) && r.host != "" && (r.host == %s || r.host.endsWith("." + %s) || (!r.host.startsWith("*.") && !(r.host == %q || r.host.endsWith(%q)))))) && (!has(object.spec.tls) || object.spec.tls.all(t, !has(t.hosts) || t.hosts.all(h, h != "" && (h == %s || h.endsWith("." + %s) || (!h.startsWith("*.") && !(h == %q || h.endsWith(%q))))))))` $A $A $rootHost (printf ".%s" $rootHost) $A $A $rootHost (printf ".%s" $rootHost) -}} --- diff --git a/packages/system/cozystack-basics/templates/route-hostname-policy.yaml b/packages/system/cozystack-basics/templates/route-hostname-policy.yaml index cde87d2494..ec39b80942 100644 --- a/packages/system/cozystack-basics/templates/route-hostname-policy.yaml +++ b/packages/system/cozystack-basics/templates/route-hostname-policy.yaml @@ -36,6 +36,8 @@ finished labelling yet — in which case the caller should retry. */}} {{- $celValidator := `(namespaceObject == null || !has(namespaceObject.metadata.labels) || !("namespace.cozystack.io/host" in namespaceObject.metadata.labels)) ? false : (!has(object.spec.hostnames) || object.spec.hostnames.all(h, h == namespaceObject.metadata.labels["namespace.cozystack.io/host"] || h.endsWith("." + namespaceObject.metadata.labels["namespace.cozystack.io/host"])))` -}} +{{- /* Render only where the ValidatingAdmissionPolicy API is served (GA since Kubernetes 1.30; the management cluster requires 1.33+). Same guard as packages/core/platform/templates/deletion-protection.yaml. */}} +{{- if .Capabilities.APIVersions.Has "admissionregistration.k8s.io/v1/ValidatingAdmissionPolicy" }} --- apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingAdmissionPolicy @@ -104,3 +106,4 @@ metadata: spec: policyName: cozystack-route-hostname-policy-tls validationActions: [Deny] +{{- end }} diff --git a/packages/system/cozystack-basics/tests/gateway-hostname-policy_test.yaml b/packages/system/cozystack-basics/tests/gateway-hostname-policy_test.yaml index 50940dee43..80749c4813 100644 --- a/packages/system/cozystack-basics/tests/gateway-hostname-policy_test.yaml +++ b/packages/system/cozystack-basics/tests/gateway-hostname-policy_test.yaml @@ -19,6 +19,13 @@ release: # Both gates are dropped; inheritance now flows via the label # selector on Gateway.spec.listeners[].allowedRoutes. +# The template now gates on the ValidatingAdmissionPolicy API (GA since +# Kubernetes 1.30); helm-unittest's default capability set omits it, so every +# rendering test declares it, exactly as the platform deletion-protection suite. +capabilities: + apiVersions: + - admissionregistration.k8s.io/v1/ValidatingAdmissionPolicy + tests: - it: renders 3 VAPs + 3 Bindings (6 documents total) asserts: diff --git a/packages/system/cozystack-basics/tests/hostname-policies-capability-gate_test.yaml b/packages/system/cozystack-basics/tests/hostname-policies-capability-gate_test.yaml new file mode 100644 index 0000000000..0c119c63d1 --- /dev/null +++ b/packages/system/cozystack-basics/tests/hostname-policies-capability-gate_test.yaml @@ -0,0 +1,30 @@ +suite: cozystack-basics hostname VAP policies are gated on the ValidatingAdmissionPolicy API +templates: + - templates/route-hostname-policy.yaml + - templates/gateway-hostname-policy.yaml + - templates/ingress-hostname-policy.yaml +release: + name: cozystack-basics + namespace: cozy-system +# root-host is set so the ingress policy's own value gate passes; with it set, +# the only reason any of these three templates render nothing is the missing +# ValidatingAdmissionPolicy API declared below. helm-unittest's per-test +# capabilities merge rather than replace the suite's, so the absent case is a +# separate suite pinned at suite level (mirrors the platform notes gate suite). +set: + _cluster: + root-host: example.org +capabilities: + apiVersions: [] +tests: + - it: none of the hostname VAP policies render when the API is unavailable + asserts: + - hasDocuments: + count: 0 + template: templates/route-hostname-policy.yaml + - hasDocuments: + count: 0 + template: templates/gateway-hostname-policy.yaml + - hasDocuments: + count: 0 + template: templates/ingress-hostname-policy.yaml diff --git a/packages/system/cozystack-basics/tests/ingress-hostname-policy_test.yaml b/packages/system/cozystack-basics/tests/ingress-hostname-policy_test.yaml index 80afe64834..407407095a 100644 --- a/packages/system/cozystack-basics/tests/ingress-hostname-policy_test.yaml +++ b/packages/system/cozystack-basics/tests/ingress-hostname-policy_test.yaml @@ -13,6 +13,13 @@ set: _cluster: root-host: example.org +# The template now also gates on the ValidatingAdmissionPolicy API (GA since +# Kubernetes 1.30); helm-unittest's default capability set omits it, so every +# rendering test declares it, exactly as the platform deletion-protection suite. +capabilities: + apiVersions: + - admissionregistration.k8s.io/v1/ValidatingAdmissionPolicy + tests: - it: renders ValidatingAdmissionPolicy + Binding for legacy Ingress # Legacy Ingress (networking.k8s.io/v1) is the default ingress path diff --git a/packages/system/cozystack-basics/tests/route-hostname-policy_test.yaml b/packages/system/cozystack-basics/tests/route-hostname-policy_test.yaml index 36f87c7a47..16ef82a67a 100644 --- a/packages/system/cozystack-basics/tests/route-hostname-policy_test.yaml +++ b/packages/system/cozystack-basics/tests/route-hostname-policy_test.yaml @@ -6,6 +6,13 @@ release: name: cozystack-basics namespace: cozy-system +# The template now gates on the ValidatingAdmissionPolicy API (GA since +# Kubernetes 1.30); helm-unittest's default capability set omits it, so every +# rendering test declares it, exactly as the platform deletion-protection suite. +capabilities: + apiVersions: + - admissionregistration.k8s.io/v1/ValidatingAdmissionPolicy + tests: - it: renders ValidatingAdmissionPolicy + Binding for HTTPRoute and TLSRoute (Layer 7) # Layer 7 (route-hostname VAP) is the only VAP rendered by THIS From 068669038176ee555b148e601e3bfd15e3baf766 Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Wed, 22 Jul 2026 22:31:00 +0500 Subject: [PATCH 02/51] fix(release): make promote-retag digest verification media-type-agnostic skopeo inspect --format '{{.Digest}}' fails on OCI artifacts: for cozystack-packages (config.mediaType application/vnd.cncf.flux.config.v1+json) it prints nothing and exits nonzero, so the v1.6.0 finalize aborted its post-copy verification right after both copies had already succeeded, leaving 32 of 43 repos without stable tags and skipping the cozy-installer publish. --dry-run never executes this branch, so rehearsals could not catch it. Compute the digest as sha256 of the raw manifest instead, which is how registries define it and works for any media type. The helper gates the hash on skopeo's exit status via a temp file rather than a pipeline: a missing tag still yields an empty string (pre-check reads that as "tag absent, proceed to copy"), never the empty-input hash sha256:e3b0c442..., and never an abort under set -eu. Verified against live GHCR: the raw-manifest hash of cozystack-packages:v1.6.0 matches the digest finalize expected (bf68208730860fa8...), and agrees byte-for-byte with the old method on regular multi-arch images. New cozytest cases cover the OCI-artifact digest, a post-copy mismatch, and the missing-tag path. Assisted-By: GPT-5 Assisted-By: Claude Signed-off-by: Myasnikov Daniil (cherry picked from commit 5d8dd51cabaa2f523fc483634e78bdcaeb369298) --- hack/promote-retag.sh | 29 +++++++--- hack/promote-retag_test.bats | 108 +++++++++++++++++++++++++++++++++++ 2 files changed, 129 insertions(+), 8 deletions(-) diff --git a/hack/promote-retag.sh b/hack/promote-retag.sh index 287bbf20a5..6c19a61d14 100755 --- a/hack/promote-retag.sh +++ b/hack/promote-retag.sh @@ -24,7 +24,7 @@ # files that covers), which is expected to be the promoted stable # digest-vendored tree (the release-X.Y.Z branch, whose digests are the rc's — # only the cosmetic tag string differs). -# Requires: yq (mikefarah), skopeo, and a registry login already done. +# Requires: yq (mikefarah), skopeo, sha256sum, and a registry login already done. # # The repo/tag split (ref_repo below) strips the :tag from the last path # component only, so registry hosts that carry a :port are preserved. @@ -49,6 +49,7 @@ command -v yq >/dev/null || { echo "yq (mikefarah) is required" >&2; exit 1; yq --version 2>&1 | grep -q mikefarah || { echo "yq (mikefarah) is required" >&2; exit 1; } # skopeo is only needed to actually copy; a --dry-run just prints the plan. [ "$DRY_RUN" -eq 1 ] || command -v skopeo >/dev/null || { echo "skopeo is required" >&2; exit 1; } +[ "$DRY_RUN" -eq 1 ] || command -v sha256sum >/dev/null || { echo "sha256sum is required" >&2; exit 1; } # Ref collection (which files are scanned, and the YAML shapes within them) is # shared with hack/nightly-mirror.sh and hack/promote-rewrite-tags.sh — see @@ -81,6 +82,17 @@ ref_repo() { } ref_digest() { printf '%s' "${1##*@}"; } # sha256:... +manifest_digest() { + _ref="$1" + _manifest="$(mktemp)" + # Hash the raw bytes so this works for images and OCI artifacts alike. A + # file preserves inspect's status without relying on non-POSIX pipefail. + if skopeo inspect --raw "docker://$_ref" >"$_manifest" 2>/dev/null; then + printf 'sha256:%s' "$(sha256sum "$_manifest" | cut -d' ' -f1)" + fi + rm -f "$_manifest" +} + copy() { _src="$1"; _dst="$2" if [ "$DRY_RUN" -eq 1 ]; then @@ -128,13 +140,14 @@ echo "$refs" | while IFS= read -r ref; do repo="${ref%@*}" digest="${ref##*@}" echo "▸ ${repo} ${digest}" - # The stable tag is write-once at the image level: inspect the destination - # before copying. No-op if it already points at this rc digest (idempotent - # re-run), fail if it points elsewhere (a partial run or manual push already - # put different bytes there) rather than mutate released bytes. :latest is - # intentionally mutable and (re)pointed below only when MOVE_LATEST=1. + # The stable tag is write-once at the image level: resolve the destination's + # raw manifest digest before copying. No-op if it already points at this rc + # digest (idempotent re-run), fail if it points elsewhere (a partial run or + # manual push already put different bytes there) rather than mutate released + # bytes. :latest is intentionally mutable and (re)pointed below only when + # MOVE_LATEST=1. if [ "$DRY_RUN" -eq 0 ]; then - cur="$(skopeo inspect --format '{{.Digest}}' "docker://${repo}:${STABLE}" 2>/dev/null || echo '')" + cur="$(manifest_digest "${repo}:${STABLE}")" if [ -n "$cur" ] && [ "$cur" != "$digest" ]; then echo "::error::${repo}:${STABLE} already exists at '${cur}'; refusing to move it to '${digest}' (stable image tags are write-once)" >&2 exit 1 @@ -150,7 +163,7 @@ echo "$refs" | while IFS= read -r ref; do [ "$MOVE_LATEST" = "1" ] && copy "$ref" "${repo}:latest" # Verify the stable tag now resolves to the exact rc digest (skip in dry-run). if [ "$DRY_RUN" -eq 0 ]; then - got="$(skopeo inspect --format '{{.Digest}}' "docker://${repo}:${STABLE}" 2>/dev/null || echo '')" + got="$(manifest_digest "${repo}:${STABLE}")" if [ "$got" != "$digest" ]; then echo "::error::${repo}:${STABLE} resolved to '${got}', expected '${digest}'" >&2 exit 1 diff --git a/hack/promote-retag_test.bats b/hack/promote-retag_test.bats index b9901755ab..2e0e6c8494 100644 --- a/hack/promote-retag_test.bats +++ b/hack/promote-retag_test.bats @@ -21,6 +21,41 @@ # (or `bats hack/promote-retag_test.bats` if the bats binary is # installed; cozytest.sh is the CI path.) +_make_registry_mocks() { + t="$1" + mkdir -p "$t/bin" + cat >"$t/bin/yq" <<'EOF' +#!/bin/sh +set -eu +if [ "${1:-}" = "--version" ]; then + echo 'yq (https://github.com/mikefarah/yq/) version v4.45.1' +else + printf '%s\n' "$MOCK_REF" +fi +EOF + cat >"$t/bin/skopeo" <<'EOF' +#!/bin/sh +set -eu +printf '%s\n' "$*" >>"$MOCK_SKOPEO_LOG" +case "$1" in + inspect) + [ "$2" = "--raw" ] + if [ "$MOCK_MISSING_ONCE" = "1" ] && [ ! -f "$MOCK_STATE" ]; then + exit 1 + fi + printf '%s' "$MOCK_MANIFEST" + ;; + copy) + : >"$MOCK_STATE" + ;; + *) + exit 2 + ;; +esac +EOF + chmod +x "$t/bin/yq" "$t/bin/skopeo" +} + @test "dry-run over the real tree retags only cozystack-owned refs" { tmp=$(mktemp -d) trap 'rm -rf "$tmp"' EXIT @@ -160,3 +195,76 @@ grep -q 'cozystack/grafana:v9.9.9' "$tmp/out" grep -q 'cozystack/multus-cni:v9.9.9' "$tmp/out" } + +@test "raw manifest digest resolves for an OCI artifact" { + tmp=$(mktemp -d) + trap 'rm -rf "$tmp"' EXIT + _make_registry_mocks "$tmp" + + manifest='{"schemaVersion":2,"config":{"mediaType":"application/vnd.cncf.flux.config.v1+json","digest":"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"},"layers":[]}' + digest="sha256:$(printf '%s' "$manifest" | sha256sum | cut -d' ' -f1)" + ref="example.com/cozystack/cozystack-packages:v1.6.0-rc.1@${digest}" + + rc=0 + REGISTRY="example.com/cozystack" MOCK_REF="$ref" MOCK_MANIFEST="$manifest" \ + MOCK_MISSING_ONCE=0 MOCK_STATE="$tmp/state" MOCK_SKOPEO_LOG="$tmp/skopeo.log" \ + PATH="$tmp/bin:/usr/bin:/bin" hack/promote-retag.sh v1.6.0 \ + >"$tmp/out" 2>"$tmp/err" || rc=$? + if [ "$rc" -ne 0 ]; then + echo "promote-retag.sh exited $rc" >&2 + echo "--- stderr ---" >&2; cat "$tmp/err" >&2 + return "$rc" + fi + + grep -q "already at ${digest}; skipping stable copy" "$tmp/out" + [ "$(grep -c '^inspect --raw ' "$tmp/skopeo.log")" -eq 2 ] + ! grep -q '^copy ' "$tmp/skopeo.log" +} + +@test "post-copy raw manifest digest mismatch fails verification" { + tmp=$(mktemp -d) + trap 'rm -rf "$tmp"' EXIT + _make_registry_mocks "$tmp" + + manifest='{"schemaVersion":2,"config":{"mediaType":"application/vnd.cncf.flux.config.v1+json","digest":"sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"},"layers":[]}' + expected_manifest='{"schemaVersion":2,"config":{"mediaType":"application/vnd.cncf.flux.config.v1+json","digest":"sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"},"layers":[]}' + actual="sha256:$(printf '%s' "$manifest" | sha256sum | cut -d' ' -f1)" + expected="sha256:$(printf '%s' "$expected_manifest" | sha256sum | cut -d' ' -f1)" + ref="example.com/cozystack/cozystack-packages:v1.6.0-rc.1@${expected}" + + rc=0 + REGISTRY="example.com/cozystack" MOCK_REF="$ref" MOCK_MANIFEST="$manifest" \ + MOCK_MISSING_ONCE=1 MOCK_STATE="$tmp/state" MOCK_SKOPEO_LOG="$tmp/skopeo.log" \ + PATH="$tmp/bin:/usr/bin:/bin" hack/promote-retag.sh v1.6.0 \ + >"$tmp/out" 2>"$tmp/err" || rc=$? + + [ "$rc" -ne 0 ] + grep -q "resolved to '${actual}', expected '${expected}'" "$tmp/err" + grep -q '^copy --multi-arch all ' "$tmp/skopeo.log" + [ "$(grep -c '^inspect --raw ' "$tmp/skopeo.log")" -eq 2 ] +} + +@test "missing stable tag remains an empty digest and proceeds to copy" { + tmp=$(mktemp -d) + trap 'rm -rf "$tmp"' EXIT + _make_registry_mocks "$tmp" + + manifest='{"schemaVersion":2,"config":{"mediaType":"application/vnd.cncf.flux.config.v1+json","digest":"sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"},"layers":[]}' + digest="sha256:$(printf '%s' "$manifest" | sha256sum | cut -d' ' -f1)" + ref="example.com/cozystack/cozystack-packages:v1.6.0-rc.1@${digest}" + + rc=0 + REGISTRY="example.com/cozystack" MOCK_REF="$ref" MOCK_MANIFEST="$manifest" \ + MOCK_MISSING_ONCE=1 MOCK_STATE="$tmp/state" MOCK_SKOPEO_LOG="$tmp/skopeo.log" \ + PATH="$tmp/bin:/usr/bin:/bin" hack/promote-retag.sh v1.6.0 \ + >"$tmp/out" 2>"$tmp/err" || rc=$? + if [ "$rc" -ne 0 ]; then + echo "promote-retag.sh exited $rc" >&2 + echo "--- stderr ---" >&2; cat "$tmp/err" >&2 + return "$rc" + fi + + grep -q '^copy --multi-arch all ' "$tmp/skopeo.log" + [ "$(grep -c '^inspect --raw ' "$tmp/skopeo.log")" -eq 2 ] + grep -q 'Retagged image refs to v1.6.0' "$tmp/out" +} From ba87460aae7d8d65ed56c5b5795533634485b7b7 Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Mon, 27 Jul 2026 13:26:15 +0500 Subject: [PATCH 03/51] fix(release): prove a stable tag is absent before copying over it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review notes from #3435, and a correction to my first attempt at them. manifest_digest gated on skopeo`s exit status alone, so a zero exit with no bytes hashed the empty string to sha256:e3b0c442... — a digest that looks real and belongs to no manifest. The first fix simply required non-empty bytes, which was worse than it looked: the caller turns an empty result into "not published, safe to copy", so a registry answering 200 with an empty body went from an accidental refusal (the bogus digest mismatched, so the write-once check refused) to an actual copy over whatever the tag really held. Verified against real skopeo 1.23: a 200 with Content-Length 0 does exit 0 with no stdout. The helper is now explicitly three-state. A digest, or empty output for a tag the registry PROVED absent, or non-zero for anything indeterminate — which under set -e aborts the promotion before it writes. Absence has to be reported as "manifest unknown"; 429, 5xx, auth and network failures all exit non-zero with no bytes too, and reading those as "unpublished" is the same fail-open one level down. finalize retags ~42 refs with no retry wrapper, so a single rate-limit was enough to reach that path. skopeo`s stderr is captured instead of discarded, so the diagnostics now name the real failure. Tests: the write-once refusal branch the review found untested (an existing stable tag at a different digest must be refused before any write), the empty- body case (refuses, writes nothing, and the empty-input hash never appears in the diagnostics), and a 429 that must not read as an absent tag. The mock now emits realistic registry errors, since a bare non-zero exit is no longer proof of absence. Both new guards are mutation-checked: accepting any non-zero exit as absence fails the 429 test, dropping the empty-bytes check fails the empty-body test. 11/11 tests green, hack/image-pin-consistency.bats still 2/2, sh -n clean. Assisted-By: Claude Signed-off-by: Myasnikov Daniil (cherry picked from commit ae394757891831132bcc791ac858ad76be93cdcc) --- hack/promote-retag.sh | 40 ++++++++++++++-- hack/promote-retag_test.bats | 93 ++++++++++++++++++++++++++++++++++++ 2 files changed, 128 insertions(+), 5 deletions(-) diff --git a/hack/promote-retag.sh b/hack/promote-retag.sh index 6c19a61d14..cbd57a323f 100755 --- a/hack/promote-retag.sh +++ b/hack/promote-retag.sh @@ -82,15 +82,45 @@ ref_repo() { } ref_digest() { printf '%s' "${1##*@}"; } # sha256:... +# manifest_digest — the digest of 's raw manifest, or empty output if +# the tag is PROVEN not to exist. Anything else is indeterminate and returns +# non-zero, which under `set -e` aborts the promotion before it writes. +# +# The three states matter because the caller turns "empty" into "not published, +# safe to copy over". Inferring that from a failure — or from a successful +# response with no bytes — is how a rate-limit or a proxy hiccup gets read as +# permission to overwrite released bytes. Absence has to be proven by the +# registry saying so. manifest_digest() { _ref="$1" _manifest="$(mktemp)" - # Hash the raw bytes so this works for images and OCI artifacts alike. A - # file preserves inspect's status without relying on non-POSIX pipefail. - if skopeo inspect --raw "docker://$_ref" >"$_manifest" 2>/dev/null; then - printf 'sha256:%s' "$(sha256sum "$_manifest" | cut -d' ' -f1)" + _mderr="$(mktemp)" + # Hash the raw bytes so this works for images and OCI artifacts alike. Files + # preserve inspect's status without relying on non-POSIX pipefail. + if skopeo inspect --raw "docker://$_ref" >"$_manifest" 2>"$_mderr"; then + if [ -s "$_manifest" ]; then + printf 'sha256:%s' "$(sha256sum "$_manifest" | cut -d' ' -f1)" + rm -f "$_manifest" "$_mderr" + return 0 + fi + # Exit 0 with no bytes — a registry answering 200 with an empty body. Not a + # digest (hashing nothing yields sha256:e3b0c442…, which looks real and + # belongs to no manifest) and not proof of absence either, so refuse to + # decide rather than let the caller copy over whatever is really there. + echo "::error::skopeo inspect of ${_ref} succeeded but returned no manifest bytes; refusing to decide whether that tag exists" >&2 + rm -f "$_manifest" "$_mderr" + return 1 + fi + # Non-zero. Only a registry that says the manifest is unknown proves absence; + # 429/5xx/auth/network failures all also exit non-zero with no bytes, and + # reading those as "not published" is exactly the fail-open this guards. + if grep -qiE 'manifest unknown|manifest not known|manifest for [^ ]* not found|not found|404' "$_mderr"; then + rm -f "$_manifest" "$_mderr" + return 0 fi - rm -f "$_manifest" + echo "::error::skopeo inspect of ${_ref} failed and did not report the manifest as unknown, so it cannot be treated as unpublished: $(tr '\n' ' ' <"$_mderr")" >&2 + rm -f "$_manifest" "$_mderr" + return 1 } copy() { diff --git a/hack/promote-retag_test.bats b/hack/promote-retag_test.bats index 2e0e6c8494..c2ac31113d 100644 --- a/hack/promote-retag_test.bats +++ b/hack/promote-retag_test.bats @@ -40,9 +40,22 @@ printf '%s\n' "$*" >>"$MOCK_SKOPEO_LOG" case "$1" in inspect) [ "$2" = "--raw" ] + # An indeterminate failure: non-zero, no bytes, and the registry never says + # the manifest is unknown. Must NOT read as "tag absent". + if [ "${MOCK_TRANSIENT_ERROR:-0}" = "1" ]; then + echo "Error: reading manifest from docker://${3:-?}: received unexpected HTTP status: 429 Too Many Requests" >&2 + exit 1 + fi if [ "$MOCK_MISSING_ONCE" = "1" ] && [ ! -f "$MOCK_STATE" ]; then + # Absence as a registry actually reports it — the script requires proof, + # not merely a non-zero exit. + echo "Error: reading manifest from docker://${3:-?}: manifest unknown" >&2 exit 1 fi + # A registry answering 200 with an empty body: exit 0, no bytes. + if [ "${MOCK_EMPTY_MANIFEST:-0}" = "1" ]; then + exit 0 + fi printf '%s' "$MOCK_MANIFEST" ;; copy) @@ -268,3 +281,83 @@ EOF [ "$(grep -c '^inspect --raw ' "$tmp/skopeo.log")" -eq 2 ] grep -q 'Retagged image refs to v1.6.0' "$tmp/out" } + +@test "existing stable tag at a different digest is refused, not moved" { + tmp=$(mktemp -d) + trap 'rm -rf "$tmp"' EXIT + _make_registry_mocks "$tmp" + + # The stable tag already exists (MOCK_MISSING_ONCE=0) but resolves to a + # manifest other than the rc's: released bytes must never be overwritten. + published='{"schemaVersion":2,"config":{"mediaType":"application/vnd.oci.image.config.v1+json","digest":"sha256:1111111111111111111111111111111111111111111111111111111111111111"},"layers":[]}' + rc_manifest='{"schemaVersion":2,"config":{"mediaType":"application/vnd.oci.image.config.v1+json","digest":"sha256:2222222222222222222222222222222222222222222222222222222222222222"},"layers":[]}' + published_digest="sha256:$(printf '%s' "$published" | sha256sum | cut -d' ' -f1)" + rc_digest="sha256:$(printf '%s' "$rc_manifest" | sha256sum | cut -d' ' -f1)" + ref="example.com/cozystack/cozystack-packages:v1.6.0-rc.1@${rc_digest}" + + rc=0 + REGISTRY="example.com/cozystack" MOCK_REF="$ref" MOCK_MANIFEST="$published" \ + MOCK_MISSING_ONCE=0 MOCK_STATE="$tmp/state" MOCK_SKOPEO_LOG="$tmp/skopeo.log" \ + PATH="$tmp/bin:/usr/bin:/bin" hack/promote-retag.sh v1.6.0 \ + >"$tmp/out" 2>"$tmp/err" || rc=$? + + [ "$rc" -ne 0 ] + grep -q "already exists at '${published_digest}'; refusing to move it to '${rc_digest}'" "$tmp/err" + # The refusal must happen BEFORE any write. + ! grep -q '^copy ' "$tmp/skopeo.log" + [ "$(grep -c '^inspect --raw ' "$tmp/skopeo.log")" -eq 1 ] +} + +@test "a zero-exit empty manifest refuses to decide and never writes" { + tmp=$(mktemp -d) + trap 'rm -rf "$tmp"' EXIT + _make_registry_mocks "$tmp" + + manifest='{"schemaVersion":2,"config":{"mediaType":"application/vnd.oci.image.config.v1+json","digest":"sha256:3333333333333333333333333333333333333333333333333333333333333333"},"layers":[]}' + digest="sha256:$(printf '%s' "$manifest" | sha256sum | cut -d' ' -f1)" + ref="example.com/cozystack/cozystack-packages:v1.6.0-rc.1@${digest}" + + rc=0 + REGISTRY="example.com/cozystack" MOCK_REF="$ref" MOCK_MANIFEST="$manifest" \ + MOCK_EMPTY_MANIFEST=1 MOCK_MISSING_ONCE=0 MOCK_STATE="$tmp/state" \ + MOCK_SKOPEO_LOG="$tmp/skopeo.log" \ + PATH="$tmp/bin:/usr/bin:/bin" hack/promote-retag.sh v1.6.0 \ + >"$tmp/out" 2>"$tmp/err" || rc=$? + + # A 200 with an empty body proves nothing: the tag may hold released bytes this + # promotion must not overwrite. So the script must abort BEFORE any copy — + # reading "no bytes" as "not published" is what would turn a proxy hiccup into + # an overwrite of a published stable tag. + [ "$rc" -ne 0 ] + grep -q 'returned no manifest bytes' "$tmp/err" + ! grep -q '^copy ' "$tmp/skopeo.log" + [ "$(grep -c '^inspect --raw ' "$tmp/skopeo.log")" -eq 1 ] + # The empty-input hash must never appear: that is the guard being absent. + ! grep -q 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855' "$tmp/err" +} + +@test "a transient registry failure is not read as an unpublished tag" { + tmp=$(mktemp -d) + trap 'rm -rf "$tmp"' EXIT + _make_registry_mocks "$tmp" + + manifest='{"schemaVersion":2,"config":{"mediaType":"application/vnd.oci.image.config.v1+json","digest":"sha256:4444444444444444444444444444444444444444444444444444444444444444"},"layers":[]}' + digest="sha256:$(printf '%s' "$manifest" | sha256sum | cut -d' ' -f1)" + ref="example.com/cozystack/cozystack-packages:v1.6.0-rc.1@${digest}" + + # 429 during finalize: non-zero exit, no bytes, no "manifest unknown". The old + # shape of this helper made that indistinguishable from an absent tag and + # proceeded to copy over it; finalize retags ~42 refs with no retry wrapper, so + # a single rate-limit was enough to reach that path. + rc=0 + REGISTRY="example.com/cozystack" MOCK_REF="$ref" MOCK_MANIFEST="$manifest" \ + MOCK_TRANSIENT_ERROR=1 MOCK_MISSING_ONCE=0 MOCK_STATE="$tmp/state" \ + MOCK_SKOPEO_LOG="$tmp/skopeo.log" \ + PATH="$tmp/bin:/usr/bin:/bin" hack/promote-retag.sh v1.6.0 \ + >"$tmp/out" 2>"$tmp/err" || rc=$? + + [ "$rc" -ne 0 ] + grep -q 'cannot be treated as unpublished' "$tmp/err" + grep -q '429' "$tmp/err" + ! grep -q '^copy ' "$tmp/skopeo.log" +} From 4eb49486752c347048663cc0f1fbc94b002b984e Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Wed, 22 Jul 2026 23:25:01 +0500 Subject: [PATCH 04/51] fix(seaweedfs): make naming audit fail closed on kubectl and payload errors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit hack/seaweedfs-naming-audit.sh was fail-open: any kubectl failure produced an empty table indistinguishable from an honestly clean fleet (namespace LIST failure = zero namespaces walked, secret/pvc/sts LIST failures = zero findings, all under 2>/dev/null with no error handling). The runbook uses this output as the gate before PVC deletion, as post-deletion verification, and as an upgrade precondition, so a transient API error could green-light destroying data (#3431). Route every kubectl call through a run_kubectl helper: non-zero exit prints a FATAL line naming the failed query and propagates the code up the whole chain (enumerations restructured to capture-then-check, since an exit inside $(...) dies with the subshell). By-name GETs distinguish legitimate absence from real errors via --ignore-not-found on a separate existence check; a Secret that exists but has no decodable release payload, or one that decodes without a chart name, is corrupt state and fails loudly instead of silently dropping the tenant. Incomplete PV-age evidence now marks the generation incomplete and falls to the safe no-direction branch, so a failed GET can no longer flip OVERLAP into a wrong Step-3 deletion candidate. A successful run with zero findings still prints the same bytes as before. Tests: 14 new cozytest cases — failure injection for every LIST and by-name GET site, corrupt-payload variants, and full-output golden diffs for the success paths (verified byte-identical to the pre-change script). 25/25 green. Fixes #3431 Assisted-By: Claude Assisted-By: GPT-5 Signed-off-by: Myasnikov Daniil (cherry picked from commit 3dc6cbd4f45aaafea795daf10b7de8162b39ee04) --- hack/seaweedfs-naming-audit.bats | 369 +++++++++++++++++++++++++++++++ hack/seaweedfs-naming-audit.sh | 204 ++++++++++++++--- 2 files changed, 540 insertions(+), 33 deletions(-) diff --git a/hack/seaweedfs-naming-audit.bats b/hack/seaweedfs-naming-audit.bats index eb6356e025..424de20a35 100644 --- a/hack/seaweedfs-naming-audit.bats +++ b/hack/seaweedfs-naming-audit.bats @@ -138,3 +138,372 @@ EOF done < "$chart/rendered" rm -rf "$chart" } + +# ----------------------------------------------------------------------------- +# Fail-closed tests (issue #3431). +# +# The classification tests above mock nothing below the classifier. These drive +# the KUBECTL layer, where the fail-open bug lived: a kubectl call that failed +# used to return empty stdout, byte-identical to a genuinely clean fleet, so the +# audit printed an empty table and exited 0 -- the exact false "nothing to do" the +# operator is told to trust before deleting PVCs. They shim `kubectl` on PATH with +# a fake and assert the audit fails LOUDLY (non-zero exit + a FATAL naming the +# query) on any real error, while still treating a genuinely-absent object as +# clean. Two families: +# +# * enumerating LISTs (get ns / secret / pvc / sts) -- a failure is always fatal; +# * by-NAME GETs (a release secret in system_releases, a PVC/PV in pv_epoch) -- +# `--ignore-not-found` splits a real error (fatal) from a legitimate absence: +# an absent release revision is a clean skip; an absent PV degrades to the +# safe "cannot establish direction" note and must NOT become a deletion +# candidate (the range-narrowing flip Codex reproduced). +# +# The two golden tests guard the reverse: on a healthy cluster the output stays +# byte-for-byte what it was on origin/main, including the MIXED path that exercises +# every pv_epoch GET. +# +# The fake is a real executable on PATH, so it exercises the actual exit-status +# handling in run_kubectl and the propagation up through every caller -- a for +# loop over `$(...)` swallows the status, so this is where a regression would hide. + +# _release_blob [chart-name] -- the value kubectl returns for a Helm release +# secret's `.data.release`: base64(base64(gzip(json))). release_json base64-decodes +# twice and gunzips it, and system_releases reads the chart name from +# chart.metadata.name (as real Helm payloads carry it). Defaults to a cozy-seaweedfs +# release so the audit confirms the tenant; pass another chart name for a non- +# SeaweedFS release, or the literal EMPTY for a chartless {} payload. +_release_blob() { + _rb_name=${1:-cozy-seaweedfs} + if [ "$_rb_name" = EMPTY ]; then + _rb_json='{}' + else + _rb_json='{"name":"seaweedfs-system","chart":{"metadata":{"name":"'"$_rb_name"'"}}}' + fi + printf '%s' "$_rb_json" | gzip | base64 | base64 | tr -d '\n' +} + +# _write_fake_kubectl -- drop a fake kubectl +# into . +# one of ns | secretlist | secretget | secretget_absent | +# secret_missing_field | secret_bad_payload | pvclist | pvcget | +# sts | pvget | "" -- the single call to make behave badly. Real- +# error targets exit non-zero with a stderr message; secretget_absent +# mirrors an absent Secret (--ignore-not-found: exit 0, empty -o +# name); secret_missing_field / secret_bad_payload keep the Secret +# present but return an empty / undecodable .data.release payload. +# a PV name (pv-legacy|pv-legacy2|pv-renamed) to report as absent +# (exit 0, empty), exercising the "bound claim, PV gone" path. +# newline-separated `get pvc -o name` LIST output. +# It otherwise walks one confirmed seaweedfs-system tenant. The by-name PVC->PV and +# PV->timestamp maps are fixed here; timestamps are chosen so the two legacy PVs +# straddle the single renamed PV (true answer OVERLAP), which is what makes the +# range-narrowing flip observable. Kept POSIX and free of a column-0 `}` so +# cozytest.sh's awk converter passes the heredoc through untouched. +_write_fake_kubectl() { + # Grouped redirect (one open of the file). The closing brace is indented, so + # cozytest.sh's awk -- which only rewrites a `}` in column 0 -- leaves it and the + # heredoc alone. + { + printf '#!/bin/sh\n' + printf "FAIL='%s'\n" "$2" + printf "ABSENT_PV='%s'\n" "$3" + printf "BLOB='%s'\n" "$4" + printf "PVCS='%s'\n" "$5" + cat <<'FAKE' +verb=${1:-}; res=${2:-}; args="$*" +fail() { echo "fake kubectl: $1 (real error)" >&2; exit 1; } +if [ "$verb $res" = "get ns" ]; then + [ "$FAIL" = ns ] && fail "get ns" + printf 'namespace/tenant-test\n'; exit 0 +fi +if [ "$verb $res" = "get secret" ]; then + case "$args" in + *sh.helm.release.v1*) + # release_json now asks two questions: existence (-o name) then payload + # (jsonpath .data.release). Mirror that split so absence, real error, and + # corrupt-payload are all reachable independently. + case "$args" in + *"-o name"*) + [ "$FAIL" = secretget ] && fail "release secret existence GET" + [ "$FAIL" = secretget_absent ] && exit 0 + printf 'secret/sh.helm.release.v1.seaweedfs-system.v1\n'; exit 0 ;; + *) + [ "$FAIL" = secret_missing_field ] && exit 0 + if [ "$FAIL" = secret_bad_payload ]; then printf '@@@not-base64@@@\n'; exit 0; fi + printf '%s\n' "$BLOB"; exit 0 ;; + esac ;; + *owner=helm*) printf '1\n'; exit 0 ;; + *) [ "$FAIL" = secretlist ] && fail "namespace secret LIST" + printf 'seaweedfs-system\n'; exit 0 ;; + esac +fi +if [ "$verb $res" = "get pvc" ]; then + case "$args" in + *"-o name"*) + [ "$FAIL" = pvclist ] && fail "pvc LIST" + [ -n "$PVCS" ] && printf '%s\n' "$PVCS" + exit 0 ;; + *data1-seaweedfs-system-volume-0*) [ "$FAIL" = pvcget ] && fail "pvc GET"; printf 'pv-renamed\n'; exit 0 ;; + *data1-seaweedfs-volume-1*) [ "$FAIL" = pvcget ] && fail "pvc GET"; printf 'pv-legacy2\n'; exit 0 ;; + *data1-seaweedfs-volume-0*) [ "$FAIL" = pvcget ] && fail "pvc GET"; printf 'pv-legacy\n'; exit 0 ;; + esac + exit 0 +fi +if [ "$verb $res" = "get sts" ]; then + [ "$FAIL" = sts ] && fail "sts LIST" + exit 0 +fi +if [ "$verb $res" = "get pv" ]; then + [ "$FAIL" = pvget ] && fail "get pv" + case "$args" in + *pv-legacy2*) [ "$ABSENT_PV" = pv-legacy2 ] && exit 0; printf '2099-01-01T00:00:00Z\n'; exit 0 ;; + *pv-legacy*) [ "$ABSENT_PV" = pv-legacy ] && exit 0; printf '2020-01-01T00:00:00Z\n'; exit 0 ;; + *pv-renamed*) [ "$ABSENT_PV" = pv-renamed ] && exit 0; printf '2020-06-01T00:00:00Z\n'; exit 0 ;; + esac + exit 0 +fi +exit 0 +FAKE + } > "$1/kubectl" + chmod +x "$1/kubectl" +} + +# _pvcs_mixed -- the `get pvc -o name` LIST for a MIXED tenant: two legacy claims +# and one release-named claim, whose PV vintages truly overlap. +_pvcs_mixed() { + printf '%s\n' \ + persistentvolumeclaim/data1-seaweedfs-volume-0 \ + persistentvolumeclaim/data1-seaweedfs-volume-1 \ + persistentvolumeclaim/data1-seaweedfs-system-volume-0 +} + +# _expected_L / _expected_mixed_overlap -- golden output built with the SAME printf +# contract the script uses, so a drift in row count, text, or padding fails the diff. +_expected_L() { + printf '%-24s %-14s %-8s %s\n' NAMESPACE RELEASE CLASS NOTE + printf '%-24s %-14s %-8s %s\n' --------- ------- ----- ---- + printf '%-24s %-14s %-8s %s\n' tenant-test seaweedfs-system L 'chart-named only; the upgrade adopts it, nothing to do' +} +_expected_mixed_overlap() { + printf '%-24s %-14s %-8s %s\n' NAMESPACE RELEASE CLASS NOTE + printf '%-24s %-14s %-8s %s\n' --------- ------- ----- ---- + printf '%-24s %-14s %-8s %s\n' tenant-test seaweedfs-system MIXED 'both generations; the chart REFUSES until one is removed' + printf '%-24s %-14s %-8s %s\n' '' '' '' 'PV vintages OVERLAP => no candidate. An interrupted Step 2 re-bind looks exactly like this (both generations on original PVs). Finish Step 2 if one is in progress; otherwise escalate. Do NOT run Step 2a.' + printf '%-24s %-14s %-8s %s\n' '' '' '' 'CANDIDATE ONLY: "original" does not mean the other set is EMPTY. A duplicate that' + printf '%-24s %-14s %-8s %s\n' '' '' '' 'served writes and later crashed looks identical here. Verify emptiness before deleting.' +} + +@test "fails closed when 'kubectl get ns' fails (whole-cluster mode)" { + # No namespace args -> main enumerates namespaces itself. If that LIST fails, + # the OLD code audited zero namespaces and still printed an empty clean table. + d=$(mktemp -d) + _write_fake_kubectl "$d" ns "" "$(_release_blob)" "" + rc=0 + out=$(PATH="$d:$PATH" SEAWEEDFS_AUDIT_LIB=0 sh "$PWD/hack/seaweedfs-naming-audit.sh" 2>&1) || rc=$? + rm -rf "$d" + echo "rc=$rc"; echo "$out" + [ "$rc" -ne 0 ] + printf '%s\n' "$out" | grep -q 'FATAL' + printf '%s\n' "$out" | grep -q 'list namespaces' +} + +@test "fails closed when the namespace-wide secret LIST fails" { + # system_releases enumerates Helm releases with a namespace-wide secret LIST -- + # the exact call that timed out in the field and reported a false clean. + d=$(mktemp -d) + _write_fake_kubectl "$d" secretlist "" "$(_release_blob)" "" + rc=0 + out=$(PATH="$d:$PATH" SEAWEEDFS_AUDIT_LIB=0 sh "$PWD/hack/seaweedfs-naming-audit.sh" tenant-test 2>&1) || rc=$? + rm -rf "$d" + echo "rc=$rc"; echo "$out" + [ "$rc" -ne 0 ] + printf '%s\n' "$out" | grep -q 'FATAL' + printf '%s\n' "$out" | grep -q 'list Helm release secrets' +} + +@test "fails closed when the by-name release-secret existence GET hits a real error" { + # Codex Finding 1: system_releases uses release_json to decide whether a release + # IS SeaweedFS. Both LISTs succeed, then a transient/forbidden by-name Secret GET + # used to leave `chart` empty -> the tenant was silently skipped -> exit 0, empty + # table. A real error here must now be fatal, not a false clean. + d=$(mktemp -d) + _write_fake_kubectl "$d" secretget "" "$(_release_blob)" "" + rc=0 + out=$(PATH="$d:$PATH" SEAWEEDFS_AUDIT_LIB=0 sh "$PWD/hack/seaweedfs-naming-audit.sh" tenant-test 2>&1) || rc=$? + rm -rf "$d" + echo "rc=$rc"; echo "$out" + [ "$rc" -ne 0 ] + printf '%s\n' "$out" | grep -q 'FATAL' + printf '%s\n' "$out" | grep -q 'Helm release secret' +} + +@test "an absent release-secret revision is a clean skip, not a failure" { + # The one legitimately-empty case for release_json: the revision secret does not + # exist (pruned by Helm history limit). The existence check (--ignore-not-found -o + # name) returns empty + exit 0, so the release is simply not confirmed as + # SeaweedFS. No error, no crash -- the audit completes and reports nothing here. + d=$(mktemp -d) + _write_fake_kubectl "$d" secretget_absent "" "$(_release_blob)" "" + rc=0 + out=$(PATH="$d:$PATH" SEAWEEDFS_AUDIT_LIB=0 sh "$PWD/hack/seaweedfs-naming-audit.sh" tenant-test 2>&1) || rc=$? + rm -rf "$d" + echo "rc=$rc"; echo "$out" + [ "$rc" -eq 0 ] + [ "$(printf '%s\n' "$out" | grep -c 'FATAL')" -eq 0 ] +} + +@test "fails closed when a present release secret has no .data.release payload" { + # Codex re-review round 2: the existence check passes (Secret EXISTS), but its + # .data.release field is empty/missing. That is CORRUPT state, not an absence -- + # and because the earlier fix used --ignore-not-found -o jsonpath, which returns + # empty+0 for BOTH, it used to read as a clean skip. It must be a loud stop. + d=$(mktemp -d) + _write_fake_kubectl "$d" secret_missing_field "" "$(_release_blob)" "" + rc=0 + out=$(PATH="$d:$PATH" SEAWEEDFS_AUDIT_LIB=0 sh "$PWD/hack/seaweedfs-naming-audit.sh" tenant-test 2>&1) || rc=$? + rm -rf "$d" + echo "rc=$rc"; echo "$out" + [ "$rc" -ne 0 ] + printf '%s\n' "$out" | grep -q 'FATAL' + printf '%s\n' "$out" | grep -q 'no .data.release payload' +} + +@test "fails closed when a present release secret has an undecodable payload" { + # Existence passes, .data.release is non-empty but is not base64(base64(gzip(...))), + # so every decode fails. The old `|| return 0` converted that decode failure into + # a clean 0 return -> silent skip. A corrupt payload must be fatal. + d=$(mktemp -d) + _write_fake_kubectl "$d" secret_bad_payload "" "$(_release_blob)" "" + rc=0 + out=$(PATH="$d:$PATH" SEAWEEDFS_AUDIT_LIB=0 sh "$PWD/hack/seaweedfs-naming-audit.sh" tenant-test 2>&1) || rc=$? + rm -rf "$d" + echo "rc=$rc"; echo "$out" + [ "$rc" -ne 0 ] + printf '%s\n' "$out" | grep -q 'FATAL' + printf '%s\n' "$out" | grep -q 'not decodable' +} + +@test "fails closed when a present release secret decodes to a chartless payload" { + # Codex round 3: the payload fetches and DECODES cleanly (valid JSON {}), so the + # decode guards all pass -- but it carries no chart name. system_releases then + # extracted chart='' and silently skipped the tenant -> exit 0, clean table. A + # decoded helm.sh/release.v1 release without a chart name is corrupt: fatal. + d=$(mktemp -d) + _write_fake_kubectl "$d" "" "" "$(_release_blob EMPTY)" "" + rc=0 + out=$(PATH="$d:$PATH" SEAWEEDFS_AUDIT_LIB=0 sh "$PWD/hack/seaweedfs-naming-audit.sh" tenant-test 2>&1) || rc=$? + rm -rf "$d" + echo "rc=$rc"; echo "$out" + [ "$rc" -ne 0 ] + printf '%s\n' "$out" | grep -q 'FATAL' + printf '%s\n' "$out" | grep -q 'no chart name' +} + +@test "a present release secret for a non-SeaweedFS chart is a silent legitimate skip" { + # The counterpart the guard must NOT break: a real, well-formed release whose + # chart is simply not cozy-seaweedfs. It has a chart name (so it is not corrupt), + # but the wrong one, so it is filtered out exactly as before -- no row, no error. + d=$(mktemp -d) + _write_fake_kubectl "$d" "" "" "$(_release_blob cozy-postgres)" "" + rc=0 + out=$(PATH="$d:$PATH" SEAWEEDFS_AUDIT_LIB=0 sh "$PWD/hack/seaweedfs-naming-audit.sh" tenant-test 2>&1) || rc=$? + rm -rf "$d" + echo "rc=$rc"; echo "$out" + [ "$rc" -eq 0 ] + [ "$(printf '%s\n' "$out" | grep -c 'FATAL')" -eq 0 ] + # Not confirmed as SeaweedFS -> no classification row for the release. + [ "$(printf '%s\n' "$out" | grep -c 'seaweedfs-system')" -eq 0 ] +} + +@test "fails closed when 'kubectl get pvc' LIST fails" { + # The secret enumeration succeeds and confirms a seaweedfs-system tenant, so the + # audit reaches the per-namespace PVC LIST; a failure there must abort, not read + # as "this tenant has no claims". + d=$(mktemp -d) + _write_fake_kubectl "$d" pvclist "" "$(_release_blob)" "" + rc=0 + out=$(PATH="$d:$PATH" SEAWEEDFS_AUDIT_LIB=0 sh "$PWD/hack/seaweedfs-naming-audit.sh" tenant-test 2>&1) || rc=$? + rm -rf "$d" + echo "rc=$rc"; echo "$out" + [ "$rc" -ne 0 ] + printf '%s\n' "$out" | grep -q 'FATAL' + printf '%s\n' "$out" | grep -q 'list PVCs' +} + +@test "fails closed when 'kubectl get sts' LIST fails" { + # PVC LIST succeeds (empty), so the audit reaches the StatefulSet LIST; a + # failure there must abort rather than read as "no StatefulSets". + d=$(mktemp -d) + _write_fake_kubectl "$d" sts "" "$(_release_blob)" "" + rc=0 + out=$(PATH="$d:$PATH" SEAWEEDFS_AUDIT_LIB=0 sh "$PWD/hack/seaweedfs-naming-audit.sh" tenant-test 2>&1) || rc=$? + rm -rf "$d" + echo "rc=$rc"; echo "$out" + [ "$rc" -ne 0 ] + printf '%s\n' "$out" | grep -q 'FATAL' + printf '%s\n' "$out" | grep -q 'StatefulSets' +} + +@test "fails closed when the by-name PV GET hits a real error" { + # Codex Finding 2, error half: pv_epoch reads each bound PV's age by name. A real + # error (RBAC/timeout) must abort, not silently drop that PV from the range. + d=$(mktemp -d) + _write_fake_kubectl "$d" pvget "" "$(_release_blob)" "$(_pvcs_mixed)" + rc=0 + out=$(PATH="$d:$PATH" SEAWEEDFS_AUDIT_LIB=0 sh "$PWD/hack/seaweedfs-naming-audit.sh" tenant-test 2>&1) || rc=$? + rm -rf "$d" + echo "rc=$rc"; echo "$out" + [ "$rc" -ne 0 ] + printf '%s\n' "$out" | grep -q 'FATAL' + printf '%s\n' "$out" | grep -q "get PV '" +} + +@test "an absent PV degrades to the safe fallback, never a wrong candidate" { + # Codex Finding 2, absence half. True vintages: legacy PVs 2020 + 2099 straddle + # the single renamed PV 2020-06 => OVERLAP. Report the newest legacy PV (2099) as + # ABSENT: the observed legacy range collapses to {2020} and, unguarded, "every + # release-named PV is strictly newer" would fire -- naming the release-named set a + # deletion candidate on incomplete evidence. The generation must instead read as + # incomplete and fall to "cannot establish direction". + d=$(mktemp -d) + _write_fake_kubectl "$d" "" pv-legacy2 "$(_release_blob)" "$(_pvcs_mixed)" + rc=0 + out=$(PATH="$d:$PATH" SEAWEEDFS_AUDIT_LIB=0 sh "$PWD/hack/seaweedfs-naming-audit.sh" tenant-test 2>&1) || rc=$? + rm -rf "$d" + echo "rc=$rc"; echo "$out" + [ "$rc" -eq 0 ] + [ "$(printf '%s\n' "$out" | grep -c 'FATAL')" -eq 0 ] + printf '%s\n' "$out" | grep -q 'direction cannot be established from PV ages' + # The whole point: incomplete evidence must NOT be reported as a deletion candidate. + [ "$(printf '%s\n' "$out" | grep -c 'candidate duplicate')" -eq 0 ] +} + +@test "success path (L) is byte-identical to the expected fixture" { + # A single chart-named claim, no release-named one, no StatefulSets => L, adopt in + # place. Full-output compare, so an extra row / warning / duplicate line fails. + d=$(mktemp -d) + _write_fake_kubectl "$d" "" "" "$(_release_blob)" "persistentvolumeclaim/data1-seaweedfs-volume-0" + rc=0 + out=$(PATH="$d:$PATH" SEAWEEDFS_AUDIT_LIB=0 sh "$PWD/hack/seaweedfs-naming-audit.sh" tenant-test 2>&1) || rc=$? + printf '%s\n' "$out" > "$d/got" + _expected_L > "$d/want" + echo "rc=$rc"; echo "--- got ---"; cat "$d/got"; echo "--- want ---"; cat "$d/want" + [ "$rc" -eq 0 ] + diff "$d/want" "$d/got" + rm -rf "$d" +} + +@test "success path (MIXED/overlap) is byte-identical and exercises pv_epoch" { + # Both generations present with all PVs readable => the MIXED path runs every + # pv_epoch GET and lands on OVERLAP. Full-output compare against the golden. + d=$(mktemp -d) + _write_fake_kubectl "$d" "" "" "$(_release_blob)" "$(_pvcs_mixed)" + rc=0 + out=$(PATH="$d:$PATH" SEAWEEDFS_AUDIT_LIB=0 sh "$PWD/hack/seaweedfs-naming-audit.sh" tenant-test 2>&1) || rc=$? + printf '%s\n' "$out" > "$d/got" + _expected_mixed_overlap > "$d/want" + echo "rc=$rc"; echo "--- got ---"; cat "$d/got"; echo "--- want ---"; cat "$d/want" + [ "$rc" -eq 0 ] + diff "$d/want" "$d/got" + rm -rf "$d" +} diff --git a/hack/seaweedfs-naming-audit.sh b/hack/seaweedfs-naming-audit.sh index a1e28c4c41..b01138a276 100755 --- a/hack/seaweedfs-naming-audit.sh +++ b/hack/seaweedfs-naming-audit.sh @@ -44,7 +44,15 @@ # # Usage: hack/seaweedfs-naming-audit.sh [namespace...] # KUBECONFIG= hack/seaweedfs-naming-audit.sh -# Exit: 0 always (audit only). Nothing is mutated. +# Exit: 0 the audit RAN TO COMPLETION; the table is the whole answer +# (an empty table then means a genuinely clean fleet). +# non-zero a kubectl query FAILED, so the audit is INCOMPLETE and the +# table must NOT be trusted -- a message naming the failed query +# is on stderr. This is fail-CLOSED by design (see run_kubectl): +# the output gates a destructive runbook step, so an unreachable +# API aborts loudly rather than printing an empty table that +# reads identically to "nothing to do". +# Nothing is ever mutated on either path -- every kubectl call is read-only. # # POSIX sh, deliberately. hack/seaweedfs-naming-audit.bats sources this file, and # cozytest.sh sources the converted test into its own /bin/sh — which is dash on @@ -55,6 +63,48 @@ # particular is not POSIX and dash 0.5.12 (Ubuntu) rejects it outright. set -u +# run_kubectl -- run ONE read-only kubectl query +# fail-CLOSED and print its stdout. This audit is the documented gate in front of +# a destructive runbook step (deleting a tenant's PVCs), so a query that FAILS +# must never be mistaken for a query that found nothing: an enumerating LIST +# returns empty stdout on a timeout or an RBAC denial, byte-identical to the same +# LIST succeeding on a genuinely clean fleet, and the old `2>/dev/null` on each +# call turned an unreachable API into a silent false "nothing to do". Every +# enumeration is routed through here so any failure is loud and terminal instead. +# +# On success prints kubectl's stdout verbatim -- which may legitimately be empty, +# because a LIST that matched nothing is honestly clean; keeping that case +# distinct from failure is the whole point. On a non-zero exit prints the failed +# query to stderr and returns that exit code; kubectl's own stderr stays on fd 2 +# and reaches the operator unfiltered. Callers MUST propagate the non-zero status +# (`|| return`/`|| exit`) -- a bare `for x in $(run_kubectl ...)` would swallow +# it, because a for-loop ignores the exit status of its word-list command, and an +# `exit` inside the `$(...)` subshell would only leave that subshell. +run_kubectl() { + _rk_what="$1"; shift + if _rk_out=$(kubectl "$@"); then + printf '%s' "$_rk_out" + return 0 + else + _rk_rc=$? + printf 'seaweedfs-naming-audit: FATAL: %s failed (kubectl %s -> exit %s). Refusing to report a clean fleet on an unreachable API.\n' \ + "$_rk_what" "$*" "$_rk_rc" >&2 + return "$_rk_rc" + fi +} + +# audit_fatal -- the counterpart to run_kubectl for corruption the API +# call itself did NOT report: the query succeeded, but what it returned is +# impossible for a healthy object (a helm.sh/release.v1 Secret with no decodable +# release payload). Same fail-CLOSED contract -- print a FATAL line naming the +# object to stderr and return non-zero; callers MUST propagate it. A silent skip +# here would be the identical false-clean this whole script guards against, just +# one field deeper than an unreachable API. +audit_fatal() { + printf 'seaweedfs-naming-audit: FATAL: %s Refusing to report a clean fleet on corrupt state.\n' "$1" >&2 + return 1 +} + # renamed_volume_prefix -- reconstruct the name 4.31 gives the volume # component of . Mirrors seaweedfs.fullname + seaweedfs.componentName # (see packages/system/seaweedfs/templates/_naming.tpl, which the chart's guard @@ -71,18 +121,51 @@ renamed_volume_prefix() { printf '%s-volume' "$(printf '%s' "$full" | cut -c1-56 | sed 's/-$//')" } -# release_json [rev] -- decode a Helm release secret. -# Helm stores base64(gzip(json)) in Secret.data.release, and Kubernetes base64s -# the data value again, hence the two decodes. +# release_json [rev] -- decode a Helm release secret's payload, or +# "" when that revision's secret does not exist. Helm stores base64(gzip(json)) in +# Secret.data.release, and Kubernetes base64s the data value again, hence the two +# decodes. +# +# This is a by-NAME GET, but it is NOT best-effort: system_releases decides whether +# a release IS SeaweedFS from the result, so a swallowed failure silently drops a +# real tenant -- the false-clean this script exists to prevent. EXISTENCE and +# EXTRACTION are therefore two separate questions, because `--ignore-not-found +# -o jsonpath` cannot tell them apart (it returns empty + exit 0 both for an absent +# Secret AND for a present Secret whose .data.release is missing): +# +# 1. Existence, via `--ignore-not-found -o name`. Empty + exit 0 = the revision +# was pruned by Helm's history limit -- a legitimate absence; return "" and +# let rev1_scheme / first_deployed treat it as "pruned". A real error (RBAC, +# timeout, apiserver down) stays non-zero and run_kubectl makes it fatal. +# 2. The Secret EXISTS, so its payload MUST decode. A helm.sh/release.v1 Secret +# with no .data.release, a payload that is not base64(base64(gzip(...))), or an +# empty JSON after decoding is CORRUPT state, not an absence -- and here that +# is safety-critical, so every such anomaly is audit_fatal, never a silent skip +# (contrast pv_epoch, where an empty field is a benign "no evidence"). +# +# The extraction GET drops --ignore-not-found deliberately: the Secret existed a +# moment ago, so a NotFound now is a mid-audit deletion race, and failing closed on +# it is correct. release_json() { - kubectl get secret -n "$1" "sh.helm.release.v1.$2.v${3:-1}" -o jsonpath='{.data.release}' 2>/dev/null \ - | base64 -d 2>/dev/null | base64 -d 2>/dev/null | gunzip 2>/dev/null + _rj_secret="sh.helm.release.v1.$2.v${3:-1}" + _rj_exists=$(run_kubectl "check Helm release secret '$_rj_secret' in namespace '$1'" \ + get secret -n "$1" "$_rj_secret" --ignore-not-found -o name) || return $? + [ -n "$_rj_exists" ] || return 0 + _rj_field=$(run_kubectl "read .data.release of secret '$_rj_secret' in namespace '$1'" \ + get secret -n "$1" "$_rj_secret" -o jsonpath='{.data.release}') || return $? + [ -n "$_rj_field" ] || { audit_fatal "secret '$_rj_secret' in namespace '$1' exists but has no .data.release payload (corrupt Helm release)."; return 1; } + _rj_json=$(printf '%s' "$_rj_field" | base64 -d 2>/dev/null | base64 -d 2>/dev/null | gunzip 2>/dev/null) \ + || { audit_fatal "secret '$_rj_secret' in namespace '$1': .data.release is not decodable base64(base64(gzip(json))) (corrupt Helm release)."; return 1; } + [ -n "$_rj_json" ] || { audit_fatal "secret '$_rj_secret' in namespace '$1': release payload decoded to nothing (corrupt Helm release)."; return 1; } + printf '%s' "$_rj_json" } # revisions -- retained revision numbers, oldest first. revisions() { - kubectl get secret -n "$1" -l "name=$2,owner=helm" \ - -o jsonpath='{range .items[*]}{.metadata.labels.version}{"\n"}{end}' 2>/dev/null | sort -n + _rev_out=$(run_kubectl "list Helm revision secrets for '$2' in namespace '$1'" \ + get secret -n "$1" -l "name=$2,owner=helm" \ + -o jsonpath='{range .items[*]}{.metadata.labels.version}{"\n"}{end}') || return $? + printf '%s\n' "$_rev_out" | sort -n } # system_releases -- the SeaweedFS -system Helm releases in a namespace. @@ -92,12 +175,29 @@ revisions() { # unrelated release in a namespace that happens to run SeaweedFS would be reported # as a SeaweedFS tenant. Confirm the chart. system_releases() { - for rel in $(kubectl get secret -n "$1" \ - -o jsonpath='{range .items[?(@.type=="helm.sh/release.v1")]}{.metadata.labels.name}{"\n"}{end}' 2>/dev/null \ - | grep -E -- '-system$' | sort -u); do - for rev in $(revisions "$1" "$rel"); do - chart=$(release_json "$1" "$rel" "$rev" | sed -n 's/.*"name":"\(cozy-seaweedfs\)".*/\1/p' | head -1) - if [ -n "$chart" ]; then printf '%s\n' "$rel"; fi + _sr_secrets=$(run_kubectl "list Helm release secrets in namespace '$1'" \ + get secret -n "$1" \ + -o jsonpath='{range .items[?(@.type=="helm.sh/release.v1")]}{.metadata.labels.name}{"\n"}{end}') || return $? + for rel in $(printf '%s\n' "$_sr_secrets" | grep -E -- '-system$' | sort -u); do + _sr_revs=$(revisions "$1" "$rel") || return $? + for rev in $_sr_revs; do + _sr_json=$(release_json "$1" "$rel" "$rev") || return $? + # Empty "" here means the revision secret was pruned (release_json's absence + # path) -- a legitimate skip. A NON-empty payload, however, was fetched and + # decoded successfully, so it MUST name its chart: a helm.sh/release.v1 release + # always carries chart.metadata.name. A payload without one ({} , or any valid + # JSON lacking it) is corrupt/unexpected, and since "no chart name" is + # indistinguishable downstream from "not SeaweedFS", silently skipping it is + # the very false-clean this guard exists to stop -- so it is FATAL. A present + # but different chart name is a real, non-SeaweedFS release and stays a + # legitimate skip, filtered exactly as before by the cozy-seaweedfs test. + # ("chart" is the release struct's only such key -- vendored subcharts nest + # under "dependencies" -- so this match is unambiguous.) + if [ -n "$_sr_json" ]; then + _sr_chart=$(printf '%s' "$_sr_json" | sed -n 's/.*"chart":{"metadata":{"name":"\([^"]*\)".*/\1/p' | head -1) + [ -n "$_sr_chart" ] || { audit_fatal "secret 'sh.helm.release.v1.$rel.v$rev' in namespace '$1' decoded to a payload with no chart name (corrupt Helm release)."; return 1; } + if [ "$_sr_chart" = cozy-seaweedfs ]; then printf '%s\n' "$rel"; fi + fi break done done @@ -106,28 +206,52 @@ system_releases() { # first_deployed -- epoch seconds of the release's first install, # from any retained revision (the field is identical on all of them). first_deployed() { - for rev in $(revisions "$1" "$2"); do - ts=$(release_json "$1" "$2" "$rev" | sed -n 's/.*"first_deployed":"\([^"]*\)".*/\1/p' | head -1) + _fd_revs=$(revisions "$1" "$2") || return $? + for rev in $_fd_revs; do + _fd_json=$(release_json "$1" "$2" "$rev") || return $? + ts=$(printf '%s' "$_fd_json" | sed -n 's/.*"first_deployed":"\([^"]*\)".*/\1/p' | head -1) if [ -n "$ts" ]; then date -u -d "$(printf '%s' "$ts" | cut -c1-19)" +%s 2>/dev/null; return; fi done } # rev1_scheme -- "legacy" | "renamed" | "" (revision 1 pruned). rev1_scheme() { - m=$(release_json "$1" "$2" 1) + m=$(release_json "$1" "$2" 1) || return $? [ -n "$m" ] || return 0 if printf '%s' "$m" | grep -q 'name: seaweedfs-master'; then printf 'legacy' elif printf '%s' "$m" | grep -qE "name: $2(-seaweedfs)?-master"; then printf 'renamed' fi } -# pv_epoch -- creation epoch of the PV the claim is BOUND to. +# pv_epoch -- creation epoch of the PV the claim is BOUND to, or "" when +# there is no usable PV age. Two by-NAME GETs. +# +# A real API error must abort (an epoch silently missing from a range can invert +# the strict-newer comparison and name the WRONG deletion candidate), so both GETs +# go through run_kubectl. But UNLIKE release_json, an EMPTY field here is NOT +# corruption and is NOT fatal -- because the consequence differs. When release_json +# returns empty for a present Secret the tenant is dropped from the report: a +# false-clean. When pv_epoch returns "" the claim merely contributes no epoch, +# which marks its whole generation INCOMPLETE (see audit_ns) and forces the safe +# "direction cannot be established -> classify by hand" branch. That degradation is +# conservative by construction: it never yields a false-clean and never names a +# candidate. So the two empty-field cases are deliberately kept benign: +# * PVC .spec.volumeName empty -- a Pending / unbound claim, a routine state; +# * PV .metadata.creationTimestamp empty -- shouldn't happen (the apiserver always +# stamps it), but if it ever did the only effect is one missing epoch, i.e. the +# same safe incomplete fallback, so a hard stop is not warranted here. +# `--ignore-not-found` keeps a genuinely-absent PVC/PV (NotFound) on that same +# benign path rather than turning it into a run_kubectl failure, and the final +# `|| return 0` keeps an unparseable timestamp there too; only a real run_kubectl +# error propagates. pv_epoch() { - pv=$(kubectl get pvc -n "$1" "$2" -o jsonpath='{.spec.volumeName}' 2>/dev/null) + pv=$(run_kubectl "get PVC '$2' in namespace '$1'" \ + get pvc -n "$1" "$2" --ignore-not-found -o jsonpath='{.spec.volumeName}') || return $? [ -n "$pv" ] || return 0 - t=$(kubectl get pv "$pv" -o jsonpath='{.metadata.creationTimestamp}' 2>/dev/null) + t=$(run_kubectl "get PV '$pv' (bound by PVC '$2' in namespace '$1')" \ + get pv "$pv" --ignore-not-found -o jsonpath='{.metadata.creationTimestamp}') || return $? [ -n "$t" ] || return 0 - date -u -d "$(printf '%s' "$t" | cut -c1-19)" +%s 2>/dev/null + date -u -d "$(printf '%s' "$t" | cut -c1-19)" +%s 2>/dev/null || return 0 } # classify_mixed_direction -- direction of a MIXED @@ -153,17 +277,21 @@ classify_mixed_direction() { audit_ns() { ns="$1" - for rel in $(system_releases "$ns"); do + _rels=$(system_releases "$ns") || return $? + for rel in $_rels; do prefix=$(renamed_volume_prefix "$rel") legacy_pvcs=""; renamed_pvcs="" - for pvc in $(kubectl get pvc -n "$ns" -o name 2>/dev/null | sed 's|persistentvolumeclaim/||'); do + _pvcs=$(run_kubectl "list PVCs in namespace '$ns'" get pvc -n "$ns" -o name) || return $? + for pvc in $(printf '%s\n' "$_pvcs" | sed 's|persistentvolumeclaim/||'); do case "$pvc" in "data1-${prefix}"*) renamed_pvcs="$renamed_pvcs $pvc" ;; data1-seaweedfs-volume*) legacy_pvcs="$legacy_pvcs $pvc" ;; esac done legacy_sts=""; renamed_sts="" - for sts in $(kubectl get sts -n "$ns" -l app.kubernetes.io/name=seaweedfs -o name 2>/dev/null | sed 's|statefulset.apps/||'); do + _sts_list=$(run_kubectl "list SeaweedFS StatefulSets in namespace '$ns'" \ + get sts -n "$ns" -l app.kubernetes.io/name=seaweedfs -o name) || return $? + for sts in $(printf '%s\n' "$_sts_list" | sed 's|statefulset.apps/||'); do case "$sts" in "${prefix}"*) renamed_sts="$renamed_sts $sts" ;; seaweedfs-volume*) legacy_sts="$legacy_sts $sts" ;; @@ -184,21 +312,30 @@ audit_ns() { fi # Both generations. Report which is ORIGINAL from durable evidence. - fd=$(first_deployed "$ns" "$rel") - scheme=$(rev1_scheme "$ns" "$rel") + fd=$(first_deployed "$ns" "$rel") || return $? + scheme=$(rev1_scheme "$ns" "$rel") || return $? printf '%-24s %-14s %-8s %s\n' "$ns" "$rel" "MIXED" "both generations; the chart REFUSES until one is removed" if [ -n "$scheme" ]; then printf '%-24s %-14s %-8s revision 1 was installed with the %s names => the %s generation is ORIGINAL\n' \ "" "" "" "$scheme" "$scheme" fi - lmin=""; lmax=""; rmin=""; rmax="" + # The direction rule is "EVERY PV of one generation strictly newer than every + # PV of the other". That holds only if we saw EVERY bound PV: a claim whose PV + # age we could not read (unbound, or PV gone) leaves the range unbounded on one + # side, and a silently-narrowed range can flip OVERLAP into a confident (wrong) + # candidate. So a generation with any unreadable claim is INCOMPLETE + # and forces the safe "cannot establish" branch -- distinct from a real API + # error, which pv_epoch has already turned into a hard failure above. + lmin=""; lmax=""; rmin=""; rmax=""; l_incomplete=0; r_incomplete=0 for p in $legacy_pvcs; do - e=$(pv_epoch "$ns" "$p"); [ -n "$e" ] || continue + e=$(pv_epoch "$ns" "$p") || return $? + if [ -z "$e" ]; then l_incomplete=1; continue; fi { [ -z "$lmin" ] || [ "$e" -lt "$lmin" ]; } && lmin=$e { [ -z "$lmax" ] || [ "$e" -gt "$lmax" ]; } && lmax=$e done for p in $renamed_pvcs; do - e=$(pv_epoch "$ns" "$p"); [ -n "$e" ] || continue + e=$(pv_epoch "$ns" "$p") || return $? + if [ -z "$e" ]; then r_incomplete=1; continue; fi { [ -z "$rmin" ] || [ "$e" -lt "$rmin" ]; } && rmin=$e { [ -z "$rmax" ] || [ "$e" -gt "$rmax" ]; } && rmax=$e done @@ -206,7 +343,7 @@ audit_ns() { printf '%-24s %-14s %-8s oldest PV vs first_deployed: chart-named +%ss, release-named +%ss (context only, not the rule)\n' \ "" "" "" "$((lmin - fd))" "$((rmin - fd))" fi - if [ -n "$lmin" ] && [ -n "$rmin" ]; then + if [ -n "$lmin" ] && [ -n "$rmin" ] && [ "$l_incomplete" = 0 ] && [ "$r_incomplete" = 0 ]; then case $(classify_mixed_direction "$lmin" "$lmax" "$rmin" "$rmax") in legacy-original) printf '%-24s %-14s %-8s every release-named PV is strictly newer => chart-named is ORIGINAL, the release-named set is the candidate duplicate (Step 3)\n' "" "" "" ;; @@ -216,7 +353,7 @@ audit_ns() { printf '%-24s %-14s %-8s PV vintages OVERLAP => no candidate. An interrupted Step 2 re-bind looks exactly like this (both generations on original PVs). Finish Step 2 if one is in progress; otherwise escalate. Do NOT run Step 2a.\n' "" "" "" ;; esac else - printf '%-24s %-14s %-8s a generation has no bound PVs (Pending claims, or StatefulSets only) => direction cannot be established from PV ages. Resolve the Pending claims or classify by hand.\n' "" "" "" + printf '%-24s %-14s %-8s direction cannot be established from PV ages: a generation has no bound PVs (Pending/unbound claims, or StatefulSets only) or a bound PV age could not be read. Resolve those claims or classify by hand. Do NOT run Step 2a.\n' "" "" "" fi printf '%-24s %-14s %-8s CANDIDATE ONLY: "original" does not mean the other set is EMPTY. A duplicate that\n' "" "" "" printf '%-24s %-14s %-8s served writes and later crashed looks identical here. Verify emptiness before deleting.\n' "" "" "" @@ -227,9 +364,10 @@ main() { printf '%-24s %-14s %-8s %s\n' NAMESPACE RELEASE CLASS NOTE printf '%-24s %-14s %-8s %s\n' --------- ------- ----- ---- if [ "$#" -gt 0 ]; then - for ns in "$@"; do audit_ns "$ns"; done + for ns in "$@"; do audit_ns "$ns" || exit $?; done else - for ns in $(kubectl get ns -o name 2>/dev/null | sed 's|namespace/||'); do audit_ns "$ns"; done + _ns_list=$(run_kubectl 'list namespaces (whole-cluster mode)' get ns -o name) || exit $? + for ns in $(printf '%s\n' "$_ns_list" | sed 's|namespace/||'); do audit_ns "$ns" || exit $?; done fi } From a4d73b2346632f11ac2e0c4b45ab745a5ebccb0f Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Mon, 27 Jul 2026 13:24:05 +0500 Subject: [PATCH 05/51] fix(seaweedfs): read the chart name first-match, fail the fake on unmodelled calls Review findings from #3436, plus what an adversarial re-check of the first attempt at them turned up. The chart-name extraction was a greedy sed, i.e. LAST match. Helm marshals "config" (the release`s values) after "chart", so a values subtree that spells chart.metadata.name shadowed the real chart name: the release then read as non-SeaweedFS, the tenant silently vanished from the report, and the script exited 0 -- the precise false clean this PR exists to prevent, reachable without any corruption at all. It now takes the FIRST match. The path stays adjacent on purpose: a looser "any name after metadata" matches chart.templates[].name, which Helm emits immediately after metadata on every healthy release, so the review`s suggested relaxation would have returned a template path for every tenant. Over-strictness fails loudly and recoverably; over-looseness deletes data quietly. Newlines are now folded before matching, so a pretty-printed payload parses at all rather than aborting, and whitespace around the punctuation is tolerated. first_deployed got the same treatment -- it had the identical greedy, single- line-only shape, and a spaced payload silently dropped the PV-vintage row from the report. The FATAL message names the path it read and the shape it expected, so a future Helm format change is diagnosable instead of looking like real corruption. Tests: three new payload shapes (spaced, pretty multi-line, and a values decoy) each byte-compared against the same golden as the compact payload, so the shape must make no difference to the report. Mutation-checked -- restoring last-match extraction fails the decoy test on its own, restoring the single-line matcher fails the pretty test. The test fake answered any unmodelled kubectl invocation with exit 0 and empty stdout, the fail-open shape this script was rewritten to reject, and enough to let a newly added query pass the goldens unnoticed. Unmodelled calls now exit 97 naming the invocation. Verified inert first by instrumenting the fake: 88 invocations across the suite, none unmodelled. Also added the missing test for the by-name PVC GET fatal path, whose FAIL mode existed with nothing behind it. The runbook now tells the operator to read the exit code: a non-zero exit means the table is incomplete and no step may be taken on it. That contract lived only in the script header, while the runbook is what the operator follows. 29/29 tests green; sh -n clean. Assisted-By: Claude Signed-off-by: Myasnikov Daniil (cherry picked from commit e02adad38f8339848344260a7cbeb9d2f58105d5) --- .../seaweedfs-431-rename-recovery.md | 4 +- hack/seaweedfs-naming-audit.bats | 114 ++++++++++++++++-- hack/seaweedfs-naming-audit.sh | 41 ++++++- 3 files changed, 144 insertions(+), 15 deletions(-) diff --git a/docs/operations/seaweedfs-431-rename-recovery.md b/docs/operations/seaweedfs-431-rename-recovery.md index 06dd2c3422..f26a671ebc 100644 --- a/docs/operations/seaweedfs-431-rename-recovery.md +++ b/docs/operations/seaweedfs-431-rename-recovery.md @@ -55,6 +55,8 @@ hack/seaweedfs-naming-audit.sh # whole cluster hack/seaweedfs-naming-audit.sh tenant-foo # or named namespaces ``` +**Read the exit code, not just the table.** The audit fails closed: any error it cannot interpret — a kubectl call that fails, a Helm release payload it cannot decode — makes it print `FATAL` and exit non-zero, and the table it printed up to that point is incomplete. A non-zero exit means you do not yet know the state of the fleet, so none of the steps below may be taken on the strength of it. Only a zero exit means the table is the whole answer; an empty table with exit 0 is a genuinely clean fleet. + It mutates nothing. Earlier revisions of this runbook inlined the classification as a shell snippet here; it is a tested script now (`hack/seaweedfs-naming-audit.bats`), because it is what the chart's refusal hands you to and acting on it deletes PVCs. Two inline versions shipped wrong — one whose selector matched both generations at once and so inverted its own primary rule, one that could not see a long instance name at all — so it is not a snippet any more. It reports one class per SeaweedFS instance, matching exactly what the chart's guard decides: @@ -297,7 +299,7 @@ Do **not** start by deleting PVCs. If the duplicate ever served, they may hold o Never delete `data1-seaweedfs-volume-*` here — those are the live data PVCs of the authoritative set. (For an `S-damaged` tenant it is the other way round; that is Step 2a's job, and it has its own precondition check.) -5. **Re-run the audit.** The tenant must now read `L` (or `S`, if you are on the Step 2 path). Only then upgrade. +5. **Re-run the audit.** The tenant must now read `L` (or `S`, if you are on the Step 2 path), and the audit must exit zero — a `FATAL` here means the re-check never completed, not that the tenant is fine. Only then upgrade. ```sh hack/seaweedfs-naming-audit.sh "$ns" diff --git a/hack/seaweedfs-naming-audit.bats b/hack/seaweedfs-naming-audit.bats index 424de20a35..eb605c5297 100644 --- a/hack/seaweedfs-naming-audit.bats +++ b/hack/seaweedfs-naming-audit.bats @@ -172,13 +172,34 @@ EOF # chart.metadata.name (as real Helm payloads carry it). Defaults to a cozy-seaweedfs # release so the audit confirms the tenant; pass another chart name for a non- # SeaweedFS release, or the literal EMPTY for a chartless {} payload. +# +# Three payload SHAPES beyond the default compact one, because the chart-name +# extraction is now fatal on a miss and every shape below is something a +# re-serializer or a user's values could produce: +# SPACED whitespace around the JSON punctuation (json.dumps' default) +# PRETTY indented and MULTI-LINE (jq . / yq -o=json), which a line-based +# matcher cannot read at all unless newlines are folded first +# DECOY a real cozy-seaweedfs chart PLUS a values subtree that spells +# chart.metadata.name with a different value. Helm marshals "config" +# (the values) AFTER "chart", so a last-match extraction returns the +# decoy and silently declares the tenant non-SeaweedFS. _release_blob() { _rb_name=${1:-cozy-seaweedfs} - if [ "$_rb_name" = EMPTY ]; then - _rb_json='{}' - else - _rb_json='{"name":"seaweedfs-system","chart":{"metadata":{"name":"'"$_rb_name"'"}}}' - fi + case "$_rb_name" in + EMPTY) _rb_json='{}' ;; + SPACED) _rb_json='{"name": "seaweedfs-system", "chart": {"metadata": {"name": "cozy-seaweedfs", "version": "1.0.0"}}}' ;; + PRETTY) _rb_json='{ + "name": "seaweedfs-system", + "chart": { + "metadata": { + "name": "cozy-seaweedfs", + "version": "1.0.0" + } + } +}' ;; + DECOY) _rb_json='{"name":"seaweedfs-system","chart":{"metadata":{"name":"cozy-seaweedfs"}},"config":{"chart":{"metadata":{"name":"decoy-not-seaweedfs"}}}}' ;; + *) _rb_json='{"name":"seaweedfs-system","chart":{"metadata":{"name":"'"$_rb_name"'"}}}' ;; + esac printf '%s' "$_rb_json" | gzip | base64 | base64 | tr -d '\n' } @@ -212,6 +233,10 @@ _write_fake_kubectl() { cat <<'FAKE' verb=${1:-}; res=${2:-}; args="$*" fail() { echo "fake kubectl: $1 (real error)" >&2; exit 1; } +# Anything this fake does not model must NOT look like a successful empty +# answer: that is the fail-open shape the audited script exists to reject, and +# it would let a new query added to the script pass these goldens unnoticed. +unmodelled() { echo "fake kubectl: unmodelled invocation: $args" >&2; exit 97; } if [ "$verb $res" = "get ns" ]; then [ "$FAIL" = ns ] && fail "get ns" printf 'namespace/tenant-test\n'; exit 0 @@ -247,7 +272,7 @@ if [ "$verb $res" = "get pvc" ]; then *data1-seaweedfs-volume-1*) [ "$FAIL" = pvcget ] && fail "pvc GET"; printf 'pv-legacy2\n'; exit 0 ;; *data1-seaweedfs-volume-0*) [ "$FAIL" = pvcget ] && fail "pvc GET"; printf 'pv-legacy\n'; exit 0 ;; esac - exit 0 + unmodelled fi if [ "$verb $res" = "get sts" ]; then [ "$FAIL" = sts ] && fail "sts LIST" @@ -260,9 +285,9 @@ if [ "$verb $res" = "get pv" ]; then *pv-legacy*) [ "$ABSENT_PV" = pv-legacy ] && exit 0; printf '2020-01-01T00:00:00Z\n'; exit 0 ;; *pv-renamed*) [ "$ABSENT_PV" = pv-renamed ] && exit 0; printf '2020-06-01T00:00:00Z\n'; exit 0 ;; esac - exit 0 + unmodelled fi -exit 0 +unmodelled FAKE } > "$1/kubectl" chmod +x "$1/kubectl" @@ -396,7 +421,10 @@ _expected_mixed_overlap() { echo "rc=$rc"; echo "$out" [ "$rc" -ne 0 ] printf '%s\n' "$out" | grep -q 'FATAL' - printf '%s\n' "$out" | grep -q 'no chart name' + printf '%s\n' "$out" | grep -q 'could not read the chart name' + # The diagnostic must name the path it looked at, so a future Helm payload + # format change is diagnosable as such instead of reading as real corruption. + printf '%s\n' "$out" | grep -q '\.chart\.metadata\.name' } @test "a present release secret for a non-SeaweedFS chart is a silent legitimate skip" { @@ -507,3 +535,71 @@ _expected_mixed_overlap() { diff "$d/want" "$d/got" rm -rf "$d" } + +@test "a whitespace-spaced Helm payload still classifies the tenant" { + # The chart-name read is FATAL on a miss, so any payload shape a re-serializer + # can produce must parse. Byte-compare against the same golden as the compact + # payload: the shape must make no difference to the report at all. + d=$(mktemp -d) + _write_fake_kubectl "$d" "" "" "$(_release_blob SPACED)" "persistentvolumeclaim/data1-seaweedfs-volume-0" + rc=0 + out=$(PATH="$d:$PATH" SEAWEEDFS_AUDIT_LIB=0 sh "$PWD/hack/seaweedfs-naming-audit.sh" tenant-test 2>&1) || rc=$? + printf '%s\n' "$out" > "$d/got" + _expected_L > "$d/want" + echo "rc=$rc"; echo "--- got ---"; cat "$d/got" + [ "$rc" -eq 0 ] + diff "$d/want" "$d/got" + rm -rf "$d" +} + +@test "a pretty-printed multi-line Helm payload still classifies the tenant" { + # Line-based sed/grep cannot see across newlines at all, so this shape is the + # one that fails hardest without the newline fold -- and jq/yq re-serialization + # is exactly how a payload would arrive indented. + d=$(mktemp -d) + _write_fake_kubectl "$d" "" "" "$(_release_blob PRETTY)" "persistentvolumeclaim/data1-seaweedfs-volume-0" + rc=0 + out=$(PATH="$d:$PATH" SEAWEEDFS_AUDIT_LIB=0 sh "$PWD/hack/seaweedfs-naming-audit.sh" tenant-test 2>&1) || rc=$? + printf '%s\n' "$out" > "$d/got" + _expected_L > "$d/want" + echo "rc=$rc"; echo "--- got ---"; cat "$d/got" + [ "$rc" -eq 0 ] + diff "$d/want" "$d/got" + rm -rf "$d" +} + +@test "a values subtree spelling chart.metadata.name cannot shadow the real chart" { + # Helm marshals "config" (the user's values) AFTER "chart", so a LAST-match + # extraction returns the decoy, the release reads as non-SeaweedFS, and the + # tenant vanishes from the report with exit 0 -- a false clean, the failure this + # whole script exists to prevent. First-match extraction is what stops it. + d=$(mktemp -d) + _write_fake_kubectl "$d" "" "" "$(_release_blob DECOY)" "persistentvolumeclaim/data1-seaweedfs-volume-0" + rc=0 + out=$(PATH="$d:$PATH" SEAWEEDFS_AUDIT_LIB=0 sh "$PWD/hack/seaweedfs-naming-audit.sh" tenant-test 2>&1) || rc=$? + printf '%s\n' "$out" > "$d/got" + _expected_L > "$d/want" + echo "rc=$rc"; echo "--- got ---"; cat "$d/got" + [ "$rc" -eq 0 ] + # The decoy name must never reach the report. + [ "$(printf '%s\n' "$out" | grep -c 'decoy-not-seaweedfs')" -eq 0 ] + diff "$d/want" "$d/got" + rm -rf "$d" +} + +@test "fails closed when the by-name PVC GET hits a real error" { + # pv_epoch resolves each claim's bound PV by name before reading the PV's age. + # A real error there (RBAC/timeout) must abort: silently dropping the claim + # shrinks the observed vintage range, which is what produces a wrong "candidate + # duplicate" verdict. The PV half of this pair was already covered; this is the + # PVC half, whose fake FAIL mode existed with no test behind it. + d=$(mktemp -d) + _write_fake_kubectl "$d" pvcget "" "$(_release_blob)" "$(_pvcs_mixed)" + rc=0 + out=$(PATH="$d:$PATH" SEAWEEDFS_AUDIT_LIB=0 sh "$PWD/hack/seaweedfs-naming-audit.sh" tenant-test 2>&1) || rc=$? + rm -rf "$d" + echo "rc=$rc"; echo "$out" + [ "$rc" -ne 0 ] + printf '%s\n' "$out" | grep -q 'FATAL' + printf '%s\n' "$out" | grep -q "get PVC '" +} diff --git a/hack/seaweedfs-naming-audit.sh b/hack/seaweedfs-naming-audit.sh index b01138a276..b66e91939e 100755 --- a/hack/seaweedfs-naming-audit.sh +++ b/hack/seaweedfs-naming-audit.sh @@ -191,11 +191,33 @@ system_releases() { # the very false-clean this guard exists to stop -- so it is FATAL. A present # but different chart name is a real, non-SeaweedFS release and stays a # legitimate skip, filtered exactly as before by the cozy-seaweedfs test. - # ("chart" is the release struct's only such key -- vendored subcharts nest - # under "dependencies" -- so this match is unambiguous.) + # + # Two properties make this match trustworthy, and both are load-bearing now + # that a miss is FATAL: + # + # * FIRST match, not last. Helm's Release marshals "chart" before + # "config" (the user's values), so a values subtree that happens to + # spell chart.metadata.name sits LATER in the payload -- and a greedy + # `sed 's/.*"chart"...'` would return that decoy instead of the real + # chart. A wrong name reads downstream as "not SeaweedFS" and drops a + # real release from the report: the exact silent false-clean this guard + # exists to stop. `grep -o | head -1` takes the leftmost match. + # * The chart -> metadata -> name key path stays ADJACENT. A looser "any + # 'name' after 'metadata'" matches chart.templates[].name, which Helm + # serializes immediately after metadata on EVERY healthy release, so it + # would return a template path for every tenant. + # + # Newlines are folded first so a pretty-printed payload parses at all (sed + # and grep are line-based), and whitespace around the punctuation is + # tolerated. Failing loudly on a payload this cannot read is the safe + # direction -- over-strictness stops the runbook, over-looseness lets it + # delete data -- so the message says what shape was expected. if [ -n "$_sr_json" ]; then - _sr_chart=$(printf '%s' "$_sr_json" | sed -n 's/.*"chart":{"metadata":{"name":"\([^"]*\)".*/\1/p' | head -1) - [ -n "$_sr_chart" ] || { audit_fatal "secret 'sh.helm.release.v1.$rel.v$rev' in namespace '$1' decoded to a payload with no chart name (corrupt Helm release)."; return 1; } + _sr_chart=$(printf '%s' "$_sr_json" | tr '\n' ' ' \ + | grep -o '"chart"[[:space:]]*:[[:space:]]*{[[:space:]]*"metadata"[[:space:]]*:[[:space:]]*{[[:space:]]*"name"[[:space:]]*:[[:space:]]*"[^"]*"' \ + | head -1 \ + | sed -n 's/.*"\([^"]*\)"$/\1/p') + [ -n "$_sr_chart" ] || { audit_fatal "could not read the chart name of secret 'sh.helm.release.v1.$rel.v$rev' in namespace '$1' at .chart.metadata.name (expected 'name' as metadata's first key; a corrupt Helm release, or a payload format this parser does not handle)."; return 1; } if [ "$_sr_chart" = cozy-seaweedfs ]; then printf '%s\n' "$rel"; fi fi break @@ -209,7 +231,16 @@ first_deployed() { _fd_revs=$(revisions "$1" "$2") || return $? for rev in $_fd_revs; do _fd_json=$(release_json "$1" "$2" "$rev") || return $? - ts=$(printf '%s' "$_fd_json" | sed -n 's/.*"first_deployed":"\([^"]*\)".*/\1/p' | head -1) + # Same extraction discipline as the chart name above: fold newlines so a + # pretty-printed payload parses at all, tolerate whitespace around the + # punctuation, and take the FIRST match (.info precedes .config, so a values + # subtree cannot shadow the real timestamp). A miss here is not fatal -- the + # caller degrades to the documented safe fallback -- but a WRONG timestamp + # would silently change a vintage verdict, which is worse than none. + ts=$(printf '%s' "$_fd_json" | tr '\n' ' ' \ + | grep -o '"first_deployed"[[:space:]]*:[[:space:]]*"[^"]*"' \ + | head -1 \ + | sed -n 's/.*"\([^"]*\)"$/\1/p') if [ -n "$ts" ]; then date -u -d "$(printf '%s' "$ts" | cut -c1-19)" +%s 2>/dev/null; return; fi done } From f7e5c5fad9bb214ba564ddfcd92898756ce16315 Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin Date: Mon, 20 Jul 2026 11:44:32 +0300 Subject: [PATCH 06/51] fix(keycloak-configure): patch HelmRelease in release namespace on teardown The pre-delete teardown Job clears the HelmRelease finalizers as its final step, but targeted a hardcoded namespace that does not match the namespace where the release and its RBAC live. The Job's Role and RoleBinding are created in the release namespace and grant patch on the HelmRelease by release name, so the ServiceAccount was Forbidden to patch the HelmRelease in the hardcoded namespace. The Job errored and retried forever, the HelmRelease stuck in Terminating, and the Helm release wedged in "uninstalling", blocking any teardown or reinstall. Target the release name and namespace so the patch matches the RBAC that grants it. Add a helm-unittest pinning the rendered namespace. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin (cherry picked from commit fd568dee87cd190c36d76a89b54a5ba88925bffc) --- .../keycloak-configure/templates/delete.yaml | 2 +- .../keycloak-configure/tests/delete_test.yaml | 58 +++++++++++++++++++ 2 files changed, 59 insertions(+), 1 deletion(-) create mode 100644 packages/system/keycloak-configure/tests/delete_test.yaml diff --git a/packages/system/keycloak-configure/templates/delete.yaml b/packages/system/keycloak-configure/templates/delete.yaml index 99abd6e67e..f499c77f6d 100644 --- a/packages/system/keycloak-configure/templates/delete.yaml +++ b/packages/system/keycloak-configure/templates/delete.yaml @@ -39,7 +39,7 @@ spec: done done - kubectl patch hr keycloak-configure -n cozy-system --type=merge -p '{"metadata":{"finalizers":[]}}' + kubectl patch hr {{ .Release.Name }} -n {{ .Release.Namespace }} --type=merge -p '{"metadata":{"finalizers":[]}}' --- diff --git a/packages/system/keycloak-configure/tests/delete_test.yaml b/packages/system/keycloak-configure/tests/delete_test.yaml new file mode 100644 index 0000000000..9e740d5b83 --- /dev/null +++ b/packages/system/keycloak-configure/tests/delete_test.yaml @@ -0,0 +1,58 @@ +suite: flux teardown job patches the HelmRelease in the release namespace + +# The pre-delete teardown Job clears the HelmRelease finalizers as its last +# step. Its RBAC (Role + RoleBinding, resourceNames: [.Release.Name]) is created +# in .Release.Namespace, so the kubectl patch MUST target that same namespace and +# release name. A hardcoded namespace that differs from the install namespace +# makes the ServiceAccount Forbidden to patch the HelmRelease, the Job retries +# forever, and the Helm release wedges in "uninstalling". +release: + name: keycloak-configure + namespace: cozy-keycloak +tests: + - it: teardown patches the HelmRelease using the release name and namespace + template: templates/delete.yaml + documentSelector: + path: kind + value: Job + set: + _cluster: + root-host: example.com + kube-root-ca: "" + asserts: + - matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: kubectl patch hr keycloak-configure -n cozy-keycloak + + - it: teardown never targets a namespace other than the release namespace + template: templates/delete.yaml + documentSelector: + path: kind + value: Job + set: + _cluster: + root-host: example.com + kube-root-ca: "" + asserts: + - notMatchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: patch hr \S+ -n cozy-system + + # Prove the patch is parameterized on .Release.Name / .Release.Namespace + # rather than matching the default install namespace by coincidence. + - it: teardown tracks a non-default release name and namespace + template: templates/delete.yaml + documentSelector: + path: kind + value: Job + release: + name: custom-release + namespace: custom-ns + set: + _cluster: + root-host: example.com + kube-root-ca: "" + asserts: + - matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: kubectl patch hr custom-release -n custom-ns From 09f4cd10da8b1d364d18c3880380c6351a2da496 Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Mon, 27 Jul 2026 21:41:57 +0500 Subject: [PATCH 07/51] fix(ci): only overlay current-main images on main-based PRs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The PR finalize job pulls cozystack-packages:main and repoints every package the PR did not rebuild at current-main images. On a main-based PR that is the whole point: it stops e2e from testing last-release images for everything outside the build matrix. On a release-line PR it is wrong — the committed refs there are not stale, they are that line`s released digests, and the charts are written against exactly those. So a release-line PR was installing main`s binaries onto its own charts, and the mismatch grows with every commit main gains. #3437 is the demonstration: a one-line change on release-1.6 that deactivates an app failed install deterministically, twice, with SchemaError(...core/v1alpha1.Option.spec): unknown model in reference: "...core~1v1alpha1.OptionSpec" from main`s cozystack-controller serving an aggregated OpenAPI that branch`s charts cannot validate against. Nothing in the PR was broken; the lane was. Left alone this makes every 1.6 backport look red, which is when release-branch PRs are busiest. Both overlay steps are now gated on `github.base_ref == main`, so a release-line PR keeps its committed digests — the behaviour that predates the overlay. Retargeting the overlay at a per-line artifact would be better but is not possible today: build-main.yaml publishes only cozystack-packages:main, and the registry carries no release-* equivalent (verified against the packages repo`s tag list: `main` plus per-PR tags, nothing else). hack/overlay-main-images_test.bats pins the wiring per step, so adding a third overlay step without the guard fails the suite. Mutation-checked by removing one guard. 12/12 green; actionlint clean. Assisted-By: Claude Signed-off-by: Myasnikov Daniil (cherry picked from commit fea40d6314424a4105d31c7ee77f818062f5bc1a) --- .github/workflows/pull-requests.yaml | 17 ++++++++++++++++ hack/overlay-main-images_test.bats | 29 ++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/.github/workflows/pull-requests.yaml b/.github/workflows/pull-requests.yaml index 7bb448248d..753ed41ec1 100644 --- a/.github/workflows/pull-requests.yaml +++ b/.github/workflows/pull-requests.yaml @@ -377,7 +377,20 @@ jobs: # pr.patch e2e applies — already reflect the overlay. A missing artifact # (build-main not yet green) is a no-op: packages keep their committed # refs (prior behaviour). See hack/overlay-main-images.sh. + # + # ONLY for PRs based on main. On a release-line PR the committed refs are + # not stale — they are that line's released digests, which is exactly what + # its charts are written against. Overlaying main's images there deploys + # main's binaries onto a release branch's charts, and the mismatch grows + # with every commit main gains: #3437 (a one-line change on release-1.6) + # failed install deterministically because main's cozystack-controller + # served an aggregated OpenAPI its charts could not validate against + # (`unknown model in reference: …v1alpha1.OptionSpec`), which reads as a + # broken PR while nothing in the PR is broken. Retargeting the overlay at a + # per-line artifact is not possible today: build-main.yaml publishes only + # cozystack-packages:main, with no release-* equivalent. - name: Pull current-main packages tree + if: github.base_ref == 'main' run: | rm -rf _out/mainpkgs mkdir -p _out/mainpkgs @@ -391,6 +404,10 @@ jobs: # so an unexpected failure must degrade to the committed refs (prior # behaviour), never block the PR. overlay-main-images.sh already handles # the expected paths (missing artifact, drift, absent unit) internally. + # Guarded on the base branch for the same reason as the pull above; the + # script would treat the absent artifact as a no-op anyway, but a + # release-line PR should not depend on that to avoid main's images. + if: github.base_ref == 'main' run: | hack/overlay-main-images.sh _out/mainpkgs "$BUILD_MATRIX" "$PR_TOUCHED" \ || echo "::warning::overlay-main-images failed; unbuilt packages keep committed refs" diff --git a/hack/overlay-main-images_test.bats b/hack/overlay-main-images_test.bats index 0f30265537..31bbe95dd5 100644 --- a/hack/overlay-main-images_test.bats +++ b/hack/overlay-main-images_test.bats @@ -147,3 +147,32 @@ grep -q 'present:main@sha256:bbbb' packages/apps/present/images/present.tag [ ! -e packages/apps/ghost/images/ghost.tag ] } + +# ── workflow wiring ───────────────────────────────────────────────────────── +# The script is only half the mechanism; WHEN the workflow invokes it decides +# which branch's images a PR gets. Both steps must be gated on the base branch, +# so a release-line PR keeps that line's committed digests instead of running +# main's binaries against its charts. + +@test "the overlay steps are gated on a main base branch" { + root=$(pwd) + wf="$root/.github/workflows/pull-requests.yaml" + [ -f "$wf" ] + + # Executable lines only: a commented-out guard must not satisfy this. + code="$(grep -v '^[[:space:]]*#' "$wf")" + + # Each overlay step name must be followed by the base-branch guard before the + # step's `run:` — assert per step, so adding a third unguarded step is caught. + for step in "Pull current-main packages tree" "Overlay current-main refs for unbuilt packages"; do + block="$(printf '%s\n' "$code" | awk -v s=" - name: $step" ' + $0 == s { inside = 1; next } + /^ - name: / { inside = 0 } + inside')" + [ -n "$block" ] || { echo "step not found in $wf: $step" >&2; exit 1; } + printf '%s\n' "$block" | grep -qF "if: github.base_ref == 'main'" || { + echo "step '$step' is not gated on the base branch; a release-line PR would" >&2 + echo "get main's images overlaid onto that line's charts." >&2 + exit 1; } + done +} From e226b85f86c053e55739a9c613218f21996147db Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Wed, 29 Jul 2026 17:29:43 +0500 Subject: [PATCH 08/51] fix(ci): overlay images from the PR base branch, and publish per-line artifacts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Supersedes the interim guard in this branch`s first commit, which skipped the overlay entirely for release-line PRs. Skipping fixed the wrong images but left those PRs testing their line`s last release: for any package the PR did not rebuild, the committed ref is the released digest, so a component changed by an earlier backport was exercised as its pre-backport binary until the next rc. The overlay now reads `cozystack-packages:` instead of always `:main`, and build-release.yaml publishes that artifact for every maintained `release-.` branch the way build-main.yaml does for main: images tagged with the branch, and the whole packages tree pushed with each reference digest-pinned to what the run just built. Each base branch therefore has its own generation to overlay from, which is what the original bug was really about — #3437 failed install because main`s cozystack-controller served an aggregated OpenAPI release-1.6`s charts could not validate against. Three deliberate choices: * The trigger matches line branches only (`release-[0-9]+.[0-9]+`). The per-release and rc staging branches promote-rc.yaml and tags.yaml create (release-1.6.1, release-1.6.0-rc.4) must not trigger a full rebuild — their images come from the tag build, and rebuilding them would be waste. * WRITE_CACHE stays 0. CACHE_REGISTRY/:buildcache is a single ref per image and build-main.yaml is deliberately its only, serialized writer so concurrent builds cannot race on the cache manifest (the 409 class #2711 fixed for image tags). A line build can overlap a main build, so writing here would reintroduce that race. Line builds read the cache. * A missing artifact still degrades to committed refs, but on a release line it now emits a ::warning:: naming the branch. Silent degradation is indistinguishable from a working overlay, which is how a mis-specified branch filter would hide for a whole release cycle. Cost: one `make build` per push to a maintained line, i.e. per merged backport. hack/overlay-main-images_test.bats pins the artifact tag to the base branch, rejects a hardcoded :main in either overlay step, and pins build-release.yaml`s branch filter, image tag and WRITE_CACHE=0. Mutation-checked: restoring :main, setting WRITE_CACHE=1, and broadening the filter to release-* each fail a test. 13/13 green; actionlint and zizmor clean. Assisted-By: Claude Signed-off-by: Myasnikov Daniil (cherry picked from commit 2330d6f3ca42306da92190b5e9dddf0bc1054fcd) --- .github/workflows/build-release.yaml | 116 +++++++++++++++++++++++++++ .github/workflows/pull-requests.yaml | 57 ++++++++----- hack/overlay-main-images_test.bats | 78 +++++++++++++----- 3 files changed, 210 insertions(+), 41 deletions(-) create mode 100644 .github/workflows/build-release.yaml diff --git a/.github/workflows/build-release.yaml b/.github/workflows/build-release.yaml new file mode 100644 index 0000000000..c70aa92d2e --- /dev/null +++ b/.github/workflows/build-release.yaml @@ -0,0 +1,116 @@ +name: Build release line + +# Publishes the per-line equivalent of what build-main.yaml publishes for main: +# every image tagged with the line's branch name, and the whole packages tree as +# the `cozystack-packages:` OCI artifact with each image reference +# digest-pinned to the images this run just pushed (`make build` -> +# packages/core/installer image-packages does the `flux push artifact`). +# +# Why it exists: pull-requests.yaml overlays image refs for every package a PR +# did NOT rebuild, so that e2e and the installer do not test last-release images +# for everything outside the PR's build matrix. That overlay had only +# `cozystack-packages:main` to read, which meant a release-line PR was handed +# MAIN's binaries to run against its own line's charts — a cross-generation mix +# that failed deterministically (see #3437: main's cozystack-controller served an +# aggregated OpenAPI release-1.6's charts could not validate against). With this +# artifact published per line, the overlay reads the PR's own base branch and the +# generations match. +# +# Cost: one full `make build` per push to a maintained line, i.e. per merged +# backport. That is the price of release-line PRs testing their line's tip rather +# than its last release. + +env: + # Same per-CI registry as pull-requests.yaml and build-main.yaml. + REGISTRY: iad.ocir.io/idyksih5sir9/cozystack + +on: + push: + # Maintained LINE branches only: `release-.`. The `+` quantifies + # the preceding character class, and `.` is literal, so this matches + # release-1.6 but NOT the per-release staging branches promote-rc.yaml creates + # (release-1.6.1) nor rc staging branches (release-1.6.0-rc.4) — building + # those would be pure waste, since their images are built by tags.yaml. + branches: ['release-[0-9]+.[0-9]+'] + paths-ignore: + - 'docs/**' + +# Per line: a newer push to the same line supersedes an in-flight build, but +# different lines build independently. +concurrency: + group: build-release-${{ github.ref_name }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + build-line: + name: Build ${{ github.ref_name }} images and packages artifact + runs-on: oracle-vm-24cpu-96gb-x86-64 + timeout-minutes: 120 + permissions: + contents: read + packages: write + + steps: + - name: Checkout code + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + fetch-depth: 0 + fetch-tags: true + # This workflow holds packages:write and an OCIR push token; it never + # pushes git, so don't leave the checkout's GITHUB_TOKEN in .git/config + # where a later build step could read it. + persist-credentials: false + + # Ephemeral runners lack the flux CLI the installer's image-packages step + # shells out to; install if absent (idempotent). + - name: Set up build toolchain + run: | + command -v flux >/dev/null \ + || curl -fsSL https://fluxcd.io/install.sh | sudo bash + + - name: Set up Docker config + run: | + if [ -d ~/.docker ]; then + cp -r ~/.docker "${{ runner.temp }}/.docker" + fi + + - name: Login to OCIR + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3 + with: + username: ${{ secrets.OCIR_USER }} + password: ${{ secrets.OCIR_TOKEN }} + registry: iad.ocir.io + env: + DOCKER_CONFIG: ${{ runner.temp }}/.docker + + # Same isolated buildkit as build-main.yaml, so a line build never contends + # with PR builds on the shared embedded builder. + - name: Set up Buildx (docker-container driver) + id: buildx + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 + with: + driver: docker-container + env: + DOCKER_CONFIG: ${{ runner.temp }}/.docker + + - name: Build line images and publish the packages artifact + run: make build + env: + DOCKER_CONFIG: ${{ runner.temp }}/.docker + BUILDER: ${{ steps.buildx.outputs.name }} + # READ the shared mode=max cache, never write it. CACHE_REGISTRY/: + # buildcache is a single ref per image and build-main.yaml is + # deliberately its only writer, serialized, precisely so concurrent + # builds cannot race on the cache manifest (the 409 collision class + # #2711 fixed for image tags). A line build can overlap a main build, so + # writing here would reintroduce exactly that race. + WRITE_CACHE: '0' + # Floating per-line handle: images and the packages artifact both land + # under the branch name (e.g. :release-1.6). Versioned and :latest tags + # belong to releases (tags.yaml / finalize), never to this workflow. + IMAGE_TAG: ${{ github.ref_name }} + PUBLISH_VERSIONED: '0' + PUBLISH_FLOATING: '0' diff --git a/.github/workflows/pull-requests.yaml b/.github/workflows/pull-requests.yaml index 753ed41ec1..a5056ca00e 100644 --- a/.github/workflows/pull-requests.yaml +++ b/.github/workflows/pull-requests.yaml @@ -378,36 +378,49 @@ jobs: # (build-main not yet green) is a no-op: packages keep their committed # refs (prior behaviour). See hack/overlay-main-images.sh. # - # ONLY for PRs based on main. On a release-line PR the committed refs are - # not stale — they are that line's released digests, which is exactly what - # its charts are written against. Overlaying main's images there deploys - # main's binaries onto a release branch's charts, and the mismatch grows - # with every commit main gains: #3437 (a one-line change on release-1.6) - # failed install deterministically because main's cozystack-controller - # served an aggregated OpenAPI its charts could not validate against - # (`unknown model in reference: …v1alpha1.OptionSpec`), which reads as a - # broken PR while nothing in the PR is broken. Retargeting the overlay at a - # per-line artifact is not possible today: build-main.yaml publishes only - # cozystack-packages:main, with no release-* equivalent. - - name: Pull current-main packages tree - if: github.base_ref == 'main' + # Read the artifact for the PR's OWN base branch, not always main. Overlaying + # main's images onto a release-line PR mixes generations: main's binaries + # against that line's charts. #3437 (a one-line change on release-1.6) failed + # install deterministically that way, because main's cozystack-controller + # served an aggregated OpenAPI release-1.6's charts could not validate + # against (`unknown model in reference: …v1alpha1.OptionSpec`) — a broken + # lane reading as a broken PR. build-release.yaml publishes + # cozystack-packages: for every maintained release-. + # branch, so each base branch has its own artifact to read. + # + # A missing artifact stays a no-op (packages keep their committed refs), which + # also covers a line whose first build-release run has not landed yet. + - name: Pull base-branch packages tree run: | rm -rf _out/mainpkgs mkdir -p _out/mainpkgs - flux pull artifact "oci://${REGISTRY}/cozystack-packages:main" \ - --output _out/mainpkgs \ - || { echo "cozystack-packages:main pull failed — keeping committed refs"; rm -rf _out/mainpkgs; } + if ! flux pull artifact "oci://${REGISTRY}/cozystack-packages:${BASE_REF}" \ + --output _out/mainpkgs; then + rm -rf _out/mainpkgs + # Degrading to committed refs is safe, but SAY so: on a release line a + # missing artifact means build-release.yaml has not published this line + # yet (first build pending, or its branch filter does not match this + # branch), and the PR is then testing the line's last release instead of + # its tip. Silent would look identical to a working overlay. + if [ "${BASE_REF}" = "main" ]; then + echo "cozystack-packages:${BASE_REF} pull failed — keeping committed refs" + else + echo "::warning title=No packages artifact for ${BASE_REF}::Keeping committed refs, so unbuilt packages use this line's last release rather than its tip. Check that build-release.yaml has run for ${BASE_REF}." + fi + fi env: DOCKER_CONFIG: ${{ runner.temp }}/.docker - - name: Overlay current-main refs for unbuilt packages + # Env-passed, not interpolated into the script: base_ref is repo-controlled + # here (a branch name in this repo), but the workflow-injection hygiene + # applies uniformly. + BASE_REF: ${{ github.base_ref }} + - name: Overlay base-branch refs for unbuilt packages # Best-effort: this only improves WHICH images e2e and the installer use, # so an unexpected failure must degrade to the committed refs (prior # behaviour), never block the PR. overlay-main-images.sh already handles - # the expected paths (missing artifact, drift, absent unit) internally. - # Guarded on the base branch for the same reason as the pull above; the - # script would treat the absent artifact as a no-op anyway, but a - # release-line PR should not depend on that to avoid main's images. - if: github.base_ref == 'main' + # the expected paths (missing artifact, drift, absent unit) internally — + # including the empty directory the step above leaves when a line has no + # artifact yet. run: | hack/overlay-main-images.sh _out/mainpkgs "$BUILD_MATRIX" "$PR_TOUCHED" \ || echo "::warning::overlay-main-images failed; unbuilt packages keep committed refs" diff --git a/hack/overlay-main-images_test.bats b/hack/overlay-main-images_test.bats index 31bbe95dd5..6953de9edb 100644 --- a/hack/overlay-main-images_test.bats +++ b/hack/overlay-main-images_test.bats @@ -149,30 +149,70 @@ } # ── workflow wiring ───────────────────────────────────────────────────────── -# The script is only half the mechanism; WHEN the workflow invokes it decides -# which branch's images a PR gets. Both steps must be gated on the base branch, -# so a release-line PR keeps that line's committed digests instead of running -# main's binaries against its charts. +# The script is only half the mechanism; WHICH artifact the workflow hands it +# decides which generation of images a PR gets. A release-line PR must read its +# own line's artifact — reading main's puts main's binaries against that line's +# charts, which failed install deterministically (#3437). -@test "the overlay steps are gated on a main base branch" { +@test "the overlay reads the artifact for the PR's own base branch" { root=$(pwd) wf="$root/.github/workflows/pull-requests.yaml" [ -f "$wf" ] - # Executable lines only: a commented-out guard must not satisfy this. + # Executable lines only: a comment mentioning the tag must not satisfy this. code="$(grep -v '^[[:space:]]*#' "$wf")" - # Each overlay step name must be followed by the base-branch guard before the - # step's `run:` — assert per step, so adding a third unguarded step is caught. - for step in "Pull current-main packages tree" "Overlay current-main refs for unbuilt packages"; do - block="$(printf '%s\n' "$code" | awk -v s=" - name: $step" ' - $0 == s { inside = 1; next } - /^ - name: / { inside = 0 } - inside')" - [ -n "$block" ] || { echo "step not found in $wf: $step" >&2; exit 1; } - printf '%s\n' "$block" | grep -qF "if: github.base_ref == 'main'" || { - echo "step '$step' is not gated on the base branch; a release-line PR would" >&2 - echo "get main's images overlaid onto that line's charts." >&2 - exit 1; } - done + block="$(printf '%s\n' "$code" | awk ' + $0 == " - name: Pull base-branch packages tree" { inside = 1; next } + /^ - name: / { inside = 0 } + inside')" + [ -n "$block" ] || { echo "the base-branch packages pull step is missing from $wf" >&2; exit 1; } + + # The artifact tag must come from the base branch, passed via env. + printf '%s\n' "$block" | grep -qF 'cozystack-packages:${BASE_REF}' || { + echo "the packages artifact tag is not the PR's base branch. Reading a fixed" >&2 + echo "tag (e.g. :main) hands a release-line PR another generation's images." >&2 + exit 1; } + printf '%s\n' "$block" | grep -qF 'BASE_REF: ${{ github.base_ref }}' || { + echo "BASE_REF is not wired to github.base_ref in the pull step's env." >&2; exit 1; } + + # A hardcoded :main anywhere in the two overlay steps defeats the point. + overlay="$(printf '%s\n' "$code" | awk ' + $0 == " - name: Overlay base-branch refs for unbuilt packages" { inside = 1; next } + /^ - name: / { inside = 0 } + inside')" + [ -n "$overlay" ] || { echo "the overlay step is missing from $wf" >&2; exit 1; } + printf '%s\n%s\n' "$block" "$overlay" | grep -qF 'cozystack-packages:main' && { + echo "an overlay step still pins cozystack-packages:main" >&2; exit 1; } + return 0 +} + +@test "every maintained release line publishes its own packages artifact" { + root=$(pwd) + wf="$root/.github/workflows/build-release.yaml" + [ -f "$wf" ] || { + echo "build-release.yaml is missing: without a per-line artifact the overlay" >&2 + echo "above degrades to a no-op on release branches, so their PRs test the" >&2 + echo "line's last release rather than its tip." >&2 + exit 1; } + + code="$(grep -v '^[[:space:]]*#' "$wf")" + + # Line branches only. The per-release and rc staging branches promote-rc.yaml + # and tags.yaml create (release-1.6.1, release-1.6.0-rc.4) must NOT trigger a + # full rebuild — their images come from the tag build. + printf '%s\n' "$code" | grep -qF "branches: ['release-[0-9]+.[0-9]+']" || { + echo "build-release.yaml does not restrict its trigger to release-." >&2; exit 1; } + + # The artifact and image tag must be the branch, or the overlay cannot find it. + printf '%s\n' "$code" | grep -qF 'IMAGE_TAG: ${{ github.ref_name }}' || { + echo "build-release.yaml does not tag images/artifact with the branch name" >&2; exit 1; } + + # It must not write the shared mode=max cache: that ref has exactly one + # serialized writer (build-main.yaml) so concurrent builds cannot race on the + # cache manifest. A line build can overlap a main build. + printf '%s\n' "$code" | grep -qF "WRITE_CACHE: '0'" || { + echo "build-release.yaml must set WRITE_CACHE: '0' — writing the shared" >&2 + echo ":buildcache ref races build-main.yaml on the cache manifest." >&2 + exit 1; } } From 042c69cb30d254cae8918fc9119b32231a4c7d8a Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Thu, 23 Jul 2026 14:01:12 +0500 Subject: [PATCH 09/51] chore(release): don't activate kubernetes-nodes on the release-1.6 line kubernetes-nodes isn't ready to launch on 1.6 yet. Remove only its include from the iaas platform bundle so the package still ships but stays inactive; restore the line when the feature is ready. The package, API types and RD package are kept intact. Assisted-By: Claude Signed-off-by: Myasnikov Daniil --- packages/core/platform/templates/bundles/iaas.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/core/platform/templates/bundles/iaas.yaml b/packages/core/platform/templates/bundles/iaas.yaml index f2c2c8b728..25215c321a 100644 --- a/packages/core/platform/templates/bundles/iaas.yaml +++ b/packages/core/platform/templates/bundles/iaas.yaml @@ -157,7 +157,6 @@ {{include "cozystack.platform.package.default" (list "cozystack.capi-provider-infra-kubevirt" $) }} {{include "cozystack.platform.package.default" (list "cozystack.bucket-application" $) }} {{include "cozystack.platform.package" (list "cozystack.kubernetes-application" "kubevirt" $) }} -{{include "cozystack.platform.package" (list "cozystack.kubernetes-nodes-application" "kubevirt" $) }} {{include "cozystack.platform.package" (list "cozystack.virtualprivatecloud-application" "kubevirt" $) }} {{include "cozystack.platform.package" (list "cozystack.vm-disk-application" "kubevirt" $) }} {{include "cozystack.platform.package" (list "cozystack.vm-instance-application" "kubevirt" $) }} From 6fe15b0bf5eb218298fb54e3b039c07027783cfd Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Tue, 4 Aug 2026 13:34:55 +0500 Subject: [PATCH 10/51] ci(release): carry the finalize fixes onto the 1.6 line MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit release-1.6 was cut at v1.6.0 on 2026-07-22, and the finalize fixes landed on main afterwards, so shipping v1.6.1 from this line would repeat two defects v1.6.0 hit. Workflows run from the ref they fire on: a promote PR based on release-1.6 runs THIS file, not main's. Carries three upstream commits, after which this file is byte-identical to main, so future backports touching it will not conflict: 01e1e7188 persist-credentials: false on the checkout f49d54a68 publish the release with the merged changelog as its body ba67fea7d drop paths-ignore from the trigger Without the first, the checkout persists GITHUB_TOKEN as an http.extraheader that silently wins over the app token injected by git remote set-url. The stable tag then pushes as GITHUB_TOKEN, which creates no workflow run, so tags.yaml never fires and its generate-changelog and update-website-docs backstops stay silent. That is exactly what happened to v1.6.0. Without the second, the release publishes with the draft's body ("Promoted from vX.Y.Z-rc.N"). The two compound on a maintenance line: a patch's changelog is committed to release-1.6 and never reaches main, and update-releasenotes.yaml only watches main, so nothing would ever sync it. v1.6.1 would ship with placeholder release notes permanently. The third is latent rather than active — a promote PR carrying only docs/changelogs/vX.Y.Z.md would be dropped by the filter, producing no finalize run, no tag and no error. v1.6.1 will carry tag-string rewrites so it would not have fired, but the filter has no remaining purpose now that the promote PR always carries a changelog. actionlint and zizmor clean; no bats suite references this workflow, and hack/promote-gate-contract.bats does not exist on this line. Assisted-By: Claude Signed-off-by: Myasnikov Daniil --- .github/workflows/pull-requests-release.yaml | 56 ++++++++++++++++++-- 1 file changed, 53 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pull-requests-release.yaml b/.github/workflows/pull-requests-release.yaml index 5abdd20ca1..fc1608a343 100644 --- a/.github/workflows/pull-requests-release.yaml +++ b/.github/workflows/pull-requests-release.yaml @@ -3,8 +3,15 @@ name: "Releasing PR" on: pull_request: types: [closed] - paths-ignore: - - 'docs/**/*' + # No paths filter. A `paths-ignore: docs/**/*` used to live here as an + # optimization, but the promote PR now carries docs/changelogs/vX.Y.Z.md — + # and `Prepare stable branch` explicitly tolerates producing no tag-string + # changes ("digests already stable"). That combination can yield a docs-only + # promote PR, which the filter would drop: no finalize run, so no tag, no + # release, and no error anywhere. Filtering is left entirely to the `if:` + # gate below (merged + `release` label + release-bot author), which is what + # actually decides whether this workflow should do anything. The cost is a + # skipped job on unrelated closed PRs. # Cancel in‑flight runs for the same PR when a new push arrives. concurrency: @@ -74,6 +81,15 @@ jobs: uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: fetch-depth: 0 + # Do NOT persist GITHUB_TOKEN as http.extraheader. Every tag/branch push + # below re-injects the app token via `git remote set-url`, but a persisted + # extraheader silently wins over the URL credential — so those pushes + # would authenticate as GITHUB_TOKEN, and a GITHUB_TOKEN push creates no + # workflow run (GitHub anti-recursion). That is exactly why v1.6.0's + # stable tag triggered no tags.yaml run (the generate-changelog and + # update-website-docs backstops never fired). With this false, the set-url + # app token authenticates, matching how promote-rc.yaml already pushes. + persist-credentials: false # Create the release tag at the merge commit — write-once. # @@ -312,6 +328,39 @@ jobs: console.log(`🏷️ ${tag} is the highest version, make_latest: true`); } + // Release body comes from the changelog that merged with THIS PR + // (promote-rc.yaml commits it onto the release-X.Y.Z branch), read + // from the merge commit already checked out here. + // + // Deliberately not left to update-releasenotes.yaml: that workflow + // triggers on pushes to main touching docs/changelogs/v*.md, so it + // would race this job on the very same push — and if it won, it + // would find no published release yet, skip, and never fire again, + // leaving the release body permanently empty. It also only watches + // main, so a patch release whose promote PR targets release-X.Y + // would never be synced at all. Reading the file here is both + // race-free and branch-agnostic; update-releasenotes.yaml remains + // for later maintainer edits and manual re-syncs. + const fs = require('fs'); + const changelogPath = `docs/changelogs/${tag}.md`; + let body; + if (fs.existsSync(changelogPath)) { + const raw = fs.readFileSync(changelogPath, 'utf8'); + // Last line of defence before the body becomes public and + // immutable-by-convention. promote-rc.yaml validates the changelog + // structurally, but it can also arrive by hand-commit, and an + // empty or whitespace-only file here would blank the release notes + // more thoroughly than publishing the draft's body would. + if (raw.trim().length === 0) { + console.log(`::warning::${changelogPath} is empty or whitespace-only — refusing to use it as the release body for ${tag}. Publishing with the draft's existing body instead.`); + } else { + body = raw; + console.log(`📝 Release body from ${changelogPath} (${body.length} bytes)`); + } + } else { + console.log(`::warning::${changelogPath} not found in the merge commit — publishing ${tag} with the draft's existing body. Add the changelog and re-run 'Update Release Notes' to backfill it.`); + } + // Publish the release await github.rest.repos.updateRelease({ owner: context.repo.owner, @@ -319,7 +368,8 @@ jobs: release_id: draft.id, draft: false, prerelease: isRc, - make_latest: makeLatest + make_latest: makeLatest, + ...(body ? { body } : {}) }); console.log(`🚀 Published release ${tag}`); From 8c4947ba448887517ac959a0cbe4ac24a90d5736 Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Tue, 4 Aug 2026 13:37:17 +0500 Subject: [PATCH 11/51] fix(kubernetes): render the talos-reconcile Job for the default md0 group The Job in templates/talos/talos-reconcile-job.yaml is the only producer of the TalosConfigTemplate that each worker MachineDeployment names in spec.template.spec.bootstrap.configRef; cluster.yaml renders the reference but deliberately not the object. Its loop read .Values.nodeGroups directly, while the MachineDeployment loop reads the effective set through the kubernetes.nodeGroups helper. For a cluster that supplies no nodeGroups the two disagree: the helper's else-branch emits the built-in md0 group, so the md0 MachineDeployment renders, but the raw map is empty and no Job renders at all. Every Machine the cluster-autoscaler adds to md0 then blocks indefinitely on a TalosConfigTemplate that nothing will ever create, and the KamajiControlPlane spec.network.certSANs patch the same Job performs is skipped too. Range over the helper so the Job set tracks the MachineDeployment set exactly. No change for a cluster that declares its own groups, and md0 stays removable: the helper's if-branch keeps a user-supplied map authoritative. The gap survived because every helm-unittest fixture and e2e suite declares md0 explicitly. tests/nodegroups_default_test.yaml does render the empty-nodeGroups case but lists only templates/cluster.yaml, so it never looked at the Job. The new suite pins both halves of the helper contract and fails without this fix. Fixes #3504 Assisted-By: Claude Signed-off-by: Myasnikov Daniil (cherry picked from commit 4e9c5ce8ca3b71fd1f4aac0beadbedb0856635b9) --- .../templates/talos/talos-reconcile-job.yaml | 9 +- .../talos_reconcile_nodegroups_test.yaml | 131 ++++++++++++++++++ 2 files changed, 139 insertions(+), 1 deletion(-) create mode 100644 packages/apps/kubernetes/tests/talos_reconcile_nodegroups_test.yaml diff --git a/packages/apps/kubernetes/templates/talos/talos-reconcile-job.yaml b/packages/apps/kubernetes/templates/talos/talos-reconcile-job.yaml index 9e2ed31448..8422f81aa4 100644 --- a/packages/apps/kubernetes/templates/talos/talos-reconcile-job.yaml +++ b/packages/apps/kubernetes/templates/talos/talos-reconcile-job.yaml @@ -485,7 +485,14 @@ spec: *management* cluster-domain, sourced from .Values._cluster (populated by cozystack-controller from the platform config). */}} {{- $mgmtClusterDomain := (index .Values._cluster "cluster-domain") | default "cozy.local" }} -{{- range $groupName, $group := .Values.nodeGroups }} +{{- /* Iterate the EFFECTIVE node groups (the kubernetes.nodeGroups helper), + never the raw .Values.nodeGroups map. The built-in md0 group lives in + the helper's else-branch, so a cluster that supplies no nodeGroups has + an empty raw map: ranging over it renders zero Jobs, leaving the md0 + MachineDeployment below pointing at a TalosConfigTemplate that only + this Job ever creates. CAPI then blocks every Machine the autoscaler + adds. Must stay in lockstep with the loop in cluster.yaml. */}} +{{- range $groupName, $group := (include "kubernetes.nodeGroups" $ | fromYaml) }} {{- /* Mirror the kubelet-reservation computation from cluster.yaml so the worker machineconfig the Job applies carries the same numbers the MachineDeployment validator was happy with. Auto-computed diff --git a/packages/apps/kubernetes/tests/talos_reconcile_nodegroups_test.yaml b/packages/apps/kubernetes/tests/talos_reconcile_nodegroups_test.yaml new file mode 100644 index 0000000000..03a6f13bcc --- /dev/null +++ b/packages/apps/kubernetes/tests/talos_reconcile_nodegroups_test.yaml @@ -0,0 +1,131 @@ +suite: talos-reconcile Job covers every effective node group + +# Regression guard for cozystack/cozystack#3504. The Job in +# templates/talos/talos-reconcile-job.yaml is the only producer of the +# TalosConfigTemplate that each worker MachineDeployment names in +# `spec.template.spec.bootstrap.configRef` (cluster.yaml renders the reference +# but not the object). Its loop must therefore iterate the same effective group +# set as the MachineDeployment loop — the kubernetes.nodeGroups helper — not the +# raw .Values.nodeGroups map. When the raw map was read, a cluster that supplied +# no nodeGroups got the helper's built-in md0 MachineDeployment and zero Jobs, so +# every Machine the autoscaler added to md0 blocked forever on a +# TalosConfigTemplate nothing would ever create, and the KamajiControlPlane +# certSANs patch the same Job performs never ran either. +# +# The two cases below pin both halves of the helper contract: the default group +# gets a Job (the bug), and a user-supplied group set stays authoritative — md0 +# must NOT reappear alongside it, which is the removability #2936 introduced. +# +# Suite-level `templates` is load-bearing: it limits the render to this one +# template. Declaring `templates` per test does not filter the render, and the +# unrelated `lookup` in templates/helmreleases/csi.yaml then fails the suite. + +templates: + - templates/talos/talos-reconcile-job.yaml + +tests: + - it: renders a Job for the built-in md0 group when nodeGroups is empty + release: + name: test-k8s + namespace: tenant-test + set: + _namespace: + etcd: etcd + ingress: nginx + host: example.com + _cluster: + cluster-domain: cozy.local + # nodeGroups intentionally left at the chart default ({}) so the helper + # emits the built-in md0 — the case that rendered no Job at all. + asserts: + # 6 scaffolding documents (ServiceAccount, Role, CiliumNetworkPolicy, + # RoleBinding, ClusterRole, ClusterRoleBinding) plus one Job for md0. + # Before the fix this was 6: the Job was missing entirely. + - hasDocuments: + count: 7 + - isKind: + of: Job + documentIndex: 6 + - matchRegex: + path: metadata.name + pattern: ^test-k8s-talos-reconcile-md0-[0-9a-f]{6}$ + documentIndex: 6 + # The Job applies TalosConfigTemplate ${RELEASE}-${GROUP_NAME}, so + # GROUP_NAME is what ties it to the MachineDeployment's configRef name + # (test-k8s-md0). A Job for the wrong group name would satisfy the + # document count above while leaving the MachineDeployment stuck. + - contains: + path: spec.template.spec.containers[0].env + content: + name: GROUP_NAME + value: md0 + documentIndex: 6 + - matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: 'kind: TalosConfigTemplate' + documentIndex: 6 + # Secondary effect of the missing Job: the KamajiControlPlane certSANs + # patch that adds the live apiserver Service ClusterIP never ran. + - matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: 'patch kamajicontrolplane' + documentIndex: 6 + + - it: user-supplied node groups stay authoritative — one Job each, no md0 + release: + name: test-k8s + namespace: tenant-test + set: + _namespace: + etcd: etcd + ingress: nginx + host: example.com + _cluster: + cluster-domain: cozy.local + nodeGroups: + worker0: + minReplicas: 1 + maxReplicas: 3 + instanceType: u1.medium + diskSize: 20Gi + storageClass: "" + roles: + - ingress-nginx + resources: {} + gpus: [] + kubelet: {} + worker1: + minReplicas: 0 + maxReplicas: 5 + instanceType: u1.medium + diskSize: 20Gi + storageClass: "" + roles: [] + resources: {} + gpus: [] + kubelet: {} + asserts: + # 6 scaffolding documents + exactly two Jobs. A fix that unconditionally + # merged md0 into the loop would render three and fail here. + - hasDocuments: + count: 8 + - matchRegex: + path: metadata.name + pattern: ^test-k8s-talos-reconcile-worker0-[0-9a-f]{6}$ + documentIndex: 6 + - matchRegex: + path: metadata.name + pattern: ^test-k8s-talos-reconcile-worker1-[0-9a-f]{6}$ + documentIndex: 7 + - contains: + path: spec.template.spec.containers[0].env + content: + name: GROUP_NAME + value: worker0 + documentIndex: 6 + - contains: + path: spec.template.spec.containers[0].env + content: + name: GROUP_NAME + value: worker1 + documentIndex: 7 From ab1ec698c54d817ad0a626e6346222e4e10cf2d0 Mon Sep 17 00:00:00 2001 From: Andrey Kolkov Date: Tue, 4 Aug 2026 12:41:43 +0400 Subject: [PATCH 12/51] chore(etcd-operator): bump etcd-operator to v0.5.4 Bump the cozystack etcd-operator packages from v0.5.3 to v0.5.4. The release is a controller bug-fix set with no API, RBAC or values changes: - fix(controllers): derive --initial-cluster-state from phase, not from the seed (cozystack/etcd-operator#355) - fix(controllers): stop exempting the bootstrap seed from self-heal (cozystack/etcd-operator#354) - fix(controllers): extend crash-loop self-heal to memory members (cozystack/etcd-operator#352) - fix(controllers): switch the PDB from maxUnavailable to minAvailable (cozystack/etcd-operator#351) Adaptations: - etcd-operator/Chart.yaml: appVersion v0.5.3 -> v0.5.4 (the manager image tag defaults to .Chart.AppVersion, so this reimages the controller). - etcd-operator/Makefile, etcd-operator-crds/Makefile: ETCD_OPERATOR_REF v0.5.3 -> v0.5.4. - etcd-operator-crds/templates/etcdmembers.yaml: re-vendored at v0.5.4 via `make update`; description-only change to the /scale replicas/selector fields tracking the PDB minAvailable fix. etcdclusters/etcdsnapshots unchanged. - templates/rbac.yaml left as-is: manager-role-rules.yaml is byte-identical between v0.5.3 and v0.5.4. Assisted-By: Claude Signed-off-by: Andrey Kolkov (cherry picked from commit 19f969b2f8ad52ffbde29ad2c051c63d02dd7f65) --- packages/system/etcd-operator-crds/Makefile | 2 +- .../templates/etcdmembers.yaml | 18 ++++++++---------- packages/system/etcd-operator/Chart.yaml | 2 +- packages/system/etcd-operator/Makefile | 2 +- 4 files changed, 11 insertions(+), 13 deletions(-) diff --git a/packages/system/etcd-operator-crds/Makefile b/packages/system/etcd-operator-crds/Makefile index e7a6e765c5..330bf9e23e 100644 --- a/packages/system/etcd-operator-crds/Makefile +++ b/packages/system/etcd-operator-crds/Makefile @@ -4,7 +4,7 @@ export NAMESPACE=cozy-etcd-operator include ../../../hack/package.mk # Pinned ref of github.com/cozystack/etcd-operator to vendor CRDs from. -ETCD_OPERATOR_REF ?= v0.5.3 +ETCD_OPERATOR_REF ?= v0.5.4 # controller-gen writes the CRDs to charts/etcd-operator/crd-bases/ upstream (there # is no config/crd kustomization). Vendor each verbatim and stamp diff --git a/packages/system/etcd-operator-crds/templates/etcdmembers.yaml b/packages/system/etcd-operator-crds/templates/etcdmembers.yaml index 5aa09d51e6..7b1d85ade8 100644 --- a/packages/system/etcd-operator-crds/templates/etcdmembers.yaml +++ b/packages/system/etcd-operator-crds/templates/etcdmembers.yaml @@ -1101,11 +1101,10 @@ spec: replicas: default: 1 description: |- - Replicas exists only because the PodDisruptionBudget controller - traverses Pods' controllerRef looking for /scale on the parent and - fails closed ("does not implement the scale subresource") if it - isn't there. Each EtcdMember represents exactly one Pod; this field - is locked to 1 by validation and cannot be tuned. + Replicas backs the /scale subresource. The operator's own PDB + (integer minAvailable) never resolves scale; kept because + maxUnavailable or percentage budgets over member Pods fail + without it. Locked to 1: an EtcdMember is exactly one Pod. format: int32 maximum: 1 minimum: 1 @@ -1677,15 +1676,14 @@ spec: replicas: description: |- Replicas exposes via /scale "this EtcdMember owns 1 Pod if it has - a PodName, 0 otherwise". Required by the PodDisruptionBudget - controller to derive expectedPods for the cluster's PDB — without - /scale on the Pod controller-ref it sets the PDB to SyncFailed. + a PodName, 0 otherwise". Unused by the operator's own PDB; + scale-resolving budgets go SyncFailed without it. format: int32 type: integer selector: description: |- - Selector exposes the label-selector that matches this member's Pod - via /scale (consumed by the PDB controller; not user-facing). + Selector exposes the label-selector matching this member's Pod via + /scale (for scale-resolving disruption budgets; not user-facing). type: string version: description: |- diff --git a/packages/system/etcd-operator/Chart.yaml b/packages/system/etcd-operator/Chart.yaml index c943505c8b..11f70e5987 100644 --- a/packages/system/etcd-operator/Chart.yaml +++ b/packages/system/etcd-operator/Chart.yaml @@ -3,4 +3,4 @@ name: cozy-etcd-operator description: Cozystack etcd-operator (etcd-operator.cozystack.io/v1alpha2) controller-manager type: application version: 0.0.0 # Placeholder, the actual version will be automatically set during the build process -appVersion: v0.5.3 +appVersion: v0.5.4 diff --git a/packages/system/etcd-operator/Makefile b/packages/system/etcd-operator/Makefile index c7d4764e85..6660aa144d 100644 --- a/packages/system/etcd-operator/Makefile +++ b/packages/system/etcd-operator/Makefile @@ -8,7 +8,7 @@ include ../../../hack/package.mk # domain, VPA), so only the RBAC role is re-vendored; refresh templates/rbac.yaml # by hand from config/rbac/role.yaml at this ref. CRDs live in the sibling # etcd-operator-crds package. -ETCD_OPERATOR_REF ?= v0.5.3 +ETCD_OPERATOR_REF ?= v0.5.4 update: @echo "etcd-operator is a cozystack-authored chart; refresh templates/rbac.yaml" From 3a7df639c960b14b1fa0d8b3a838e4b9853ee035 Mon Sep 17 00:00:00 2001 From: Andrey Kolkov Date: Tue, 4 Aug 2026 13:09:25 +0400 Subject: [PATCH 13/51] test(etcd-operator): pin deployment unittest to v0.5.4 The manager image tag derives from .Chart.AppVersion, so bumping appVersion to v0.5.4 without updating the two image literals in the deployment unittest broke the suite (helm-unittest gates the PR via `make unit-tests`). Update both asserts (manager container image and the agent OPERATOR_IMAGE env) to ghcr.io/cozystack/etcd-operator:v0.5.4. Suite is green again (18/18). Assisted-By: Claude Signed-off-by: Andrey Kolkov (cherry picked from commit 22c1df8b9464921543bd458825c04299bfa6af37) --- packages/system/etcd-operator/tests/deployment_test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/system/etcd-operator/tests/deployment_test.yaml b/packages/system/etcd-operator/tests/deployment_test.yaml index a91d207d3d..893bb8506d 100644 --- a/packages/system/etcd-operator/tests/deployment_test.yaml +++ b/packages/system/etcd-operator/tests/deployment_test.yaml @@ -20,12 +20,12 @@ tests: asserts: - equal: path: spec.template.spec.containers[0].image - value: ghcr.io/cozystack/etcd-operator:v0.5.3 + value: ghcr.io/cozystack/etcd-operator:v0.5.4 - contains: path: spec.template.spec.containers[0].env content: name: OPERATOR_IMAGE - value: ghcr.io/cozystack/etcd-operator:v0.5.3 + value: ghcr.io/cozystack/etcd-operator:v0.5.4 any: true - it: "image.tag overrides the appVersion default for both the container and the agent env" From 2258d2c640409de9b8fd2a2138637f672a7101fc Mon Sep 17 00:00:00 2001 From: Alexey Artamonov Date: Mon, 3 Aug 2026 14:59:15 +0300 Subject: [PATCH 14/51] fix(postgres-operator): align CNPG operator and CRDs to 1.28.2 for PVC resize-deadlock fix Raises the CloudNativePG operator image and its CRDs together to 1.28.2, which carries the PVC resize-deadlock fix (cloudnative-pg#9980 / #9981): after a simultaneous resources+size change the operator deletes the sole primary Pod, leaves the PVC in the resizing class, and never recreates the Pod, wedging the cluster. Upstream publishes a chart only per minor .0/.1, so there is no 1.28.2 chart on the 1.28 line. Following #3526/#3528, patches/cloudnative-pg-1.28.2.patch (applied by make update) raises the vendored chart's appVersion and CRDs from 1.28.1 to 1.28.2 in lockstep; the operator image follows appVersion, so no image.tag pin. Verified #3479-safe: no CRD status field changes between 1.28.1 and 1.28.2 (only the extensions spec grows). Backportable to release-1.6. Signed-off-by: Alexey Artamonov (cherry picked from commit bcb36262efa9f32e472d063871cf1e8af17f88e9) --- packages/system/postgres-operator/Makefile | 1 + .../charts/cloudnative-pg/Chart.yaml | 2 +- .../cloudnative-pg/templates/crds/crds.yaml | 284 +++++++++++- .../patches/cloudnative-pg-1.28.2.patch | 414 ++++++++++++++++++ .../tests/cnpg-version_test.yaml | 38 ++ packages/system/postgres-operator/values.yaml | 18 +- 6 files changed, 734 insertions(+), 23 deletions(-) create mode 100644 packages/system/postgres-operator/patches/cloudnative-pg-1.28.2.patch create mode 100644 packages/system/postgres-operator/tests/cnpg-version_test.yaml diff --git a/packages/system/postgres-operator/Makefile b/packages/system/postgres-operator/Makefile index 9b8f454597..9df330da39 100644 --- a/packages/system/postgres-operator/Makefile +++ b/packages/system/postgres-operator/Makefile @@ -12,5 +12,6 @@ update: helm repo update cnpg helm pull cnpg/cloudnative-pg --untar --untardir charts --version 0.27.1 rm -rf charts/cloudnative-pg/charts + patch --no-backup-if-mismatch -p4 < patches/cloudnative-pg-1.28.2.patch helm pull cnpg/plugin-barman-cloud --untar --untardir charts --version 0.7.0 rm -rf charts/plugin-barman-cloud/charts diff --git a/packages/system/postgres-operator/charts/cloudnative-pg/Chart.yaml b/packages/system/postgres-operator/charts/cloudnative-pg/Chart.yaml index 8ea6169ef8..62490487d8 100644 --- a/packages/system/postgres-operator/charts/cloudnative-pg/Chart.yaml +++ b/packages/system/postgres-operator/charts/cloudnative-pg/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v2 -appVersion: 1.28.1 +appVersion: 1.28.2 dependencies: - alias: monitoring condition: monitoring.grafanaDashboard.create diff --git a/packages/system/postgres-operator/charts/cloudnative-pg/templates/crds/crds.yaml b/packages/system/postgres-operator/charts/cloudnative-pg/templates/crds/crds.yaml index 7982b5e122..1b040e0bc1 100644 --- a/packages/system/postgres-operator/charts/cloudnative-pg/templates/crds/crds.yaml +++ b/packages/system/postgres-operator/charts/cloudnative-pg/templates/crds/crds.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.20.0 + controller-gen.kubebuilder.io/version: v0.20.1 helm.sh/resource-policy: keep name: backups.postgresql.cnpg.io spec: @@ -466,7 +466,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.20.0 + controller-gen.kubebuilder.io/version: v0.20.1 helm.sh/resource-policy: keep name: clusterimagecatalogs.postgresql.cnpg.io spec: @@ -515,6 +515,112 @@ spec: items: description: CatalogImage defines the image and major version properties: + extensions: + description: The configuration of the extensions to be added + items: + description: |- + ExtensionConfiguration is the configuration used to add + PostgreSQL extensions to the Cluster. + properties: + bin_path: + description: |- + A list of directories within the image to be appended to the + PostgreSQL process's `PATH` environment variable. + items: + type: string + type: array + dynamic_library_path: + description: |- + The list of directories inside the image which should be added to dynamic_library_path. + If not defined, defaults to "/lib". + items: + type: string + type: array + env: + description: |- + Env is a list of custom environment variables to be set in the + PostgreSQL process for this extension. It is the responsibility of the + cluster administrator to ensure the variables are correct for the + specific extension. Note that changes to these variables require + a manual cluster restart to take effect. + items: + description: |- + ExtensionEnvVar defines an environment variable for a specific extension + image volume. + properties: + name: + description: |- + Name of the environment variable to be injected into the + PostgreSQL process. + minLength: 1 + pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ + type: string + value: + description: |- + Value of the environment variable. CloudNativePG performs a direct + replacement of this value, with support for placeholder expansion. + The ${`image_root`} placeholder resolves to the absolute mount path + of the extension's volume (e.g., `/extensions/my-extension`). This + is particularly useful for allowing applications or libraries to + locate specific directories within the mounted image. + Unrecognized placeholders are rejected. To include a literal ${...} + in the value, escape it as $${...}. + minLength: 1 + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + extension_control_path: + description: |- + The list of directories inside the image which should be added to extension_control_path. + If not defined, defaults to "/share". + items: + type: string + type: array + image: + description: The image containing the extension. + properties: + pullPolicy: + description: |- + Policy for pulling OCI objects. Possible values are: + Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. + Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. + IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + type: string + reference: + description: |- + Required: Image or artifact reference to be used. + Behaves in the same way as pod.spec.containers[*].image. + Pull secrets will be assembled in the same way as for the container image by looking up node credentials, SA image pull secrets, and pod spec image pull secrets. + More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management to default or override + container images in workload controllers like Deployments and StatefulSets. + type: string + type: object + ld_library_path: + description: The list of directories inside the image + which should be added to ld_library_path. + items: + type: string + type: array + name: + description: The name of the extension, required + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9_]*[a-z0-9])?$ + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map image: description: The image reference type: string @@ -548,7 +654,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.20.0 + controller-gen.kubebuilder.io/version: v0.20.1 helm.sh/resource-policy: keep name: clusters.postgresql.cnpg.io spec: @@ -4913,6 +5019,13 @@ spec: ExtensionConfiguration is the configuration used to add PostgreSQL extensions to the Cluster. properties: + bin_path: + description: |- + A list of directories within the image to be appended to the + PostgreSQL process's `PATH` environment variable. + items: + type: string + type: array dynamic_library_path: description: |- The list of directories inside the image which should be added to dynamic_library_path. @@ -4920,6 +5033,45 @@ spec: items: type: string type: array + env: + description: |- + Env is a list of custom environment variables to be set in the + PostgreSQL process for this extension. It is the responsibility of the + cluster administrator to ensure the variables are correct for the + specific extension. Note that changes to these variables require + a manual cluster restart to take effect. + items: + description: |- + ExtensionEnvVar defines an environment variable for a specific extension + image volume. + properties: + name: + description: |- + Name of the environment variable to be injected into the + PostgreSQL process. + minLength: 1 + pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ + type: string + value: + description: |- + Value of the environment variable. CloudNativePG performs a direct + replacement of this value, with support for placeholder expansion. + The ${`image_root`} placeholder resolves to the absolute mount path + of the extension's volume (e.g., `/extensions/my-extension`). This + is particularly useful for allowing applications or libraries to + locate specific directories within the mounted image. + Unrecognized placeholders are rejected. To include a literal ${...} + in the value, escape it as $${...}. + minLength: 1 + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map extension_control_path: description: |- The list of directories inside the image which should be added to extension_control_path. @@ -4928,7 +5080,7 @@ spec: type: string type: array image: - description: The image containing the extension, required + description: The image containing the extension. properties: pullPolicy: description: |- @@ -4948,9 +5100,6 @@ spec: container images in workload controllers like Deployments and StatefulSets. type: string type: object - x-kubernetes-validations: - - message: An image reference is required - rule: has(self.reference) ld_library_path: description: The list of directories inside the image which should be added to ld_library_path. @@ -4963,7 +5112,6 @@ spec: pattern: ^[a-z0-9]([-a-z0-9_]*[a-z0-9])?$ type: string required: - - image - name type: object type: array @@ -7787,7 +7935,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.20.0 + controller-gen.kubebuilder.io/version: v0.20.1 helm.sh/resource-policy: keep name: databases.postgresql.cnpg.io spec: @@ -8382,7 +8530,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.20.0 + controller-gen.kubebuilder.io/version: v0.20.1 helm.sh/resource-policy: keep name: failoverquorums.postgresql.cnpg.io spec: @@ -8460,7 +8608,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.20.0 + controller-gen.kubebuilder.io/version: v0.20.1 helm.sh/resource-policy: keep name: imagecatalogs.postgresql.cnpg.io spec: @@ -8508,6 +8656,112 @@ spec: items: description: CatalogImage defines the image and major version properties: + extensions: + description: The configuration of the extensions to be added + items: + description: |- + ExtensionConfiguration is the configuration used to add + PostgreSQL extensions to the Cluster. + properties: + bin_path: + description: |- + A list of directories within the image to be appended to the + PostgreSQL process's `PATH` environment variable. + items: + type: string + type: array + dynamic_library_path: + description: |- + The list of directories inside the image which should be added to dynamic_library_path. + If not defined, defaults to "/lib". + items: + type: string + type: array + env: + description: |- + Env is a list of custom environment variables to be set in the + PostgreSQL process for this extension. It is the responsibility of the + cluster administrator to ensure the variables are correct for the + specific extension. Note that changes to these variables require + a manual cluster restart to take effect. + items: + description: |- + ExtensionEnvVar defines an environment variable for a specific extension + image volume. + properties: + name: + description: |- + Name of the environment variable to be injected into the + PostgreSQL process. + minLength: 1 + pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ + type: string + value: + description: |- + Value of the environment variable. CloudNativePG performs a direct + replacement of this value, with support for placeholder expansion. + The ${`image_root`} placeholder resolves to the absolute mount path + of the extension's volume (e.g., `/extensions/my-extension`). This + is particularly useful for allowing applications or libraries to + locate specific directories within the mounted image. + Unrecognized placeholders are rejected. To include a literal ${...} + in the value, escape it as $${...}. + minLength: 1 + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + extension_control_path: + description: |- + The list of directories inside the image which should be added to extension_control_path. + If not defined, defaults to "/share". + items: + type: string + type: array + image: + description: The image containing the extension. + properties: + pullPolicy: + description: |- + Policy for pulling OCI objects. Possible values are: + Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. + Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. + IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + type: string + reference: + description: |- + Required: Image or artifact reference to be used. + Behaves in the same way as pod.spec.containers[*].image. + Pull secrets will be assembled in the same way as for the container image by looking up node credentials, SA image pull secrets, and pod spec image pull secrets. + More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management to default or override + container images in workload controllers like Deployments and StatefulSets. + type: string + type: object + ld_library_path: + description: The list of directories inside the image + which should be added to ld_library_path. + items: + type: string + type: array + name: + description: The name of the extension, required + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9_]*[a-z0-9])?$ + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map image: description: The image reference type: string @@ -8541,7 +8795,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.20.0 + controller-gen.kubebuilder.io/version: v0.20.1 helm.sh/resource-policy: keep name: poolers.postgresql.cnpg.io spec: @@ -17910,7 +18164,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.20.0 + controller-gen.kubebuilder.io/version: v0.20.1 helm.sh/resource-policy: keep name: publications.postgresql.cnpg.io spec: @@ -18106,7 +18360,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.20.0 + controller-gen.kubebuilder.io/version: v0.20.1 helm.sh/resource-policy: keep name: scheduledbackups.postgresql.cnpg.io spec: @@ -18298,7 +18552,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.20.0 + controller-gen.kubebuilder.io/version: v0.20.1 helm.sh/resource-policy: keep name: subscriptions.postgresql.cnpg.io spec: diff --git a/packages/system/postgres-operator/patches/cloudnative-pg-1.28.2.patch b/packages/system/postgres-operator/patches/cloudnative-pg-1.28.2.patch new file mode 100644 index 0000000000..d535809e80 --- /dev/null +++ b/packages/system/postgres-operator/patches/cloudnative-pg-1.28.2.patch @@ -0,0 +1,414 @@ +--- a/packages/system/postgres-operator/charts/cloudnative-pg/Chart.yaml ++++ b/packages/system/postgres-operator/charts/cloudnative-pg/Chart.yaml +@@ -1,5 +1,5 @@ + apiVersion: v2 +-appVersion: 1.28.1 ++appVersion: 1.28.2 + dependencies: + - alias: monitoring + condition: monitoring.grafanaDashboard.create +--- a/packages/system/postgres-operator/charts/cloudnative-pg/templates/crds/crds.yaml ++++ b/packages/system/postgres-operator/charts/cloudnative-pg/templates/crds/crds.yaml +@@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 + kind: CustomResourceDefinition + metadata: + annotations: +- controller-gen.kubebuilder.io/version: v0.20.0 ++ controller-gen.kubebuilder.io/version: v0.20.1 + helm.sh/resource-policy: keep + name: backups.postgresql.cnpg.io + spec: +@@ -466,7 +466,7 @@ apiVersion: apiextensions.k8s.io/v1 + kind: CustomResourceDefinition + metadata: + annotations: +- controller-gen.kubebuilder.io/version: v0.20.0 ++ controller-gen.kubebuilder.io/version: v0.20.1 + helm.sh/resource-policy: keep + name: clusterimagecatalogs.postgresql.cnpg.io + spec: +@@ -515,6 +515,112 @@ spec: + items: + description: CatalogImage defines the image and major version + properties: ++ extensions: ++ description: The configuration of the extensions to be added ++ items: ++ description: |- ++ ExtensionConfiguration is the configuration used to add ++ PostgreSQL extensions to the Cluster. ++ properties: ++ bin_path: ++ description: |- ++ A list of directories within the image to be appended to the ++ PostgreSQL process's `PATH` environment variable. ++ items: ++ type: string ++ type: array ++ dynamic_library_path: ++ description: |- ++ The list of directories inside the image which should be added to dynamic_library_path. ++ If not defined, defaults to "/lib". ++ items: ++ type: string ++ type: array ++ env: ++ description: |- ++ Env is a list of custom environment variables to be set in the ++ PostgreSQL process for this extension. It is the responsibility of the ++ cluster administrator to ensure the variables are correct for the ++ specific extension. Note that changes to these variables require ++ a manual cluster restart to take effect. ++ items: ++ description: |- ++ ExtensionEnvVar defines an environment variable for a specific extension ++ image volume. ++ properties: ++ name: ++ description: |- ++ Name of the environment variable to be injected into the ++ PostgreSQL process. ++ minLength: 1 ++ pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ ++ type: string ++ value: ++ description: |- ++ Value of the environment variable. CloudNativePG performs a direct ++ replacement of this value, with support for placeholder expansion. ++ The ${`image_root`} placeholder resolves to the absolute mount path ++ of the extension's volume (e.g., `/extensions/my-extension`). This ++ is particularly useful for allowing applications or libraries to ++ locate specific directories within the mounted image. ++ Unrecognized placeholders are rejected. To include a literal ${...} ++ in the value, escape it as $${...}. ++ minLength: 1 ++ type: string ++ required: ++ - name ++ - value ++ type: object ++ type: array ++ x-kubernetes-list-map-keys: ++ - name ++ x-kubernetes-list-type: map ++ extension_control_path: ++ description: |- ++ The list of directories inside the image which should be added to extension_control_path. ++ If not defined, defaults to "/share". ++ items: ++ type: string ++ type: array ++ image: ++ description: The image containing the extension. ++ properties: ++ pullPolicy: ++ description: |- ++ Policy for pulling OCI objects. Possible values are: ++ Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. ++ Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. ++ IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. ++ Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. ++ type: string ++ reference: ++ description: |- ++ Required: Image or artifact reference to be used. ++ Behaves in the same way as pod.spec.containers[*].image. ++ Pull secrets will be assembled in the same way as for the container image by looking up node credentials, SA image pull secrets, and pod spec image pull secrets. ++ More info: https://kubernetes.io/docs/concepts/containers/images ++ This field is optional to allow higher level config management to default or override ++ container images in workload controllers like Deployments and StatefulSets. ++ type: string ++ type: object ++ ld_library_path: ++ description: The list of directories inside the image ++ which should be added to ld_library_path. ++ items: ++ type: string ++ type: array ++ name: ++ description: The name of the extension, required ++ minLength: 1 ++ pattern: ^[a-z0-9]([-a-z0-9_]*[a-z0-9])?$ ++ type: string ++ required: ++ - name ++ type: object ++ type: array ++ x-kubernetes-list-map-keys: ++ - name ++ x-kubernetes-list-type: map + image: + description: The image reference + type: string +@@ -548,7 +654,7 @@ apiVersion: apiextensions.k8s.io/v1 + kind: CustomResourceDefinition + metadata: + annotations: +- controller-gen.kubebuilder.io/version: v0.20.0 ++ controller-gen.kubebuilder.io/version: v0.20.1 + helm.sh/resource-policy: keep + name: clusters.postgresql.cnpg.io + spec: +@@ -4913,6 +5019,13 @@ spec: + ExtensionConfiguration is the configuration used to add + PostgreSQL extensions to the Cluster. + properties: ++ bin_path: ++ description: |- ++ A list of directories within the image to be appended to the ++ PostgreSQL process's `PATH` environment variable. ++ items: ++ type: string ++ type: array + dynamic_library_path: + description: |- + The list of directories inside the image which should be added to dynamic_library_path. +@@ -4920,6 +5033,45 @@ spec: + items: + type: string + type: array ++ env: ++ description: |- ++ Env is a list of custom environment variables to be set in the ++ PostgreSQL process for this extension. It is the responsibility of the ++ cluster administrator to ensure the variables are correct for the ++ specific extension. Note that changes to these variables require ++ a manual cluster restart to take effect. ++ items: ++ description: |- ++ ExtensionEnvVar defines an environment variable for a specific extension ++ image volume. ++ properties: ++ name: ++ description: |- ++ Name of the environment variable to be injected into the ++ PostgreSQL process. ++ minLength: 1 ++ pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ ++ type: string ++ value: ++ description: |- ++ Value of the environment variable. CloudNativePG performs a direct ++ replacement of this value, with support for placeholder expansion. ++ The ${`image_root`} placeholder resolves to the absolute mount path ++ of the extension's volume (e.g., `/extensions/my-extension`). This ++ is particularly useful for allowing applications or libraries to ++ locate specific directories within the mounted image. ++ Unrecognized placeholders are rejected. To include a literal ${...} ++ in the value, escape it as $${...}. ++ minLength: 1 ++ type: string ++ required: ++ - name ++ - value ++ type: object ++ type: array ++ x-kubernetes-list-map-keys: ++ - name ++ x-kubernetes-list-type: map + extension_control_path: + description: |- + The list of directories inside the image which should be added to extension_control_path. +@@ -4928,7 +5080,7 @@ spec: + type: string + type: array + image: +- description: The image containing the extension, required ++ description: The image containing the extension. + properties: + pullPolicy: + description: |- +@@ -4948,9 +5100,6 @@ spec: + container images in workload controllers like Deployments and StatefulSets. + type: string + type: object +- x-kubernetes-validations: +- - message: An image reference is required +- rule: has(self.reference) + ld_library_path: + description: The list of directories inside the image which + should be added to ld_library_path. +@@ -4963,7 +5112,6 @@ spec: + pattern: ^[a-z0-9]([-a-z0-9_]*[a-z0-9])?$ + type: string + required: +- - image + - name + type: object + type: array +@@ -7787,7 +7935,7 @@ apiVersion: apiextensions.k8s.io/v1 + kind: CustomResourceDefinition + metadata: + annotations: +- controller-gen.kubebuilder.io/version: v0.20.0 ++ controller-gen.kubebuilder.io/version: v0.20.1 + helm.sh/resource-policy: keep + name: databases.postgresql.cnpg.io + spec: +@@ -8382,7 +8530,7 @@ apiVersion: apiextensions.k8s.io/v1 + kind: CustomResourceDefinition + metadata: + annotations: +- controller-gen.kubebuilder.io/version: v0.20.0 ++ controller-gen.kubebuilder.io/version: v0.20.1 + helm.sh/resource-policy: keep + name: failoverquorums.postgresql.cnpg.io + spec: +@@ -8460,7 +8608,7 @@ apiVersion: apiextensions.k8s.io/v1 + kind: CustomResourceDefinition + metadata: + annotations: +- controller-gen.kubebuilder.io/version: v0.20.0 ++ controller-gen.kubebuilder.io/version: v0.20.1 + helm.sh/resource-policy: keep + name: imagecatalogs.postgresql.cnpg.io + spec: +@@ -8508,6 +8656,112 @@ spec: + items: + description: CatalogImage defines the image and major version + properties: ++ extensions: ++ description: The configuration of the extensions to be added ++ items: ++ description: |- ++ ExtensionConfiguration is the configuration used to add ++ PostgreSQL extensions to the Cluster. ++ properties: ++ bin_path: ++ description: |- ++ A list of directories within the image to be appended to the ++ PostgreSQL process's `PATH` environment variable. ++ items: ++ type: string ++ type: array ++ dynamic_library_path: ++ description: |- ++ The list of directories inside the image which should be added to dynamic_library_path. ++ If not defined, defaults to "/lib". ++ items: ++ type: string ++ type: array ++ env: ++ description: |- ++ Env is a list of custom environment variables to be set in the ++ PostgreSQL process for this extension. It is the responsibility of the ++ cluster administrator to ensure the variables are correct for the ++ specific extension. Note that changes to these variables require ++ a manual cluster restart to take effect. ++ items: ++ description: |- ++ ExtensionEnvVar defines an environment variable for a specific extension ++ image volume. ++ properties: ++ name: ++ description: |- ++ Name of the environment variable to be injected into the ++ PostgreSQL process. ++ minLength: 1 ++ pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ ++ type: string ++ value: ++ description: |- ++ Value of the environment variable. CloudNativePG performs a direct ++ replacement of this value, with support for placeholder expansion. ++ The ${`image_root`} placeholder resolves to the absolute mount path ++ of the extension's volume (e.g., `/extensions/my-extension`). This ++ is particularly useful for allowing applications or libraries to ++ locate specific directories within the mounted image. ++ Unrecognized placeholders are rejected. To include a literal ${...} ++ in the value, escape it as $${...}. ++ minLength: 1 ++ type: string ++ required: ++ - name ++ - value ++ type: object ++ type: array ++ x-kubernetes-list-map-keys: ++ - name ++ x-kubernetes-list-type: map ++ extension_control_path: ++ description: |- ++ The list of directories inside the image which should be added to extension_control_path. ++ If not defined, defaults to "/share". ++ items: ++ type: string ++ type: array ++ image: ++ description: The image containing the extension. ++ properties: ++ pullPolicy: ++ description: |- ++ Policy for pulling OCI objects. Possible values are: ++ Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. ++ Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. ++ IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. ++ Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. ++ type: string ++ reference: ++ description: |- ++ Required: Image or artifact reference to be used. ++ Behaves in the same way as pod.spec.containers[*].image. ++ Pull secrets will be assembled in the same way as for the container image by looking up node credentials, SA image pull secrets, and pod spec image pull secrets. ++ More info: https://kubernetes.io/docs/concepts/containers/images ++ This field is optional to allow higher level config management to default or override ++ container images in workload controllers like Deployments and StatefulSets. ++ type: string ++ type: object ++ ld_library_path: ++ description: The list of directories inside the image ++ which should be added to ld_library_path. ++ items: ++ type: string ++ type: array ++ name: ++ description: The name of the extension, required ++ minLength: 1 ++ pattern: ^[a-z0-9]([-a-z0-9_]*[a-z0-9])?$ ++ type: string ++ required: ++ - name ++ type: object ++ type: array ++ x-kubernetes-list-map-keys: ++ - name ++ x-kubernetes-list-type: map + image: + description: The image reference + type: string +@@ -8541,7 +8795,7 @@ apiVersion: apiextensions.k8s.io/v1 + kind: CustomResourceDefinition + metadata: + annotations: +- controller-gen.kubebuilder.io/version: v0.20.0 ++ controller-gen.kubebuilder.io/version: v0.20.1 + helm.sh/resource-policy: keep + name: poolers.postgresql.cnpg.io + spec: +@@ -17910,7 +18164,7 @@ apiVersion: apiextensions.k8s.io/v1 + kind: CustomResourceDefinition + metadata: + annotations: +- controller-gen.kubebuilder.io/version: v0.20.0 ++ controller-gen.kubebuilder.io/version: v0.20.1 + helm.sh/resource-policy: keep + name: publications.postgresql.cnpg.io + spec: +@@ -18106,7 +18360,7 @@ apiVersion: apiextensions.k8s.io/v1 + kind: CustomResourceDefinition + metadata: + annotations: +- controller-gen.kubebuilder.io/version: v0.20.0 ++ controller-gen.kubebuilder.io/version: v0.20.1 + helm.sh/resource-policy: keep + name: scheduledbackups.postgresql.cnpg.io + spec: +@@ -18298,7 +18552,7 @@ apiVersion: apiextensions.k8s.io/v1 + kind: CustomResourceDefinition + metadata: + annotations: +- controller-gen.kubebuilder.io/version: v0.20.0 ++ controller-gen.kubebuilder.io/version: v0.20.1 + helm.sh/resource-policy: keep + name: subscriptions.postgresql.cnpg.io + spec: diff --git a/packages/system/postgres-operator/tests/cnpg-version_test.yaml b/packages/system/postgres-operator/tests/cnpg-version_test.yaml new file mode 100644 index 0000000000..eeb22077fa --- /dev/null +++ b/packages/system/postgres-operator/tests/cnpg-version_test.yaml @@ -0,0 +1,38 @@ +suite: CNPG operator and CRDs stay aligned + +templates: + - charts/cloudnative-pg/templates/deployment.yaml + - charts/cloudnative-pg/templates/rbac.yaml + - charts/cloudnative-pg/templates/config.yaml + - charts/cloudnative-pg/templates/monitoring-configmap.yaml + - charts/cloudnative-pg/templates/crds/crds.yaml + +release: + name: postgres-operator + namespace: cozy-postgres-operator + +tests: + - it: runs the CNPG 1.28.2 operator image + template: charts/cloudnative-pg/templates/deployment.yaml + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: ghcr.io/cloudnative-pg/cloudnative-pg:1.28.2 + - equal: + path: spec.template.spec.containers[0].env[0].value + value: ghcr.io/cloudnative-pg/cloudnative-pg:1.28.2 + - equal: + path: metadata.labels["app.kubernetes.io/version"] + value: 1.28.2 + + - it: ships the CNPG 1.28.2 CRDs patched in lockstep with the operator + template: charts/cloudnative-pg/templates/crds/crds.yaml + asserts: + - documentIndex: 0 + equal: + path: metadata.annotations["controller-gen.kubebuilder.io/version"] + value: v0.20.1 + - documentIndex: 0 + equal: + path: spec.versions[0].schema.openAPIV3Schema.properties.status.properties.instanceID.properties.sessionID.type + value: string diff --git a/packages/system/postgres-operator/values.yaml b/packages/system/postgres-operator/values.yaml index 1e3fa9c118..a2211c6d03 100644 --- a/packages/system/postgres-operator/values.yaml +++ b/packages/system/postgres-operator/values.yaml @@ -1,13 +1,17 @@ cloudnative-pg: crds: create: true - # Image tag intentionally left to the chart's appVersion (1.28.1). Do NOT pin - # image.tag to a version the vendored chart's CRDs don't match: an operator - # newer than its CRDs writes status fields the CRD prunes (e.g. - # status.instanceID.sessionID, added in CNPG 1.27.3), which makes the operator - # believe the instance manager restarted and fail EVERY backup cluster-wide - # ("instance manager was restarted during backup"). Bump the chart version in - # the Makefile to move the operator + CRDs together. + # The vendored chart is 0.27.1 (appVersion 1.28.1). patches/cloudnative-pg-1.28.2.patch, + # applied by `make update`, raises the operator image AND its CRDs together to 1.28.2 for the + # PVC resize-deadlock fix (cloudnative-pg#9980 / #9981), which ships in the 1.28.2 operator + # binary. Upstream publishes a chart only per minor .0/.1, so there is no 1.28.2 chart to bump + # to on the 1.28 line without jumping a minor - hence the patch keeps operator and CRDs aligned. + # + # Do NOT pin image.tag here: the tag follows the chart's appVersion, so operator and CRDs stay + # in lockstep. Pinning image.tag to a version the CRDs do not match makes the operator write + # status fields the CRD prunes (e.g. status.instanceID.sessionID, added in CNPG 1.27.3), + # failing every backup cluster-wide. To move the operator + CRDs, edit the patch (or bump the + # chart version in the Makefile). # CloudNativePG Barman Cloud Plugin (github.com/cloudnative-pg/plugin-barman-cloud). # Native spec.backup.barmanObjectStore is deprecated in CNPG 1.27 and removed in 1.29, From 70d0c15ba7e64184ebc8f028d268f162416e75d6 Mon Sep 17 00:00:00 2001 From: "cozystack-ci[bot]" <274107086+cozystack-ci[bot]@users.noreply.github.com> Date: Tue, 4 Aug 2026 18:45:25 +0000 Subject: [PATCH 15/51] Prepare release v1.6.1-rc.1 Signed-off-by: cozystack-ci[bot] <274107086+cozystack-ci[bot]@users.noreply.github.com> --- .../images/altinity-clickhouse-backup.tag | 2 +- .../clickhouse/images/clickhouse-backup.tag | 2 +- .../apps/http-cache/images/nginx-cache.tag | 2 +- .../kubernetes/images/cluster-autoscaler.tag | 2 +- .../images/kubevirt-cloud-provider.tag | 2 +- .../kubernetes/images/kubevirt-csi-driver.tag | 2 +- .../kubernetes/images/talos-csr-signer.tag | 2 +- .../apps/mariadb/images/mariadb-backup.tag | 2 +- packages/core/installer/values.yaml | 4 ++-- packages/core/platform/values.yaml | 2 +- packages/core/testing/values.yaml | 2 +- packages/extra/bootbox/images/matchbox.tag | 2 +- .../images/objectstorage-sidecar.tag | 2 +- packages/system/backup-controller/values.yaml | 2 +- .../backupstrategy-controller/values.yaml | 4 ++-- packages/system/bucket/images/s3manager.tag | 2 +- .../files/components.gz | Bin 126108 -> 126107 bytes .../files/control-plane-components.yaml | 2 +- ...ster-api-control-plane-provider-kamaji.tag | 2 +- packages/system/cilium/values.yaml | 2 +- packages/system/cozystack-api/values.yaml | 2 +- .../system/cozystack-controller/values.yaml | 2 +- packages/system/dashboard/values.yaml | 4 ++-- .../system/flux-shard-operator/values.yaml | 2 +- .../images/grafana-dashboards.tag | 2 +- packages/system/kamaji/values.yaml | 4 ++-- packages/system/kubeovn-plunger/values.yaml | 2 +- packages/system/kubeovn-webhook/values.yaml | 2 +- packages/system/kubevirt-csi-node/values.yaml | 2 +- .../lineage-controller-webhook/values.yaml | 2 +- packages/system/linstor-gui/values.yaml | 2 +- packages/system/linstor/values.yaml | 4 ++-- packages/system/metallb/values.yaml | 4 ++-- packages/system/monitoring/images/grafana.tag | 2 +- .../templates/multus-daemonset-thick.yml | 4 ++-- .../objectstorage-controller/values.yaml | 2 +- packages/system/seaweedfs/values.yaml | 2 +- .../securitygroup-controller/values.yaml | 2 +- 38 files changed, 44 insertions(+), 44 deletions(-) diff --git a/packages/apps/clickhouse/images/altinity-clickhouse-backup.tag b/packages/apps/clickhouse/images/altinity-clickhouse-backup.tag index 73c1764be0..985ca92072 100644 --- a/packages/apps/clickhouse/images/altinity-clickhouse-backup.tag +++ b/packages/apps/clickhouse/images/altinity-clickhouse-backup.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/altinity-clickhouse-backup:v1.6.0@sha256:e890d78cd83968bdff9a36c9ca4312afc05e138af486e476da0a935f81f9b7a7 +ghcr.io/cozystack/cozystack/altinity-clickhouse-backup:v1.6.1-rc.1@sha256:e890d78cd83968bdff9a36c9ca4312afc05e138af486e476da0a935f81f9b7a7 diff --git a/packages/apps/clickhouse/images/clickhouse-backup.tag b/packages/apps/clickhouse/images/clickhouse-backup.tag index 2ba1bf36f4..46e0fa3646 100644 --- a/packages/apps/clickhouse/images/clickhouse-backup.tag +++ b/packages/apps/clickhouse/images/clickhouse-backup.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/clickhouse-backup:v1.6.0@sha256:8efef62d59fb82d39544823f81a797c2b9ac130a50241fee48e701161ce2670d +ghcr.io/cozystack/cozystack/clickhouse-backup:v1.6.1-rc.1@sha256:8efef62d59fb82d39544823f81a797c2b9ac130a50241fee48e701161ce2670d diff --git a/packages/apps/http-cache/images/nginx-cache.tag b/packages/apps/http-cache/images/nginx-cache.tag index 8f54917c2f..d9da31e076 100644 --- a/packages/apps/http-cache/images/nginx-cache.tag +++ b/packages/apps/http-cache/images/nginx-cache.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/nginx-cache:v1.6.0@sha256:bd042a8a9789e8b21b12cb266e390484a3a48f4af6ca6e2d2864a8a83c74bd25 +ghcr.io/cozystack/cozystack/nginx-cache:v1.6.1-rc.1@sha256:bd042a8a9789e8b21b12cb266e390484a3a48f4af6ca6e2d2864a8a83c74bd25 diff --git a/packages/apps/kubernetes/images/cluster-autoscaler.tag b/packages/apps/kubernetes/images/cluster-autoscaler.tag index c9a4369e23..6e27c6dc48 100644 --- a/packages/apps/kubernetes/images/cluster-autoscaler.tag +++ b/packages/apps/kubernetes/images/cluster-autoscaler.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/cluster-autoscaler:v1.6.0@sha256:d188f395a31e37ea09bfa91e45d7ce1606b7459192a437fa66bb760ffb5f17d2 +ghcr.io/cozystack/cozystack/cluster-autoscaler:v1.6.1-rc.1@sha256:e77ff02b328e4e119439efd1edd3aa71e9325fabf10a4a84b03a87d12d90b308 diff --git a/packages/apps/kubernetes/images/kubevirt-cloud-provider.tag b/packages/apps/kubernetes/images/kubevirt-cloud-provider.tag index 4c0395201a..2fbad18297 100644 --- a/packages/apps/kubernetes/images/kubevirt-cloud-provider.tag +++ b/packages/apps/kubernetes/images/kubevirt-cloud-provider.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/kubevirt-cloud-provider:v1.6.0@sha256:f005b99041d191eed9dde98f9f6691ba2d0cbfa1b3b0b2fe4b32cc0fdba74acf +ghcr.io/cozystack/cozystack/kubevirt-cloud-provider:v1.6.1-rc.1@sha256:f005b99041d191eed9dde98f9f6691ba2d0cbfa1b3b0b2fe4b32cc0fdba74acf diff --git a/packages/apps/kubernetes/images/kubevirt-csi-driver.tag b/packages/apps/kubernetes/images/kubevirt-csi-driver.tag index cd94f98ea7..a16e26f0cd 100644 --- a/packages/apps/kubernetes/images/kubevirt-csi-driver.tag +++ b/packages/apps/kubernetes/images/kubevirt-csi-driver.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/kubevirt-csi-driver:v1.6.0@sha256:d6326e343c7932ce3d375c6742b9b9745090a4a26bf5bbc5ad19f5504ea93d65 +ghcr.io/cozystack/cozystack/kubevirt-csi-driver:v1.6.1-rc.1@sha256:7a47bb5aa46fccfe33dc0e0d857a6ec2ce2c6284a213e37beff5edcf6f37c673 diff --git a/packages/apps/kubernetes/images/talos-csr-signer.tag b/packages/apps/kubernetes/images/talos-csr-signer.tag index 48d18c892d..8f521fc7cc 100644 --- a/packages/apps/kubernetes/images/talos-csr-signer.tag +++ b/packages/apps/kubernetes/images/talos-csr-signer.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/talos-csr-signer:v1.6.0@sha256:85b41e16e5e40398b1272fe0a69de72e205532c8034d0e6a9fc4a1d510f17818 +ghcr.io/cozystack/cozystack/talos-csr-signer:v1.6.1-rc.1@sha256:7051e90245cb446ce4deb857d4337e2fd69ce84436243ba60939d374defc2a07 diff --git a/packages/apps/mariadb/images/mariadb-backup.tag b/packages/apps/mariadb/images/mariadb-backup.tag index 3dafe6c5e1..2d4f90b455 100644 --- a/packages/apps/mariadb/images/mariadb-backup.tag +++ b/packages/apps/mariadb/images/mariadb-backup.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/mariadb-backup:v1.6.0@sha256:1de944b5c4fbeef94004b8620b0865aa7886a692dacb23d50c8f9b31fbeece07 +ghcr.io/cozystack/cozystack/mariadb-backup:v1.6.1-rc.1@sha256:1de944b5c4fbeef94004b8620b0865aa7886a692dacb23d50c8f9b31fbeece07 diff --git a/packages/core/installer/values.yaml b/packages/core/installer/values.yaml index 6817be8c0b..218d869848 100644 --- a/packages/core/installer/values.yaml +++ b/packages/core/installer/values.yaml @@ -14,9 +14,9 @@ cozystackOperator: # to the value baked at build time. The Makefile sets this for the # kubectl-apply release artifacts (_out/assets/cozystack-operator-*.yaml). platformVersion: "" - image: ghcr.io/cozystack/cozystack/cozystack-operator:v1.6.0@sha256:93b305f78823b070c59ca631b213e39bd9d8a256d5c520f156a03f5f72f57a5d + image: ghcr.io/cozystack/cozystack/cozystack-operator:v1.6.1-rc.1@sha256:ab2c0c8f87dc70f675bf36ce55f8615f810bbe3d0840d3a7d1a412f38abdf78a platformSourceUrl: 'oci://ghcr.io/cozystack/cozystack/cozystack-packages' - platformSourceRef: 'digest=sha256:bf68208730860fa8e47f378a0260e79a0262e0783fa63ba4f7f97f57a48b2d23' + platformSourceRef: 'digest=sha256:2d7e07e4cf3d49a97f5b9c62ad732cfb93586483723788eed489b32643a1a8df' # When non-empty, overrides the operator's --helmrelease-interval flag # (operator default: 5m). E2E sets this to 30s; production should leave empty. helmReleaseInterval: "" diff --git a/packages/core/platform/values.yaml b/packages/core/platform/values.yaml index 69c0de5596..63abfb386e 100644 --- a/packages/core/platform/values.yaml +++ b/packages/core/platform/values.yaml @@ -13,7 +13,7 @@ migrations: # below therefore lags targetVersion in-tree by design; run-migrations.sh # refuses to advance past a migration file missing from the image (exit 1), so # a stale pin fails loudly rather than silently skipping the etcd adoption. - image: ghcr.io/cozystack/cozystack/platform-migrations:v1.6.0@sha256:6777042e3e9c6bad76e7d96fb64baa115061975f39061cb3ede84e21d0e2213f + image: ghcr.io/cozystack/cozystack/platform-migrations:v1.6.1-rc.1@sha256:a34e7156a52dc55b27662917e5f9d5bbea671188d8db233296d0a98c800dc6c6 targetVersion: 54 # Adopt legacy etcd clusters onto the v1alpha2 operator WITHOUT the mandatory # pre-adoption safety snapshot (migration 50). Leave false unless migration 50 diff --git a/packages/core/testing/values.yaml b/packages/core/testing/values.yaml index 7e3c1739ae..b42b13545c 100644 --- a/packages/core/testing/values.yaml +++ b/packages/core/testing/values.yaml @@ -1,2 +1,2 @@ e2e: - image: ghcr.io/cozystack/cozystack/e2e-sandbox:v1.6.0@sha256:a9980e8c48d6e50ed2b5d8245a4e0aae00970777b6e2255577fdb66035a44c68 + image: ghcr.io/cozystack/cozystack/e2e-sandbox:v1.6.1-rc.1@sha256:a9980e8c48d6e50ed2b5d8245a4e0aae00970777b6e2255577fdb66035a44c68 diff --git a/packages/extra/bootbox/images/matchbox.tag b/packages/extra/bootbox/images/matchbox.tag index 5d086145d8..c321112344 100644 --- a/packages/extra/bootbox/images/matchbox.tag +++ b/packages/extra/bootbox/images/matchbox.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/matchbox:v1.6.0@sha256:7805917462ad1288272a6324c0f32a4f9a250e1723a2759f7afebce181400f96 +ghcr.io/cozystack/cozystack/matchbox:v1.6.1-rc.1@sha256:3f492d2bd2d14de94c88dc995e3092911e2ca8ed7e17d305398e7fd49135d351 diff --git a/packages/extra/seaweedfs/images/objectstorage-sidecar.tag b/packages/extra/seaweedfs/images/objectstorage-sidecar.tag index a3cd453be9..14122ad411 100644 --- a/packages/extra/seaweedfs/images/objectstorage-sidecar.tag +++ b/packages/extra/seaweedfs/images/objectstorage-sidecar.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/objectstorage-sidecar:v1.6.0@sha256:4d665becd3399c2fe1e8a66e15d14d16e4a5a75395fb0ae65cff46679a8989f1 +ghcr.io/cozystack/cozystack/objectstorage-sidecar:v1.6.1-rc.1@sha256:ef3c154c1a6dd1ac9fa1b0763a2bd23707d68f2f16f16a97ebf331f3646b53ce diff --git a/packages/system/backup-controller/values.yaml b/packages/system/backup-controller/values.yaml index a62272f454..6b0e3d475b 100644 --- a/packages/system/backup-controller/values.yaml +++ b/packages/system/backup-controller/values.yaml @@ -1,5 +1,5 @@ backupController: - image: "ghcr.io/cozystack/cozystack/backup-controller:v1.6.0@sha256:fc4571354413d86ed11fa38dee8fd3d2d879e11c48d7d92c28630b2d54a2bcbb" + image: "ghcr.io/cozystack/cozystack/backup-controller:v1.6.1-rc.1@sha256:ef3c45ac4b23d6231a4166b4331a622374960caeeef3e4a901b0ee1dabcb6922" replicas: 2 debug: false metrics: diff --git a/packages/system/backupstrategy-controller/values.yaml b/packages/system/backupstrategy-controller/values.yaml index 225a8f8d14..253290bb58 100644 --- a/packages/system/backupstrategy-controller/values.yaml +++ b/packages/system/backupstrategy-controller/values.yaml @@ -1,5 +1,5 @@ backupStrategyController: - image: "ghcr.io/cozystack/cozystack/backupstrategy-controller:v1.6.0@sha256:e8bf0521d1d1d8b90ebe66e9013010887033ba2653901799d57f2df47c9d879e" + image: "ghcr.io/cozystack/cozystack/backupstrategy-controller:v1.6.1-rc.1@sha256:7cb5af7bd8fae2ef5d6e58d1eb6eb7d9579bff8df228ac3d936c6e14ae5e063f" # chBackupClientImage is the image rendered into the Altinity strategy # Pod: it drives clickhouse-backup's HTTP API via curl + jq. # @@ -35,7 +35,7 @@ backupStrategyController: # retag both to the same stable tag, and the write-once guard in # hack/promote-retag.sh fails the release. hack/image-pin-consistency.bats # asserts no repository is pinned at more than one digest. - chBackupClientImage: "ghcr.io/cozystack/cozystack/platform-migrations:v1.6.0@sha256:6777042e3e9c6bad76e7d96fb64baa115061975f39061cb3ede84e21d0e2213f" + chBackupClientImage: "ghcr.io/cozystack/cozystack/platform-migrations:v1.6.1-rc.1@sha256:a34e7156a52dc55b27662917e5f9d5bbea671188d8db233296d0a98c800dc6c6" replicas: 2 debug: false metrics: diff --git a/packages/system/bucket/images/s3manager.tag b/packages/system/bucket/images/s3manager.tag index b5234c0e78..5e9239a41a 100644 --- a/packages/system/bucket/images/s3manager.tag +++ b/packages/system/bucket/images/s3manager.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/s3manager:v1.6.0@sha256:73665aaf7f5406b4a1e456d14060fe874e0ae2a436ba7fd3058fa73abff8a927 +ghcr.io/cozystack/cozystack/s3manager:v1.6.1-rc.1@sha256:73665aaf7f5406b4a1e456d14060fe874e0ae2a436ba7fd3058fa73abff8a927 diff --git a/packages/system/capi-providers-cpprovider/files/components.gz b/packages/system/capi-providers-cpprovider/files/components.gz index b96f984eb09d5bd4489c458b1c142aa4c572d748..7fa19e3df7ac13404b65ac59925c06b659eefbb4 100644 GIT binary patch delta 4522 zcmV;b5moM-*$1212Y|Ez1=}0jS5ubkG_gIC>AAL&YHGVW$;sVK+snWbWOJ7w+W{bd z_&ul{i1mj;4xCmRiqBq zZ8MHvL#-F)z4_tL9TASz?gCuh%i{?spnk0jNo|yMm z;)Du`E-Q*hhEBn5lTwuyo<+SXD_OtQZQm2iNk^VlvTRgFS87g3 zE>H}scSc9vM-zEyXn@4tEg$v4(vrf_(|%Ia!onnBz|m04AAEW4OcP$^Ij(F)Lh*J&{lu7XhZyFMLmB>}PE#S(m3m}u(Ffk{8 zdVAvF+oj&ruFBNTU4Ptyy9ITj_B~-LCc+$YUBsmEVJFidP&b~0==l~7yw|V2DC~E| zpA|%{E7QKf-PuYc?sQIH;6ozl+*9dRT5WlMr+jH3frz|0Bq8VPaprAT$3 z)vYddo`gqVEGP^&#WaC~NaX{tm1T8NanIJhC%S0Sy$V_bFXV$%Cugu)tCLdbz!NL< z08AA;9u==F>TT9*_;069y26vRQK?_#y$RS<1FRe@`Gb>B4JNYgH}En?AxGj;bteQds0% zGu2!Gh_0-}h0(o6TjsXVnjNYbk2>KuE5SJCzVy5TcR7hc7L>`n_pNr>r4Qta!~%r! zLcqAlg2;jQce~Vgk4St?C+fCo5nH?B zSOIfp9^BIwj>%cKjomHAP|!-6JR5(zX2~WPKDFs>!Y=mm-NAbhPhE1c5jjfHUOfq)m4Cc_K?`ts!u`lHcg>nlL&Mh=6G|6NH ze?aoG$tVp9cfOf*?jZL!ft-UI3YIe7?Jo<5Jq_jCZ)X$2`KpwbrMxTpy%4y-uNvC{ zduX9a^AK5y`4yB$Ex4vPE0+GWcC!yT6ABtoO|4&lgvA`|sF2LXPj_Uh5{{udu+FxL zsZn7&SeUQW^L`TMl{yO5VJNa_DGOr%3PM%;Vqv=t;s~AR=Yt}wFeXPZF=woOxlo=S zwqTI@90%$)juy@wOHA#d8Po#Om)|W^Gr3!c{$((d2UI38_q14`t0^Yabj3%HJ!?@a zBk=rx3wjzw(q~leE|tj3_O6#tF*NeI3eo<2#QAg)wf9jt8S*no^|TBW-H`1Jwq@cn z4--qlY{n`d@~hIFr-~z~kQTE_s28HtUL>fW=uM8X|5hwaCgKp=VTumjA~CX1NQoHa z7xm62q%;N|dQ&U#-p1j^(t}wftpst|70&d3Rgo;;j{uTFb?%X>o`Jth^L(tn0`FMu zTL%aDq<$b1OvVjWi(6%yXZHfW=%-5Ear9;f=i#&50M1pBlyWVkA2%5>7QPouZPXf7 z%2amXASw~dONX=mFPK)`R10Vg+B(}zgvB?SBRHV#Pa|&v4saCGT-3MjBwRM7l-oIf z-T>4iiE+z<%d%v`=PK?)>!_t;8Y_mZtk$qp9@vks6Usvf!?t|&F&0YQ%8$1>U}2JH zF0mx`iN#k0L#Nhd6os6XCqIevg1joaP0yGtYs<%Lj;bM}snSE4&b}5(gj!g`j48aQ zmvFNuGSYh6PD!mw4G#rL<&ACM%w3Crt&8<0g-0q$TQ%w8BSE6n2l~d~%CF*kAm?&b zieyBtWg$!XJ~I`T>-AwhZmUvE&iTUSecqy#TjdNrQiu5=7^ORf|7_e0YNX!E(ciXn zWU8V|Xfo>qf|Ox^2?OV(kG*o?=rG?i62=(ucOOfpKFHyLKO8VE!Z5 z?`dgnNpRd&dgnZa>8Wa#;mTq`-xT}?^huruO%M$nWw{JhtG+T|K{$zj`Neh2#Y7KD z)gtogHFF?z>@7sr3WFpKNI4oUWJF7_YD_&ex#yOZYh~87h?u}nf~2afs_85|o&KES zRs~Ph)CEu`Helfi57~W}3mWu^6?Gi`w5Fk$>MHbdz8E{7sOD^tDqdvfA?zfy?RUuq zEvYOrEJe33M3%hDJb-0?^_8$pOuZoLFV3vC-5v-6hgXpx_Xu6C6a&fF9 zr9eXiC)#AJR?jEOQrJyKlXUUPa^V1Qm71llh~$<^EwhCc9|Np^fj1#qm5+?bmFl~& zez1U*)6wk;%pAhVO!sR$XKm-K?VQ}RwsVTHU)worJ4Zi{j-_im2cE}b=O~e?XytU3 zAQl|uD8TEJ9B*ymDt19=W$p##1Bo?&o;9fgY_SDtiMenBN;GSAjdF~TF>X4mPLH{q4K=qtT_QG+nTBfm zC)t=%gVuynPpsVSwktCMh7<4&Wx90wX+qnRdoiw{r@4I;>WX6~N*cUptD<4Y^w+$W z^63W$CFNu@z`A<6f^GREB?m3flwo9t?Qa0SJ3&b4@Da(xmHNn&nqb=Ix1-!V!`v(rS*DDWZUbLIP2V0+i6*ldS{NLxzUwQL) zB9N`EAHE?V_rP)l3njNf-sjA~A-lu<6_XCiVnQp!Fte0&q8Q5TK8rBT{2)3kD(Ed# ze2T9&6Kf;wm@nh1I!A|^*NJ&6@E z5_2zqRb?J58wY*DZFh4uNwMMpL)(GhK_R&ywc=I!l+B6d{ErFk-DGIK-u+;BZ7su9 zB@p0uJ;+fh(fL1_`J@j^KPvoSzNz9P_oiOYg*F=!)V&sbWVwy|Oej+?Wf?qcVkK!5 zV;%{7+mfc>-jgB)^H9iosjQPhuA~5R4HBYoF?*D z+5L?Q@8@&A#r>U5F2WQ&W`8mvGca0y+Ja#G(fPv0wO^}oL=VaX1e~!)XeJG2~r@QZeMN!;WsU@=XB{_V=SQEZ_TxO9%Y^O(Qu4@av z1iHXW#Y}9bCq1|dBA~!NCYB(G|9=wgoHy4z z{UkPJ&O0%~BGE3I^qtqBxo?9wbzY7pVmS%!oOjWf!z$Ceao!3e=7zhC#SFX~MsDLY z7~Kfk@Gx{<-zHm|`N3zLoT7pKlRM~{>>n=)L(qHSoOe|z4U)8{!QIT>WnE923YhA4dbTgNpZQVQ8c)Ljn`QAY+YozX7Fh9+=kImIXC02m zuUaw-WI}vpZE?watwdHF#r1vWDalvardyWH(~|D>_QKs>Xhav3W!u7kU0`ohO}J-@ zGXAm_TH#MaqfeFY6HD4k?7dV)pc;6`OJ*(^SmesZVc{fr!6Tq-@!fiq{E z4;2Tvz&S)?7H;HNBK=1bGz=n@7zll`-0BFl0nmn^Z+}#0me?W9XH}8_CiK!U#|(An zm|e8F^5}|jl~K^f;dz>Wn7W0$?bC0>rozdt>wCUqhdDneXGb5L9h29}DYE74h?Hh& z19x73^T+Ygal8HVhm-%DADy+1KeW!@bdI_gop;9fy7$EWYQ7tPLx_R+=hPlvBD z7vG=T%{#b8XQ5V7+FL6F*Zj9=b$)be0Cbvfnw{qPakCW2aU98ivXCfv6p$Z|JHK?h zn0Be#KC-&#-UMSD{OmDze{QwqfM<`>p=@8$(s#F)zTO{ZP-j!Hd-&S>(~^s!5P-hT z)E!smJek?~(7U?s#Zp^-y2l-<{TEex9NyMzje}a{Tt^meW}3)AaQ?oUxH)`b1~zU`AfD;mjbd~bc>2ETZ_ zS%{6&c4qdOA4~v$C7hwv{h}xrVBDx%h|TbHpXIUE8&3p%%zAz3ydCmk6pQ7mvAcg3 zSij76_dytqIUruXupsD?Pmmm*-#OGl&!~1B+y)`P;?1PTLYX{SCh)Yr6sr!q48maA zgQy=zxfn;^J32jmR?>Mq8673((WnzespBh?I9AX(ZOwy6reQ|>-^M}MKPDwb Ir9;FJ02G?nbpQYW delta 4523 zcmV;c5mfG**$1522Y|Ez1=}0T*HD&h$8|K5>AA9!YHGVW$;sVK+snWbWpS4v+W{bd zxP7P{i1mj8>1fK*@oRwU3e4R5wd5{Im^~64I3g4c_eV%(DtlRyJAvcx$jx3Ks6NCf zMyGdjQE+ZWy@TXH0oW$)96e*Z&zF9beMfQb0_}Mnyrfu7 zn`WFhf?6-kdGpWH3lYqAadNqajqaL%gD(b?Seozc&RV&|&G;0seI3ky(t8H}W04rk zY5>hJ@}MPWx~Ru#tSsPBWOt|*Je*k&e$J;ARXwOA-=6iTPZ)La+f@QNL$Q#TASz?gCw&ii{eUpnk0gMo|yAi z;Dic^jdnOIg3=ZQoPFNk^PjvUF5hS1L{j zE>H}scUni@hcj_#sDVVztv4Bfr6q);ul%H_g@sANfTN+7Klt+8nJ2u=$(^A&0SPKs zN)S(Fo!CU2nmKaS&jTb4)l61@Wr*eo5f@u9tt<`A87_@USVl^jIq;1A$(c8~PDVxd zLGY9ik`-uflGD_fCRqt&EKB%JQXrnP{yNEk@PHPS8NGrKZiW`L&84g7P{C)!Pmopw zgQ8H4@Zqg{oA^+RV&b#~Qjg7?=m#v0FCI4c)zF#%OY2klcI~SOX)gbN#jGS-xK^~1 zFi#LBwYog(?e%(R&Xn_0UYFuEhDeN|*)1eL@V4eut~s@-iSW41sq!Lenp0VQ`PA&G zwXUEy+pmn_Kmq$_TcoN>exn@%R;04g@tOL~@)GRv4Xc36fsA@4oZ za`$rt5sN6S)liQEYoN@3g2bN&9^$2vAkH~RP$tD2ylIHVRiY=;Z2?ynT>$CKhOs{R zQ`-{@-yZd*c2lNy=KA9n+%2dJwd-(GG3Mrw?II?O4>Oqtfx6KwK+m^e=)8XIgu$T4 z|12SDJ(2bW?#?zMac4{V0v{4i=e|t0((Z`+JLyXU2}H!rA#vA#VF?GYQA>!OFGZ;P zv~G2w^CUd#VnJe9;nM^TBBc+&Zkp9a#yuPNp6H@Q_bO-)oxmF=DmjDETAq|b2M%AM zhhVDU@hE>~P_I&};=jEb=?YKKMy7rd_a1inpjM(eXBDi;doX#RtwG_l#A`@Q2`Qv&0)qb^pbX=|TQSTpu2SQyD;rRKVg|NuE zX3Dt$5M5b`3!{CFw$5##F*{^29(lsASAuEexx(`b+~ouYX;3C}-nTnNmp+gy5(^ND z3jysS3nB;J-6b*dye!F8S3W2w zvVwLRfG-b7_(GbD!a#UWU*Fs zSOGI;9^BLBj>%fLwcX9fkkd+-JUf3IvA6<;Pj!CVuW7246XLqM8leAsAfH`#O`&!IF>%*|KoI6raoN*;x(FceX=l=+c+4WX(7zOdc;QHaj-%Rv!V7@Z@Sm^0SATqsTt zn=?ptjstZYCo5-;C8qXJ3~B@Ei|-c7ncQte|H_|;11g=EdtNNi)fAIy+Tx?ep0+5I z5qSQ81wBnd;WH|Bmr~?Kd)vz=9~${wg=l|1;(WS@+WRn=jlFY7^|THY?U3yhwqfEj z2NO%dY{n`d;;X`)r;H;hkruN|s28HtStY2S=uM8X|3)lKD&i2^p^FaPA~DiXNP!sS z7j@2Oq%;N&dQ(gA&Q4=z?ZGU9R-Cx(8fW@{rbw3UhXYBVI`>3Y&%ob>c|Mh2fp@I( zt%C!6Qa_LhCgp~##Vs?GfefzL2Gutn-Dd`>aJPw#q4bqz|HZgz)JVM*qyNIp zktvHVp~V4HK{Z8bcdJZoxEEA4uw?#h7DEOrsxTIa0O+oN;w4Np zBXxYO1Y$QHp0*U(yhk7B8A@sU56E zbSGYJ6PI3>ozz6tmZsFOSenjjiD%3>KRSAAv3{9qP;d6%Qe;}bm~ zRg1``*UW%Wv9}Oe%MFsmC*^1~kl`)Cs?qh(#GYGJt`%9+LSh0x0g|e+%BHh$RQhv@ zTNONIQx`y)Sf2%_Ucl~q9;d;8SW(B~PkSDUuC78a=ZmrPv24zUiR49U9>PvS*?w0Z zrzMp|nx*LWg~*aunFp|cthN!B@u}yBgVmYUG1~({;P5Ks>MdlC9RyE z6vUjP90_=PlH+VGob0o;aEhL-g_Do{*239ZIO=(HEZtf-@H`F+hxnnc8d~D&tt;Df z#d?lO=raXjW%<{C!m}n>fGsj0tuYslPl;wNuTh!^8RMqY>hze)SyS_cqe{g3BGXV+ z{UjPwYS8LX>Y0(d-E?Ipz_0?op-h)bKTT+Rde6t@_Z7EK0#$KLM@fVCY*{qSnEtBM zmOlO9pd_7ahFDimRj@6dq~xIGg))rnF#Qd{cZZ8%L`j%`hjrCDe02u~TM$VayqsZA zcT$CCwRO@l@`R^8ttywfCJhXG8V;%p1eqYnAI&_`7%p)-EdZ;;LxCfFuLM*14d=g=-@g0o z?*34z{91Eg{r+9$w<`Ye_uU_MKdb+Ix4X-K|M~oX?PaI+yTAMSS8mhZi2r@A{OYxS zCj!~s{oxw|at|y=uu^gx-g+-kQCn@N!C!J3$YptMBFKY=yosO|B4TnR-4kCS zLq7L^5?SWKuyN2Q+;lgWlN2isFti=KJ18XQCq}%=fU-F;oc}SQy`2orRl6SyudSxI z$^-)5T_18(3UvNYW1a+@DA6aJOz7WdXOHl^Tm{@TV zMwmwe-?*;(j7|Oboai{4ZtKr({K-Hkw>jy5<0Zkfv6rv`OBi4l7vdV?Fl<(;6;;r4sTrRERxfOBK=;EomZt z72V&+@P2Q}x46Gk$wio?$Kp??EaKX1SawJ%-7r^-#{!Q0h$E-I5?J&Bpm^sl9Bi6@ zUt2bH<9iN!JjZC@O_j}(Q5eb01~Qezo)&>_$hUG&gYkjK;A<}s6`0*943@F8JnM6S zeNl!l8RSxyLuwb!Etli%zm3A#6j|pw&!0cboG`l5GoSAz;1rkdC$HrDYb;po;;9<` z@e$4+L=)sB{yCct)SpxR`vA&fd-L6Y_row6$kY;P`jQMje5|pzd0ZBef^VlMNv3N9 zz5u$ybdpKAp3tn4Y6S)D3S#}F?YWLgC9y09{bhCPyndB^;AC4iqvWR|vrD$;c?lC8 zn2m}gJ(XlbyoLC00GT~*C9;5XFv{R1JKsc?T;`J55;Dw#x0fgb|MfaLKXqO8H*U-Owamo6+}RReT)r3;Q#-`+gWc$ zUh;`=%B**MhK0OcI2%~6L37{wQDVKE%=mKR-&ya%sRye}|HgXDjhGwHZp>%k-8i%x zC;sGy)5Z$|>-B9sg&PZ^Sf`|aTp~TCDVZI7%E>tzm_ONrUdaCGnlJ>t7aUkb!&g~O zte_5rBv&Cxd*a_M>|NCLq^W?pZs%wFYWtZVrLFNa46sEO|GEvaM`nTL4|(}shkw@f z@c30pW{ynouZ%4&d9M|Vilex`FFYmrD%*6!vUyt4z1?2e+Y5#0in45fTDU9hZORGv zLQ%$Fwu1Z@5adN8-jLW@+pDjjfjOn)R7Ov+XdK+AtR|cJ$I?&kRD6(<W=%G)BW9l!<}RC(EvmKpOyU2EuA9U){aPO zmJV>|^*4VsPnw<1&mT_zb8&LsZhmNAyy>3wF1zoVm+!i*56!c8y~|ejL+9kO`BURH z=HmPFMe7c((OICBl=9X}!L|O)TV0%-X#m~Un^w1V(QFk0X-1)cC<}>#M*;bv+5M&0 z!?a7i&WX`Q@5Z0v;1`d%_j9`=2E2Hj7G(#5mae_Mbk+Vag}Ru6y~bMxBi?BEw? zKMk>w+D^|t{euqRFNHI-x?dIL9E|HV1F;33?u$Iu`qLSHhgqwQt+!)u97cS(s_!44 z`^GPe-F*;7eF=!8Lj!^;`2@+~y*rCK=o!_C{98ZpuDw>=XMsqbEE0GcU-DIlUHO4O z??E((!c2^#>ztgOJuB!unoUmPi*V8n!^HBHi7hMWg0|+tBhoM<{%<3$g*tPL`X9Y= JL8n8+5C9 Date: Wed, 5 Aug 2026 09:26:07 +0000 Subject: [PATCH 16/51] Prepare release v1.6.1 (promoted from v1.6.1-rc.1) Signed-off-by: cozystack-ci[bot] <274107086+cozystack-ci[bot]@users.noreply.github.com> --- .release-tooling | 1 + .../apps/clickhouse/images/altinity-clickhouse-backup.tag | 2 +- packages/apps/clickhouse/images/clickhouse-backup.tag | 2 +- packages/apps/http-cache/images/nginx-cache.tag | 2 +- packages/apps/kubernetes/images/cluster-autoscaler.tag | 2 +- packages/apps/kubernetes/images/kubevirt-cloud-provider.tag | 2 +- packages/apps/kubernetes/images/kubevirt-csi-driver.tag | 2 +- packages/apps/kubernetes/images/talos-csr-signer.tag | 2 +- packages/apps/mariadb/images/mariadb-backup.tag | 2 +- packages/core/installer/values.yaml | 2 +- packages/core/platform/values.yaml | 2 +- packages/core/testing/values.yaml | 2 +- packages/extra/bootbox/images/matchbox.tag | 2 +- packages/extra/seaweedfs/images/objectstorage-sidecar.tag | 2 +- packages/system/backup-controller/values.yaml | 2 +- packages/system/backupstrategy-controller/values.yaml | 4 ++-- packages/system/bucket/images/s3manager.tag | 2 +- packages/system/cilium/values.yaml | 2 +- packages/system/cozystack-api/values.yaml | 2 +- packages/system/cozystack-controller/values.yaml | 2 +- packages/system/dashboard/values.yaml | 4 ++-- packages/system/flux-shard-operator/values.yaml | 2 +- .../system/grafana-operator/images/grafana-dashboards.tag | 2 +- packages/system/kamaji/values.yaml | 4 ++-- packages/system/kubeovn-plunger/values.yaml | 2 +- packages/system/kubeovn-webhook/values.yaml | 2 +- packages/system/kubevirt-csi-node/values.yaml | 2 +- packages/system/lineage-controller-webhook/values.yaml | 2 +- packages/system/linstor-gui/values.yaml | 2 +- packages/system/linstor/values.yaml | 4 ++-- packages/system/metallb/values.yaml | 4 ++-- packages/system/monitoring/images/grafana.tag | 2 +- packages/system/multus/templates/multus-daemonset-thick.yml | 4 ++-- packages/system/objectstorage-controller/values.yaml | 2 +- packages/system/seaweedfs/values.yaml | 2 +- packages/system/securitygroup-controller/values.yaml | 2 +- 36 files changed, 42 insertions(+), 41 deletions(-) create mode 160000 .release-tooling diff --git a/.release-tooling b/.release-tooling new file mode 160000 index 0000000000..ee26a50f0e --- /dev/null +++ b/.release-tooling @@ -0,0 +1 @@ +Subproject commit ee26a50f0e6a9dfa30254f663ecaddbe9be97664 diff --git a/packages/apps/clickhouse/images/altinity-clickhouse-backup.tag b/packages/apps/clickhouse/images/altinity-clickhouse-backup.tag index 985ca92072..6ee6e2547a 100644 --- a/packages/apps/clickhouse/images/altinity-clickhouse-backup.tag +++ b/packages/apps/clickhouse/images/altinity-clickhouse-backup.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/altinity-clickhouse-backup:v1.6.1-rc.1@sha256:e890d78cd83968bdff9a36c9ca4312afc05e138af486e476da0a935f81f9b7a7 +ghcr.io/cozystack/cozystack/altinity-clickhouse-backup:v1.6.1@sha256:e890d78cd83968bdff9a36c9ca4312afc05e138af486e476da0a935f81f9b7a7 diff --git a/packages/apps/clickhouse/images/clickhouse-backup.tag b/packages/apps/clickhouse/images/clickhouse-backup.tag index 46e0fa3646..e709b9c75b 100644 --- a/packages/apps/clickhouse/images/clickhouse-backup.tag +++ b/packages/apps/clickhouse/images/clickhouse-backup.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/clickhouse-backup:v1.6.1-rc.1@sha256:8efef62d59fb82d39544823f81a797c2b9ac130a50241fee48e701161ce2670d +ghcr.io/cozystack/cozystack/clickhouse-backup:v1.6.1@sha256:8efef62d59fb82d39544823f81a797c2b9ac130a50241fee48e701161ce2670d diff --git a/packages/apps/http-cache/images/nginx-cache.tag b/packages/apps/http-cache/images/nginx-cache.tag index d9da31e076..94a6a92cab 100644 --- a/packages/apps/http-cache/images/nginx-cache.tag +++ b/packages/apps/http-cache/images/nginx-cache.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/nginx-cache:v1.6.1-rc.1@sha256:bd042a8a9789e8b21b12cb266e390484a3a48f4af6ca6e2d2864a8a83c74bd25 +ghcr.io/cozystack/cozystack/nginx-cache:v1.6.1@sha256:bd042a8a9789e8b21b12cb266e390484a3a48f4af6ca6e2d2864a8a83c74bd25 diff --git a/packages/apps/kubernetes/images/cluster-autoscaler.tag b/packages/apps/kubernetes/images/cluster-autoscaler.tag index 6e27c6dc48..4bff49114f 100644 --- a/packages/apps/kubernetes/images/cluster-autoscaler.tag +++ b/packages/apps/kubernetes/images/cluster-autoscaler.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/cluster-autoscaler:v1.6.1-rc.1@sha256:e77ff02b328e4e119439efd1edd3aa71e9325fabf10a4a84b03a87d12d90b308 +ghcr.io/cozystack/cozystack/cluster-autoscaler:v1.6.1@sha256:e77ff02b328e4e119439efd1edd3aa71e9325fabf10a4a84b03a87d12d90b308 diff --git a/packages/apps/kubernetes/images/kubevirt-cloud-provider.tag b/packages/apps/kubernetes/images/kubevirt-cloud-provider.tag index 2fbad18297..35cbec44e4 100644 --- a/packages/apps/kubernetes/images/kubevirt-cloud-provider.tag +++ b/packages/apps/kubernetes/images/kubevirt-cloud-provider.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/kubevirt-cloud-provider:v1.6.1-rc.1@sha256:f005b99041d191eed9dde98f9f6691ba2d0cbfa1b3b0b2fe4b32cc0fdba74acf +ghcr.io/cozystack/cozystack/kubevirt-cloud-provider:v1.6.1@sha256:f005b99041d191eed9dde98f9f6691ba2d0cbfa1b3b0b2fe4b32cc0fdba74acf diff --git a/packages/apps/kubernetes/images/kubevirt-csi-driver.tag b/packages/apps/kubernetes/images/kubevirt-csi-driver.tag index a16e26f0cd..97c2e3ab13 100644 --- a/packages/apps/kubernetes/images/kubevirt-csi-driver.tag +++ b/packages/apps/kubernetes/images/kubevirt-csi-driver.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/kubevirt-csi-driver:v1.6.1-rc.1@sha256:7a47bb5aa46fccfe33dc0e0d857a6ec2ce2c6284a213e37beff5edcf6f37c673 +ghcr.io/cozystack/cozystack/kubevirt-csi-driver:v1.6.1@sha256:7a47bb5aa46fccfe33dc0e0d857a6ec2ce2c6284a213e37beff5edcf6f37c673 diff --git a/packages/apps/kubernetes/images/talos-csr-signer.tag b/packages/apps/kubernetes/images/talos-csr-signer.tag index 8f521fc7cc..b4a6f53f08 100644 --- a/packages/apps/kubernetes/images/talos-csr-signer.tag +++ b/packages/apps/kubernetes/images/talos-csr-signer.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/talos-csr-signer:v1.6.1-rc.1@sha256:7051e90245cb446ce4deb857d4337e2fd69ce84436243ba60939d374defc2a07 +ghcr.io/cozystack/cozystack/talos-csr-signer:v1.6.1@sha256:7051e90245cb446ce4deb857d4337e2fd69ce84436243ba60939d374defc2a07 diff --git a/packages/apps/mariadb/images/mariadb-backup.tag b/packages/apps/mariadb/images/mariadb-backup.tag index 2d4f90b455..77268f30d2 100644 --- a/packages/apps/mariadb/images/mariadb-backup.tag +++ b/packages/apps/mariadb/images/mariadb-backup.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/mariadb-backup:v1.6.1-rc.1@sha256:1de944b5c4fbeef94004b8620b0865aa7886a692dacb23d50c8f9b31fbeece07 +ghcr.io/cozystack/cozystack/mariadb-backup:v1.6.1@sha256:1de944b5c4fbeef94004b8620b0865aa7886a692dacb23d50c8f9b31fbeece07 diff --git a/packages/core/installer/values.yaml b/packages/core/installer/values.yaml index 218d869848..7c9ea260ae 100644 --- a/packages/core/installer/values.yaml +++ b/packages/core/installer/values.yaml @@ -14,7 +14,7 @@ cozystackOperator: # to the value baked at build time. The Makefile sets this for the # kubectl-apply release artifacts (_out/assets/cozystack-operator-*.yaml). platformVersion: "" - image: ghcr.io/cozystack/cozystack/cozystack-operator:v1.6.1-rc.1@sha256:ab2c0c8f87dc70f675bf36ce55f8615f810bbe3d0840d3a7d1a412f38abdf78a + image: ghcr.io/cozystack/cozystack/cozystack-operator:v1.6.1@sha256:ab2c0c8f87dc70f675bf36ce55f8615f810bbe3d0840d3a7d1a412f38abdf78a platformSourceUrl: 'oci://ghcr.io/cozystack/cozystack/cozystack-packages' platformSourceRef: 'digest=sha256:2d7e07e4cf3d49a97f5b9c62ad732cfb93586483723788eed489b32643a1a8df' # When non-empty, overrides the operator's --helmrelease-interval flag diff --git a/packages/core/platform/values.yaml b/packages/core/platform/values.yaml index 63abfb386e..278665c641 100644 --- a/packages/core/platform/values.yaml +++ b/packages/core/platform/values.yaml @@ -13,7 +13,7 @@ migrations: # below therefore lags targetVersion in-tree by design; run-migrations.sh # refuses to advance past a migration file missing from the image (exit 1), so # a stale pin fails loudly rather than silently skipping the etcd adoption. - image: ghcr.io/cozystack/cozystack/platform-migrations:v1.6.1-rc.1@sha256:a34e7156a52dc55b27662917e5f9d5bbea671188d8db233296d0a98c800dc6c6 + image: ghcr.io/cozystack/cozystack/platform-migrations:v1.6.1@sha256:a34e7156a52dc55b27662917e5f9d5bbea671188d8db233296d0a98c800dc6c6 targetVersion: 54 # Adopt legacy etcd clusters onto the v1alpha2 operator WITHOUT the mandatory # pre-adoption safety snapshot (migration 50). Leave false unless migration 50 diff --git a/packages/core/testing/values.yaml b/packages/core/testing/values.yaml index b42b13545c..4f3beb63cb 100644 --- a/packages/core/testing/values.yaml +++ b/packages/core/testing/values.yaml @@ -1,2 +1,2 @@ e2e: - image: ghcr.io/cozystack/cozystack/e2e-sandbox:v1.6.1-rc.1@sha256:a9980e8c48d6e50ed2b5d8245a4e0aae00970777b6e2255577fdb66035a44c68 + image: ghcr.io/cozystack/cozystack/e2e-sandbox:v1.6.1@sha256:a9980e8c48d6e50ed2b5d8245a4e0aae00970777b6e2255577fdb66035a44c68 diff --git a/packages/extra/bootbox/images/matchbox.tag b/packages/extra/bootbox/images/matchbox.tag index c321112344..7a3ea6c975 100644 --- a/packages/extra/bootbox/images/matchbox.tag +++ b/packages/extra/bootbox/images/matchbox.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/matchbox:v1.6.1-rc.1@sha256:3f492d2bd2d14de94c88dc995e3092911e2ca8ed7e17d305398e7fd49135d351 +ghcr.io/cozystack/cozystack/matchbox:v1.6.1@sha256:3f492d2bd2d14de94c88dc995e3092911e2ca8ed7e17d305398e7fd49135d351 diff --git a/packages/extra/seaweedfs/images/objectstorage-sidecar.tag b/packages/extra/seaweedfs/images/objectstorage-sidecar.tag index 14122ad411..4f903c420a 100644 --- a/packages/extra/seaweedfs/images/objectstorage-sidecar.tag +++ b/packages/extra/seaweedfs/images/objectstorage-sidecar.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/objectstorage-sidecar:v1.6.1-rc.1@sha256:ef3c154c1a6dd1ac9fa1b0763a2bd23707d68f2f16f16a97ebf331f3646b53ce +ghcr.io/cozystack/cozystack/objectstorage-sidecar:v1.6.1@sha256:ef3c154c1a6dd1ac9fa1b0763a2bd23707d68f2f16f16a97ebf331f3646b53ce diff --git a/packages/system/backup-controller/values.yaml b/packages/system/backup-controller/values.yaml index 6b0e3d475b..1ddce6ae32 100644 --- a/packages/system/backup-controller/values.yaml +++ b/packages/system/backup-controller/values.yaml @@ -1,5 +1,5 @@ backupController: - image: "ghcr.io/cozystack/cozystack/backup-controller:v1.6.1-rc.1@sha256:ef3c45ac4b23d6231a4166b4331a622374960caeeef3e4a901b0ee1dabcb6922" + image: "ghcr.io/cozystack/cozystack/backup-controller:v1.6.1@sha256:ef3c45ac4b23d6231a4166b4331a622374960caeeef3e4a901b0ee1dabcb6922" replicas: 2 debug: false metrics: diff --git a/packages/system/backupstrategy-controller/values.yaml b/packages/system/backupstrategy-controller/values.yaml index 253290bb58..a404d00a0b 100644 --- a/packages/system/backupstrategy-controller/values.yaml +++ b/packages/system/backupstrategy-controller/values.yaml @@ -1,5 +1,5 @@ backupStrategyController: - image: "ghcr.io/cozystack/cozystack/backupstrategy-controller:v1.6.1-rc.1@sha256:7cb5af7bd8fae2ef5d6e58d1eb6eb7d9579bff8df228ac3d936c6e14ae5e063f" + image: "ghcr.io/cozystack/cozystack/backupstrategy-controller:v1.6.1@sha256:7cb5af7bd8fae2ef5d6e58d1eb6eb7d9579bff8df228ac3d936c6e14ae5e063f" # chBackupClientImage is the image rendered into the Altinity strategy # Pod: it drives clickhouse-backup's HTTP API via curl + jq. # @@ -35,7 +35,7 @@ backupStrategyController: # retag both to the same stable tag, and the write-once guard in # hack/promote-retag.sh fails the release. hack/image-pin-consistency.bats # asserts no repository is pinned at more than one digest. - chBackupClientImage: "ghcr.io/cozystack/cozystack/platform-migrations:v1.6.1-rc.1@sha256:a34e7156a52dc55b27662917e5f9d5bbea671188d8db233296d0a98c800dc6c6" + chBackupClientImage: "ghcr.io/cozystack/cozystack/platform-migrations:v1.6.1@sha256:a34e7156a52dc55b27662917e5f9d5bbea671188d8db233296d0a98c800dc6c6" replicas: 2 debug: false metrics: diff --git a/packages/system/bucket/images/s3manager.tag b/packages/system/bucket/images/s3manager.tag index 5e9239a41a..83fec5eec0 100644 --- a/packages/system/bucket/images/s3manager.tag +++ b/packages/system/bucket/images/s3manager.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/s3manager:v1.6.1-rc.1@sha256:73665aaf7f5406b4a1e456d14060fe874e0ae2a436ba7fd3058fa73abff8a927 +ghcr.io/cozystack/cozystack/s3manager:v1.6.1@sha256:73665aaf7f5406b4a1e456d14060fe874e0ae2a436ba7fd3058fa73abff8a927 diff --git a/packages/system/cilium/values.yaml b/packages/system/cilium/values.yaml index 5903609d0e..9700f73a79 100644 --- a/packages/system/cilium/values.yaml +++ b/packages/system/cilium/values.yaml @@ -23,7 +23,7 @@ cilium: mode: "kubernetes" image: repository: ghcr.io/cozystack/cozystack/cilium - tag: v1.6.1-rc.1 + tag: v1.6.1 digest: "sha256:136a7dfff4d6adcd448424c2354144160028a8e96eb6558b917fd3ee6d7b39fe" envoy: enabled: true diff --git a/packages/system/cozystack-api/values.yaml b/packages/system/cozystack-api/values.yaml index cab825f252..31e9e253e7 100644 --- a/packages/system/cozystack-api/values.yaml +++ b/packages/system/cozystack-api/values.yaml @@ -1,3 +1,3 @@ cozystackAPI: - image: ghcr.io/cozystack/cozystack/cozystack-api:v1.6.1-rc.1@sha256:6d1db0a4fc1f21282f7825fed13807f80e40998f6703a087f372f19cc4197173 + image: ghcr.io/cozystack/cozystack/cozystack-api:v1.6.1@sha256:6d1db0a4fc1f21282f7825fed13807f80e40998f6703a087f372f19cc4197173 replicas: 2 diff --git a/packages/system/cozystack-controller/values.yaml b/packages/system/cozystack-controller/values.yaml index d76f85b957..a3444ceaef 100644 --- a/packages/system/cozystack-controller/values.yaml +++ b/packages/system/cozystack-controller/values.yaml @@ -1,4 +1,4 @@ cozystackController: - image: ghcr.io/cozystack/cozystack/cozystack-controller:v1.6.1-rc.1@sha256:1b60f18b37c140fa9e50baacd62a8fe9ed8d549c26746b081d5b460c7ca98a04 + image: ghcr.io/cozystack/cozystack/cozystack-controller:v1.6.1@sha256:1b60f18b37c140fa9e50baacd62a8fe9ed8d549c26746b081d5b460c7ca98a04 debug: false disableTelemetry: false diff --git a/packages/system/dashboard/values.yaml b/packages/system/dashboard/values.yaml index 704ccc27ea..1b232d9e98 100644 --- a/packages/system/dashboard/values.yaml +++ b/packages/system/dashboard/values.yaml @@ -1,4 +1,4 @@ console: - image: ghcr.io/cozystack/cozystack/cozystack-ui:v1.6.1-rc.1@sha256:e8a7f00cf8f970c2d8e598570ca4ed98c5448e2362489f4470a2a89543f760ed + image: ghcr.io/cozystack/cozystack/cozystack-ui:v1.6.1@sha256:e8a7f00cf8f970c2d8e598570ca4ed98c5448e2362489f4470a2a89543f760ed tokenProxy: - image: ghcr.io/cozystack/cozystack/token-proxy:v1.6.1-rc.1@sha256:97124c012246c33cdcac60e8f884097b48c0081624716fe8161efc2b3ceeb2a8 + image: ghcr.io/cozystack/cozystack/token-proxy:v1.6.1@sha256:97124c012246c33cdcac60e8f884097b48c0081624716fe8161efc2b3ceeb2a8 diff --git a/packages/system/flux-shard-operator/values.yaml b/packages/system/flux-shard-operator/values.yaml index 460d518995..1a6a201ec4 100644 --- a/packages/system/flux-shard-operator/values.yaml +++ b/packages/system/flux-shard-operator/values.yaml @@ -1,5 +1,5 @@ fluxShardOperator: - image: ghcr.io/cozystack/cozystack/flux-shard-operator:v1.6.1-rc.1@sha256:5774b606d182ebc70319f1adf6641c52afe417e968bdd64540eb32fb8f28605f + image: ghcr.io/cozystack/cozystack/flux-shard-operator:v1.6.1@sha256:5774b606d182ebc70319f1adf6641c52afe417e968bdd64540eb32fb8f28605f debug: false replicas: 2 ## Number of helm-controller shards to provision and distribute tenants diff --git a/packages/system/grafana-operator/images/grafana-dashboards.tag b/packages/system/grafana-operator/images/grafana-dashboards.tag index 42048cb4f0..49327da8d4 100644 --- a/packages/system/grafana-operator/images/grafana-dashboards.tag +++ b/packages/system/grafana-operator/images/grafana-dashboards.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/grafana-dashboards:v1.6.1-rc.1@sha256:fbbcfed72e9d23f223d990b9cd38a8c49ea3f985d7da65defc79eb730c0861e3 +ghcr.io/cozystack/cozystack/grafana-dashboards:v1.6.1@sha256:fbbcfed72e9d23f223d990b9cd38a8c49ea3f985d7da65defc79eb730c0861e3 diff --git a/packages/system/kamaji/values.yaml b/packages/system/kamaji/values.yaml index c95e9a6e76..858b9e8d1f 100644 --- a/packages/system/kamaji/values.yaml +++ b/packages/system/kamaji/values.yaml @@ -30,7 +30,7 @@ kamaji: topologyKey: kubernetes.io/hostname image: pullPolicy: IfNotPresent - tag: v1.6.1-rc.1@sha256:dd81f5b76208e11665f2d2eed00396653c32182079ee53f44169074ac35d62ed + tag: v1.6.1@sha256:dd81f5b76208e11665f2d2eed00396653c32182079ee53f44169074ac35d62ed repository: ghcr.io/cozystack/cozystack/kamaji resources: limits: @@ -63,4 +63,4 @@ kamaji: periodSeconds: 10 timeoutSeconds: 1 extraArgs: - - --migrate-image=ghcr.io/cozystack/cozystack/kamaji:v1.6.1-rc.1@sha256:dd81f5b76208e11665f2d2eed00396653c32182079ee53f44169074ac35d62ed + - --migrate-image=ghcr.io/cozystack/cozystack/kamaji:v1.6.1@sha256:dd81f5b76208e11665f2d2eed00396653c32182079ee53f44169074ac35d62ed diff --git a/packages/system/kubeovn-plunger/values.yaml b/packages/system/kubeovn-plunger/values.yaml index d98bda2b1d..ed38af48a6 100644 --- a/packages/system/kubeovn-plunger/values.yaml +++ b/packages/system/kubeovn-plunger/values.yaml @@ -1,4 +1,4 @@ portSecurity: true routes: "" -image: ghcr.io/cozystack/cozystack/kubeovn-plunger:v1.6.1-rc.1@sha256:6f38f72c16f86a5937630ba58ee42afca2fd8f14dfe45bb79670f9a4a8447b60 +image: ghcr.io/cozystack/cozystack/kubeovn-plunger:v1.6.1@sha256:6f38f72c16f86a5937630ba58ee42afca2fd8f14dfe45bb79670f9a4a8447b60 ovnCentralName: ovn-central diff --git a/packages/system/kubeovn-webhook/values.yaml b/packages/system/kubeovn-webhook/values.yaml index fd17dc8ad9..c356efd936 100644 --- a/packages/system/kubeovn-webhook/values.yaml +++ b/packages/system/kubeovn-webhook/values.yaml @@ -1,3 +1,3 @@ portSecurity: true routes: "" -image: ghcr.io/cozystack/cozystack/kubeovn-webhook:v1.6.1-rc.1@sha256:593c324a38db59495497c5b68b90cad25dfe6753c73b7a287b3280e26f70a15f +image: ghcr.io/cozystack/cozystack/kubeovn-webhook:v1.6.1@sha256:593c324a38db59495497c5b68b90cad25dfe6753c73b7a287b3280e26f70a15f diff --git a/packages/system/kubevirt-csi-node/values.yaml b/packages/system/kubevirt-csi-node/values.yaml index 848417ba50..98c80c4d78 100644 --- a/packages/system/kubevirt-csi-node/values.yaml +++ b/packages/system/kubevirt-csi-node/values.yaml @@ -9,4 +9,4 @@ storageClasses: infraStorageClass: replicated default: false csiDriver: - image: ghcr.io/cozystack/cozystack/kubevirt-csi-driver:v1.6.1-rc.1@sha256:7a47bb5aa46fccfe33dc0e0d857a6ec2ce2c6284a213e37beff5edcf6f37c673 + image: ghcr.io/cozystack/cozystack/kubevirt-csi-driver:v1.6.1@sha256:7a47bb5aa46fccfe33dc0e0d857a6ec2ce2c6284a213e37beff5edcf6f37c673 diff --git a/packages/system/lineage-controller-webhook/values.yaml b/packages/system/lineage-controller-webhook/values.yaml index 78f3aa7d4c..e535d4ca8a 100644 --- a/packages/system/lineage-controller-webhook/values.yaml +++ b/packages/system/lineage-controller-webhook/values.yaml @@ -1,5 +1,5 @@ lineageControllerWebhook: - image: ghcr.io/cozystack/cozystack/lineage-controller-webhook:v1.6.1-rc.1@sha256:b8bfe1573ef75f74e32eff6fe3c57d16413860836299d2f64a8a28fd482a65ab + image: ghcr.io/cozystack/cozystack/lineage-controller-webhook:v1.6.1@sha256:b8bfe1573ef75f74e32eff6fe3c57d16413860836299d2f64a8a28fd482a65ab debug: false replicas: 2 # DEPRECATED. Injects KUBERNETES_SERVICE_HOST=status.hostIP and diff --git a/packages/system/linstor-gui/values.yaml b/packages/system/linstor-gui/values.yaml index 6b35b14868..b690912057 100644 --- a/packages/system/linstor-gui/values.yaml +++ b/packages/system/linstor-gui/values.yaml @@ -3,7 +3,7 @@ ## @param image.tag LINSTOR GUI container image tag (digest recommended) image: repository: ghcr.io/cozystack/cozystack/linstor-gui - tag: v1.6.1-rc.1@sha256:6e11829de86709f5e21636cdd1a1ada1239a7dcc1741f91e9d95d9f6ade603a2 + tag: v1.6.1@sha256:6e11829de86709f5e21636cdd1a1ada1239a7dcc1741f91e9d95d9f6ade603a2 ## @section Deployment ## @param replicas Number of linstor-gui replicas replicas: 1 diff --git a/packages/system/linstor/values.yaml b/packages/system/linstor/values.yaml index 8cd531bebe..be08bf00fc 100644 --- a/packages/system/linstor/values.yaml +++ b/packages/system/linstor/values.yaml @@ -1,7 +1,7 @@ piraeusServer: image: repository: ghcr.io/cozystack/cozystack/piraeus-server - tag: v1.6.1-rc.1@sha256:82d2234181ac8d5942e57286101b242a08fbcc49a9bf37d5df529921624fdca9 + tag: v1.6.1@sha256:82d2234181ac8d5942e57286101b242a08fbcc49a9bf37d5df529921624fdca9 # Talos-specific workarounds (disable for generic Linux like Ubuntu/Debian) talos: enabled: true @@ -13,7 +13,7 @@ linstor: linstorCSI: image: repository: ghcr.io/cozystack/cozystack/linstor-csi - tag: v1.6.1-rc.1@sha256:3228487ae861ddf02cac984289f8c377fc9e66a71fb3c387e9af9f72c1dbc78b + tag: v1.6.1@sha256:3228487ae861ddf02cac984289f8c377fc9e66a71fb3c387e9af9f72c1dbc78b controller: # Resource requests for the linstor-controller JVM. Requests (not limits) are # set on purpose: a CPU request guarantees CPU shares so the JVM can answer its diff --git a/packages/system/metallb/values.yaml b/packages/system/metallb/values.yaml index 0ce1c9722d..9404c13e60 100644 --- a/packages/system/metallb/values.yaml +++ b/packages/system/metallb/values.yaml @@ -37,11 +37,11 @@ metallb: controller: image: repository: ghcr.io/cozystack/cozystack/metallb-controller - tag: v1.6.1-rc.1@sha256:9d8ba76cdb9c7c6221334ad05d706dee22b138b3e90c1fe8fc884925b7480c02 + tag: v1.6.1@sha256:9d8ba76cdb9c7c6221334ad05d706dee22b138b3e90c1fe8fc884925b7480c02 speaker: image: repository: ghcr.io/cozystack/cozystack/metallb-speaker - tag: v1.6.1-rc.1@sha256:87df3c82d0b6ea223b26fd5d6fbba6e940c13e56418dfc1cd90863d145795be9 + tag: v1.6.1@sha256:87df3c82d0b6ea223b26fd5d6fbba6e940c13e56418dfc1cd90863d145795be9 # The vendored metallb chart's values.yaml leaves `frr-k8s.prometheus:` as # a YAML block containing only commented examples — it parses to null. # Helm-controller's deep-merge then writes that null over the frr-k8s diff --git a/packages/system/monitoring/images/grafana.tag b/packages/system/monitoring/images/grafana.tag index 1737139776..7221f07092 100644 --- a/packages/system/monitoring/images/grafana.tag +++ b/packages/system/monitoring/images/grafana.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/grafana:v1.6.1-rc.1@sha256:d29c2306b96dfc47aab05e6afd1bce581c552f02d022084c11737632d8839029 +ghcr.io/cozystack/cozystack/grafana:v1.6.1@sha256:d29c2306b96dfc47aab05e6afd1bce581c552f02d022084c11737632d8839029 diff --git a/packages/system/multus/templates/multus-daemonset-thick.yml b/packages/system/multus/templates/multus-daemonset-thick.yml index 72a02d4d1a..da69f773b8 100644 --- a/packages/system/multus/templates/multus-daemonset-thick.yml +++ b/packages/system/multus/templates/multus-daemonset-thick.yml @@ -156,7 +156,7 @@ spec: serviceAccountName: multus containers: - name: kube-multus - image: ghcr.io/cozystack/cozystack/multus-cni:v1.6.1-rc.1@sha256:f94fdd68d674a1f8c8a2b45fa8fd4b478fc632e66fa147aea73a6b60b721920c + image: ghcr.io/cozystack/cozystack/multus-cni:v1.6.1@sha256:f94fdd68d674a1f8c8a2b45fa8fd4b478fc632e66fa147aea73a6b60b721920c command: [ "/usr/src/multus-cni/bin/multus-daemon" ] resources: requests: @@ -202,7 +202,7 @@ spec: fieldPath: spec.nodeName initContainers: - name: install-multus-binary - image: ghcr.io/cozystack/cozystack/multus-cni:v1.6.1-rc.1@sha256:f94fdd68d674a1f8c8a2b45fa8fd4b478fc632e66fa147aea73a6b60b721920c + image: ghcr.io/cozystack/cozystack/multus-cni:v1.6.1@sha256:f94fdd68d674a1f8c8a2b45fa8fd4b478fc632e66fa147aea73a6b60b721920c command: - "/usr/src/multus-cni/bin/install_multus" - "-d" diff --git a/packages/system/objectstorage-controller/values.yaml b/packages/system/objectstorage-controller/values.yaml index 8ee02ab44b..c1dc68cc36 100644 --- a/packages/system/objectstorage-controller/values.yaml +++ b/packages/system/objectstorage-controller/values.yaml @@ -1,3 +1,3 @@ objectstorage: controller: - image: "ghcr.io/cozystack/cozystack/objectstorage-controller:v1.6.1-rc.1@sha256:0d9ff6c2a3453fdc7e72894bbfdd3e5a292be78342c7c5352f655c976c3b7f0b" + image: "ghcr.io/cozystack/cozystack/objectstorage-controller:v1.6.1@sha256:0d9ff6c2a3453fdc7e72894bbfdd3e5a292be78342c7c5352f655c976c3b7f0b" diff --git a/packages/system/seaweedfs/values.yaml b/packages/system/seaweedfs/values.yaml index 257affa121..253dae4f81 100644 --- a/packages/system/seaweedfs/values.yaml +++ b/packages/system/seaweedfs/values.yaml @@ -259,7 +259,7 @@ seaweedfs: bucketClassName: "seaweedfs" region: "" sidecar: - image: "ghcr.io/cozystack/cozystack/objectstorage-sidecar:v1.6.1-rc.1@sha256:ef3c154c1a6dd1ac9fa1b0763a2bd23707d68f2f16f16a97ebf331f3646b53ce" + image: "ghcr.io/cozystack/cozystack/objectstorage-sidecar:v1.6.1@sha256:ef3c154c1a6dd1ac9fa1b0763a2bd23707d68f2f16f16a97ebf331f3646b53ce" certificates: commonName: "SeaweedFS CA" ipAddresses: [] diff --git a/packages/system/securitygroup-controller/values.yaml b/packages/system/securitygroup-controller/values.yaml index cfc25635e3..987e944e98 100644 --- a/packages/system/securitygroup-controller/values.yaml +++ b/packages/system/securitygroup-controller/values.yaml @@ -1,5 +1,5 @@ securityGroupController: - image: "ghcr.io/cozystack/cozystack/securitygroup-controller:v1.6.1-rc.1@sha256:1eaaed61b595ed79460709a410d1da830031e82926b0a2fd8cfd917e0e120844" + image: "ghcr.io/cozystack/cozystack/securitygroup-controller:v1.6.1@sha256:1eaaed61b595ed79460709a410d1da830031e82926b0a2fd8cfd917e0e120844" replicas: 2 debug: false resources: From 781328ebed8e078871e18a6b1b4054a769378ba4 Mon Sep 17 00:00:00 2001 From: "cozystack-ci[bot]" <274107086+cozystack-ci[bot]@users.noreply.github.com> Date: Wed, 5 Aug 2026 09:35:27 +0000 Subject: [PATCH 17/51] docs: add changelog for v1.6.1 Signed-off-by: cozystack-ci[bot] <274107086+cozystack-ci[bot]@users.noreply.github.com> --- docs/changelogs/v1.6.1.md | 72 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 docs/changelogs/v1.6.1.md diff --git a/docs/changelogs/v1.6.1.md b/docs/changelogs/v1.6.1.md new file mode 100644 index 0000000000..94f690f505 --- /dev/null +++ b/docs/changelogs/v1.6.1.md @@ -0,0 +1,72 @@ + + +# v1.6.1 (2026-08-05) + +A patch release with seven fixes covering PostgreSQL, etcd, managed Kubernetes, Keycloak, `cozystack-basics`, and SeaweedFS, plus release-pipeline reliability fixes and a `talm` update adding declarative Talos preset knobs. + +## Fixes + +* **fix(postgres-operator): align CNPG operator and CRDs to 1.28.2 for PVC resize-deadlock fix**: A simultaneous `resources` + `size` change on a single-instance PostgreSQL cluster could make the CloudNativePG operator delete the sole primary Pod, classify the PVC as `resizing`, and never recreate the Pod — wedging the cluster with zero instances and leaving the filesystem resize incomplete. Bumping the operator image and CRDs together to 1.28.2 (which carries upstream's fix, cloudnative-pg#9980 / cloudnative-pg#9981) resolves the deadlock ([**@scooby87**](https://github.com/scooby87) in #3510, backport #3542). + +* **chore(etcd-operator): bump etcd-operator to v0.5.4**: Rolls up four upstream controller bug fixes: the operator no longer exempts the bootstrap seed from crash-loop self-heal, self-heal now also covers memory-backed etcd members, `--initial-cluster-state` is derived from cluster phase instead of the seed, and each EtcdCluster's PodDisruptionBudget switches from `maxUnavailable` to `minAvailable` (existing clusters are reconciled onto the new field automatically on upgrade). Tenant etcd clusters recover more reliably from member crashes and are less likely to have their PDB block a node drain ([**@androndo**](https://github.com/androndo) in #3529, backport #3538). + +* **fix(kubernetes): render the talos-reconcile Job for the default md0 group**: On a managed Kubernetes cluster left with the default (undeclared) `nodeGroups`, the Job that creates each worker's `TalosConfigTemplate` and patches the control-plane's cert SANs only iterated the user-supplied node-group map, so it silently skipped the implicit `md0` group. Nothing failed at install time, since `md0` defaults to zero replicas, but the first scale-up (for example, autoscaler-driven growth after enabling ingress-nginx) left new Machines permanently blocked with no matching `TalosConfigTemplate`. The Job now iterates the same helper that produces the `MachineDeployment`, so `md0` gets its reconcile Job like any explicitly declared group ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #3535, backport #3536). + +* **fix(keycloak-configure): patch HelmRelease in release namespace on teardown**: The `keycloak-configure` pre-delete Job cleared the Flux `HelmRelease` finalizer in a hardcoded namespace that did not match where the release actually installs (`cozy-keycloak`), so its `ServiceAccount` was forbidden to patch it, the Job retried forever, and the `HelmRelease` stuck in `Terminating` — blocking any uninstall or reinstall of Keycloak. The teardown Job now templates both the release name and namespace from the Helm release itself, so teardown completes correctly ([**@lexfrei**](https://github.com/lexfrei) in #3372, backport #3478). + +* **fix(cozystack-basics): gate the hostname VAP policies on the VAP API**: The hostname `ValidatingAdmissionPolicy` templates rendered unconditionally, so a first install on a cluster where the `ValidatingAdmissionPolicy` API is unavailable dropped the policies permanently — a later cluster upgrade that gains the API would not bring them back. The templates are now gated on `.Capabilities.APIVersions.Has`, so they render only where the API exists and are picked up automatically once it becomes available ([**@lexfrei**](https://github.com/lexfrei) in #3409, backport #3442). + +* **fix(seaweedfs): make naming audit fail closed on kubectl and payload errors**: `hack/seaweedfs-naming-audit.sh`, used by operators to classify SeaweedFS instances before the naming-migration cleanup, was fail-open — any `kubectl` failure or unreadable Helm release payload produced an empty result table indistinguishable from a genuinely clean fleet. Since the runbook uses this script as the gate before deleting old PVCs, a transient API error could have green-lit destroying live data. Every query now fails loudly on error instead of silently reporting "nothing found," and incomplete evidence now falls back to a conservative "direction cannot be established" verdict rather than a wrong deletion candidate ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #3436, backport #3474). + +* **chore(release): don't activate `kubernetes-nodes` on the release-1.6 line**: The `kubernetes-nodes` app package was not ready to ship on the 1.6 line, so its single include is removed from the `iaas` platform bundle — the platform stops activating the package on this line while its code, API types, and `kubernetes-nodes-rd` system package are all kept intact for when it is ready ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #3437). + +## Development, Testing, and CI/CD + +* **ci(release): carry the finalize fixes onto the 1.6 line**: `release-1.6` was cut before three release-pipeline fixes landed on `main`: dropping `persist-credentials` on checkout (which had let a stale `GITHUB_TOKEN` silently win over the app token, so the stable tag push created no workflow run and the automated changelog/docs backstops never fired, as happened for v1.6.0), publishing the GitHub release with the merged changelog as its body instead of a placeholder, and dropping a `paths-ignore` filter that could drop a changelog-only promotion PR. All three are backported so v1.6.1 releases correctly with real release notes ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #3530). + +* **fix(ci): overlay images from the PR base branch, and publish per-line artifacts**: PR validation always overlaid unbuilt packages from `cozystack-packages:main`, so a `release-1.6` PR was tested against `main`'s controller binaries against its own line's charts — which is exactly what made #3437 above fail install deterministically with a schema-validation error. Each maintained `release-X.Y` branch now builds and publishes its own packages artifact, and PR validation overlays from the artifact matching the PR's own base branch ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #3471, backport #3514). + +* **fix(release): make promote-retag digest verification media-type-agnostic**: The v1.6.0 finalize run aborted partway through promoting rc images to stable tags because its post-copy digest check used `skopeo inspect --format '{{.Digest}}'`, which prints nothing for OCI artifacts like `cozystack-packages`, leaving most repositories without a stable tag and skipping the installer publish. The digest is now computed as the sha256 of the raw manifest, which works identically for container images and OCI artifacts, so promotion can no longer abort mid-way on this class of artifact ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #3435, backport #3473). + +## Other repositories + +### talm v0.34.0 + +* **[talm] feat(charts): add preset value knobs**: Exposes `timeServers`, control-plane component `extraArgs`, `registryMirrors`, per-host `registryTLS`, multiple Layer2 `vips`, `network.preserveExisting`, and `network.extraLinks` (bonds, VLANs, extra addresses and routes) as values on the cozystack, generic, and talm presets, so a node's Talos machine config can be described declaratively instead of via a template fork. Every knob defaults empty and a stock render stays byte-identical; each input Talos would reject fails fast at render time with a hinted error ([**@lexfrei**](https://github.com/lexfrei) in cozystack/talm#232). + +* **[talm] chore(deps): migrate to Helm 4 and drop the cozystack/talos fork**: Moves talm's vendored Helm template engine from v3 to v4 and drops the `cozystack/talos` fork (carried solely for a `--skip-verify` flag, now reimplemented locally), tracking stock upstream Talos v1.13.7. Golden render snapshots confirm the generated machine config is unchanged for users ([**@lexfrei**](https://github.com/lexfrei) in cozystack/talm#231). + +## Documentation + +* **[website] feat(blog): add Cozystack 1.6 release and Blockstor announcement**: Publishes the v1.6.0 release-announcement blog post — covering Talos Linux tenant workers, tenant-controlled OIDC, the `SecurityGroup` API, hierarchical quotas, and in-place etcd-operator adoption — alongside a companion post announcing the open-sourcing of Blockstor, the LINSTOR-compatible storage control plane ([**@tym83**](https://github.com/tym83) in cozystack/website#641). + +* **[website] chore(blog): repair front matter, links and bundle names**: Follow-up cleanup on the two new blog posts, fixing front matter, internal links, and page bundle names ([**@tym83**](https://github.com/tym83) in cozystack/website@4b5d1ad). + +* **[website] chore: upgrade Hugo to 0.164.0 and convert HTML content to markdown**: Upgrades the site generator to Hugo 0.164.0, requiring Node 22 under its new node permission model, and converts remaining raw-HTML content to markdown along the way, keeping the site buildable on current tooling ([**@tym83**](https://github.com/tym83) in cozystack/website#636). + +* **[website] chore(blog): make the two taxonomy axes disjoint**: Cleans up the blog's tagging so its two taxonomy axes no longer overlap, making blog post categorization and filtering more consistent ([**@tym83**](https://github.com/tym83) in cozystack/website#635). + +* **[website] docs(talm): describe the preset value knobs for network and registries**: Documents the new `talm` preset value knobs (cozystack/talm#232) for time servers, control-plane extra args, registry mirrors and TLS, VIPs, and network links, so operators can find the declarative equivalents to a template fork ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#633). + +* **[website] feat(hack): generate docs from a pre-tag ref and fail loudly on fetch errors**: Hardens the docs-generation tooling to build from a pinned pre-tag ref and to fail loudly instead of silently on a fetch error, reducing the chance of publishing docs generated from the wrong ref ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in cozystack/website#631). + +* **[website] docs(backups): document PostgreSQL point-in-time recovery (PITR)**: Adds documentation for PostgreSQL point-in-time recovery, covering how to configure and perform a PITR restore for managed PostgreSQL clusters ([**@androndo**](https://github.com/androndo) in cozystack/website#629). + +* **[website] docs: update managed apps reference for v1.6.0**: Refreshes the managed applications reference pages to match what shipped in v1.6.0 ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in cozystack/website#628). + +* **[website] docs(monitoring): add OIDC authentication guide for Grafana**: Adds a guide for configuring OIDC single sign-on authentication for per-instance Grafana, complementing v1.6.0's tenant OIDC support ([**@IvanHunters**](https://github.com/IvanHunters) in cozystack/website#597). + +## Contributors + +Thanks to everyone who contributed to this patch release: + +* [**@IvanHunters**](https://github.com/IvanHunters) +* [**@androndo**](https://github.com/androndo) +* [**@lexfrei**](https://github.com/lexfrei) +* [**@myasnikovdaniil**](https://github.com/myasnikovdaniil) +* [**@scooby87**](https://github.com/scooby87) +* [**@tym83**](https://github.com/tym83) + +**Full Changelog**: https://github.com/cozystack/cozystack/compare/v1.6.0...v1.6.1 From 8a43376ed0ae5eee735f94a74a3d9fd4f7dd216b Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Wed, 5 Aug 2026 14:53:38 +0500 Subject: [PATCH 18/51] chore(release): remove accidental tooling gitlink Assisted-By: GPT-5 Signed-off-by: Myasnikov Daniil --- .release-tooling | 1 - 1 file changed, 1 deletion(-) delete mode 160000 .release-tooling diff --git a/.release-tooling b/.release-tooling deleted file mode 160000 index ee26a50f0e..0000000000 --- a/.release-tooling +++ /dev/null @@ -1 +0,0 @@ -Subproject commit ee26a50f0e6a9dfa30254f663ecaddbe9be97664 From e97379eaf44387f9e2cf5231f11a1a017fec9ebd Mon Sep 17 00:00:00 2001 From: Aleksei Sviridkin <3811295@gmail.com> Date: Mon, 10 Aug 2026 12:23:16 +0300 Subject: [PATCH 19/51] fix(velero): apply CRD updates on upgrade via CreateReplace velero evolves its CRD schema between versions; v1.18 adds the Queued and ReadyToStart backup phases. Helm never upgrades CRDs shipped in crds/, the generated HelmRelease left spec.upgrade.crds at the Skip default, and the package disables the chart's upgrade-crds Job, so live CRDs stayed frozen at first-install state: the server upgrades, the apiserver rejects the new phases, and backups sit in New while the HelmRelease stays green. Opt the velero PackageSource into CreateReplace, consistent with mongodb-operator, add a platform test pinning the policy, and correct the stale Job-disable rationale in the package. Assisted-By: Claude Signed-off-by: Aleksei Sviridkin <3811295@gmail.com> (cherry picked from commit f4d812cfb919496dfdfbca2ca8a3047a702d96cb) --- packages/core/platform/sources/velero.yaml | 1 + .../tests/sources_velero_crds_test.yaml | 22 +++++++++++++++++++ packages/system/velero/tests/velero_test.yaml | 9 ++++---- packages/system/velero/values.yaml | 8 +++---- 4 files changed, 31 insertions(+), 9 deletions(-) create mode 100644 packages/core/platform/tests/sources_velero_crds_test.yaml diff --git a/packages/core/platform/sources/velero.yaml b/packages/core/platform/sources/velero.yaml index 2374e84733..acfb4d2bdd 100644 --- a/packages/core/platform/sources/velero.yaml +++ b/packages/core/platform/sources/velero.yaml @@ -21,3 +21,4 @@ spec: privileged: true namespace: cozy-velero releaseName: velero + upgradeCRDs: CreateReplace diff --git a/packages/core/platform/tests/sources_velero_crds_test.yaml b/packages/core/platform/tests/sources_velero_crds_test.yaml new file mode 100644 index 0000000000..a48ad53c87 --- /dev/null +++ b/packages/core/platform/tests/sources_velero_crds_test.yaml @@ -0,0 +1,22 @@ +suite: velero applies CRD updates on upgrade +# velero evolves its CRD schema between versions (v1.18 adds the Queued and +# ReadyToStart backup phases). Helm never upgrades CRDs shipped in crds/, and +# the velero package disables the chart's upgrade-crds Job, so the PackageSource +# must opt into CreateReplace; without it the server binary upgrades while the +# live CRDs stay at first-install state, the apiserver rejects the new phases, +# and backups sit in New while the HelmRelease stays green. Pin the policy so an +# upgrade cannot silently regress. +templates: + - templates/sources.yaml +release: + name: cozystack + namespace: cozy-system +tests: + - it: velero PackageSource sets upgradeCRDs to CreateReplace + documentSelector: + path: metadata.name + value: cozystack.velero + asserts: + - equal: + path: spec.variants[0].components[0].install.upgradeCRDs + value: CreateReplace diff --git a/packages/system/velero/tests/velero_test.yaml b/packages/system/velero/tests/velero_test.yaml index 182bd4027c..44177de012 100644 --- a/packages/system/velero/tests/velero_test.yaml +++ b/packages/system/velero/tests/velero_test.yaml @@ -26,11 +26,10 @@ tests: path: spec.template.spec.initContainers[1].image value: quay.io/kubevirt/kubevirt-velero-plugin:v0.9.0 - # Cozystack disables the upgrade-crds Job (upgradeCRDs: false) because CRDs - # ship via the chart's crds/ directory, making the Job redundant. (The old - # kubectl-image-compatibility rationale is obsolete in velero 12.x, where the - # Job now runs the velero image natively.) Pin that invariant — it silently - # regressed once. + # Cozystack disables the upgrade-crds Job (upgradeCRDs: false): CRD lifecycle + # is owned by the generated HelmRelease (upgradeCRDs: CreateReplace on the + # velero PackageSource), making the Job redundant. Pin that invariant — it + # silently regressed once. - it: does not emit the upgrade-crds Job when upgradeCRDs is disabled template: charts/velero/templates/upgrade-crds/upgrade-crds.yaml asserts: diff --git a/packages/system/velero/values.yaml b/packages/system/velero/values.yaml index 07476feed1..a2c54e615b 100644 --- a/packages/system/velero/values.yaml +++ b/packages/system/velero/values.yaml @@ -1,8 +1,8 @@ velero: - # Disable the upgrade-crds Job: CRDs ship via the chart's crds/ directory, so - # the Job is redundant. (The old kubectl-image-compatibility rationale is - # obsolete in velero 12.x — the Job now runs the velero image natively, not a - # kubectl one — but it stays disabled because cozystack manages CRDs via crds/.) + # Disable the upgrade-crds Job: CRD lifecycle is owned by the generated + # HelmRelease instead (upgradeCRDs: CreateReplace on the velero PackageSource), + # which applies the chart's crds/ directory on install and on every upgrade, + # so the Job is redundant. upgradeCRDs: false initContainers: - name: velero-plugin-for-aws From a6107e6135885a1f4a2570860689c007eab9a8c3 Mon Sep 17 00:00:00 2001 From: IvanHunters Date: Wed, 5 Aug 2026 15:04:35 +0300 Subject: [PATCH 20/51] fix(kubeovn-webhook): reload serving certificate on cert-manager renewal The webhook loaded its TLS key pair once at startup and served it as a static tls.Config.Certificates entry for the lifetime of the process. A cert-manager renewal of the backing Secret was never picked up by a running pod, so once the certificate expired (about a year after install) the pod kept presenting the expired certificate. The kube-apiserver's TLS call to the webhook then failed and, because the MutatingWebhookConfiguration uses failurePolicy: Fail, every pod creation in tenant namespaces was rejected (including virt-launcher pods, blocking all VMIs). Serve the certificate through a reloading tls.Config.GetCertificate callback that re-reads the key pair when the mounted files change and keeps serving the last good certificate on a failed reload. Add a regression test that fails against the static implementation and passes once the certificate is reloaded. Signed-off-by: IvanHunters (cherry picked from commit 44d9f534a9bef729a9650b61c929828236b2a510) --- .../images/kubeovn-webhook/cert.go | 97 ++++++++++ .../images/kubeovn-webhook/cert_test.go | 181 ++++++++++++++++++ .../images/kubeovn-webhook/main.go | 11 +- 3 files changed, 282 insertions(+), 7 deletions(-) create mode 100644 packages/system/kubeovn-webhook/images/kubeovn-webhook/cert.go create mode 100644 packages/system/kubeovn-webhook/images/kubeovn-webhook/cert_test.go diff --git a/packages/system/kubeovn-webhook/images/kubeovn-webhook/cert.go b/packages/system/kubeovn-webhook/images/kubeovn-webhook/cert.go new file mode 100644 index 0000000000..9e15ec302b --- /dev/null +++ b/packages/system/kubeovn-webhook/images/kubeovn-webhook/cert.go @@ -0,0 +1,97 @@ +package main + +import ( + "crypto/tls" + "fmt" + "log" + "os" + "sync" + "time" +) + +// certReloader keeps the webhook serving certificate in sync with the key pair +// files written by cert-manager. +// +// The certificate is served through tls.Config.GetCertificate, so a cert-manager +// renewal of the backing Secret is picked up by the running process without a +// restart. Without this, the key pair is read exactly once at startup and cached +// for the lifetime of the process: after the certificate expires (roughly one year +// after install) the pod keeps presenting the expired certificate, the +// kube-apiserver's TLS call to the webhook fails, and because the +// MutatingWebhookConfiguration uses failurePolicy: Fail, every pod creation in +// tenant namespaces is rejected. +type certReloader struct { + certFile string + keyFile string + + mu sync.RWMutex + cert *tls.Certificate + modTime time.Time +} + +// newCertReloader loads the initial key pair and returns a reloader for it. +func newCertReloader(certFile, keyFile string) (*certReloader, error) { + cr := &certReloader{certFile: certFile, keyFile: keyFile} + if err := cr.reload(); err != nil { + return nil, err + } + return cr, nil +} + +// reload reads the key pair from disk and atomically swaps the cached certificate. +func (cr *certReloader) reload() error { + cert, err := tls.LoadX509KeyPair(cr.certFile, cr.keyFile) + if err != nil { + return err + } + + var modTime time.Time + if fi, statErr := os.Stat(cr.certFile); statErr == nil { + modTime = fi.ModTime() + } + + cr.mu.Lock() + cr.cert = &cert + cr.modTime = modTime + cr.mu.Unlock() + return nil +} + +// changed reports whether the certificate file was modified since the last load. +func (cr *certReloader) changed() bool { + fi, err := os.Stat(cr.certFile) + if err != nil { + return false + } + cr.mu.RLock() + defer cr.mu.RUnlock() + return fi.ModTime().After(cr.modTime) +} + +// GetCertificate is a tls.Config.GetCertificate callback. It reloads the key pair +// when the file changes and keeps serving the last good certificate if a reload +// fails (for example a torn read while the mounted Secret is being updated). +func (cr *certReloader) GetCertificate(*tls.ClientHelloInfo) (*tls.Certificate, error) { + if cr.changed() { + if err := cr.reload(); err != nil { + log.Printf("certificate reload failed, keeping previous certificate: %v", err) + } + } + + cr.mu.RLock() + defer cr.mu.RUnlock() + if cr.cert == nil { + return nil, fmt.Errorf("no certificate loaded") + } + return cr.cert, nil +} + +// newReloadingTLSConfig builds a tls.Config that serves the key pair via a +// certReloader, so cert-manager renewals are honoured without a restart. +func newReloadingTLSConfig(certFile, keyFile string) (*tls.Config, error) { + cr, err := newCertReloader(certFile, keyFile) + if err != nil { + return nil, err + } + return &tls.Config{GetCertificate: cr.GetCertificate}, nil +} diff --git a/packages/system/kubeovn-webhook/images/kubeovn-webhook/cert_test.go b/packages/system/kubeovn-webhook/images/kubeovn-webhook/cert_test.go new file mode 100644 index 0000000000..1a670e4fd8 --- /dev/null +++ b/packages/system/kubeovn-webhook/images/kubeovn-webhook/cert_test.go @@ -0,0 +1,181 @@ +package main + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "math/big" + "os" + "path/filepath" + "testing" + "time" +) + +// genSelfSigned returns a PEM-encoded self-signed cert/key pair carrying the given +// serial number, so tests can tell two generations of the certificate apart. +func genSelfSigned(t *testing.T, serial int64) (certPEM, keyPEM []byte) { + t.Helper() + + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatalf("generate key: %v", err) + } + + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(serial), + Subject: pkix.Name{CommonName: "kube-ovn-webhook-test"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + DNSNames: []string{"localhost"}, + BasicConstraintsValid: true, + } + + der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) + if err != nil { + t.Fatalf("create certificate: %v", err) + } + keyDER, err := x509.MarshalPKCS8PrivateKey(key) + if err != nil { + t.Fatalf("marshal key: %v", err) + } + + certPEM = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}) + keyPEM = pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: keyDER}) + return certPEM, keyPEM +} + +// writeKeyPair writes the cert/key files and bumps their mtime strictly forward, +// mirroring how cert-manager replaces the mounted Secret on renewal. +func writeKeyPair(t *testing.T, certFile, keyFile string, certPEM, keyPEM []byte, mtime time.Time) { + t.Helper() + if err := os.WriteFile(certFile, certPEM, 0o600); err != nil { + t.Fatalf("write cert: %v", err) + } + if err := os.WriteFile(keyFile, keyPEM, 0o600); err != nil { + t.Fatalf("write key: %v", err) + } + if err := os.Chtimes(certFile, mtime, mtime); err != nil { + t.Fatalf("chtimes cert: %v", err) + } + if err := os.Chtimes(keyFile, mtime, mtime); err != nil { + t.Fatalf("chtimes key: %v", err) + } +} + +// servedSerial completes a TLS handshake against addr and returns the serial number +// of the leaf certificate the server actually presented. +func servedSerial(t *testing.T, addr string) int64 { + t.Helper() + conn, err := tls.Dial("tcp", addr, &tls.Config{InsecureSkipVerify: true}) //nolint:gosec // test-only, inspecting the served cert + if err != nil { + t.Fatalf("dial: %v", err) + } + defer conn.Close() + certs := conn.ConnectionState().PeerCertificates + if len(certs) == 0 { + t.Fatalf("server presented no certificate") + } + return certs[0].SerialNumber.Int64() +} + +// TestReloadingTLSConfigServesRenewedCertificate is the regression test for the +// cert-manager renewal not being picked up: it fails against the old code that +// loads the key pair once into tls.Config.Certificates, and passes once the +// certificate is served via a reloading GetCertificate callback. +func TestReloadingTLSConfigServesRenewedCertificate(t *testing.T) { + dir := t.TempDir() + certFile := filepath.Join(dir, "tls.crt") + keyFile := filepath.Join(dir, "tls.key") + + certA, keyA := genSelfSigned(t, 1) + writeKeyPair(t, certFile, keyFile, certA, keyA, time.Now().Add(-2*time.Second)) + + tlsConfig, err := newReloadingTLSConfig(certFile, keyFile) + if err != nil { + t.Fatalf("newReloadingTLSConfig: %v", err) + } + + ln, err := tls.Listen("tcp", "127.0.0.1:0", tlsConfig) + if err != nil { + t.Fatalf("listen: %v", err) + } + defer ln.Close() + go func() { + for { + conn, err := ln.Accept() + if err != nil { + return + } + go func() { + if tc, ok := conn.(*tls.Conn); ok { + _ = tc.Handshake() + } + conn.Close() + }() + } + }() + + addr := ln.Addr().String() + + if got := servedSerial(t, addr); got != 1 { + t.Fatalf("before renewal: expected serial 1, got %d", got) + } + + // cert-manager renews the Secret: the mounted files are replaced in place. + certB, keyB := genSelfSigned(t, 2) + writeKeyPair(t, certFile, keyFile, certB, keyB, time.Now().Add(2*time.Second)) + + if got := servedSerial(t, addr); got != 2 { + t.Fatalf("after renewal: expected renewed serial 2, got %d (certificate was not reloaded)", got) + } +} + +// TestCertReloaderKeepsLastGoodCertificate verifies that a failed reload (e.g. a +// torn read while the volume is updated) does not take the webhook down: it keeps +// serving the previously loaded certificate instead of erroring the handshake. +func TestCertReloaderKeepsLastGoodCertificate(t *testing.T) { + dir := t.TempDir() + certFile := filepath.Join(dir, "tls.crt") + keyFile := filepath.Join(dir, "tls.key") + + certA, keyA := genSelfSigned(t, 1) + writeKeyPair(t, certFile, keyFile, certA, keyA, time.Now().Add(-2*time.Second)) + + cr, err := newCertReloader(certFile, keyFile) + if err != nil { + t.Fatalf("newCertReloader: %v", err) + } + + // Corrupt the cert file and advance its mtime so a reload is attempted and fails. + if err := os.WriteFile(certFile, []byte("not a certificate"), 0o600); err != nil { + t.Fatalf("corrupt cert: %v", err) + } + if err := os.Chtimes(certFile, time.Now().Add(2*time.Second), time.Now().Add(2*time.Second)); err != nil { + t.Fatalf("chtimes: %v", err) + } + + got, err := cr.GetCertificate(&tls.ClientHelloInfo{}) + if err != nil { + t.Fatalf("GetCertificate returned error instead of serving last good cert: %v", err) + } + if got == nil || got.Leaf == nil { + // Leaf may be nil depending on the Go version; fall back to parsing. + if got == nil || len(got.Certificate) == 0 { + t.Fatalf("GetCertificate returned no certificate") + } + leaf, perr := x509.ParseCertificate(got.Certificate[0]) + if perr != nil { + t.Fatalf("parse served cert: %v", perr) + } + got.Leaf = leaf + } + if got.Leaf.SerialNumber.Int64() != 1 { + t.Fatalf("expected to keep serving serial 1 after failed reload, got %d", got.Leaf.SerialNumber.Int64()) + } +} diff --git a/packages/system/kubeovn-webhook/images/kubeovn-webhook/main.go b/packages/system/kubeovn-webhook/images/kubeovn-webhook/main.go index 70185961d7..a85776c605 100644 --- a/packages/system/kubeovn-webhook/images/kubeovn-webhook/main.go +++ b/packages/system/kubeovn-webhook/images/kubeovn-webhook/main.go @@ -1,7 +1,6 @@ package main import ( - "crypto/tls" "flag" "log" "net/http" @@ -28,17 +27,15 @@ func main() { mux := http.NewServeMux() mux.HandleFunc("/mutate-pods", HandleMutatePods) - tlsCert, err := tls.LoadX509KeyPair(tlsCertFile, tlsKeyFile) + tlsConfig, err := newReloadingTLSConfig(tlsCertFile, tlsKeyFile) if err != nil { log.Fatalf("Failed to load key pair: %v", err) } server := &http.Server{ - Addr: ":8443", - TLSConfig: &tls.Config{ - Certificates: []tls.Certificate{tlsCert}, - }, - Handler: mux, + Addr: ":8443", + TLSConfig: tlsConfig, + Handler: mux, } log.Printf("Starting webhook server on %s", server.Addr) From 8a61ddb0a7fc5dd8514406a844f6b697cb685d9e Mon Sep 17 00:00:00 2001 From: IvanHunters Date: Wed, 5 Aug 2026 15:04:43 +0300 Subject: [PATCH 21/51] fix(kubeovn-webhook): widen serving cert renewBefore to 720h renewBefore: 24h on a one-year certificate leaves a one-day window for cert-manager to rotate the Secret and for the webhook to pick it up. Widen it to 720h (30 days) so renewal happens well ahead of expiry. Signed-off-by: IvanHunters (cherry picked from commit 791d99d8e5afda091719005b742cbbd43532f66d) --- packages/system/kubeovn-webhook/templates/certmanager.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/system/kubeovn-webhook/templates/certmanager.yaml b/packages/system/kubeovn-webhook/templates/certmanager.yaml index f8eee740e4..cb2770360a 100644 --- a/packages/system/kubeovn-webhook/templates/certmanager.yaml +++ b/packages/system/kubeovn-webhook/templates/certmanager.yaml @@ -38,7 +38,7 @@ metadata: spec: secretName: {{ include "namespace-annotation-webhook.fullname" . }}-tls duration: 8760h - renewBefore: 24h + renewBefore: 720h issuerRef: name: {{ include "namespace-annotation-webhook.fullname" . }}-ca-issuer commonName: {{ include "namespace-annotation-webhook.fullname" . }}-tls From a9a13b3d0d378d277a5dec65b4d3c7395220ef39 Mon Sep 17 00:00:00 2001 From: IvanHunters Date: Wed, 5 Aug 2026 15:24:17 +0300 Subject: [PATCH 22/51] fix(kubeovn-webhook): harden certificate reloader Address review of the reload path: - Observe the cert file mtime before reading the key pair, so a Secret swap racing the read is retried on the next handshake instead of being cached under a newer mtime and missed. - Record the load attempt regardless of outcome, so a persistently unreadable file is retried only once its mtime advances, not on every handshake. - Detect change with mtime inequality instead of a strictly-newer comparison, catching equal-mtime replacements and backward clock steps. - Log stat failures instead of swallowing them, so a broken mount is not invisible until expiry. Signed-off-by: IvanHunters (cherry picked from commit a537bb647bf3b20a0a7bd7d1fd9b2dfcab076cf0) --- .../images/kubeovn-webhook/cert.go | 67 ++++++++++--------- 1 file changed, 37 insertions(+), 30 deletions(-) diff --git a/packages/system/kubeovn-webhook/images/kubeovn-webhook/cert.go b/packages/system/kubeovn-webhook/images/kubeovn-webhook/cert.go index 9e15ec302b..56abdbb877 100644 --- a/packages/system/kubeovn-webhook/images/kubeovn-webhook/cert.go +++ b/packages/system/kubeovn-webhook/images/kubeovn-webhook/cert.go @@ -24,56 +24,63 @@ type certReloader struct { certFile string keyFile string - mu sync.RWMutex - cert *tls.Certificate - modTime time.Time + mu sync.RWMutex + cert *tls.Certificate + loadedModTime time.Time } // newCertReloader loads the initial key pair and returns a reloader for it. func newCertReloader(certFile, keyFile string) (*certReloader, error) { cr := &certReloader{certFile: certFile, keyFile: keyFile} - if err := cr.reload(); err != nil { + if err := cr.reload(cr.certModTime()); err != nil { return nil, err } return cr, nil } -// reload reads the key pair from disk and atomically swaps the cached certificate. -func (cr *certReloader) reload() error { - cert, err := tls.LoadX509KeyPair(cr.certFile, cr.keyFile) +// certModTime returns the modification time of the certificate file, or the zero +// time if it cannot be stat'd (logged so a broken mount is not silently invisible). +func (cr *certReloader) certModTime() time.Time { + fi, err := os.Stat(cr.certFile) if err != nil { - return err + log.Printf("certificate stat failed, keeping previous certificate: %v", err) + return time.Time{} } + return fi.ModTime() +} - var modTime time.Time - if fi, statErr := os.Stat(cr.certFile); statErr == nil { - modTime = fi.ModTime() - } +// reload reads the key pair from disk and atomically swaps the cached certificate. +// +// modTime is the certificate file's modification time observed BEFORE the read, so a +// Secret swap racing the read leaves modTime older than the file on disk and is caught +// on the next handshake instead of being cached under a newer mtime and missed. The +// attempt is recorded regardless of outcome, so a persistently unreadable file is +// retried only once its mtime advances, not on every handshake. +func (cr *certReloader) reload(modTime time.Time) error { + cert, err := tls.LoadX509KeyPair(cr.certFile, cr.keyFile) cr.mu.Lock() - cr.cert = &cert - cr.modTime = modTime + cr.loadedModTime = modTime + if err == nil { + cr.cert = &cert + } cr.mu.Unlock() - return nil + return err } -// changed reports whether the certificate file was modified since the last load. -func (cr *certReloader) changed() bool { - fi, err := os.Stat(cr.certFile) - if err != nil { - return false - } +// GetCertificate is a tls.Config.GetCertificate callback. It reloads the key pair when +// the certificate file's modification time changes and keeps serving the last good +// certificate if a reload fails (for example a torn read while the mounted Secret is +// being updated). +func (cr *certReloader) GetCertificate(*tls.ClientHelloInfo) (*tls.Certificate, error) { + modTime := cr.certModTime() + cr.mu.RLock() - defer cr.mu.RUnlock() - return fi.ModTime().After(cr.modTime) -} + stale := !modTime.IsZero() && !modTime.Equal(cr.loadedModTime) + cr.mu.RUnlock() -// GetCertificate is a tls.Config.GetCertificate callback. It reloads the key pair -// when the file changes and keeps serving the last good certificate if a reload -// fails (for example a torn read while the mounted Secret is being updated). -func (cr *certReloader) GetCertificate(*tls.ClientHelloInfo) (*tls.Certificate, error) { - if cr.changed() { - if err := cr.reload(); err != nil { + if stale { + if err := cr.reload(modTime); err != nil { log.Printf("certificate reload failed, keeping previous certificate: %v", err) } } From 60b42e219d7350d46811e6461347342ebc298e37 Mon Sep 17 00:00:00 2001 From: IvanHunters Date: Wed, 5 Aug 2026 15:52:30 +0300 Subject: [PATCH 23/51] fix(kubeovn-webhook): retry certificate reload after a transient failure A previous revision recorded the certificate file mtime even when the reload failed, to avoid re-reading a persistently bad file on every handshake. That conflated two failure modes: a transient, content- independent error (fd exhaustion, a torn read) on a VALID renewed file advanced the recorded mtime, so the staleness check never fired again and the renewed certificate was never loaded. The cached certificate then expired within renewBefore, and failurePolicy: Fail blocked all tenant pod creation - exactly the outage this reloader prevents. Advance the recorded mtime only on a successful load, and bound retries of a failed load with a wall-clock interval instead. A transient failure is now retried until it succeeds, while a persistently unreadable file is still retried only once per interval, not on every handshake. Keep a single reload in flight to avoid a thundering herd, and rate-limit the failure log. Add a regression test that fails when the mtime is advanced on failure. Detect change with mtime inequality, so a backward clock step is also picked up. Signed-off-by: IvanHunters (cherry picked from commit 5b6bb47caf7aa879f75d0297c4cdb979bff500c3) --- .../images/kubeovn-webhook/cert.go | 121 +++++++++++++----- .../images/kubeovn-webhook/cert_test.go | 120 +++++++++++++---- 2 files changed, 178 insertions(+), 63 deletions(-) diff --git a/packages/system/kubeovn-webhook/images/kubeovn-webhook/cert.go b/packages/system/kubeovn-webhook/images/kubeovn-webhook/cert.go index 56abdbb877..ebc3c3e654 100644 --- a/packages/system/kubeovn-webhook/images/kubeovn-webhook/cert.go +++ b/packages/system/kubeovn-webhook/images/kubeovn-webhook/cert.go @@ -24,66 +24,117 @@ type certReloader struct { certFile string keyFile string + // retryInterval bounds how often a failed reload (or a repeated stat failure) is + // retried, so a persistently broken file does not read/parse/log on every handshake. + retryInterval time.Duration + mu sync.RWMutex cert *tls.Certificate loadedModTime time.Time + reloading bool + nextRetry time.Time + nextLog time.Time } +const defaultReloadRetryInterval = 30 * time.Second + // newCertReloader loads the initial key pair and returns a reloader for it. func newCertReloader(certFile, keyFile string) (*certReloader, error) { - cr := &certReloader{certFile: certFile, keyFile: keyFile} - if err := cr.reload(cr.certModTime()); err != nil { + cr := &certReloader{ + certFile: certFile, + keyFile: keyFile, + retryInterval: defaultReloadRetryInterval, + } + modTime, err := fileModTime(certFile) + if err != nil { + return nil, err + } + if err := cr.load(modTime); err != nil { return nil, err } return cr, nil } -// certModTime returns the modification time of the certificate file, or the zero -// time if it cannot be stat'd (logged so a broken mount is not silently invisible). -func (cr *certReloader) certModTime() time.Time { - fi, err := os.Stat(cr.certFile) +// fileModTime returns the modification time of the given file. +func fileModTime(name string) (time.Time, error) { + fi, err := os.Stat(name) if err != nil { - log.Printf("certificate stat failed, keeping previous certificate: %v", err) - return time.Time{} + return time.Time{}, err } - return fi.ModTime() + return fi.ModTime(), nil } -// reload reads the key pair from disk and atomically swaps the cached certificate. -// -// modTime is the certificate file's modification time observed BEFORE the read, so a -// Secret swap racing the read leaves modTime older than the file on disk and is caught -// on the next handshake instead of being cached under a newer mtime and missed. The -// attempt is recorded regardless of outcome, so a persistently unreadable file is -// retried only once its mtime advances, not on every handshake. -func (cr *certReloader) reload(modTime time.Time) error { +// load reads the key pair and, on success, caches it tagged with modTime. On failure +// the cached certificate and its recorded mtime are left unchanged, so a transient +// failure on a valid file is retried rather than the renewal being abandoned. +func (cr *certReloader) load(modTime time.Time) error { cert, err := tls.LoadX509KeyPair(cr.certFile, cr.keyFile) - + if err != nil { + return err + } cr.mu.Lock() + cr.cert = &cert cr.loadedModTime = modTime - if err == nil { - cr.cert = &cert - } cr.mu.Unlock() - return err + return nil } -// GetCertificate is a tls.Config.GetCertificate callback. It reloads the key pair when -// the certificate file's modification time changes and keeps serving the last good -// certificate if a reload fails (for example a torn read while the mounted Secret is -// being updated). -func (cr *certReloader) GetCertificate(*tls.ClientHelloInfo) (*tls.Certificate, error) { - modTime := cr.certModTime() +// refresh reloads the key pair when the certificate file's modification time changes. +// +// A failed reload keeps the last good certificate and does NOT advance the recorded +// mtime, so a transient error (fd exhaustion, a torn read) is retried, at most once per +// retryInterval, until it succeeds: a transient failure must never permanently pin an +// expiring certificate. A single reload is kept in flight so a burst of handshakes does +// not trigger a thundering herd of reads. +func (cr *certReloader) refresh() { + modTime, err := fileModTime(cr.certFile) + if err != nil { + cr.logRateLimited("certificate stat failed, keeping previous certificate: %v", err) + return + } - cr.mu.RLock() - stale := !modTime.IsZero() && !modTime.Equal(cr.loadedModTime) - cr.mu.RUnlock() + cr.mu.Lock() + if modTime.Equal(cr.loadedModTime) || cr.reloading || time.Now().Before(cr.nextRetry) { + cr.mu.Unlock() + return + } + cr.reloading = true + cr.mu.Unlock() + + loadErr := cr.load(modTime) + + cr.mu.Lock() + cr.reloading = false + if loadErr == nil { + cr.nextRetry = time.Time{} + } else { + cr.nextRetry = time.Now().Add(cr.retryInterval) + } + cr.mu.Unlock() - if stale { - if err := cr.reload(modTime); err != nil { - log.Printf("certificate reload failed, keeping previous certificate: %v", err) - } + if loadErr != nil { + cr.logRateLimited("certificate reload failed, keeping previous certificate: %v", loadErr) } +} + +// logRateLimited emits a log line at most once per retryInterval, so a persistently +// broken mount does not spam the log on every TLS handshake. +func (cr *certReloader) logRateLimited(format string, args ...any) { + cr.mu.Lock() + if time.Now().Before(cr.nextLog) { + cr.mu.Unlock() + return + } + cr.nextLog = time.Now().Add(cr.retryInterval) + cr.mu.Unlock() + log.Printf(format, args...) +} + +// GetCertificate is a tls.Config.GetCertificate callback. It refreshes the key pair +// when the certificate file changes and keeps serving the last good certificate if a +// reload fails. +func (cr *certReloader) GetCertificate(*tls.ClientHelloInfo) (*tls.Certificate, error) { + cr.refresh() cr.mu.RLock() defer cr.mu.RUnlock() diff --git a/packages/system/kubeovn-webhook/images/kubeovn-webhook/cert_test.go b/packages/system/kubeovn-webhook/images/kubeovn-webhook/cert_test.go index 1a670e4fd8..c3b1d4baf9 100644 --- a/packages/system/kubeovn-webhook/images/kubeovn-webhook/cert_test.go +++ b/packages/system/kubeovn-webhook/images/kubeovn-webhook/cert_test.go @@ -50,8 +50,8 @@ func genSelfSigned(t *testing.T, serial int64) (certPEM, keyPEM []byte) { return certPEM, keyPEM } -// writeKeyPair writes the cert/key files and bumps their mtime strictly forward, -// mirroring how cert-manager replaces the mounted Secret on renewal. +// writeKeyPair writes the cert/key files and sets their mtime, mirroring how +// cert-manager replaces the mounted Secret on renewal. func writeKeyPair(t *testing.T, certFile, keyFile string, certPEM, keyPEM []byte, mtime time.Time) { t.Helper() if err := os.WriteFile(certFile, certPEM, 0o600); err != nil { @@ -60,12 +60,36 @@ func writeKeyPair(t *testing.T, certFile, keyFile string, certPEM, keyPEM []byte if err := os.WriteFile(keyFile, keyPEM, 0o600); err != nil { t.Fatalf("write key: %v", err) } - if err := os.Chtimes(certFile, mtime, mtime); err != nil { - t.Fatalf("chtimes cert: %v", err) + setModTime(t, certFile, mtime) + setModTime(t, keyFile, mtime) +} + +func setModTime(t *testing.T, name string, mtime time.Time) { + t.Helper() + if err := os.Chtimes(name, mtime, mtime); err != nil { + t.Fatalf("chtimes %s: %v", name, err) } - if err := os.Chtimes(keyFile, mtime, mtime); err != nil { - t.Fatalf("chtimes key: %v", err) +} + +// certSerial returns the serial number of a *tls.Certificate leaf without mutating the +// shared value returned by GetCertificate (it parses into a local when Leaf is unset). +func certSerial(t *testing.T, c *tls.Certificate) int64 { + t.Helper() + if c == nil { + t.Fatal("nil certificate") + } + leaf := c.Leaf + if leaf == nil { + if len(c.Certificate) == 0 { + t.Fatal("certificate has no DER data") + } + parsed, err := x509.ParseCertificate(c.Certificate[0]) + if err != nil { + t.Fatalf("parse served cert: %v", err) + } + leaf = parsed } + return leaf.SerialNumber.Int64() } // servedSerial completes a TLS handshake against addr and returns the serial number @@ -84,10 +108,9 @@ func servedSerial(t *testing.T, addr string) int64 { return certs[0].SerialNumber.Int64() } -// TestReloadingTLSConfigServesRenewedCertificate is the regression test for the -// cert-manager renewal not being picked up: it fails against the old code that -// loads the key pair once into tls.Config.Certificates, and passes once the -// certificate is served via a reloading GetCertificate callback. +// TestReloadingTLSConfigServesRenewedCertificate is the core regression test: it fails +// against the old code that loads the key pair once into tls.Config.Certificates, and +// passes once the certificate is served via a reloading GetCertificate callback. func TestReloadingTLSConfigServesRenewedCertificate(t *testing.T) { dir := t.TempDir() certFile := filepath.Join(dir, "tls.crt") @@ -136,9 +159,9 @@ func TestReloadingTLSConfigServesRenewedCertificate(t *testing.T) { } } -// TestCertReloaderKeepsLastGoodCertificate verifies that a failed reload (e.g. a -// torn read while the volume is updated) does not take the webhook down: it keeps -// serving the previously loaded certificate instead of erroring the handshake. +// TestCertReloaderKeepsLastGoodCertificate verifies that a failed reload (e.g. a torn +// read while the volume is updated) does not take the webhook down: it keeps serving +// the previously loaded certificate instead of erroring the handshake. func TestCertReloaderKeepsLastGoodCertificate(t *testing.T) { dir := t.TempDir() certFile := filepath.Join(dir, "tls.crt") @@ -156,26 +179,67 @@ func TestCertReloaderKeepsLastGoodCertificate(t *testing.T) { if err := os.WriteFile(certFile, []byte("not a certificate"), 0o600); err != nil { t.Fatalf("corrupt cert: %v", err) } - if err := os.Chtimes(certFile, time.Now().Add(2*time.Second), time.Now().Add(2*time.Second)); err != nil { - t.Fatalf("chtimes: %v", err) - } + setModTime(t, certFile, time.Now().Add(2*time.Second)) got, err := cr.GetCertificate(&tls.ClientHelloInfo{}) if err != nil { t.Fatalf("GetCertificate returned error instead of serving last good cert: %v", err) } - if got == nil || got.Leaf == nil { - // Leaf may be nil depending on the Go version; fall back to parsing. - if got == nil || len(got.Certificate) == 0 { - t.Fatalf("GetCertificate returned no certificate") - } - leaf, perr := x509.ParseCertificate(got.Certificate[0]) - if perr != nil { - t.Fatalf("parse served cert: %v", perr) - } - got.Leaf = leaf + if certSerial(t, got) != 1 { + t.Fatalf("expected to keep serving serial 1 after failed reload, got %d", certSerial(t, got)) + } +} + +// TestCertReloaderRetriesAfterTransientFailure is the regression test for the reloader +// abandoning a renewal on a transient error. A failed reload must NOT advance the +// recorded mtime, so once the (unchanged-mtime) file becomes readable the renewed +// certificate is still picked up. Against the code that advanced the mtime on failure, +// the renewal is permanently missed and this test fails. +func TestCertReloaderRetriesAfterTransientFailure(t *testing.T) { + dir := t.TempDir() + certFile := filepath.Join(dir, "tls.crt") + keyFile := filepath.Join(dir, "tls.key") + + certA, keyA := genSelfSigned(t, 1) + writeKeyPair(t, certFile, keyFile, certA, keyA, time.Now().Add(-2*time.Second)) + + cr, err := newCertReloader(certFile, keyFile) + if err != nil { + t.Fatalf("newCertReloader: %v", err) + } + cr.retryInterval = 0 // retry immediately, no backoff wait in the test + + // Renewal advances the mtime, but the first load fails (simulating a transient, + // content-independent error) by writing an unreadable cert at the new mtime. + renewMod := time.Now().Add(2 * time.Second) + if err := os.WriteFile(certFile, []byte("transiently unreadable"), 0o600); err != nil { + t.Fatalf("write bad cert: %v", err) + } + setModTime(t, certFile, renewMod) + + if got, gerr := cr.GetCertificate(&tls.ClientHelloInfo{}); gerr != nil { + t.Fatalf("GetCertificate errored during transient failure: %v", gerr) + } else if certSerial(t, got) != 1 { + t.Fatalf("during transient failure: expected to keep serial 1, got %d", certSerial(t, got)) + } + + // The renewed, valid certificate is now readable at the SAME mtime as the failed + // attempt. The reloader must still pick it up on retry. + certB, keyB := genSelfSigned(t, 2) + if err := os.WriteFile(certFile, certB, 0o600); err != nil { + t.Fatalf("write renewed cert: %v", err) + } + if err := os.WriteFile(keyFile, keyB, 0o600); err != nil { + t.Fatalf("write renewed key: %v", err) + } + setModTime(t, certFile, renewMod) + setModTime(t, keyFile, renewMod) + + got, gerr := cr.GetCertificate(&tls.ClientHelloInfo{}) + if gerr != nil { + t.Fatalf("GetCertificate errored after retry: %v", gerr) } - if got.Leaf.SerialNumber.Int64() != 1 { - t.Fatalf("expected to keep serving serial 1 after failed reload, got %d", got.Leaf.SerialNumber.Int64()) + if certSerial(t, got) != 2 { + t.Fatalf("after transient failure: expected renewed serial 2 on retry, got %d (mtime advanced on failure, renewal permanently missed)", certSerial(t, got)) } } From d2dc85f7124be348da7b619cc2f584bc569cfdf5 Mon Sep 17 00:00:00 2001 From: IvanHunters Date: Fri, 7 Aug 2026 16:36:42 +0300 Subject: [PATCH 24/51] refactor(kube-ovn): reload webhook cert via a plain GetCertificate closure Collapse the certReloader (mtime tracking, serialized refresh, retry backoff, rate-limited logging, last-good-certificate fallback) into a GetCertificate callback that simply re-reads the key pair from disk on every handshake. The signature is unchanged, so main.go is untouched. The extra machinery guarded scenarios this deployment cannot produce: - Torn reads are unreachable: the cert/key files come from a plain Secret volume with no subPath, and Kubernetes' atomic writer swaps the whole ..data directory via a single symlink flip, so a reader always sees a complete old or complete new generation of both files. - The per-handshake LoadX509KeyPair cost is negligible next to the asymmetric crypto the handshake already performs, so no cache is needed. - The webhook configures no mTLS, so GetCertificate not refreshing client CA pools does not apply. If the mounted files genuinely become unreadable (an operator error, not a renewal) the handshake now fails loudly instead of silently serving a stale certificate. The core regression test is retained and remains non-vacuous: it fails against a static tls.Config.Certificates and passes with the reloading callback. Signed-off-by: IvanHunters (cherry picked from commit 8baa449f53f6410a348ce2e7e7ea89393f1c431d) --- .../images/kubeovn-webhook/cert.go | 182 ++++-------------- .../images/kubeovn-webhook/cert_test.go | 106 ---------- 2 files changed, 36 insertions(+), 252 deletions(-) diff --git a/packages/system/kubeovn-webhook/images/kubeovn-webhook/cert.go b/packages/system/kubeovn-webhook/images/kubeovn-webhook/cert.go index ebc3c3e654..1fcbf573b4 100644 --- a/packages/system/kubeovn-webhook/images/kubeovn-webhook/cert.go +++ b/packages/system/kubeovn-webhook/images/kubeovn-webhook/cert.go @@ -1,155 +1,45 @@ package main -import ( - "crypto/tls" - "fmt" - "log" - "os" - "sync" - "time" -) +import "crypto/tls" -// certReloader keeps the webhook serving certificate in sync with the key pair -// files written by cert-manager. +// newReloadingTLSConfig returns a tls.Config that reloads the certificate and +// key from disk on every TLS handshake, so a cert-manager renewal of the mounted +// Secret is picked up by the running process without a restart. // -// The certificate is served through tls.Config.GetCertificate, so a cert-manager -// renewal of the backing Secret is picked up by the running process without a -// restart. Without this, the key pair is read exactly once at startup and cached -// for the lifetime of the process: after the certificate expires (roughly one year -// after install) the pod keeps presenting the expired certificate, the -// kube-apiserver's TLS call to the webhook fails, and because the -// MutatingWebhookConfiguration uses failurePolicy: Fail, every pod creation in -// tenant namespaces is rejected. -type certReloader struct { - certFile string - keyFile string - - // retryInterval bounds how often a failed reload (or a repeated stat failure) is - // retried, so a persistently broken file does not read/parse/log on every handshake. - retryInterval time.Duration - - mu sync.RWMutex - cert *tls.Certificate - loadedModTime time.Time - reloading bool - nextRetry time.Time - nextLog time.Time -} - -const defaultReloadRetryInterval = 30 * time.Second - -// newCertReloader loads the initial key pair and returns a reloader for it. -func newCertReloader(certFile, keyFile string) (*certReloader, error) { - cr := &certReloader{ - certFile: certFile, - keyFile: keyFile, - retryInterval: defaultReloadRetryInterval, - } - modTime, err := fileModTime(certFile) - if err != nil { - return nil, err - } - if err := cr.load(modTime); err != nil { - return nil, err - } - return cr, nil -} - -// fileModTime returns the modification time of the given file. -func fileModTime(name string) (time.Time, error) { - fi, err := os.Stat(name) - if err != nil { - return time.Time{}, err - } - return fi.ModTime(), nil -} - -// load reads the key pair and, on success, caches it tagged with modTime. On failure -// the cached certificate and its recorded mtime are left unchanged, so a transient -// failure on a valid file is retried rather than the renewal being abandoned. -func (cr *certReloader) load(modTime time.Time) error { - cert, err := tls.LoadX509KeyPair(cr.certFile, cr.keyFile) - if err != nil { - return err - } - cr.mu.Lock() - cr.cert = &cert - cr.loadedModTime = modTime - cr.mu.Unlock() - return nil -} - -// refresh reloads the key pair when the certificate file's modification time changes. +// Without this, the key pair is read exactly once at startup and cached for the +// lifetime of the process: after the certificate expires (roughly one year after +// install) the pod keeps presenting the expired certificate, the kube-apiserver's +// TLS call to the webhook fails, and because the MutatingWebhookConfiguration uses +// failurePolicy: Fail, every pod creation in tenant namespaces is rejected. // -// A failed reload keeps the last good certificate and does NOT advance the recorded -// mtime, so a transient error (fd exhaustion, a torn read) is retried, at most once per -// retryInterval, until it succeeds: a transient failure must never permanently pin an -// expiring certificate. A single reload is kept in flight so a burst of handshakes does -// not trigger a thundering herd of reads. -func (cr *certReloader) refresh() { - modTime, err := fileModTime(cr.certFile) - if err != nil { - cr.logRateLimited("certificate stat failed, keeping previous certificate: %v", err) - return - } - - cr.mu.Lock() - if modTime.Equal(cr.loadedModTime) || cr.reloading || time.Now().Before(cr.nextRetry) { - cr.mu.Unlock() - return - } - cr.reloading = true - cr.mu.Unlock() - - loadErr := cr.load(modTime) - - cr.mu.Lock() - cr.reloading = false - if loadErr == nil { - cr.nextRetry = time.Time{} - } else { - cr.nextRetry = time.Now().Add(cr.retryInterval) - } - cr.mu.Unlock() - - if loadErr != nil { - cr.logRateLimited("certificate reload failed, keeping previous certificate: %v", loadErr) - } -} - -// logRateLimited emits a log line at most once per retryInterval, so a persistently -// broken mount does not spam the log on every TLS handshake. -func (cr *certReloader) logRateLimited(format string, args ...any) { - cr.mu.Lock() - if time.Now().Before(cr.nextLog) { - cr.mu.Unlock() - return - } - cr.nextLog = time.Now().Add(cr.retryInterval) - cr.mu.Unlock() - log.Printf(format, args...) -} - -// GetCertificate is a tls.Config.GetCertificate callback. It refreshes the key pair -// when the certificate file changes and keeps serving the last good certificate if a -// reload fails. -func (cr *certReloader) GetCertificate(*tls.ClientHelloInfo) (*tls.Certificate, error) { - cr.refresh() - - cr.mu.RLock() - defer cr.mu.RUnlock() - if cr.cert == nil { - return nil, fmt.Errorf("no certificate loaded") - } - return cr.cert, nil -} - -// newReloadingTLSConfig builds a tls.Config that serves the key pair via a -// certReloader, so cert-manager renewals are honoured without a restart. +// tls.Config.GetCertificate is invoked on every handshake whenever Certificates is +// left unset, so re-reading the files there is enough; no watcher, mtime tracking +// or cache is needed: +// - The cert/key files are mounted from a plain Secret volume with no subPath. +// Kubernetes' atomic writer swaps the whole ..data directory via a single +// symlink flip, so a reader always sees a complete old or complete new +// generation of both files, never a torn mid-write mixture. A per-handshake +// LoadX509KeyPair therefore cannot observe a partial renewal. +// - The per-handshake cost (a few KB of file I/O plus a PEM/key parse) is +// negligible next to the asymmetric crypto the handshake already performs. +// - The webhook configures no mTLS (no ClientCAs), so GetCertificate's +// limitation of not refreshing client CA pools does not apply. +// +// If the mounted files genuinely become unreadable (Secret deleted, permissions +// broken: an operator error, not a normal renewal) the handshake fails loudly +// rather than silently serving a stale certificate. func newReloadingTLSConfig(certFile, keyFile string) (*tls.Config, error) { - cr, err := newCertReloader(certFile, keyFile) - if err != nil { + // Fail fast at startup if the initial key pair is missing or malformed. + if _, err := tls.LoadX509KeyPair(certFile, keyFile); err != nil { return nil, err } - return &tls.Config{GetCertificate: cr.GetCertificate}, nil + return &tls.Config{ + GetCertificate: func(*tls.ClientHelloInfo) (*tls.Certificate, error) { + cert, err := tls.LoadX509KeyPair(certFile, keyFile) + if err != nil { + return nil, err + } + return &cert, nil + }, + }, nil } diff --git a/packages/system/kubeovn-webhook/images/kubeovn-webhook/cert_test.go b/packages/system/kubeovn-webhook/images/kubeovn-webhook/cert_test.go index c3b1d4baf9..2d39fe5999 100644 --- a/packages/system/kubeovn-webhook/images/kubeovn-webhook/cert_test.go +++ b/packages/system/kubeovn-webhook/images/kubeovn-webhook/cert_test.go @@ -71,27 +71,6 @@ func setModTime(t *testing.T, name string, mtime time.Time) { } } -// certSerial returns the serial number of a *tls.Certificate leaf without mutating the -// shared value returned by GetCertificate (it parses into a local when Leaf is unset). -func certSerial(t *testing.T, c *tls.Certificate) int64 { - t.Helper() - if c == nil { - t.Fatal("nil certificate") - } - leaf := c.Leaf - if leaf == nil { - if len(c.Certificate) == 0 { - t.Fatal("certificate has no DER data") - } - parsed, err := x509.ParseCertificate(c.Certificate[0]) - if err != nil { - t.Fatalf("parse served cert: %v", err) - } - leaf = parsed - } - return leaf.SerialNumber.Int64() -} - // servedSerial completes a TLS handshake against addr and returns the serial number // of the leaf certificate the server actually presented. func servedSerial(t *testing.T, addr string) int64 { @@ -158,88 +137,3 @@ func TestReloadingTLSConfigServesRenewedCertificate(t *testing.T) { t.Fatalf("after renewal: expected renewed serial 2, got %d (certificate was not reloaded)", got) } } - -// TestCertReloaderKeepsLastGoodCertificate verifies that a failed reload (e.g. a torn -// read while the volume is updated) does not take the webhook down: it keeps serving -// the previously loaded certificate instead of erroring the handshake. -func TestCertReloaderKeepsLastGoodCertificate(t *testing.T) { - dir := t.TempDir() - certFile := filepath.Join(dir, "tls.crt") - keyFile := filepath.Join(dir, "tls.key") - - certA, keyA := genSelfSigned(t, 1) - writeKeyPair(t, certFile, keyFile, certA, keyA, time.Now().Add(-2*time.Second)) - - cr, err := newCertReloader(certFile, keyFile) - if err != nil { - t.Fatalf("newCertReloader: %v", err) - } - - // Corrupt the cert file and advance its mtime so a reload is attempted and fails. - if err := os.WriteFile(certFile, []byte("not a certificate"), 0o600); err != nil { - t.Fatalf("corrupt cert: %v", err) - } - setModTime(t, certFile, time.Now().Add(2*time.Second)) - - got, err := cr.GetCertificate(&tls.ClientHelloInfo{}) - if err != nil { - t.Fatalf("GetCertificate returned error instead of serving last good cert: %v", err) - } - if certSerial(t, got) != 1 { - t.Fatalf("expected to keep serving serial 1 after failed reload, got %d", certSerial(t, got)) - } -} - -// TestCertReloaderRetriesAfterTransientFailure is the regression test for the reloader -// abandoning a renewal on a transient error. A failed reload must NOT advance the -// recorded mtime, so once the (unchanged-mtime) file becomes readable the renewed -// certificate is still picked up. Against the code that advanced the mtime on failure, -// the renewal is permanently missed and this test fails. -func TestCertReloaderRetriesAfterTransientFailure(t *testing.T) { - dir := t.TempDir() - certFile := filepath.Join(dir, "tls.crt") - keyFile := filepath.Join(dir, "tls.key") - - certA, keyA := genSelfSigned(t, 1) - writeKeyPair(t, certFile, keyFile, certA, keyA, time.Now().Add(-2*time.Second)) - - cr, err := newCertReloader(certFile, keyFile) - if err != nil { - t.Fatalf("newCertReloader: %v", err) - } - cr.retryInterval = 0 // retry immediately, no backoff wait in the test - - // Renewal advances the mtime, but the first load fails (simulating a transient, - // content-independent error) by writing an unreadable cert at the new mtime. - renewMod := time.Now().Add(2 * time.Second) - if err := os.WriteFile(certFile, []byte("transiently unreadable"), 0o600); err != nil { - t.Fatalf("write bad cert: %v", err) - } - setModTime(t, certFile, renewMod) - - if got, gerr := cr.GetCertificate(&tls.ClientHelloInfo{}); gerr != nil { - t.Fatalf("GetCertificate errored during transient failure: %v", gerr) - } else if certSerial(t, got) != 1 { - t.Fatalf("during transient failure: expected to keep serial 1, got %d", certSerial(t, got)) - } - - // The renewed, valid certificate is now readable at the SAME mtime as the failed - // attempt. The reloader must still pick it up on retry. - certB, keyB := genSelfSigned(t, 2) - if err := os.WriteFile(certFile, certB, 0o600); err != nil { - t.Fatalf("write renewed cert: %v", err) - } - if err := os.WriteFile(keyFile, keyB, 0o600); err != nil { - t.Fatalf("write renewed key: %v", err) - } - setModTime(t, certFile, renewMod) - setModTime(t, keyFile, renewMod) - - got, gerr := cr.GetCertificate(&tls.ClientHelloInfo{}) - if gerr != nil { - t.Fatalf("GetCertificate errored after retry: %v", gerr) - } - if certSerial(t, got) != 2 { - t.Fatalf("after transient failure: expected renewed serial 2 on retry, got %d (mtime advanced on failure, renewal permanently missed)", certSerial(t, got)) - } -} From 90409213f8bc558bbbe162f79e71a2c485ae3e09 Mon Sep 17 00:00:00 2001 From: IvanHunters Date: Fri, 7 Aug 2026 16:36:49 +0300 Subject: [PATCH 25/51] fix(kube-ovn): bound webhook server request timeouts The webhook http.Server set no read, write, or idle timeout, so a client that opens a connection and sends headers slowly could hold a goroutine and file descriptor indefinitely. Add ReadHeaderTimeout, ReadTimeout, WriteTimeout, and IdleTimeout. Signed-off-by: IvanHunters (cherry picked from commit 4151885871d917f20d3840616045f9c4d45d1f4c) --- .../kubeovn-webhook/images/kubeovn-webhook/main.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/packages/system/kubeovn-webhook/images/kubeovn-webhook/main.go b/packages/system/kubeovn-webhook/images/kubeovn-webhook/main.go index a85776c605..7f1d8ec7ab 100644 --- a/packages/system/kubeovn-webhook/images/kubeovn-webhook/main.go +++ b/packages/system/kubeovn-webhook/images/kubeovn-webhook/main.go @@ -4,6 +4,7 @@ import ( "flag" "log" "net/http" + "time" ) var ( @@ -33,9 +34,13 @@ func main() { } server := &http.Server{ - Addr: ":8443", - TLSConfig: tlsConfig, - Handler: mux, + Addr: ":8443", + TLSConfig: tlsConfig, + Handler: mux, + ReadHeaderTimeout: 10 * time.Second, + ReadTimeout: 30 * time.Second, + WriteTimeout: 30 * time.Second, + IdleTimeout: 60 * time.Second, } log.Printf("Starting webhook server on %s", server.Addr) From 9244be9aa0befe29c187adf80be977db23827704 Mon Sep 17 00:00:00 2001 From: Mattia Eleuteri Date: Mon, 3 Aug 2026 17:09:40 +0200 Subject: [PATCH 26/51] fix(backupstrategy-controller): repair lookup-gated backup objects MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The default Strategy CRs and the Velero BackupStorageLocation are Helm-templated behind a `lookup` of the BucketClaim the same chart creates, and the --credentials Secret behind a `lookup` of the COSI Secret. When those lookups are empty at install time the objects were silently skipped, and helm-controller does not re-render a release whose chart and values did not change (drift detection is off on operator-generated HelmReleases), so the skip was permanent: clusters ran for months with only BackupClass cozy-default and no Strategy CRs at all. That also fail-closes the pre-adoption snapshot in the v1.6.0 etcd migration, which reads the projected credentials Secret. Add a DefaultObjectsGate runnable to backupstrategy-controller. Once the bucket name is resolvable from the projector's source Secret, it checks that every object cozy-default routes to exists and forces one real Helm upgrade (reconcile.fluxcd.io/forceAt + requestedAt) when any is missing. Helm remains the objects' only author; the gate only makes sure the render that produces them actually happens. It cannot be a render-time `fail` in this chart: the chart is the producer of the Bucket its own lookup reads, so a failed render would deadlock the condition. The bucket chart's user-credentials template CAN fail safely — the BucketAccess that produces the COSI Secret belongs to the parent release and that release retries forever — so make it fail loudly instead of skipping, with requireUserCredentials=false as the offline-render escape. Also correct the chart and doc comments that claimed Flux re-renders on its interval, and document the two-release manual recovery for clusters already affected. Signed-off-by: Mattia Eleuteri (cherry picked from commit 51ad22670684b7bc3b7b13bb54403e1aecedc997) --- cmd/backupstrategy-controller/main.go | 24 + docs/operations/backup-classes.md | 46 +- .../backupcontroller/default_objects_gate.go | 356 +++++++++++++++ .../default_objects_gate_test.go | 416 ++++++++++++++++++ .../templates/_helpers.tpl | 31 +- .../templates/backupclass-default.yaml | 12 +- .../templates/deployment.yaml | 19 + .../templates/rbac.yaml | 16 +- .../tests/default_objects_gate_test.yaml | 91 ++++ .../backupstrategy-controller/values.yaml | 28 +- .../bucket/templates/user-credentials.yaml | 35 +- .../bucket/tests/user_credentials_test.yaml | 56 +++ packages/system/bucket/values.yaml | 9 + 13 files changed, 1122 insertions(+), 17 deletions(-) create mode 100644 internal/backupcontroller/default_objects_gate.go create mode 100644 internal/backupcontroller/default_objects_gate_test.go create mode 100644 packages/system/backupstrategy-controller/tests/default_objects_gate_test.yaml create mode 100644 packages/system/bucket/tests/user_credentials_test.yaml diff --git a/cmd/backupstrategy-controller/main.go b/cmd/backupstrategy-controller/main.go index d5995af04d..b8338c2c0c 100644 --- a/cmd/backupstrategy-controller/main.go +++ b/cmd/backupstrategy-controller/main.go @@ -27,6 +27,7 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" @@ -203,6 +204,29 @@ func main() { } } + // The default Strategy CRs and the Velero BSL are Helm-templated behind + // a lookup of the BucketClaim the same chart creates, so a fresh install + // renders them empty — permanently, because helm-controller does not + // re-render an unchanged, successful release. The gate detects that and + // forces the one real Helm upgrade that materialises them. Disabled when + // the HelmRelease coordinates are not plumbed (e.g. a chart-less local run). + if hrName := os.Getenv("BACKUP_DEFAULT_OBJECTS_HELMRELEASE_NAME"); hrName != "" { + gate := &backupcontroller.DefaultObjectsGate{ + Client: mgr.GetClient(), + Config: credentialsConfig, + BackupClassName: os.Getenv("BACKUP_DEFAULT_OBJECTS_BACKUPCLASS"), + HelmRelease: types.NamespacedName{ + Namespace: os.Getenv("BACKUP_DEFAULT_OBJECTS_HELMRELEASE_NAMESPACE"), + Name: hrName, + }, + VeleroNamespace: os.Getenv("BACKUP_DEFAULT_OBJECTS_VELERO_NAMESPACE"), + } + if err := gate.SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to add DefaultObjectsGate runnable") + os.Exit(1) + } + } + if err = (&backupcontroller.BackupJobReconciler{ Client: mgr.GetClient(), Scheme: mgr.GetScheme(), diff --git a/docs/operations/backup-classes.md b/docs/operations/backup-classes.md index e96b406f2a..1effdacf5a 100644 --- a/docs/operations/backup-classes.md +++ b/docs/operations/backup-classes.md @@ -93,9 +93,46 @@ On a fresh-cluster install, the Velero `BackupStorageLocation` `cozy-default` is ### Cozy-default Bucket bootstrap -`cozy-default` ships an `apps.cozystack.io/Bucket cozy-backups` CR in `tenant-root`, which the bucket-application chart turns into a `BucketClaim`; the COSI driver then assigns the real S3 bucket name and writes it to the BucketClaim's `.status.bucketName`. The strategy templates and the Velero BSL all read that real bucket name (Helm `lookup` against the BucketClaim). On a fresh install the BucketClaim takes a short reconcile cycle to populate its status — until it does, the strategy templates render empty and only the `Bucket` CR + `BackupClass` are present in the cluster. The HelmRelease re-reconciles on its interval (5 minutes by default — set by the cozystack operator's `helmrelease-interval` flag, not a Flux default), at which point the populated BucketClaim status causes the missing strategy templates to materialise. +`cozy-default` ships an `apps.cozystack.io/Bucket cozy-backups` CR in `tenant-root`, which the bucket-application chart turns into a `BucketClaim`; the COSI driver then assigns the real S3 bucket name (`bucket-`, so it cannot be computed in advance) and writes it to the BucketClaim's `.status.bucketName`. The strategy templates and the Velero BSL all read that real bucket name (Helm `lookup` against the BucketClaim). On a fresh install the BucketClaim takes a reconcile cycle to populate its status — until it does, the strategy templates render empty and only the `Bucket` CR + `BackupClass` are present in the cluster. -If you need the BackupClass functional immediately (e.g. an e2e), trigger a Flux reconcile (`flux reconcile helmrelease backupstrategy-controller -n cozy-backup-controller`) once you see `kubectl get bucketclaim -n tenant-root bucket-cozy-backups -o jsonpath='{.status.bucketName}'` non-empty. +**That skip does not repair itself on a reconcile.** helm-controller re-renders a release only when its chart or values change; the interval reconcile is a no-op for a healthy release, and drift detection is off on operator-generated HelmReleases. A cluster that lost the race at install time therefore keeps `BackupClass cozy-default` with **no** `Strategy` CRs and **no** Velero BSL indefinitely — which also fail-closes the pre-adoption snapshot in the v1.6.0 etcd migration. + +Convergence is driven instead by the controller's default-objects gate (`backupStorage.reconcileDefaultObjects`, on by default). Once the bucket name is resolvable it checks that every object `cozy-default` routes to exists, and stamps `reconcile.fluxcd.io/forceAt` + `requestedAt` on the `backupstrategy-controller` HelmRelease to force the real Helm upgrade that re-runs the lookups. Expect the objects within one minute of the bucket becoming ready. Watch it with: + +```bash +kubectl -n cozy-backup-controller logs deploy/backupstrategy-controller | grep default-objects-gate +``` + +#### Manual recovery on an affected cluster + +Only needed on a cluster running a version without the gate (or with `reconcileDefaultObjects: false`). A plain `flux reconcile helmrelease` does **nothing** here — it does not re-render. You need a forced upgrade, and **both** annotations: `forceAt` is what makes helm-controller run a real Helm upgrade, and it is only honoured together with `requestedAt`. + +First confirm the bucket name is actually resolvable — forcing before that just re-runs the same empty lookup: + +```bash +kubectl -n tenant-root get bucketclaim bucket-cozy-backups -o jsonpath='{.status.bucketName}' +``` + +Then force the two releases, **in this order**: + +```bash +# 1. The credentials Secret. It is rendered by the -system release, not by +# bucket-cozy-backups — easy to miss, and the projector (hence every +# strategy and Velero) has no source without it. +ts=$(date -u +%Y-%m-%dT%H:%M:%SZ) +kubectl -n tenant-root annotate helmrelease bucket-cozy-backups-system \ + reconcile.fluxcd.io/forceAt="$ts" reconcile.fluxcd.io/requestedAt="$ts" --overwrite +kubectl -n tenant-root get secret bucket-cozy-backups-system-credentials + +# 2. The Strategy CRs and the Velero BSL. +ts=$(date -u +%Y-%m-%dT%H:%M:%SZ) +kubectl -n cozy-backup-controller annotate helmrelease backupstrategy-controller \ + reconcile.fluxcd.io/forceAt="$ts" reconcile.fluxcd.io/requestedAt="$ts" --overwrite +kubectl get $(kubectl get crd -o name | grep strategy.backups.cozystack.io) 2>/dev/null +kubectl -n cozy-velero get backupstoragelocation cozy-default +``` + +The race is nested: step 2's `lookup` resolves from the BucketClaim status, but the projector — and the v1.6.0 etcd migration — read the Secret from step 1, so a cluster missing both needs both. ### Observability @@ -106,6 +143,11 @@ The credentials projector emits two Prometheus counters labelled by `namespace` Alert on `rate(cozystack_backup_credentials_projection_failures_total[5m]) > 0` or `absent_over_time(cozystack_backup_credentials_projection_successes_total[10m])` to catch a stale BSL credential or a malformed source Secret without log scraping. +The default-objects gate emits two more: + +- `cozystack_backup_default_objects_missing{backupclass="cozy-default"}` — how many objects `cozy-default` routes to are absent. **This is the alert that would have caught the missing Strategy CRs**: it is non-zero whatever the HelmRelease's `Ready` condition says. Alert on `min_over_time(cozystack_backup_default_objects_missing[15m]) > 0`. +- `cozystack_backup_default_objects_force_reconciles_total` — forced Helm upgrades issued. A counter that keeps climbing means the forced render is not producing the objects (a missing CRD, for instance), which is a different problem from the install-time race. + ## Admin overrides for `cozy-default` `cozy-default` is rendered by the `backupstrategy-controller` chart and owned by Flux's helm-controller. **Direct `kubectl edit backupclass cozy-default` is overwritten on the next helm reconcile** — the same applies to its companion `strategy.backups.cozystack.io/*` CRs (`cozy-default-cnpg`, `cozy-default-etcd`, `cozy-default-mariadb`, `cozy-default-altinity`, `cozy-default-foundationdb`, the two `cozy-default-velero-*`). The supported override path is the `backupStorage` block on the **`platform` component** of the `cozystack.cozystack-platform` Package CR: diff --git a/internal/backupcontroller/default_objects_gate.go b/internal/backupcontroller/default_objects_gate.go new file mode 100644 index 0000000000..3e66931097 --- /dev/null +++ b/internal/backupcontroller/default_objects_gate.go @@ -0,0 +1,356 @@ +package backupcontroller + +import ( + "context" + "fmt" + "net/http" + "sort" + "time" + + "github.com/go-logr/logr" + "github.com/prometheus/client_golang/prometheus" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/rest" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/apiutil" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/metrics" + + backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1" +) + +// defaultObjectsMissing reports how many of the objects the platform +// BackupClass depends on are absent from the cluster. It is the signal an +// operator alerts on: a non-zero value that does not return to zero means +// the platform default backups are not usable, whatever the HelmRelease's +// Ready condition says. +var defaultObjectsMissing = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "cozystack_backup_default_objects_missing", + Help: "Number of objects referenced by the platform BackupClass that do not exist in the cluster.", + }, + []string{"backupclass"}, +) + +// defaultObjectsForceReconciles counts the forced Helm upgrades the gate +// issued to materialise those objects. A counter that keeps climbing means +// the forced render is not producing the objects (e.g. a CRD is missing), +// which is a different failure than the install-time race this gate closes. +var defaultObjectsForceReconciles = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "cozystack_backup_default_objects_force_reconciles_total", + Help: "Number of forced HelmRelease reconciles issued to create missing platform backup objects.", + }, + []string{"namespace", "name"}, +) + +func init() { + metrics.Registry.MustRegister(defaultObjectsMissing, defaultObjectsForceReconciles) +} + +const ( + // strategyAPIGroup is the API group of the strategy.backups.cozystack.io + // CRs a BackupClass routes to. BackupClassStrategy.StrategyRef is a + // TypedLocalObjectReference and carries no version, so the group+kind + // are mapped to a resource through the RESTMapper. + strategyAPIGroup = "strategy.backups.cozystack.io" + + // defaultBackupStorageLocationName is the Velero BSL the cozy-default + // Velero strategies reference via storageLocation. It is rendered by the + // same lookup-gated template as the Strategy CRs, so it belongs to the + // same existence check. + defaultBackupStorageLocationName = "cozy-default" +) + +var backupStorageLocationGVR = schema.GroupVersionResource{ + Group: "velero.io", + Version: "v1", + Resource: "backupstoragelocations", +} + +// DefaultObjectsGate closes a race the chart's templates cannot close by +// themselves. +// +// The default Strategy CRs and the Velero BackupStorageLocation are +// Helm-templated behind a `lookup` of the BucketClaim the SAME chart +// creates, because the S3 bucket name is assigned by the COSI driver +// (`bucket-`) and is therefore unknowable at render time. On a +// fresh install the lookup is empty and those templates render nothing. +// helm-controller does not re-render a release that succeeded and whose +// chart and values did not change (drift detection is off on +// operator-generated HelmReleases), so the skip is PERMANENT: the objects +// are never created, and the only recovery is a forced Helm upgrade +// (reconcile.fluxcd.io/forceAt + requestedAt — a plain reconcile request +// does not re-render). +// +// This runnable performs that recovery automatically. Once the bucket name +// is resolvable — read from the same platform credentials Secret the +// projector already consumes, so no new dependency and no `lookup` — it +// checks that every object the platform BackupClass routes to actually +// exists, and forces one Helm upgrade when any is missing. A forced +// upgrade re-runs the lookups, so the gated templates materialise. +// +// It deliberately does not create the objects itself: their bodies are +// values-driven (endpoint, region, forcePathStyle, the Altinity strategy's +// whole PodTemplateSpec) and Helm remains their single owner. The gate only +// makes sure the render that produces them actually happens. +type DefaultObjectsGate struct { + // Client reads the BackupClass (cached) and the source credentials + // Secret (uncached — the manager disables the Secret cache). + Client client.Client + // Interface is a dynamic client used for the per-object existence + // checks and the HelmRelease annotation patch. Going through the + // dynamic client keeps these reads off the controller-runtime cache, + // so they need only `get` RBAC and start no cluster-wide informers. + dynamic.Interface + meta.RESTMapper + + // Config supplies the source credentials Secret coordinates. Reused + // verbatim from the credentials projector. + Config BackupCredentialsConfig + // BackupClassName is the platform BackupClass whose strategyRefs + // enumerate the objects that must exist. Empty disables the gate. + BackupClassName string + // HelmRelease is the release that renders those objects and is forced + // when they are missing. Empty name disables the gate. + HelmRelease types.NamespacedName + // VeleroNamespace is where the cozy-default BackupStorageLocation + // lives. Empty skips the BSL check (velero.bslEnabled=false). + VeleroNamespace string + + // Period is the check interval. Defaults to 1 minute. + Period time.Duration + // MinForceInterval throttles forced upgrades so a condition the forced + // render cannot fix (missing CRD, wrong bucket) does not turn into a + // hot loop against helm-controller. Defaults to 5 minutes. + MinForceInterval time.Duration + + // now is a test seam for the throttle clock. + now func() time.Time + lastForce time.Time +} + +var ( + _ manager.Runnable = (*DefaultObjectsGate)(nil) + _ manager.LeaderElectionRunnable = (*DefaultObjectsGate)(nil) +) + +// NeedLeaderElection keeps a single replica forcing the release. Two +// replicas racing the same annotation patch would double the Helm upgrades +// for no benefit. +func (g *DefaultObjectsGate) NeedLeaderElection() bool { return true } + +// SetupWithManager wires the dynamic client and the RESTMapper from the +// manager's rest.Config, mirroring RestoreJobReconciler, and registers the +// gate as a manager Runnable. +func (g *DefaultObjectsGate) SetupWithManager(mgr ctrl.Manager) error { + cfg := mgr.GetConfig() + var err error + if g.Interface, err = dynamic.NewForConfig(cfg); err != nil { + return err + } + var h *http.Client + if h, err = rest.HTTPClientFor(cfg); err != nil { + return err + } + if g.RESTMapper, err = apiutil.NewDynamicRESTMapper(cfg, h); err != nil { + return err + } + return mgr.Add(g) +} + +func (g *DefaultObjectsGate) Start(ctx context.Context) error { + logger := log.FromContext(ctx).WithName("default-objects-gate") + if g.BackupClassName == "" || g.HelmRelease.Name == "" || g.HelmRelease.Namespace == "" || !g.Config.IsEnabled() { + logger.V(1).Info("default backup objects gate disabled", + "backupClass", g.BackupClassName, + "helmRelease", g.HelmRelease.String(), + "credentialsConfigured", g.Config.IsEnabled()) + return nil + } + if g.Period == 0 { + g.Period = time.Minute + } + if g.MinForceInterval == 0 { + g.MinForceInterval = 5 * time.Minute + } + if g.now == nil { + g.now = time.Now + } + + tick := time.NewTicker(g.Period) + defer tick.Stop() + g.checkAndLog(ctx, logger) + for { + select { + case <-ctx.Done(): + return nil + case <-tick.C: + g.checkAndLog(ctx, logger) + } + } +} + +func (g *DefaultObjectsGate) checkAndLog(ctx context.Context, logger logr.Logger) { + missing, forced, err := g.Check(ctx) + switch { + case err != nil: + // Info, not Error: every branch here is either transient (the + // bucket is still being provisioned) or already exposed as a + // metric, and this runs every minute for the life of the cluster. + logger.Info("default backup objects check failed", "error", err.Error()) + case len(missing) > 0 && forced: + logger.Info("forced a Helm upgrade to create missing default backup objects", + "helmRelease", g.HelmRelease.String(), "missing", missing) + case len(missing) > 0: + logger.Info("default backup objects still missing, force throttled", + "helmRelease", g.HelmRelease.String(), "missing", missing, + "minForceInterval", g.MinForceInterval.String()) + default: + logger.V(1).Info("all default backup objects present") + } +} + +// Check resolves the platform bucket name, verifies that every object the +// BackupClass routes to exists, and forces one Helm upgrade when any is +// missing. It returns the missing objects (formatted for logs) and whether +// a forced upgrade was issued on this call. +// +// It is a no-op while the bucket name is unresolvable: forcing then would +// re-render the same empty lookup. The bucket name is read from the +// projector's source Secret rather than from the BucketClaim, so the gate +// needs no objectstorage RBAC and works for the external-S3 path too. +func (g *DefaultObjectsGate) Check(ctx context.Context) ([]string, bool, error) { + src := &corev1.Secret{} + if err := g.Client.Get(ctx, types.NamespacedName{Namespace: g.Config.SourceNamespace, Name: g.Config.SourceSecretName}, src); err != nil { + return nil, false, fmt.Errorf("get source credentials Secret %s/%s: %w", g.Config.SourceNamespace, g.Config.SourceSecretName, err) + } + creds, err := parseSourceSecret(src) + if err != nil { + return nil, false, err + } + if creds.bucket == "" { + // The bucket is not provisioned yet (or an admin-managed Secret + // omits the name). Nothing a re-render could resolve. + return nil, false, nil + } + + backupClass := &backupsv1alpha1.BackupClass{} + if err := g.Client.Get(ctx, client.ObjectKey{Name: g.BackupClassName}, backupClass); err != nil { + return nil, false, fmt.Errorf("get BackupClass %s: %w", g.BackupClassName, err) + } + + missing, err := g.missingObjects(ctx, backupClass) + if err != nil { + return nil, false, err + } + defaultObjectsMissing.WithLabelValues(g.BackupClassName).Set(float64(len(missing))) + if len(missing) == 0 { + return nil, false, nil + } + + now := g.now() + if !g.lastForce.IsZero() && now.Sub(g.lastForce) < g.MinForceInterval { + return missing, false, nil + } + if err := g.forceHelmRelease(ctx, now); err != nil { + if apierrors.IsNotFound(err) { + // Not a Flux-managed install (a plain `helm install` for local + // development). There is nothing to force; stay quiet rather + // than logging an error every tick. + g.lastForce = now + return missing, false, nil + } + return missing, false, fmt.Errorf("force HelmRelease %s: %w", g.HelmRelease.String(), err) + } + g.lastForce = now + defaultObjectsForceReconciles.WithLabelValues(g.HelmRelease.Namespace, g.HelmRelease.Name).Inc() + return missing, true, nil +} + +// missingObjects returns a sorted, de-duplicated list of the +// "/" objects the BackupClass references that do not exist, +// plus the Velero BSL when a Velero namespace is configured. +// +// An unmappable group/kind (the CRD is not installed) is NOT reported as +// missing: no Helm re-render can create an object whose CRD is absent, and +// counting it would keep the gate forcing upgrades forever. +func (g *DefaultObjectsGate) missingObjects(ctx context.Context, backupClass *backupsv1alpha1.BackupClass) ([]string, error) { + seen := map[string]struct{}{} + var missing []string + + for _, strategy := range backupClass.Spec.Strategies { + name := strategy.StrategyRef.Name + kind := strategy.StrategyRef.Kind + if name == "" || kind == "" { + continue + } + group := strategyAPIGroup + if strategy.StrategyRef.APIGroup != nil && *strategy.StrategyRef.APIGroup != "" { + group = *strategy.StrategyRef.APIGroup + } + key := fmt.Sprintf("%s/%s", kind, name) + if _, dup := seen[key]; dup { + continue + } + seen[key] = struct{}{} + + mapping, err := g.RESTMapping(schema.GroupKind{Group: group, Kind: kind}) + if err != nil { + if meta.IsNoMatchError(err) { + continue + } + return nil, fmt.Errorf("map %s/%s: %w", group, kind, err) + } + // Strategy CRs are cluster-scoped; Namespace("") is correct for + // both scopes on the dynamic client. + _, err = g.Resource(mapping.Resource).Get(ctx, name, metav1.GetOptions{}) + switch { + case err == nil: + case apierrors.IsNotFound(err): + missing = append(missing, key) + default: + return nil, fmt.Errorf("get %s %s: %w", kind, name, err) + } + } + + if g.VeleroNamespace != "" { + _, err := g.Resource(backupStorageLocationGVR).Namespace(g.VeleroNamespace).Get(ctx, defaultBackupStorageLocationName, metav1.GetOptions{}) + switch { + case err == nil: + case apierrors.IsNotFound(err): + missing = append(missing, fmt.Sprintf("BackupStorageLocation/%s", defaultBackupStorageLocationName)) + case meta.IsNoMatchError(err): + // Velero CRDs absent: nothing to materialise. + default: + return nil, fmt.Errorf("get BackupStorageLocation %s: %w", defaultBackupStorageLocationName, err) + } + } + + sort.Strings(missing) + return missing, nil +} + +// forceHelmRelease stamps BOTH reconcile.fluxcd.io/forceAt and +// reconcile.fluxcd.io/requestedAt. requestedAt alone only asks +// helm-controller to reconcile, which is a no-op for a release whose chart +// and values are unchanged — it is forceAt that makes it run a real Helm +// upgrade, and forceAt is only honoured together with requestedAt. +func (g *DefaultObjectsGate) forceHelmRelease(ctx context.Context, now time.Time) error { + stamp := now.UTC().Format(time.RFC3339Nano) + patch := fmt.Sprintf( + `{"metadata":{"annotations":{"reconcile.fluxcd.io/forceAt":%q,"reconcile.fluxcd.io/requestedAt":%q}}}`, + stamp, stamp, + ) + _, err := g.Resource(helmReleaseGVR).Namespace(g.HelmRelease.Namespace). + Patch(ctx, g.HelmRelease.Name, types.MergePatchType, []byte(patch), metav1.PatchOptions{}) + return err +} diff --git a/internal/backupcontroller/default_objects_gate_test.go b/internal/backupcontroller/default_objects_gate_test.go new file mode 100644 index 0000000000..f13577164e --- /dev/null +++ b/internal/backupcontroller/default_objects_gate_test.go @@ -0,0 +1,416 @@ +package backupcontroller + +import ( + "context" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + dynamicfake "k8s.io/client-go/dynamic/fake" + k8stesting "k8s.io/client-go/testing" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1" +) + +// gateGVKs are the kinds the gate touches in these tests: the +// strategy.backups.cozystack.io CRs cozy-default routes to, the Velero BSL, +// and the HelmRelease it patches. The RESTMapper and the dynamic fake's +// resource registry are both derived from this one list so the plural the +// gate resolves and the plural the fake serves cannot drift apart. +var gateGVKs = []schema.GroupVersionKind{ + {Group: strategyAPIGroup, Version: "v1alpha1", Kind: "CNPG"}, + {Group: strategyAPIGroup, Version: "v1alpha1", Kind: "Etcd"}, + {Group: strategyAPIGroup, Version: "v1alpha1", Kind: "Velero"}, + {Group: "velero.io", Version: "v1", Kind: "BackupStorageLocation"}, + {Group: "helm.toolkit.fluxcd.io", Version: "v2", Kind: "HelmRelease"}, +} + +func gateScheme() *runtime.Scheme { + scheme := runtime.NewScheme() + _ = corev1.AddToScheme(scheme) + _ = backupsv1alpha1.AddToScheme(scheme) + return scheme +} + +func gateListKinds() map[schema.GroupVersionResource]string { + out := map[schema.GroupVersionResource]string{} + for _, gvk := range gateGVKs { + gvr, _ := meta.UnsafeGuessKindToResource(gvk) + out[gvr] = gvk.Kind + "List" + } + return out +} + +// gateRESTMapper resolves group+kind (the only coordinates a +// BackupClass strategyRef carries) to a resource, which is what the gate +// does in production through the dynamic RESTMapper. The group versions must +// be passed as defaults: RESTMapping is called without a version. +func gateRESTMapper() meta.RESTMapper { + groupVersions := map[schema.GroupVersion]struct{}{} + for _, gvk := range gateGVKs { + groupVersions[gvk.GroupVersion()] = struct{}{} + } + var gvs []schema.GroupVersion + for gv := range groupVersions { + gvs = append(gvs, gv) + } + m := meta.NewDefaultRESTMapper(gvs) + for _, gvk := range gateGVKs { + scope := meta.RESTScopeRoot + if gvk.Kind == "BackupStorageLocation" || gvk.Kind == "HelmRelease" { + scope = meta.RESTScopeNamespace + } + m.Add(gvk, scope) + } + return m +} + +func apiGroup(s string) *string { return &s } + +// cozyDefaultBackupClass mirrors the routes the chart's +// backupclass-default.yaml renders unconditionally: it is the manifest of +// what must exist, which is exactly why the gate reads it instead of +// hard-coding a list of Strategy names. +func cozyDefaultBackupClass() *backupsv1alpha1.BackupClass { + ref := func(kind, name string) backupsv1alpha1.BackupClassStrategy { + return backupsv1alpha1.BackupClassStrategy{ + Application: backupsv1alpha1.ApplicationSelector{ + APIGroup: apiGroup("apps.cozystack.io"), + Kind: kind + "App", + }, + StrategyRef: corev1.TypedLocalObjectReference{ + APIGroup: apiGroup(strategyAPIGroup), + Kind: kind, + Name: name, + }, + } + } + return &backupsv1alpha1.BackupClass{ + ObjectMeta: metav1.ObjectMeta{Name: "cozy-default"}, + Spec: backupsv1alpha1.BackupClassSpec{ + Strategies: []backupsv1alpha1.BackupClassStrategy{ + ref("CNPG", "cozy-default-cnpg"), + ref("Etcd", "cozy-default-etcd"), + ref("Velero", "cozy-default-velero-vminstance"), + ref("Velero", "cozy-default-velero-vmdisk"), + }, + }, + } +} + +func strategyObject(kind, name string) *unstructured.Unstructured { + u := &unstructured.Unstructured{} + u.SetAPIVersion(strategyAPIGroup + "/v1alpha1") + u.SetKind(kind) + u.SetName(name) + return u +} + +func helmReleaseObject() *unstructured.Unstructured { + u := &unstructured.Unstructured{} + u.SetAPIVersion("helm.toolkit.fluxcd.io/v2") + u.SetKind("HelmRelease") + u.SetNamespace("cozy-backup-controller") + u.SetName("backupstrategy-controller") + return u +} + +func newGate(t *testing.T, ctrlObjs []client.Object, dynObjs ...runtime.Object) (*DefaultObjectsGate, *dynamicfake.FakeDynamicClient) { + t.Helper() + dyn := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(runtime.NewScheme(), gateListKinds(), dynObjs...) + g := &DefaultObjectsGate{ + Client: fake.NewClientBuilder().WithScheme(gateScheme()).WithObjects(ctrlObjs...).Build(), + Config: BackupCredentialsConfig{ + SourceNamespace: "tenant-root", + SourceSecretName: "bucket-cozy-backups-system-credentials", + TargetSecretName: "cozy-backups-creds", + }, + BackupClassName: "cozy-default", + HelmRelease: types.NamespacedName{ + Namespace: "cozy-backup-controller", + Name: "backupstrategy-controller", + }, + VeleroNamespace: "cozy-velero", + MinForceInterval: 5 * time.Minute, + now: func() time.Time { return time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) }, + } + g.Interface = dyn + g.RESTMapper = gateRESTMapper() + return g, dyn +} + +func sourceSecret(bucket string) *corev1.Secret { + data := map[string][]byte{ + "accessKey": []byte("AK"), + "secretKey": []byte("SK"), + "endpoint": []byte("s3.example.com"), + } + if bucket != "" { + data["bucketName"] = []byte(bucket) + } + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant-root", Name: "bucket-cozy-backups-system-credentials"}, + Data: data, + } +} + +func bslObject() *unstructured.Unstructured { + u := &unstructured.Unstructured{} + u.SetAPIVersion("velero.io/v1") + u.SetKind("BackupStorageLocation") + u.SetNamespace("cozy-velero") + u.SetName(defaultBackupStorageLocationName) + return u +} + +func forceAnnotations(t *testing.T, dyn *dynamicfake.FakeDynamicClient) map[string]string { + t.Helper() + hr, err := dyn.Resource(helmReleaseGVR).Namespace("cozy-backup-controller"). + Get(context.Background(), "backupstrategy-controller", metav1.GetOptions{}) + if err != nil { + t.Fatalf("get HelmRelease: %v", err) + } + return hr.GetAnnotations() +} + +// TestCheckForcesWhenObjectsMissing is the bug this gate exists for: the +// bucket name resolves, so a Helm re-render WOULD produce the gated +// templates, but the objects are absent because the install-time lookup was +// empty and helm-controller never re-rendered. The gate must force a real +// Helm upgrade — BOTH annotations, because requestedAt alone is a no-op for +// an unchanged release. +func TestCheckForcesWhenObjectsMissing(t *testing.T) { + g, dyn := newGate(t, + []client.Object{sourceSecret("bucket-1a2b"), cozyDefaultBackupClass()}, + helmReleaseObject(), + ) + + missing, forced, err := g.Check(context.Background()) + if err != nil { + t.Fatalf("Check: %v", err) + } + if !forced { + t.Fatal("expected a forced Helm upgrade when default objects are missing") + } + // 4 strategyRefs (Velero twice under different names) + the BSL. + if len(missing) != 5 { + t.Fatalf("missing = %v, want 5 entries", missing) + } + ann := forceAnnotations(t, dyn) + forceAt, ok := ann["reconcile.fluxcd.io/forceAt"] + if !ok || forceAt == "" { + t.Errorf("reconcile.fluxcd.io/forceAt not set: %v", ann) + } + requestedAt, ok := ann["reconcile.fluxcd.io/requestedAt"] + if !ok || requestedAt == "" { + t.Errorf("reconcile.fluxcd.io/requestedAt not set: %v", ann) + } + if forceAt != requestedAt { + t.Errorf("forceAt (%q) and requestedAt (%q) must carry the same stamp", forceAt, requestedAt) + } +} + +// TestCheckNoopWhenAllPresent pins the steady state: no forced upgrades +// once the objects exist, otherwise the gate would rewrite the annotation +// forever and re-run a Helm upgrade every MinForceInterval for the life of +// the cluster. +func TestCheckNoopWhenAllPresent(t *testing.T) { + g, dyn := newGate(t, + []client.Object{sourceSecret("bucket-1a2b"), cozyDefaultBackupClass()}, + helmReleaseObject(), + strategyObject("CNPG", "cozy-default-cnpg"), + strategyObject("Etcd", "cozy-default-etcd"), + strategyObject("Velero", "cozy-default-velero-vminstance"), + strategyObject("Velero", "cozy-default-velero-vmdisk"), + bslObject(), + ) + + missing, forced, err := g.Check(context.Background()) + if err != nil { + t.Fatalf("Check: %v", err) + } + if len(missing) != 0 || forced { + t.Fatalf("missing = %v, forced = %v, want none", missing, forced) + } + if ann := forceAnnotations(t, dyn); len(ann) != 0 { + t.Errorf("HelmRelease annotated in the steady state: %v", ann) + } +} + +// TestCheckSkipsWhileBucketUnresolved pins the precondition. Forcing a +// re-render before the COSI driver has published a bucket name would just +// re-run the same empty lookup and burn a Helm upgrade, so the gate waits. +func TestCheckSkipsWhileBucketUnresolved(t *testing.T) { + g, dyn := newGate(t, + []client.Object{sourceSecret(""), cozyDefaultBackupClass()}, + helmReleaseObject(), + ) + + missing, forced, err := g.Check(context.Background()) + if err != nil { + t.Fatalf("Check: %v", err) + } + if len(missing) != 0 || forced { + t.Fatalf("missing = %v, forced = %v, want none while the bucket is unresolved", missing, forced) + } + if ann := forceAnnotations(t, dyn); len(ann) != 0 { + t.Errorf("HelmRelease forced before the bucket resolved: %v", ann) + } +} + +// TestCheckThrottlesRepeatedForces pins the throttle. A condition a +// re-render cannot fix (missing CRD, unreachable bucket) must not turn into +// a hot loop of Helm upgrades against helm-controller. +func TestCheckThrottlesRepeatedForces(t *testing.T) { + g, _ := newGate(t, + []client.Object{sourceSecret("bucket-1a2b"), cozyDefaultBackupClass()}, + helmReleaseObject(), + ) + base := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) + g.now = func() time.Time { return base } + + if _, forced, err := g.Check(context.Background()); err != nil || !forced { + t.Fatalf("first Check: forced = %v, err = %v; want forced", forced, err) + } + + // Well inside MinForceInterval: still missing, but no second upgrade. + g.now = func() time.Time { return base.Add(time.Minute) } + missing, forced, err := g.Check(context.Background()) + if err != nil { + t.Fatalf("second Check: %v", err) + } + if forced { + t.Error("second Check forced inside MinForceInterval") + } + if len(missing) == 0 { + t.Error("second Check should still report the objects as missing") + } + + // Past MinForceInterval: force again, because the objects are still gone. + g.now = func() time.Time { return base.Add(6 * time.Minute) } + if _, forced, err := g.Check(context.Background()); err != nil || !forced { + t.Fatalf("third Check: forced = %v, err = %v; want forced", forced, err) + } +} + +// TestMissingObjectsIgnoresUnmappedKinds pins that a strategyRef whose CRD +// is not installed is not counted as missing. No Helm re-render can create +// such an object, so counting it would force upgrades forever. +func TestMissingObjectsIgnoresUnmappedKinds(t *testing.T) { + bc := cozyDefaultBackupClass() + bc.Spec.Strategies = append(bc.Spec.Strategies, backupsv1alpha1.BackupClassStrategy{ + Application: backupsv1alpha1.ApplicationSelector{Kind: "Nonexistent"}, + StrategyRef: corev1.TypedLocalObjectReference{ + APIGroup: apiGroup(strategyAPIGroup), + Kind: "NotInstalled", + Name: "cozy-default-notinstalled", + }, + }) + g, _ := newGate(t, []client.Object{sourceSecret("b"), bc}, + helmReleaseObject(), + strategyObject("CNPG", "cozy-default-cnpg"), + strategyObject("Etcd", "cozy-default-etcd"), + strategyObject("Velero", "cozy-default-velero-vminstance"), + strategyObject("Velero", "cozy-default-velero-vmdisk"), + bslObject(), + ) + + missing, err := g.missingObjects(context.Background(), bc) + if err != nil { + t.Fatalf("missingObjects: %v", err) + } + if len(missing) != 0 { + t.Fatalf("missing = %v, want none (unmapped kind must be ignored)", missing) + } +} + +// TestMissingObjectsSkipsBSLWhenVeleroNamespaceEmpty covers +// velero.bslEnabled=false: the chart renders no BSL, so its absence is not +// a defect and must not drive forced upgrades. +func TestMissingObjectsSkipsBSLWhenVeleroNamespaceEmpty(t *testing.T) { + bc := cozyDefaultBackupClass() + g, _ := newGate(t, []client.Object{sourceSecret("b"), bc}, + helmReleaseObject(), + strategyObject("CNPG", "cozy-default-cnpg"), + strategyObject("Etcd", "cozy-default-etcd"), + strategyObject("Velero", "cozy-default-velero-vminstance"), + strategyObject("Velero", "cozy-default-velero-vmdisk"), + ) + g.VeleroNamespace = "" + + missing, err := g.missingObjects(context.Background(), bc) + if err != nil { + t.Fatalf("missingObjects: %v", err) + } + if len(missing) != 0 { + t.Fatalf("missing = %v, want none when the BSL is disabled", missing) + } +} + +// TestCheckSurfacesPatchFailure pins that a failed force is reported rather +// than silently recorded as done — and that lastForce is NOT advanced, so +// the next tick retries instead of waiting out MinForceInterval. +func TestCheckSurfacesPatchFailure(t *testing.T) { + g, dyn := newGate(t, + []client.Object{sourceSecret("bucket-1a2b"), cozyDefaultBackupClass()}, + helmReleaseObject(), + ) + dyn.PrependReactor("patch", "helmreleases", func(k8stesting.Action) (bool, runtime.Object, error) { + return true, nil, apierrors.NewForbidden(schema.GroupResource{Group: "helm.toolkit.fluxcd.io", Resource: "helmreleases"}, "backupstrategy-controller", nil) + }) + + if _, forced, err := g.Check(context.Background()); err == nil || forced { + t.Fatalf("forced = %v, err = %v; want an error and forced=false", forced, err) + } + if !g.lastForce.IsZero() { + t.Error("lastForce advanced despite the patch failing; the next tick would be throttled") + } +} + +// TestStartDisabledWithoutHelmRelease pins the opt-out: without release +// coordinates the runnable must return immediately instead of ticking with +// a nil target. +func TestStartDisabledWithoutHelmRelease(t *testing.T) { + g, _ := newGate(t, []client.Object{sourceSecret("b"), cozyDefaultBackupClass()}, helmReleaseObject()) + g.HelmRelease = types.NamespacedName{} + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + done := make(chan error, 1) + go func() { done <- g.Start(ctx) }() + select { + case err := <-done: + if err != nil { + t.Fatalf("Start: %v", err) + } + case <-ctx.Done(): + t.Fatal("Start did not return immediately when disabled") + } +} + +// TestCheckToleratesAbsentHelmRelease covers a plain `helm install` (local +// development): there is no HelmRelease to force, which is not an error to +// report every tick. +func TestCheckToleratesAbsentHelmRelease(t *testing.T) { + g, _ := newGate(t, []client.Object{sourceSecret("bucket-1a2b"), cozyDefaultBackupClass()}) + + missing, forced, err := g.Check(context.Background()) + if err != nil { + t.Fatalf("Check: %v", err) + } + if forced { + t.Error("forced reported true with no HelmRelease to patch") + } + if len(missing) == 0 { + t.Error("objects should still be reported as missing") + } +} diff --git a/packages/system/backupstrategy-controller/templates/_helpers.tpl b/packages/system/backupstrategy-controller/templates/_helpers.tpl index 0739e420dd..a9f8165b80 100644 --- a/packages/system/backupstrategy-controller/templates/_helpers.tpl +++ b/packages/system/backupstrategy-controller/templates/_helpers.tpl @@ -27,9 +27,22 @@ cozystack.bucket-application + cozystack.objectstorage-controller ensures the controllers exist before this chart installs, but the BucketClaim status is reconciled asynchronously, so the first render - sees an unpopulated status and skips. Flux re-renders the - HelmRelease on its next reconcile (spec.interval); once COSI has - populated status.bucketName, the gated templates materialise. + sees an unpopulated status and skips. + + The render CANNOT fail here instead: this chart is the producer of + the very Bucket the lookup reads, so a failed render would never + apply templates/bucket.yaml and the condition could never resolve. + + Nor does Flux repair the skip on its own. helm-controller does not + re-render a release whose chart and values did not change — the + interval reconcile is a no-op for a healthy release, and drift + detection is off on operator-generated HelmReleases — so the skip is + PERMANENT, not a bootstrap window. Convergence is driven instead by + the controller's DefaultObjectsGate + (internal/backupcontroller/default_objects_gate.go), which forces a + real Helm upgrade (reconcile.fluxcd.io/forceAt + requestedAt) once + the bucket name is resolvable and any gated object is missing. See + backupStorage.reconcileDefaultObjects in values.yaml. - bucketNameOverride set → bypass the lookup and use it directly. This is the escape hatch for offline `helm template` / `--dry-run` renders (CI / local diffs), where lookup returns nil and no apiserver is @@ -53,11 +66,13 @@ {{- end -}} {{/* When neither path produces a value, emit the empty string. Strategy/BSL templates that include this helper must gate - themselves on a non-empty result and skip rendering until Flux - re-reconciles the HelmRelease (driven by spec.interval) once the - BucketClaim's COSI-assigned status.bucketName is populated. The - accompanying templates/bucket.yaml ALWAYS renders so the - BucketClaim CAN come into existence even on the first install. */}} + themselves on a non-empty result and skip rendering until a real + Helm upgrade re-runs this lookup with the BucketClaim's + COSI-assigned status.bucketName populated. That upgrade is forced by + the controller's DefaultObjectsGate — it does NOT happen on the + HelmRelease's interval reconcile. The accompanying + templates/bucket.yaml ALWAYS renders so the BucketClaim CAN come + into existence even on the first install. */}} {{- end -}} {{- end -}} diff --git a/packages/system/backupstrategy-controller/templates/backupclass-default.yaml b/packages/system/backupstrategy-controller/templates/backupclass-default.yaml index f635c69c89..366b34ad36 100644 --- a/packages/system/backupstrategy-controller/templates/backupclass-default.yaml +++ b/packages/system/backupstrategy-controller/templates/backupclass-default.yaml @@ -6,9 +6,15 @@ from Day 1; the Strategy CRs it references stay gated on a populated $bucketName because they DO need the COSI-assigned bucket name embedded in their template. A BackupJob fired during the bootstrap - window surfaces a transient Ready=False/StrategyNotReady in the - driver and self-heals once Flux re-renders the chart with the - populated BucketClaim status. + window surfaces a transient Ready=False/StrategyNotReady in the driver. + + It does NOT self-heal on a Flux reconcile: helm-controller re-renders + only when the chart or values change, so the gated Strategy CRs would + stay absent forever. The controller's DefaultObjectsGate reads THIS + object's strategyRefs as the manifest of what must exist and forces the + real Helm upgrade that materialises them (see + backupStorage.reconcileDefaultObjects). Adding a route here therefore + also adds the referenced Strategy CR to that existence check. */}} apiVersion: backups.cozystack.io/v1alpha1 kind: BackupClass diff --git a/packages/system/backupstrategy-controller/templates/deployment.yaml b/packages/system/backupstrategy-controller/templates/deployment.yaml index 7c89f416fa..d56652d0ec 100644 --- a/packages/system/backupstrategy-controller/templates/deployment.yaml +++ b/packages/system/backupstrategy-controller/templates/deployment.yaml @@ -50,6 +50,25 @@ spec: value: {{ .Values.backupStorage.forcePathStyle | quote }} - name: BACKUP_STORAGE_SYSTEM_NAMESPACES value: {{ .Values.backupStorage.systemNamespaces | join "," | quote }} + {{- if .Values.backupStorage.reconcileDefaultObjects }} + # DefaultObjectsGate: the Strategy CRs and the Velero BSL are gated + # on a `lookup` of the BucketClaim this same chart creates, so a + # fresh install renders them empty — permanently, because + # helm-controller does not re-render an unchanged, successful + # release. The gate detects the missing objects once the bucket name + # is resolvable and forces exactly one real Helm upgrade on THIS + # release, which is the only thing that re-runs the lookups. + - name: BACKUP_DEFAULT_OBJECTS_BACKUPCLASS + value: cozy-default + - name: BACKUP_DEFAULT_OBJECTS_HELMRELEASE_NAME + value: {{ .Release.Name | quote }} + - name: BACKUP_DEFAULT_OBJECTS_HELMRELEASE_NAMESPACE + value: {{ .Release.Namespace | quote }} + {{- if .Values.velero.bslEnabled }} + - name: BACKUP_DEFAULT_OBJECTS_VELERO_NAMESPACE + value: {{ .Values.velero.namespace | quote }} + {{- end }} + {{- end }} ports: {{- if .Values.backupStrategyController.metrics.enabled }} - name: metrics diff --git a/packages/system/backupstrategy-controller/templates/rbac.yaml b/packages/system/backupstrategy-controller/templates/rbac.yaml index 24d9f8d8ad..b61c7eb566 100644 --- a/packages/system/backupstrategy-controller/templates/rbac.yaml +++ b/packages/system/backupstrategy-controller/templates/rbac.yaml @@ -53,10 +53,15 @@ rules: - apiGroups: ["cdi.kubevirt.io"] resources: ["datavolumes"] verbs: ["delete"] -# HelmReleases: pre-restore suspends HRs; post-restore rename creates new HR and deletes old +# HelmReleases: pre-restore suspends HRs; post-restore rename creates new HR +# and deletes old. patch additionally lets the DefaultObjectsGate stamp +# reconcile.fluxcd.io/forceAt + requestedAt on THIS chart's own release to +# force the Helm upgrade that materialises the lookup-gated default backup +# objects (no list/watch: every access is a point Get through the dynamic +# client, so no cluster-wide HelmRelease informer is started). - apiGroups: ["helm.toolkit.fluxcd.io"] resources: ["helmreleases"] - verbs: ["get", "create", "update", "delete"] + verbs: ["get", "create", "update", "patch", "delete"] # PVCs and PVs: pre-restore renames PVCs (delete old, create new) and patches PV reclaim policy - apiGroups: [""] resources: ["persistentvolumeclaims"] @@ -68,6 +73,13 @@ rules: - apiGroups: ["velero.io"] resources: ["backups", "restores", "datauploads"] verbs: ["create", "get", "list", "watch", "update", "patch", "delete", "deletecollection"] +# Velero BackupStorageLocation: the DefaultObjectsGate checks that the +# cozy-default BSL — rendered by the same lookup-gated template as the +# Strategy CRs — actually exists. Read-only, and a point Get through the +# dynamic client, so `get` alone is enough (no informer, no list/watch). +- apiGroups: ["velero.io"] + resources: ["backupstoragelocations"] + verbs: ["get"] # Velero DeleteBackupRequest: used by BackupReconciler to delete Velero backups with their storage data - apiGroups: ["velero.io"] resources: ["deletebackuprequests"] diff --git a/packages/system/backupstrategy-controller/tests/default_objects_gate_test.yaml b/packages/system/backupstrategy-controller/tests/default_objects_gate_test.yaml new file mode 100644 index 0000000000..7622822aa8 --- /dev/null +++ b/packages/system/backupstrategy-controller/tests/default_objects_gate_test.yaml @@ -0,0 +1,91 @@ +suite: DefaultObjectsGate wiring — the release the gate forces, and its RBAC + +# The Strategy CRs and the Velero BSL are rendered behind a `lookup` of the +# BucketClaim this same chart creates, so a fresh install renders them empty +# and helm-controller never re-renders (unchanged chart + values, drift +# detection off) — the skip is permanent. The controller's DefaultObjectsGate +# repairs that by forcing a real Helm upgrade on THIS release, so the release +# coordinates it is given must be this chart's own release, and it must hold +# the verbs to do it. + +templates: + - templates/deployment.yaml + - templates/rbac.yaml + +release: + name: backupstrategy-controller + namespace: cozy-backup-controller + +tests: + - it: "gate targets this chart's own HelmRelease and the cozy-default BackupClass" + template: templates/deployment.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_DEFAULT_OBJECTS_HELMRELEASE_NAME + value: backupstrategy-controller + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_DEFAULT_OBJECTS_HELMRELEASE_NAMESPACE + value: cozy-backup-controller + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_DEFAULT_OBJECTS_BACKUPCLASS + value: cozy-default + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_DEFAULT_OBJECTS_VELERO_NAMESPACE + value: cozy-velero + + - it: "velero.bslEnabled=false: the BSL is not rendered, so the gate must not look for it" + template: templates/deployment.yaml + set: + velero.bslEnabled: false + asserts: + - notContains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_DEFAULT_OBJECTS_VELERO_NAMESPACE + value: cozy-velero + # The rest of the gate stays wired: the Strategy CRs still need it. + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_DEFAULT_OBJECTS_HELMRELEASE_NAME + value: backupstrategy-controller + + - it: "reconcileDefaultObjects=false: the gate is fully unwired (main.go disables it on an empty release name)" + template: templates/deployment.yaml + set: + backupStorage.reconcileDefaultObjects: false + asserts: + - notContains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_DEFAULT_OBJECTS_HELMRELEASE_NAME + value: backupstrategy-controller + - notContains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_DEFAULT_OBJECTS_BACKUPCLASS + value: cozy-default + + - it: "ClusterRole grants patch on helmreleases (forceAt stamp) and get on backupstoragelocations" + template: templates/rbac.yaml + asserts: + - contains: + path: rules + content: + apiGroups: ["helm.toolkit.fluxcd.io"] + resources: ["helmreleases"] + verbs: ["get", "create", "update", "patch", "delete"] + - contains: + path: rules + content: + apiGroups: ["velero.io"] + resources: ["backupstoragelocations"] + verbs: ["get"] diff --git a/packages/system/backupstrategy-controller/values.yaml b/packages/system/backupstrategy-controller/values.yaml index a404d00a0b..d9f8cb1a2f 100644 --- a/packages/system/backupstrategy-controller/values.yaml +++ b/packages/system/backupstrategy-controller/values.yaml @@ -118,6 +118,30 @@ backupStorage: # rendering. Tenants are projected lazily during BackupJob reconcile. systemNamespaces: - cozy-velero + # reconcileDefaultObjects enables the controller's DefaultObjectsGate. + # + # The Strategy CRs and the Velero BSL below are Helm-templated behind a + # `lookup` of the BucketClaim THIS chart creates, because the S3 bucket + # name is assigned by the COSI driver (bucket-) and cannot be + # computed at render time. On a fresh install the lookup is empty and those + # templates render nothing — and helm-controller does not re-render a + # release whose chart and values did not change (drift detection is off on + # operator-generated HelmReleases), so the skip is PERMANENT. Clusters have + # run for months with only the BackupClass present and no Strategy CRs at + # all; recovery needed a hand-stamped reconcile.fluxcd.io/forceAt + + # requestedAt, because a plain reconcile request does not re-render. + # + # With this enabled the controller resolves the bucket name from the + # projector's source Secret (no new dependency, no lookup), checks that + # every object cozy-default routes to exists, and forces one Helm upgrade + # when any is missing — throttled, and a no-op in the steady state. It does + # NOT create the objects itself: their bodies are values-driven and Helm + # stays their single owner. + # + # Turn this off only if you manage the default Strategy CRs out of band and + # accept that a missed install-time render is never repaired. Watch + # cozystack_backup_default_objects_missing if you do. + reconcileDefaultObjects: true # velero — BSL rendering. The cozy-default BackupStorageLocation is # created in cozy-velero by templates/velero-bsl.yaml so endpoint/bucket/ # region come from the same backupStorage block used by Strategy CRs and @@ -129,7 +153,9 @@ backupStorage: # Velero CRDs are guaranteed to exist by the time this chart installs. # The BSL template additionally gates itself on a populated BucketClaim # status via the bucketName helper, so on a bootstrap install the BSL -# materialises only after the platform Bucket reconciles. Setting this +# materialises only after the platform Bucket reconciles AND the +# DefaultObjectsGate has forced a real Helm upgrade (an interval reconcile +# alone does not re-render). Setting this # to false is an explicit opt-out for clusters that disable VM backups # entirely and want to avoid the cluster-default BSL — in that case the # cozy-default-velero-vminstance / -vmdisk strategy CRs still ship but diff --git a/packages/system/bucket/templates/user-credentials.yaml b/packages/system/bucket/templates/user-credentials.yaml index da65cf94a4..a0195d3999 100644 --- a/packages/system/bucket/templates/user-credentials.yaml +++ b/packages/system/bucket/templates/user-credentials.yaml @@ -1,7 +1,40 @@ +{{/* + The human-friendly per-user credentials Secret is derived from the COSI + Secret the objectstorage sidecar writes for the matching BucketAccess + (packages/apps/bucket/templates/bucketclaim.yaml). That Secret carries a + single BucketInfo JSON document with the driver-assigned bucket name and + the S3 keys — none of it is knowable at render time, so a `lookup` is + unavoidable here. + + What is NOT acceptable is silently skipping the Secret when the lookup is + empty. helm-controller does not re-render a release whose chart and values + did not change, and drift detection is off by default, so a skip at + install time is PERMANENT: the Secret is never created, and every consumer + that reads it (the dashboard, and for the platform cozy-backups bucket the + backupstrategy-controller credentials projector — hence every default + BackupClass strategy and Velero itself) stays broken until somebody forces + a real Helm upgrade by hand. + + So fail the render instead. This release retries forever + (install/upgrade.remediation.retries: -1 on the -system + HelmRelease, see packages/apps/bucket/templates/helmrelease.yaml) and + every retry re-runs the lookup, so it converges as soon as COSI publishes + the Secret. Failing cannot deadlock its own precondition: the BucketAccess + that produces the COSI Secret is rendered by the PARENT release, not by + this one. + + requireUserCredentials: false is the escape hatch for offline renders + (`helm template`, `make show`, CI diffs) where lookup always returns nil + and no apiserver is reachable. +*/}} {{- range $name, $user := .Values.users }} {{- $secretName := printf "%s-%s" $.Values.bucketName $name }} {{- $existingSecret := lookup "v1" "Secret" $.Release.Namespace $secretName }} -{{- if $existingSecret }} +{{- if not $existingSecret }} +{{- if $.Values.requireUserCredentials }} +{{- fail (printf "COSI credentials Secret %s/%s does not exist yet, so %s-credentials cannot be rendered. This is expected while the bucket is being provisioned: the release retries and converges once the objectstorage sidecar has written the Secret for BucketAccess %s. If it never appears, inspect the BucketClaim/BucketAccess in this namespace and the objectstorage-controller logs. Set requireUserCredentials=false to render this chart offline." $.Release.Namespace $secretName $secretName $secretName) }} +{{- end }} +{{- else }} {{- $bucketInfo := fromJson (b64dec (index $existingSecret.data "BucketInfo")) }} --- apiVersion: v1 diff --git a/packages/system/bucket/tests/user_credentials_test.yaml b/packages/system/bucket/tests/user_credentials_test.yaml new file mode 100644 index 0000000000..09e6268f7f --- /dev/null +++ b/packages/system/bucket/tests/user_credentials_test.yaml @@ -0,0 +1,56 @@ +suite: per-user credentials Secret — loud failure instead of a permanent silent skip + +# The --credentials Secret is derived from the COSI Secret the +# objectstorage sidecar writes, which is only knowable through a `lookup`. +# helm-unittest renders with no live cluster, so lookup always returns nil — +# exactly the pre-reconcile install window these cases pin. +# +# The old template silently emitted nothing in that window. Because +# helm-controller does not re-render a release whose chart and values did not +# change, that skip was permanent: the Secret was never created, and the +# platform credentials projector (hence every default BackupClass strategy) +# had no source to read. + +templates: + - templates/user-credentials.yaml + +release: + name: bucket-cozy-backups + namespace: tenant-root + +tests: + - it: "fails the render when a declared user's COSI Secret does not exist yet" + set: + bucketName: bucket-cozy-backups + users: + system: + readonly: false + asserts: + - failedTemplate: + errorMessage: >- + COSI credentials Secret tenant-root/bucket-cozy-backups-system does not exist yet, + so bucket-cozy-backups-system-credentials cannot be rendered. This is expected while + the bucket is being provisioned: the release retries and converges once the + objectstorage sidecar has written the Secret for BucketAccess + bucket-cozy-backups-system. If it never appears, inspect the BucketClaim/BucketAccess + in this namespace and the objectstorage-controller logs. Set + requireUserCredentials=false to render this chart offline. + + - it: "requireUserCredentials=false keeps offline renders working (no Secret, no failure)" + set: + bucketName: bucket-cozy-backups + requireUserCredentials: false + users: + system: + readonly: false + asserts: + - hasDocuments: + count: 0 + + - it: "no declared users: nothing to render and nothing to fail on" + set: + bucketName: bucket-cozy-backups + users: {} + asserts: + - hasDocuments: + count: 0 diff --git a/packages/system/bucket/values.yaml b/packages/system/bucket/values.yaml index 739c497724..d750671322 100644 --- a/packages/system/bucket/values.yaml +++ b/packages/system/bucket/values.yaml @@ -1,2 +1,11 @@ bucketName: "cozystack" users: {} +# requireUserCredentials makes the render FAIL while a declared user's COSI +# credentials Secret does not exist yet, instead of silently emitting no +# --credentials Secret. Keep it true on a live cluster: the +# release retries (install/upgrade.remediation.retries: -1) and converges, +# whereas a silent skip is permanent — helm-controller does not re-render an +# unchanged release, so the Secret would never be created at all. +# Set to false only for offline renders (`helm template`, `make show`, CI +# diffs), where `lookup` always returns nil because no apiserver is reachable. +requireUserCredentials: true From 0580534295fe26ce3084337bb381cabe0c14a04b Mon Sep 17 00:00:00 2001 From: Mattia Eleuteri Date: Mon, 3 Aug 2026 17:47:36 +0200 Subject: [PATCH 27/51] fix(backupstrategy-controller): bound each gate check and tolerate an absent Secret Two defects found in review of the periodic recovery loop. checkAndLog passed the manager's root context straight into Check, which makes several sequential API calls. That context is only cancelled at shutdown, so an API server stalling on any one of them blocked the call indefinitely. The ticker loop is sequential, so one stuck call stopped every later check for the life of the pod: the recovery this gate exists to provide went silent, leaving a stale gauge and no further log line, with only a restart to bring it back. Each check now runs under a timeout of half the Period, capped at 30s, so it can never overlap the next tick. Check also propagated a NotFound on the projector's source Secret, although the very next branch treats an empty bucket name as an expected no-op. Those are the same bootstrap state seen a moment apart, so the absent Secret logged a check failure on every tick of the whole bootstrap window for a condition the doc comment frames as graceful. Both are pinned by tests that fail against the previous behaviour: one reverts to an error on the absent Secret, the other to a timeout equal to Period. Signed-off-by: Mattia Eleuteri (cherry picked from commit e85501e9f5d3dc6effe5e8721b5db073af59bdb4) --- .../backupcontroller/default_objects_gate.go | 40 +++++++++++++-- .../default_objects_gate_test.go | 50 +++++++++++++++++++ 2 files changed, 86 insertions(+), 4 deletions(-) diff --git a/internal/backupcontroller/default_objects_gate.go b/internal/backupcontroller/default_objects_gate.go index 3e66931097..7904cf8e4e 100644 --- a/internal/backupcontroller/default_objects_gate.go +++ b/internal/backupcontroller/default_objects_gate.go @@ -199,8 +199,32 @@ func (g *DefaultObjectsGate) Start(ctx context.Context) error { } } +// checkTimeout bounds a single check. It stays below Period so a slow check +// cannot overlap the next tick, and is capped so a long Period does not mean a +// correspondingly long stall. +func (g *DefaultObjectsGate) checkTimeout() time.Duration { + const maxTimeout = 30 * time.Second + if g.Period <= 0 { + return maxTimeout + } + if half := g.Period / 2; half < maxTimeout { + return half + } + return maxTimeout +} + func (g *DefaultObjectsGate) checkAndLog(ctx context.Context, logger logr.Logger) { - missing, forced, err := g.Check(ctx) + // Bound every tick. Check makes several sequential API calls (Secret get, + // BackupClass get, one Get per routed strategy, HelmRelease patch) and the + // manager context is only cancelled at shutdown, so an API server that + // stalls on any one of them would block here indefinitely. The loop that + // drives this is sequential, so that single stuck call would stop all + // later checks for the life of the pod: the recovery this gate exists to + // provide would go silent, with a stale gauge and no further log line, and + // only a restart would bring it back. + checkCtx, cancel := context.WithTimeout(ctx, g.checkTimeout()) + defer cancel() + missing, forced, err := g.Check(checkCtx) switch { case err != nil: // Info, not Error: every branch here is either transient (the @@ -225,12 +249,20 @@ func (g *DefaultObjectsGate) checkAndLog(ctx context.Context, logger logr.Logger // a forced upgrade was issued on this call. // // It is a no-op while the bucket name is unresolvable: forcing then would -// re-render the same empty lookup. The bucket name is read from the -// projector's source Secret rather than from the BucketClaim, so the gate -// needs no objectstorage RBAC and works for the external-S3 path too. +// re-render the same empty lookup. That covers both an absent source Secret +// and one whose bucket name is still empty, which are the same bootstrap state +// seen a moment apart. The bucket name is read from the projector's source +// Secret rather than from the BucketClaim, so the gate needs no objectstorage +// RBAC and works for the external-S3 path too. func (g *DefaultObjectsGate) Check(ctx context.Context) ([]string, bool, error) { src := &corev1.Secret{} if err := g.Client.Get(ctx, types.NamespacedName{Namespace: g.Config.SourceNamespace, Name: g.Config.SourceSecretName}, src); err != nil { + if apierrors.IsNotFound(err) { + // The projector has not created the Secret yet. Same no-op as an + // unresolved bucket name: without it, every tick of the whole + // bootstrap window logs a check failure for an expected state. + return nil, false, nil + } return nil, false, fmt.Errorf("get source credentials Secret %s/%s: %w", g.Config.SourceNamespace, g.Config.SourceSecretName, err) } creds, err := parseSourceSecret(src) diff --git a/internal/backupcontroller/default_objects_gate_test.go b/internal/backupcontroller/default_objects_gate_test.go index f13577164e..6ff6e9209b 100644 --- a/internal/backupcontroller/default_objects_gate_test.go +++ b/internal/backupcontroller/default_objects_gate_test.go @@ -267,6 +267,56 @@ func TestCheckSkipsWhileBucketUnresolved(t *testing.T) { } } +// TestCheckSkipsWhileSourceSecretAbsent pins the state one moment earlier than +// TestCheckSkipsWhileBucketUnresolved: the projector has not created the Secret +// at all yet. Returning an error here would log a check failure on every tick +// of the entire bootstrap window, for a state the gate documents as a no-op. +func TestCheckSkipsWhileSourceSecretAbsent(t *testing.T) { + g, dyn := newGate(t, + []client.Object{cozyDefaultBackupClass()}, + helmReleaseObject(), + ) + + missing, forced, err := g.Check(context.Background()) + if err != nil { + t.Fatalf("Check returned an error for an absent source Secret, want a silent no-op: %v", err) + } + if len(missing) != 0 || forced { + t.Fatalf("missing = %v, forced = %v, want none while the source Secret is absent", missing, forced) + } + if ann := forceAnnotations(t, dyn); len(ann) != 0 { + t.Errorf("HelmRelease forced before the source Secret existed: %v", ann) + } +} + +// TestCheckTimeoutStaysBelowPeriod pins the bound on a single check. The ticker +// loop is sequential and the manager context is only cancelled at shutdown, so +// an unbounded check that hangs on a stalled API call stops every later check +// for the life of the pod. +func TestCheckTimeoutStaysBelowPeriod(t *testing.T) { + for _, tc := range []struct { + name string + period time.Duration + want time.Duration + }{ + {"unset falls back to the cap", 0, 30 * time.Second}, + {"default period halves", time.Minute, 30 * time.Second}, + {"short period halves", 10 * time.Second, 5 * time.Second}, + {"long period is capped", time.Hour, 30 * time.Second}, + } { + t.Run(tc.name, func(t *testing.T) { + g := &DefaultObjectsGate{Period: tc.period} + got := g.checkTimeout() + if got != tc.want { + t.Fatalf("checkTimeout() = %v, want %v", got, tc.want) + } + if tc.period > 0 && got >= tc.period { + t.Fatalf("checkTimeout() = %v, must stay below Period %v", got, tc.period) + } + }) + } +} + // TestCheckThrottlesRepeatedForces pins the throttle. A condition a // re-render cannot fix (missing CRD, unreachable bucket) must not turn into // a hot loop of Helm upgrades against helm-controller. From f79cad4b46a2129face94b597a435b8d8576b128 Mon Sep 17 00:00:00 2001 From: IvanHunters Date: Wed, 5 Aug 2026 09:52:18 +0300 Subject: [PATCH 28/51] fix(backupstrategy-controller): stop the default-objects gate looping when velero.bslEnabled=false The cozy-default BackupClass routes VMInstance/VMDisk to the Velero Strategy CRs unconditionally, but the chart gates those CRs on velero.bslEnabled. With the BSL disabled they never render, so the gate counted them as permanently missing and forced a Helm upgrade every MinForceInterval forever, climbing force_reconciles_total and pinning the missing gauge. Skip Velero-kind strategyRefs when the BSL is disabled (empty VeleroNamespace), the same flag that gates their render. The regression test now omits the manually-created Velero objects so it fails without the skip. Correct the values.yaml comment that wrongly claimed the Strategy CRs still ship in this mode. Signed-off-by: IvanHunters (cherry picked from commit aa3accd12800fd977678d771c719e89fd064ff5c) --- .../backupcontroller/default_objects_gate.go | 9 +++++++++ .../default_objects_gate_test.go | 18 +++++++++++------- .../backupstrategy-controller/values.yaml | 11 +++++++---- 3 files changed, 27 insertions(+), 11 deletions(-) diff --git a/internal/backupcontroller/default_objects_gate.go b/internal/backupcontroller/default_objects_gate.go index 7904cf8e4e..837e018946 100644 --- a/internal/backupcontroller/default_objects_gate.go +++ b/internal/backupcontroller/default_objects_gate.go @@ -329,6 +329,15 @@ func (g *DefaultObjectsGate) missingObjects(ctx context.Context, backupClass *ba if strategy.StrategyRef.APIGroup != nil && *strategy.StrategyRef.APIGroup != "" { group = *strategy.StrategyRef.APIGroup } + // When the Velero BSL is disabled (velero.bslEnabled=false, surfaced + // here as an empty VeleroNamespace) the chart gates the Velero + // Strategy CRs off the SAME flag, so they are never rendered. The + // BackupClass still routes to them unconditionally, so counting them + // as missing would force a Helm upgrade every MinForceInterval + // forever against a render that can never produce them. + if g.VeleroNamespace == "" && group == strategyAPIGroup && kind == "Velero" { + continue + } key := fmt.Sprintf("%s/%s", kind, name) if _, dup := seen[key]; dup { continue diff --git a/internal/backupcontroller/default_objects_gate_test.go b/internal/backupcontroller/default_objects_gate_test.go index 6ff6e9209b..19aa9fe388 100644 --- a/internal/backupcontroller/default_objects_gate_test.go +++ b/internal/backupcontroller/default_objects_gate_test.go @@ -383,17 +383,21 @@ func TestMissingObjectsIgnoresUnmappedKinds(t *testing.T) { } } -// TestMissingObjectsSkipsBSLWhenVeleroNamespaceEmpty covers -// velero.bslEnabled=false: the chart renders no BSL, so its absence is not -// a defect and must not drive forced upgrades. -func TestMissingObjectsSkipsBSLWhenVeleroNamespaceEmpty(t *testing.T) { +// TestMissingObjectsSkipsVeleroWhenNamespaceEmpty covers +// velero.bslEnabled=false: the chart gates BOTH the BSL and the Velero +// Strategy CRs off the same flag, so neither is rendered and their absence +// is not a defect. The BackupClass still routes to the Velero strategies +// unconditionally, which is the trap: without the skip the gate would count +// the never-rendered Velero CRs as missing and force a Helm upgrade every +// MinForceInterval forever. The Velero objects are deliberately NOT +// pre-created here, because with the BSL disabled the chart never renders +// them on a real cluster. +func TestMissingObjectsSkipsVeleroWhenNamespaceEmpty(t *testing.T) { bc := cozyDefaultBackupClass() g, _ := newGate(t, []client.Object{sourceSecret("b"), bc}, helmReleaseObject(), strategyObject("CNPG", "cozy-default-cnpg"), strategyObject("Etcd", "cozy-default-etcd"), - strategyObject("Velero", "cozy-default-velero-vminstance"), - strategyObject("Velero", "cozy-default-velero-vmdisk"), ) g.VeleroNamespace = "" @@ -402,7 +406,7 @@ func TestMissingObjectsSkipsBSLWhenVeleroNamespaceEmpty(t *testing.T) { t.Fatalf("missingObjects: %v", err) } if len(missing) != 0 { - t.Fatalf("missing = %v, want none when the BSL is disabled", missing) + t.Fatalf("missing = %v, want none when Velero is disabled", missing) } } diff --git a/packages/system/backupstrategy-controller/values.yaml b/packages/system/backupstrategy-controller/values.yaml index d9f8cb1a2f..15985983c8 100644 --- a/packages/system/backupstrategy-controller/values.yaml +++ b/packages/system/backupstrategy-controller/values.yaml @@ -157,10 +157,13 @@ backupStorage: # DefaultObjectsGate has forced a real Helm upgrade (an interval reconcile # alone does not re-render). Setting this # to false is an explicit opt-out for clusters that disable VM backups -# entirely and want to avoid the cluster-default BSL — in that case the -# cozy-default-velero-vminstance / -vmdisk strategy CRs still ship but -# their `storageLocation: cozy-default` reference will not resolve, so -# VM BackupJobs against cozy-default will fail. +# entirely and want to avoid the cluster-default BSL. In that case the +# cozy-default-velero-vminstance / -vmdisk Strategy CRs are gated off the +# same flag and are NOT rendered either, so the DefaultObjectsGate does +# not expect them (it would otherwise force a Helm upgrade forever against +# a render that can never produce them). VM BackupJobs against cozy-default +# still fail, because the BackupClass keeps routing VMInstance / VMDisk to +# those absent strategies. velero: bslEnabled: true namespace: cozy-velero From 3176eef3087b323e05f2aff2b5181843e721b11a Mon Sep 17 00:00:00 2001 From: IvanHunters Date: Wed, 5 Aug 2026 09:55:12 +0300 Subject: [PATCH 29/51] fix(backupstrategy-controller): resolve the default BSL through the RESTMapper The BSL existence check used a hardcoded GVR and the dynamic client, whose Get returns a plain 404 (IsNotFound) for an unserved API group. The IsNoMatchError branch meant to skip an absent Velero API was therefore dead code, and if Velero were uninstalled after bootstrap while bslEnabled=true the BSL would be counted missing and force a Helm upgrade forever. Route the lookup through the RESTMapper like the strategy loop, so an absent velero.io API is a NoMatch we skip. A regression test fails without the change. Signed-off-by: IvanHunters (cherry picked from commit bf152ba8640da08fd77f486eaa7c4714c6ba6a9d) --- .../backupcontroller/default_objects_gate.go | 23 +++++++-- .../default_objects_gate_test.go | 49 +++++++++++++++++++ 2 files changed, 67 insertions(+), 5 deletions(-) diff --git a/internal/backupcontroller/default_objects_gate.go b/internal/backupcontroller/default_objects_gate.go index 837e018946..14a435b63b 100644 --- a/internal/backupcontroller/default_objects_gate.go +++ b/internal/backupcontroller/default_objects_gate.go @@ -364,15 +364,28 @@ func (g *DefaultObjectsGate) missingObjects(ctx context.Context, backupClass *ba } if g.VeleroNamespace != "" { - _, err := g.Resource(backupStorageLocationGVR).Namespace(g.VeleroNamespace).Get(ctx, defaultBackupStorageLocationName, metav1.GetOptions{}) + // Resolve through the RESTMapper, like the strategy loop above, so a + // genuinely absent Velero API is a NoMatch we skip rather than a + // NotFound we count. The dynamic client's Get bypasses the mapper and + // returns a plain 404 for an unserved group, which IsNotFound would + // catch first, making the "CRDs absent" branch dead code and looping + // the gate forever if Velero were removed while bslEnabled=true. + mapping, err := g.RESTMapping(schema.GroupKind{Group: backupStorageLocationGVR.Group, Kind: "BackupStorageLocation"}) switch { case err == nil: - case apierrors.IsNotFound(err): - missing = append(missing, fmt.Sprintf("BackupStorageLocation/%s", defaultBackupStorageLocationName)) + _, getErr := g.Resource(mapping.Resource).Namespace(g.VeleroNamespace).Get(ctx, defaultBackupStorageLocationName, metav1.GetOptions{}) + switch { + case getErr == nil: + case apierrors.IsNotFound(getErr): + missing = append(missing, fmt.Sprintf("BackupStorageLocation/%s", defaultBackupStorageLocationName)) + default: + return nil, fmt.Errorf("get BackupStorageLocation %s: %w", defaultBackupStorageLocationName, getErr) + } case meta.IsNoMatchError(err): - // Velero CRDs absent: nothing to materialise. + // Velero API not served (e.g. Velero uninstalled after bootstrap): + // no Helm re-render can create the BSL, so it is not "missing". default: - return nil, fmt.Errorf("get BackupStorageLocation %s: %w", defaultBackupStorageLocationName, err) + return nil, fmt.Errorf("map BackupStorageLocation: %w", err) } } diff --git a/internal/backupcontroller/default_objects_gate_test.go b/internal/backupcontroller/default_objects_gate_test.go index 19aa9fe388..d80d13a177 100644 --- a/internal/backupcontroller/default_objects_gate_test.go +++ b/internal/backupcontroller/default_objects_gate_test.go @@ -410,6 +410,55 @@ func TestMissingObjectsSkipsVeleroWhenNamespaceEmpty(t *testing.T) { } } +// TestMissingObjectsSkipsBSLWhenVeleroAPIAbsent covers Velero being removed +// after bootstrap while bslEnabled=true: the velero.io API is no longer +// served, so the BSL cannot be re-rendered and must not be counted as +// missing. Resolving through the RESTMapper turns that into a NoMatch skip; +// the previous hardcoded-GVR Get returned a NotFound that would have looped +// the gate forever. +func TestMissingObjectsSkipsBSLWhenVeleroAPIAbsent(t *testing.T) { + bc := cozyDefaultBackupClass() + g, _ := newGate(t, []client.Object{sourceSecret("b"), bc}, + helmReleaseObject(), + strategyObject("CNPG", "cozy-default-cnpg"), + strategyObject("Etcd", "cozy-default-etcd"), + strategyObject("Velero", "cozy-default-velero-vminstance"), + strategyObject("Velero", "cozy-default-velero-vmdisk"), + ) + // Rebuild the RESTMapper without BackupStorageLocation: the velero.io API + // is not served on this cluster. + gvset := map[schema.GroupVersion]struct{}{} + var kept []schema.GroupVersionKind + for _, gvk := range gateGVKs { + if gvk.Kind == "BackupStorageLocation" { + continue + } + gvset[gvk.GroupVersion()] = struct{}{} + kept = append(kept, gvk) + } + var gvs []schema.GroupVersion + for gv := range gvset { + gvs = append(gvs, gv) + } + m := meta.NewDefaultRESTMapper(gvs) + for _, gvk := range kept { + scope := meta.RESTScopeRoot + if gvk.Kind == "HelmRelease" { + scope = meta.RESTScopeNamespace + } + m.Add(gvk, scope) + } + g.RESTMapper = m + + missing, err := g.missingObjects(context.Background(), bc) + if err != nil { + t.Fatalf("missingObjects: %v", err) + } + if len(missing) != 0 { + t.Fatalf("missing = %v, want none when the Velero API is absent", missing) + } +} + // TestCheckSurfacesPatchFailure pins that a failed force is reported rather // than silently recorded as done — and that lastForce is NOT advanced, so // the next tick retries instead of waiting out MinForceInterval. From a67ef93c5c2466a255fad40ce5c3b64e589cff24 Mon Sep 17 00:00:00 2001 From: IvanHunters Date: Wed, 5 Aug 2026 10:09:04 +0300 Subject: [PATCH 30/51] test(backupstrategy-controller): mirror all six cozy-default routes in the gate fixture The gate reads the BackupClass as the manifest of what must exist, but the Go fixture listed only CNPG, Etcd and the two Velero routes while the shipped backupclass-default.yaml also routes MariaDB and Altinity. Add both to the fixture, and to the RESTMapper and list-kinds derived from it, so the expected-object set the tests exercise equals the real route set. Signed-off-by: IvanHunters (cherry picked from commit 8b006391c2ba89280096dcee8fd7f85dd3a41c7d) --- .../default_objects_gate_test.go | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/internal/backupcontroller/default_objects_gate_test.go b/internal/backupcontroller/default_objects_gate_test.go index d80d13a177..1236140050 100644 --- a/internal/backupcontroller/default_objects_gate_test.go +++ b/internal/backupcontroller/default_objects_gate_test.go @@ -28,7 +28,9 @@ import ( // gate resolves and the plural the fake serves cannot drift apart. var gateGVKs = []schema.GroupVersionKind{ {Group: strategyAPIGroup, Version: "v1alpha1", Kind: "CNPG"}, + {Group: strategyAPIGroup, Version: "v1alpha1", Kind: "MariaDB"}, {Group: strategyAPIGroup, Version: "v1alpha1", Kind: "Etcd"}, + {Group: strategyAPIGroup, Version: "v1alpha1", Kind: "Altinity"}, {Group: strategyAPIGroup, Version: "v1alpha1", Kind: "Velero"}, {Group: "velero.io", Version: "v1", Kind: "BackupStorageLocation"}, {Group: "helm.toolkit.fluxcd.io", Version: "v2", Kind: "HelmRelease"}, @@ -99,7 +101,9 @@ func cozyDefaultBackupClass() *backupsv1alpha1.BackupClass { Spec: backupsv1alpha1.BackupClassSpec{ Strategies: []backupsv1alpha1.BackupClassStrategy{ ref("CNPG", "cozy-default-cnpg"), + ref("MariaDB", "cozy-default-mariadb"), ref("Etcd", "cozy-default-etcd"), + ref("Altinity", "cozy-default-altinity"), ref("Velero", "cozy-default-velero-vminstance"), ref("Velero", "cozy-default-velero-vmdisk"), }, @@ -201,9 +205,9 @@ func TestCheckForcesWhenObjectsMissing(t *testing.T) { if !forced { t.Fatal("expected a forced Helm upgrade when default objects are missing") } - // 4 strategyRefs (Velero twice under different names) + the BSL. - if len(missing) != 5 { - t.Fatalf("missing = %v, want 5 entries", missing) + // 6 strategyRefs (Velero twice under different names) + the BSL. + if len(missing) != 7 { + t.Fatalf("missing = %v, want 7 entries", missing) } ann := forceAnnotations(t, dyn) forceAt, ok := ann["reconcile.fluxcd.io/forceAt"] @@ -228,7 +232,9 @@ func TestCheckNoopWhenAllPresent(t *testing.T) { []client.Object{sourceSecret("bucket-1a2b"), cozyDefaultBackupClass()}, helmReleaseObject(), strategyObject("CNPG", "cozy-default-cnpg"), + strategyObject("MariaDB", "cozy-default-mariadb"), strategyObject("Etcd", "cozy-default-etcd"), + strategyObject("Altinity", "cozy-default-altinity"), strategyObject("Velero", "cozy-default-velero-vminstance"), strategyObject("Velero", "cozy-default-velero-vmdisk"), bslObject(), @@ -368,7 +374,9 @@ func TestMissingObjectsIgnoresUnmappedKinds(t *testing.T) { g, _ := newGate(t, []client.Object{sourceSecret("b"), bc}, helmReleaseObject(), strategyObject("CNPG", "cozy-default-cnpg"), + strategyObject("MariaDB", "cozy-default-mariadb"), strategyObject("Etcd", "cozy-default-etcd"), + strategyObject("Altinity", "cozy-default-altinity"), strategyObject("Velero", "cozy-default-velero-vminstance"), strategyObject("Velero", "cozy-default-velero-vmdisk"), bslObject(), @@ -397,7 +405,9 @@ func TestMissingObjectsSkipsVeleroWhenNamespaceEmpty(t *testing.T) { g, _ := newGate(t, []client.Object{sourceSecret("b"), bc}, helmReleaseObject(), strategyObject("CNPG", "cozy-default-cnpg"), + strategyObject("MariaDB", "cozy-default-mariadb"), strategyObject("Etcd", "cozy-default-etcd"), + strategyObject("Altinity", "cozy-default-altinity"), ) g.VeleroNamespace = "" @@ -421,7 +431,9 @@ func TestMissingObjectsSkipsBSLWhenVeleroAPIAbsent(t *testing.T) { g, _ := newGate(t, []client.Object{sourceSecret("b"), bc}, helmReleaseObject(), strategyObject("CNPG", "cozy-default-cnpg"), + strategyObject("MariaDB", "cozy-default-mariadb"), strategyObject("Etcd", "cozy-default-etcd"), + strategyObject("Altinity", "cozy-default-altinity"), strategyObject("Velero", "cozy-default-velero-vminstance"), strategyObject("Velero", "cozy-default-velero-vmdisk"), ) From c6fc034fed5c6fe02f031b5ea3e15309ff41b669 Mon Sep 17 00:00:00 2001 From: Mattia Eleuteri Date: Fri, 7 Aug 2026 11:10:22 +0200 Subject: [PATCH 31/51] fix(backupstrategy-controller): repair the bucket credentials Secret from the gate, not a chart fail MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses the four findings on #3524. **The `fail` in the bucket chart (MAJOR).** The --credentials Secret was made to `fail` the render while its COSI lookup was empty, so the release would retry and converge instead of skipping the Secret permanently. Helm cannot render a partial set, so that fail aborts the WHOLE -system release: the other users' Secrets, and the bucket UI Deployment/Service/Ingress/HTTPRoute with them. One declared user whose BucketAccess never provisions takes down every other user of that bucket, and on an already-installed release parks it in Failed, blocking every later upgrade. **No per-bucket escape (MAJOR).** Rather than add one, remove the need for it. The gate already forces the release that renders the lookup-gated Strategy CRs; the credentials Secret is the same trap one release earlier, and the gate reads that Secret to resolve the bucket name in the first place. So while it is absent, or carries no bucket name, force the bucket's -system release instead of doing nothing. That cannot deadlock its own precondition: the BucketClaim and the BucketAccess whose COSI Secret the lookup reads are rendered unconditionally by the parent release, not by the one being forced. The two releases are throttled independently — they are forced in sequence on a bootstrap, so a shared timestamp would delay the second by a full MinForceInterval for no reason. Coordinates come from the chart under provisionBucket, and are omitted on external S3 where the Secret is admin-managed and no release renders it. No RBAC delta: patch on helmreleases is already cluster-scoped. The chart goes back to a partial render, with the comment now explaining why the skip is not self-healing and why failing is the wrong lever, and a test that renders the UI with every user unresolvable so a reintroduced `fail` fails the suite. Tenant buckets keep their pre-existing behaviour; generalising the repair to every bucket needs a controller that owns them and is tracked separately. **Stale gauge (MINOR).** cozystack_backup_default_objects_missing was only written on the happy path, so the state the gate exists to catch reported 0 and the documented alert never fired. It is now written on the unresolved path too — the credentials Secret counts as one of the objects the default backups depend on, which is what it always was. On an API error the gauge is deliberately left alone rather than flapping, so a new cozystack_backup_default_objects_check_errors_total marks it stale, and the doc pairs the two instead of overstating the gauge. **Suspended releases (MINOR).** forceHelmRelease now does a point Get and skips a release with spec.suspend: true. helm-controller ignores forceAt and requestedAt while suspended (`cozyhr suspend` sets exactly that), so the gate was re-stamping every MinForceInterval for the whole suspension and climbing the force counter — which the runbook attributes to a render that is not producing the objects, the wrong diagnosis. The skip is logged and not counted, so a climbing counter keeps its documented meaning. Reported-by: IvanHunters Signed-off-by: Mattia Eleuteri Assisted-By: Claude (cherry picked from commit f859b710a45d7df4bd255991ab1d980b048d352a) --- cmd/backupstrategy-controller/main.go | 10 + docs/operations/backup-classes.md | 15 +- .../backupcontroller/default_objects_gate.go | 212 ++++++++++++--- .../default_objects_gate_test.go | 257 ++++++++++++++++-- .../templates/deployment.yaml | 24 ++ .../tests/default_objects_gate_test.yaml | 65 +++++ .../backupstrategy-controller/values.yaml | 24 +- .../bucket/templates/user-credentials.yaml | 49 ++-- .../bucket/tests/user_credentials_test.yaml | 86 +++--- packages/system/bucket/values.yaml | 9 - 10 files changed, 622 insertions(+), 129 deletions(-) diff --git a/cmd/backupstrategy-controller/main.go b/cmd/backupstrategy-controller/main.go index b8338c2c0c..eeec8bf010 100644 --- a/cmd/backupstrategy-controller/main.go +++ b/cmd/backupstrategy-controller/main.go @@ -220,6 +220,16 @@ func main() { Name: hrName, }, VeleroNamespace: os.Getenv("BACKUP_DEFAULT_OBJECTS_VELERO_NAMESPACE"), + // The platform bucket's -system release renders the + // credentials Secret the projector reads, behind the same kind + // of install-time lookup — and while that Secret is missing + // nothing downstream can resolve. Empty when the bucket is not + // provisioned by Cozystack (external S3), where the Secret is + // admin-managed and no release renders it. + CredentialsHelmRelease: types.NamespacedName{ + Namespace: os.Getenv("BACKUP_CREDENTIALS_HELMRELEASE_NAMESPACE"), + Name: os.Getenv("BACKUP_CREDENTIALS_HELMRELEASE_NAME"), + }, } if err := gate.SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to add DefaultObjectsGate runnable") diff --git a/docs/operations/backup-classes.md b/docs/operations/backup-classes.md index 1effdacf5a..de9fb63332 100644 --- a/docs/operations/backup-classes.md +++ b/docs/operations/backup-classes.md @@ -97,7 +97,13 @@ On a fresh-cluster install, the Velero `BackupStorageLocation` `cozy-default` is **That skip does not repair itself on a reconcile.** helm-controller re-renders a release only when its chart or values change; the interval reconcile is a no-op for a healthy release, and drift detection is off on operator-generated HelmReleases. A cluster that lost the race at install time therefore keeps `BackupClass cozy-default` with **no** `Strategy` CRs and **no** Velero BSL indefinitely — which also fail-closes the pre-adoption snapshot in the v1.6.0 etcd migration. -Convergence is driven instead by the controller's default-objects gate (`backupStorage.reconcileDefaultObjects`, on by default). Once the bucket name is resolvable it checks that every object `cozy-default` routes to exists, and stamps `reconcile.fluxcd.io/forceAt` + `requestedAt` on the `backupstrategy-controller` HelmRelease to force the real Helm upgrade that re-runs the lookups. Expect the objects within one minute of the bucket becoming ready. Watch it with: +The same trap sits one release earlier, on the Secret everything else depends on. `bucket-cozy-backups-system-credentials` is rendered by the `bucket-cozy-backups-system` release behind a `lookup` of the COSI Secret, and is skipped just as permanently when that lookup is empty. Without it the credentials projector has no source at all, so no strategy, no Velero and no v1.6.0 etcd migration can resolve — and the gate cannot even determine the bucket name, which it reads from that Secret. + +Convergence is driven instead by the controller's default-objects gate (`backupStorage.reconcileDefaultObjects`, on by default), which repairs both, in order. While the credentials Secret is absent or carries no bucket name, the gate stamps `reconcile.fluxcd.io/forceAt` + `requestedAt` on the **bucket's** `bucket--system` HelmRelease. Once the bucket name resolves it checks that every object `cozy-default` routes to exists and stamps the same pair on the **`backupstrategy-controller`** HelmRelease, forcing the real Helm upgrade that re-runs the lookups. The two are throttled independently (one force per 5 minutes each), so the second is not delayed by the first. Expect the objects within a minute or two of the bucket becoming ready. + +The gate does not force a suspended HelmRelease — helm-controller ignores both annotations while `spec.suspend` is true, so it logs the skip and leaves the force counter alone. A release left suspended (`cozyhr suspend`) is therefore never repaired until it is resumed. + +Watch it with: ```bash kubectl -n cozy-backup-controller logs deploy/backupstrategy-controller | grep default-objects-gate @@ -143,10 +149,11 @@ The credentials projector emits two Prometheus counters labelled by `namespace` Alert on `rate(cozystack_backup_credentials_projection_failures_total[5m]) > 0` or `absent_over_time(cozystack_backup_credentials_projection_successes_total[10m])` to catch a stale BSL credential or a malformed source Secret without log scraping. -The default-objects gate emits two more: +The default-objects gate emits three more: -- `cozystack_backup_default_objects_missing{backupclass="cozy-default"}` — how many objects `cozy-default` routes to are absent. **This is the alert that would have caught the missing Strategy CRs**: it is non-zero whatever the HelmRelease's `Ready` condition says. Alert on `min_over_time(cozystack_backup_default_objects_missing[15m]) > 0`. -- `cozystack_backup_default_objects_force_reconciles_total` — forced Helm upgrades issued. A counter that keeps climbing means the forced render is not producing the objects (a missing CRD, for instance), which is a different problem from the install-time race. +- `cozystack_backup_default_objects_missing{backupclass="cozy-default"}` — how many of the objects the platform default backups depend on are absent: the credentials Secret, the Strategy CRs `cozy-default` routes to, and the Velero BSL. **This is the alert that would have caught the missing Strategy CRs**: it is non-zero whatever the HelmRelease's `Ready` condition says. Alert on `min_over_time(cozystack_backup_default_objects_missing[15m]) > 0`. +- `cozystack_backup_default_objects_check_errors_total{backupclass="cozy-default"}` — checks that could not reach a conclusion (an API error reading the source Secret, the BackupClass, or one of the routed objects). The gauge above is deliberately **not** written on those ticks, so that it does not flap on a transient API error — which means that while this counter climbs, the gauge is stale and a `0` on it proves nothing. Pair the two: `min_over_time(cozystack_backup_default_objects_missing[15m]) > 0 or rate(cozystack_backup_default_objects_check_errors_total[15m]) > 0`. +- `cozystack_backup_default_objects_force_reconciles_total{namespace,name}` — forced Helm upgrades issued, labelled by the release forced (this chart's own, or the bucket's `-system` release). A counter that keeps climbing means the forced render is not producing the objects (a missing CRD, for instance), which is a different problem from the install-time race. A suspended release is skipped before the patch and is **not** counted here, so a paused release cannot masquerade as a render that keeps failing — look for the `skipped forcing a suspended HelmRelease` log line instead. ## Admin overrides for `cozy-default` diff --git a/internal/backupcontroller/default_objects_gate.go b/internal/backupcontroller/default_objects_gate.go index 14a435b63b..023682036a 100644 --- a/internal/backupcontroller/default_objects_gate.go +++ b/internal/backupcontroller/default_objects_gate.go @@ -2,6 +2,7 @@ package backupcontroller import ( "context" + "errors" "fmt" "net/http" "sort" @@ -13,6 +14,7 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/dynamic" @@ -27,15 +29,21 @@ import ( backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1" ) -// defaultObjectsMissing reports how many of the objects the platform -// BackupClass depends on are absent from the cluster. It is the signal an -// operator alerts on: a non-zero value that does not return to zero means -// the platform default backups are not usable, whatever the HelmRelease's -// Ready condition says. +// defaultObjectsMissing reports how many of the objects the platform default +// backups depend on are absent from the cluster: the bucket credentials +// Secret the projector reads, the Strategy CRs the BackupClass routes to, +// and the Velero BSL. It is the signal an operator alerts on: a non-zero +// value that does not return to zero means the platform default backups are +// not usable, whatever the HelmRelease's Ready condition says. +// +// It is only written when a check reached a conclusion. An API error mid-check +// leaves the previous value in place rather than flapping the alert, so the +// gauge alone cannot distinguish "healthy" from "not evaluated" — that is what +// defaultObjectsCheckErrors is for, and the runbook pairs the two. var defaultObjectsMissing = prometheus.NewGaugeVec( prometheus.GaugeOpts{ Name: "cozystack_backup_default_objects_missing", - Help: "Number of objects referenced by the platform BackupClass that do not exist in the cluster.", + Help: "Number of objects the platform default backups depend on (bucket credentials Secret, BackupClass strategy CRs, Velero BSL) that do not exist in the cluster.", }, []string{"backupclass"}, ) @@ -44,6 +52,8 @@ var defaultObjectsMissing = prometheus.NewGaugeVec( // issued to materialise those objects. A counter that keeps climbing means // the forced render is not producing the objects (e.g. a CRD is missing), // which is a different failure than the install-time race this gate closes. +// A suspended release is NOT counted here: it is skipped before the patch, +// so a paused release cannot masquerade as a render that keeps failing. var defaultObjectsForceReconciles = prometheus.NewCounterVec( prometheus.CounterOpts{ Name: "cozystack_backup_default_objects_force_reconciles_total", @@ -52,8 +62,21 @@ var defaultObjectsForceReconciles = prometheus.NewCounterVec( []string{"namespace", "name"}, ) +// defaultObjectsCheckErrors counts the checks that could not reach a +// conclusion (API error reading the source Secret, the BackupClass, or one +// of the routed objects). While this climbs, defaultObjectsMissing is stale: +// alerting on the gauge alone would silently keep reporting the last known +// state, including a 0 that is no longer true. +var defaultObjectsCheckErrors = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "cozystack_backup_default_objects_check_errors_total", + Help: "Number of default backup object checks that failed before reaching a conclusion, leaving cozystack_backup_default_objects_missing stale.", + }, + []string{"backupclass"}, +) + func init() { - metrics.Registry.MustRegister(defaultObjectsMissing, defaultObjectsForceReconciles) + metrics.Registry.MustRegister(defaultObjectsMissing, defaultObjectsForceReconciles, defaultObjectsCheckErrors) } const ( @@ -122,6 +145,18 @@ type DefaultObjectsGate struct { // HelmRelease is the release that renders those objects and is forced // when they are missing. Empty name disables the gate. HelmRelease types.NamespacedName + // CredentialsHelmRelease is the platform bucket's -system + // release, which renders the user-credentials Secret named by + // Config.SourceSecretName behind a `lookup` of the COSI Secret — the + // same permanent-skip trap as the Strategy CRs, one release earlier. + // Forcing it is what unblocks everything downstream, since the gate + // itself cannot resolve the bucket name without that Secret. + // + // Empty when the platform bucket is not provisioned by Cozystack + // (backupStorage.provisionBucket=false, external S3): the Secret is + // then admin-managed and no release renders it, so there is nothing + // to force. + CredentialsHelmRelease types.NamespacedName // VeleroNamespace is where the cozy-default BackupStorageLocation // lives. Empty skips the BSL check (velero.bslEnabled=false). VeleroNamespace string @@ -134,10 +169,22 @@ type DefaultObjectsGate struct { MinForceInterval time.Duration // now is a test seam for the throttle clock. - now func() time.Time - lastForce time.Time + now func() time.Time + // lastForce and lastCredentialsForce throttle the two releases + // independently: the credentials release is forced during the window in + // which the objects release cannot even be evaluated, so sharing one + // timestamp would make the first force delay the second by + // MinForceInterval for no reason. + lastForce time.Time + lastCredentialsForce time.Time } +// errHelmReleaseSuspended reports that the target release has +// spec.suspend: true. helm-controller ignores reconcile.fluxcd.io/forceAt on +// a suspended release, so patching it would be a no-op repeated every +// MinForceInterval for as long as the suspension lasts. +var errHelmReleaseSuspended = errors.New("HelmRelease is suspended") + var ( _ manager.Runnable = (*DefaultObjectsGate)(nil) _ manager.LeaderElectionRunnable = (*DefaultObjectsGate)(nil) @@ -185,6 +232,9 @@ func (g *DefaultObjectsGate) Start(ctx context.Context) error { if g.now == nil { g.now = time.Now } + // force() logs the suspended-release skip from deep in the call stack; + // without this it would land on a bare logger with no name. + ctx = log.IntoContext(ctx, logger) tick := time.NewTicker(g.Period) defer tick.Stop() @@ -232,56 +282,78 @@ func (g *DefaultObjectsGate) checkAndLog(ctx context.Context, logger logr.Logger // metric, and this runs every minute for the life of the cluster. logger.Info("default backup objects check failed", "error", err.Error()) case len(missing) > 0 && forced: + // The release named here is whichever one renders what is missing: + // the credentials Secret comes from the bucket's -system + // release, everything else from this chart's own. logger.Info("forced a Helm upgrade to create missing default backup objects", - "helmRelease", g.HelmRelease.String(), "missing", missing) + "helmRelease", g.forcedReleaseFor(missing).String(), "missing", missing) case len(missing) > 0: - logger.Info("default backup objects still missing, force throttled", - "helmRelease", g.HelmRelease.String(), "missing", missing, + logger.Info("default backup objects still missing, no force issued on this tick", + "helmRelease", g.forcedReleaseFor(missing).String(), "missing", missing, "minForceInterval", g.MinForceInterval.String()) default: logger.V(1).Info("all default backup objects present") } } +// forcedReleaseFor reports which release renders the given missing set, for +// logging only. reconcileCredentials returns the credentials Secret as the +// sole missing object, and it is never mixed with the routed objects — +// resolving the bucket name is a precondition for checking those at all. +func (g *DefaultObjectsGate) forcedReleaseFor(missing []string) types.NamespacedName { + credsKey := fmt.Sprintf("Secret/%s", g.Config.SourceSecretName) + if len(missing) == 1 && missing[0] == credsKey && g.CredentialsHelmRelease.Name != "" { + return g.CredentialsHelmRelease + } + return g.HelmRelease +} + // Check resolves the platform bucket name, verifies that every object the // BackupClass routes to exists, and forces one Helm upgrade when any is // missing. It returns the missing objects (formatted for logs) and whether // a forced upgrade was issued on this call. // -// It is a no-op while the bucket name is unresolvable: forcing then would -// re-render the same empty lookup. That covers both an absent source Secret -// and one whose bucket name is still empty, which are the same bootstrap state -// seen a moment apart. The bucket name is read from the projector's source -// Secret rather than from the BucketClaim, so the gate needs no objectstorage -// RBAC and works for the external-S3 path too. +// While the source Secret is absent or carries no bucket name, the gate +// cannot evaluate the routed objects — their templates gate on the same +// unresolved bucket — so it instead forces the release that renders that +// Secret (see reconcileCredentials). The bucket name is read from the +// projector's source Secret rather than from the BucketClaim, so the gate +// needs no objectstorage RBAC and works for the external-S3 path too. func (g *DefaultObjectsGate) Check(ctx context.Context) ([]string, bool, error) { src := &corev1.Secret{} if err := g.Client.Get(ctx, types.NamespacedName{Namespace: g.Config.SourceNamespace, Name: g.Config.SourceSecretName}, src); err != nil { if apierrors.IsNotFound(err) { - // The projector has not created the Secret yet. Same no-op as an - // unresolved bucket name: without it, every tick of the whole - // bootstrap window logs a check failure for an expected state. - return nil, false, nil + // The Secret the projector reads does not exist. On a fresh + // install that is the bootstrap window; months later it is the + // permanent skip this gate exists to repair, and the two are + // indistinguishable from here — so treat both the same way and + // let the throttle bound the cost. + return g.reconcileCredentials(ctx) } + defaultObjectsCheckErrors.WithLabelValues(g.BackupClassName).Inc() return nil, false, fmt.Errorf("get source credentials Secret %s/%s: %w", g.Config.SourceNamespace, g.Config.SourceSecretName, err) } creds, err := parseSourceSecret(src) if err != nil { + defaultObjectsCheckErrors.WithLabelValues(g.BackupClassName).Inc() return nil, false, err } if creds.bucket == "" { - // The bucket is not provisioned yet (or an admin-managed Secret - // omits the name). Nothing a re-render could resolve. - return nil, false, nil + // The Secret exists but carries no bucket name: a partially rendered + // or hand-written Secret. Same treatment — a re-render of the + // producing release is the only thing that can complete it. + return g.reconcileCredentials(ctx) } backupClass := &backupsv1alpha1.BackupClass{} if err := g.Client.Get(ctx, client.ObjectKey{Name: g.BackupClassName}, backupClass); err != nil { + defaultObjectsCheckErrors.WithLabelValues(g.BackupClassName).Inc() return nil, false, fmt.Errorf("get BackupClass %s: %w", g.BackupClassName, err) } missing, err := g.missingObjects(ctx, backupClass) if err != nil { + defaultObjectsCheckErrors.WithLabelValues(g.BackupClassName).Inc() return nil, false, err } defaultObjectsMissing.WithLabelValues(g.BackupClassName).Set(float64(len(missing))) @@ -289,22 +361,73 @@ func (g *DefaultObjectsGate) Check(ctx context.Context) ([]string, bool, error) return nil, false, nil } + return g.force(ctx, g.HelmRelease, &g.lastForce, missing) +} + +// reconcileCredentials handles the one object the gate cannot check the same +// way as the others: the bucket user-credentials Secret it reads to resolve +// the bucket name in the first place. +// +// That Secret is rendered by the platform bucket's -system release +// behind a `lookup` of the COSI Secret, so it is subject to the identical +// permanent-skip trap — and when it is missing, nothing downstream can +// resolve: the projector has no source, every Strategy CR and the Velero BSL +// stay gated off, and migration 50 cannot find its snapshot target. +// +// The chart cannot repair this itself. Making the render `fail` instead of +// skip would abort the whole -system release — the other users' +// Secrets and the bucket UI Deployment/Service/Ingress with it — and one +// user whose BucketAccess never provisions would park the release in Failed, +// blocking every later upgrade of that bucket with no per-bucket way out. So +// the repair belongs here, where it is per-object and costs a throttled +// annotation patch: exactly the mechanism the gate already applies to the +// Strategy CRs, one release earlier in the chain. +// +// Forcing that release cannot deadlock its own precondition: the BucketClaim +// and the BucketAccess whose COSI Secret the lookup reads are rendered +// unconditionally by the PARENT release (packages/apps/bucket/templates/ +// bucketclaim.yaml), not by the one being forced. +func (g *DefaultObjectsGate) reconcileCredentials(ctx context.Context) ([]string, bool, error) { + missing := []string{fmt.Sprintf("Secret/%s", g.Config.SourceSecretName)} + defaultObjectsMissing.WithLabelValues(g.BackupClassName).Set(float64(len(missing))) + if g.CredentialsHelmRelease.Name == "" || g.CredentialsHelmRelease.Namespace == "" { + // External S3: the Secret is admin-managed and no release renders + // it. Report it missing, force nothing. + return missing, false, nil + } + return g.force(ctx, g.CredentialsHelmRelease, &g.lastCredentialsForce, missing) +} + +// force issues at most one throttled forced Helm upgrade of target and +// reports whether it went through. last is the caller's throttle timestamp, +// advanced on every outcome that must not be retried immediately. +func (g *DefaultObjectsGate) force(ctx context.Context, target types.NamespacedName, last *time.Time, missing []string) ([]string, bool, error) { now := g.now() - if !g.lastForce.IsZero() && now.Sub(g.lastForce) < g.MinForceInterval { + if !last.IsZero() && now.Sub(*last) < g.MinForceInterval { return missing, false, nil } - if err := g.forceHelmRelease(ctx, now); err != nil { - if apierrors.IsNotFound(err) { + if err := g.forceHelmRelease(ctx, target, now); err != nil { + switch { + case apierrors.IsNotFound(err): // Not a Flux-managed install (a plain `helm install` for local // development). There is nothing to force; stay quiet rather // than logging an error every tick. - g.lastForce = now + *last = now + return missing, false, nil + case errors.Is(err, errHelmReleaseSuspended): + // Deliberate operator state (cozyhr suspend, maintenance). The + // annotations would be ignored, so skip the patch entirely and + // leave the force counter alone — a climbing counter must keep + // meaning "the forced render is not producing the objects". + log.FromContext(ctx).Info("skipped forcing a suspended HelmRelease; missing default backup objects will not be repaired until it is resumed", + "helmRelease", target.String(), "missing", missing) + *last = now return missing, false, nil } - return missing, false, fmt.Errorf("force HelmRelease %s: %w", g.HelmRelease.String(), err) + return missing, false, fmt.Errorf("force HelmRelease %s: %w", target.String(), err) } - g.lastForce = now - defaultObjectsForceReconciles.WithLabelValues(g.HelmRelease.Namespace, g.HelmRelease.Name).Inc() + *last = now + defaultObjectsForceReconciles.WithLabelValues(target.Namespace, target.Name).Inc() return missing, true, nil } @@ -398,13 +521,32 @@ func (g *DefaultObjectsGate) missingObjects(ctx context.Context, backupClass *ba // helm-controller to reconcile, which is a no-op for a release whose chart // and values are unchanged — it is forceAt that makes it run a real Helm // upgrade, and forceAt is only honoured together with requestedAt. -func (g *DefaultObjectsGate) forceHelmRelease(ctx context.Context, now time.Time) error { +// +// A point Get precedes the patch so a suspended release is skipped rather +// than re-stamped forever: helm-controller ignores both annotations while +// spec.suspend is true, and `cozyhr suspend` sets exactly that. Without the +// check the gate would patch every MinForceInterval for the whole suspension +// and inflate the force counter, which the runbook reads as a render that +// keeps failing — the wrong diagnosis. The Get needs no RBAC beyond the +// `get` on helmreleases the chart already grants. +func (g *DefaultObjectsGate) forceHelmRelease(ctx context.Context, target types.NamespacedName, now time.Time) error { + hr, err := g.Resource(helmReleaseGVR).Namespace(target.Namespace).Get(ctx, target.Name, metav1.GetOptions{}) + if err != nil { + return err + } + // A malformed spec.suspend (wrong type) is reported by NestedBool as an + // error; treat it as not suspended and let the patch proceed, rather + // than silently disabling the repair on a field we could not parse. + if suspended, found, err := unstructured.NestedBool(hr.Object, "spec", "suspend"); err == nil && found && suspended { + return errHelmReleaseSuspended + } + stamp := now.UTC().Format(time.RFC3339Nano) patch := fmt.Sprintf( `{"metadata":{"annotations":{"reconcile.fluxcd.io/forceAt":%q,"reconcile.fluxcd.io/requestedAt":%q}}}`, stamp, stamp, ) - _, err := g.Resource(helmReleaseGVR).Namespace(g.HelmRelease.Namespace). - Patch(ctx, g.HelmRelease.Name, types.MergePatchType, []byte(patch), metav1.PatchOptions{}) + _, err = g.Resource(helmReleaseGVR).Namespace(target.Namespace). + Patch(ctx, target.Name, types.MergePatchType, []byte(patch), metav1.PatchOptions{}) return err } diff --git a/internal/backupcontroller/default_objects_gate_test.go b/internal/backupcontroller/default_objects_gate_test.go index 1236140050..d04de2b665 100644 --- a/internal/backupcontroller/default_objects_gate_test.go +++ b/internal/backupcontroller/default_objects_gate_test.go @@ -5,6 +5,7 @@ import ( "testing" "time" + "github.com/prometheus/client_golang/prometheus/testutil" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" @@ -128,6 +129,23 @@ func helmReleaseObject() *unstructured.Unstructured { return u } +// credentialsHelmReleaseObject is the platform bucket's -system +// release: the one that renders the user-credentials Secret the gate reads +// to resolve the bucket name, behind its own install-time lookup. +func credentialsHelmReleaseObject() *unstructured.Unstructured { + u := &unstructured.Unstructured{} + u.SetAPIVersion("helm.toolkit.fluxcd.io/v2") + u.SetKind("HelmRelease") + u.SetNamespace("tenant-root") + u.SetName("bucket-cozy-backups-system") + return u +} + +func suspended(u *unstructured.Unstructured) *unstructured.Unstructured { + _ = unstructured.SetNestedField(u.Object, true, "spec", "suspend") + return u +} + func newGate(t *testing.T, ctrlObjs []client.Object, dynObjs ...runtime.Object) (*DefaultObjectsGate, *dynamicfake.FakeDynamicClient) { t.Helper() dyn := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(runtime.NewScheme(), gateListKinds(), dynObjs...) @@ -143,6 +161,10 @@ func newGate(t *testing.T, ctrlObjs []client.Object, dynObjs ...runtime.Object) Namespace: "cozy-backup-controller", Name: "backupstrategy-controller", }, + CredentialsHelmRelease: types.NamespacedName{ + Namespace: "tenant-root", + Name: "bucket-cozy-backups-system", + }, VeleroNamespace: "cozy-velero", MinForceInterval: 5 * time.Minute, now: func() time.Time { return time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) }, @@ -178,10 +200,20 @@ func bslObject() *unstructured.Unstructured { func forceAnnotations(t *testing.T, dyn *dynamicfake.FakeDynamicClient) map[string]string { t.Helper() - hr, err := dyn.Resource(helmReleaseGVR).Namespace("cozy-backup-controller"). - Get(context.Background(), "backupstrategy-controller", metav1.GetOptions{}) + return annotationsOf(t, dyn, "cozy-backup-controller", "backupstrategy-controller") +} + +func credentialsForceAnnotations(t *testing.T, dyn *dynamicfake.FakeDynamicClient) map[string]string { + t.Helper() + return annotationsOf(t, dyn, "tenant-root", "bucket-cozy-backups-system") +} + +func annotationsOf(t *testing.T, dyn *dynamicfake.FakeDynamicClient, namespace, name string) map[string]string { + t.Helper() + hr, err := dyn.Resource(helmReleaseGVR).Namespace(namespace). + Get(context.Background(), name, metav1.GetOptions{}) if err != nil { - t.Fatalf("get HelmRelease: %v", err) + t.Fatalf("get HelmRelease %s/%s: %v", namespace, name, err) } return hr.GetAnnotations() } @@ -252,46 +284,184 @@ func TestCheckNoopWhenAllPresent(t *testing.T) { } } -// TestCheckSkipsWhileBucketUnresolved pins the precondition. Forcing a -// re-render before the COSI driver has published a bucket name would just -// re-run the same empty lookup and burn a Helm upgrade, so the gate waits. -func TestCheckSkipsWhileBucketUnresolved(t *testing.T) { +// TestCheckForcesCredentialsReleaseWhenSourceSecretAbsent is the second half +// of the bug. The Secret the gate reads to resolve the bucket name is itself +// rendered behind an install-time lookup, by the bucket's -system +// release — so it is subject to the identical permanent skip, one release +// earlier. While it is absent nothing downstream can resolve: no projected +// credentials, no Strategy CRs, no Velero, and migration 50 has no snapshot +// target. Forcing THIS chart's release would be useless (its own lookups +// depend on that Secret); the bucket release is the one to force. +func TestCheckForcesCredentialsReleaseWhenSourceSecretAbsent(t *testing.T) { g, dyn := newGate(t, - []client.Object{sourceSecret(""), cozyDefaultBackupClass()}, + []client.Object{cozyDefaultBackupClass()}, helmReleaseObject(), + credentialsHelmReleaseObject(), ) missing, forced, err := g.Check(context.Background()) if err != nil { t.Fatalf("Check: %v", err) } - if len(missing) != 0 || forced { - t.Fatalf("missing = %v, forced = %v, want none while the bucket is unresolved", missing, forced) + if !forced { + t.Fatal("expected the bucket release to be forced while the credentials Secret is absent") + } + if len(missing) != 1 || missing[0] != "Secret/bucket-cozy-backups-system-credentials" { + t.Fatalf("missing = %v, want the credentials Secret", missing) } + ann := credentialsForceAnnotations(t, dyn) + if ann["reconcile.fluxcd.io/forceAt"] == "" || ann["reconcile.fluxcd.io/requestedAt"] == "" { + t.Errorf("bucket release not forced with both annotations: %v", ann) + } + // This chart's own release must be left alone: its templates gate on the + // same unresolved bucket, so forcing it would burn a Helm upgrade that + // re-runs the same empty lookup. if ann := forceAnnotations(t, dyn); len(ann) != 0 { - t.Errorf("HelmRelease forced before the bucket resolved: %v", ann) + t.Errorf("own HelmRelease forced before the bucket resolved: %v", ann) + } +} + +// TestCheckForcesCredentialsReleaseWhenBucketNameEmpty pins the same +// treatment one state later: the Secret exists but carries no bucket name (a +// partial render, or a hand-written Secret missing the key). Only a +// re-render of the producing release can complete it. +func TestCheckForcesCredentialsReleaseWhenBucketNameEmpty(t *testing.T) { + g, dyn := newGate(t, + []client.Object{sourceSecret(""), cozyDefaultBackupClass()}, + helmReleaseObject(), + credentialsHelmReleaseObject(), + ) + + missing, forced, err := g.Check(context.Background()) + if err != nil { + t.Fatalf("Check: %v", err) + } + if !forced || len(missing) != 1 { + t.Fatalf("missing = %v, forced = %v; want the credentials Secret forced", missing, forced) + } + if ann := credentialsForceAnnotations(t, dyn); ann["reconcile.fluxcd.io/forceAt"] == "" { + t.Errorf("bucket release not forced: %v", ann) } } -// TestCheckSkipsWhileSourceSecretAbsent pins the state one moment earlier than -// TestCheckSkipsWhileBucketUnresolved: the projector has not created the Secret -// at all yet. Returning an error here would log a check failure on every tick -// of the entire bootstrap window, for a state the gate documents as a no-op. -func TestCheckSkipsWhileSourceSecretAbsent(t *testing.T) { +// TestCheckReportsCredentialsSecretWithoutBucketRelease covers external S3 +// (backupStorage.provisionBucket=false): the Secret is admin-managed and no +// release renders it, so there is nothing to force — but it must still be +// reported missing, because that is the state an operator has to alert on. +func TestCheckReportsCredentialsSecretWithoutBucketRelease(t *testing.T) { g, dyn := newGate(t, []client.Object{cozyDefaultBackupClass()}, helmReleaseObject(), + credentialsHelmReleaseObject(), ) + g.CredentialsHelmRelease = types.NamespacedName{} missing, forced, err := g.Check(context.Background()) if err != nil { - t.Fatalf("Check returned an error for an absent source Secret, want a silent no-op: %v", err) + t.Fatalf("Check: %v", err) } - if len(missing) != 0 || forced { - t.Fatalf("missing = %v, forced = %v, want none while the source Secret is absent", missing, forced) + if forced { + t.Error("forced a release on the external-S3 path, where none renders the Secret") + } + if len(missing) != 1 { + t.Fatalf("missing = %v, want the credentials Secret reported", missing) + } + if ann := credentialsForceAnnotations(t, dyn); len(ann) != 0 { + t.Errorf("bucket release annotated with no coordinates configured: %v", ann) + } +} + +// TestCheckThrottlesTheTwoReleasesIndependently pins that forcing the bucket +// release does not eat the objects release's throttle budget. The two happen +// in sequence on a real bootstrap — credentials first, then the objects the +// resolved bucket unblocks — so a shared timestamp would delay the second +// force by a full MinForceInterval for no reason. +func TestCheckThrottlesTheTwoReleasesIndependently(t *testing.T) { + base := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) + g, dyn := newGate(t, + []client.Object{cozyDefaultBackupClass()}, + helmReleaseObject(), + credentialsHelmReleaseObject(), + ) + g.now = func() time.Time { return base } + + // No source Secret yet: the bucket release is forced. + if _, forced, err := g.Check(context.Background()); err != nil || !forced { + t.Fatalf("first Check: forced = %v, err = %v; want the bucket release forced", forced, err) + } + + // A minute later the Secret lands. The objects are still missing, and + // this force must go through despite being well inside MinForceInterval + // of the previous one. + g.now = func() time.Time { return base.Add(time.Minute) } + if err := g.Client.Create(context.Background(), sourceSecret("bucket-1a2b")); err != nil { + t.Fatalf("create source Secret: %v", err) + } + missing, forced, err := g.Check(context.Background()) + if err != nil { + t.Fatalf("second Check: %v", err) + } + if !forced { + t.Fatal("objects release throttled by the earlier credentials force") + } + if len(missing) != 7 { + t.Fatalf("missing = %v, want the 6 strategies + the BSL", missing) + } + if ann := forceAnnotations(t, dyn); ann["reconcile.fluxcd.io/forceAt"] == "" { + t.Errorf("own HelmRelease not forced: %v", ann) + } +} + +// TestForceSkipsSuspendedHelmRelease pins the suspend guard. helm-controller +// ignores forceAt/requestedAt while spec.suspend is true (`cozyhr suspend` is +// a standard dev workflow), so patching anyway would re-stamp every +// MinForceInterval for the whole suspension and climb +// cozystack_backup_default_objects_force_reconciles_total — which the runbook +// reads as "the forced render is not producing the objects", the wrong +// diagnosis. The objects must still be reported missing. +func TestForceSkipsSuspendedHelmRelease(t *testing.T) { + g, dyn := newGate(t, + []client.Object{sourceSecret("bucket-1a2b"), cozyDefaultBackupClass()}, + suspended(helmReleaseObject()), + ) + + missing, forced, err := g.Check(context.Background()) + if err != nil { + t.Fatalf("Check: %v", err) + } + if forced { + t.Error("forced reported true for a suspended HelmRelease that ignores the annotations") + } + if len(missing) != 7 { + t.Fatalf("missing = %v, want the objects still reported", missing) } if ann := forceAnnotations(t, dyn); len(ann) != 0 { - t.Errorf("HelmRelease forced before the source Secret existed: %v", ann) + t.Errorf("suspended HelmRelease annotated: %v", ann) + } +} + +// TestForceSkipsSuspendedCredentialsHelmRelease pins the same guard on the +// bucket release, which an operator is far more likely to have suspended: +// it lives in a tenant namespace and is not part of this chart. +func TestForceSkipsSuspendedCredentialsHelmRelease(t *testing.T) { + g, dyn := newGate(t, + []client.Object{cozyDefaultBackupClass()}, + helmReleaseObject(), + suspended(credentialsHelmReleaseObject()), + ) + + missing, forced, err := g.Check(context.Background()) + if err != nil { + t.Fatalf("Check: %v", err) + } + if forced { + t.Error("forced reported true for a suspended bucket HelmRelease") + } + if len(missing) != 1 { + t.Fatalf("missing = %v, want the credentials Secret still reported", missing) + } + if ann := credentialsForceAnnotations(t, dyn); len(ann) != 0 { + t.Errorf("suspended bucket HelmRelease annotated: %v", ann) } } @@ -491,6 +661,53 @@ func TestCheckSurfacesPatchFailure(t *testing.T) { } } +// TestMissingGaugeCoversTheUnresolvedPath pins the alerting contract. The +// gauge used to be written only after the bucket resolved, so the state the +// gate exists to catch — no credentials Secret, therefore no strategies — +// reported 0 or nothing at all, and an alert on it never fired. Deleting the +// source Secret later (credential rotation) had the same effect: the gauge +// froze at its last value. +func TestMissingGaugeCoversTheUnresolvedPath(t *testing.T) { + defaultObjectsMissing.Reset() + g, _ := newGate(t, + []client.Object{cozyDefaultBackupClass()}, + helmReleaseObject(), + credentialsHelmReleaseObject(), + ) + + if _, _, err := g.Check(context.Background()); err != nil { + t.Fatalf("Check: %v", err) + } + if got := testutil.ToFloat64(defaultObjectsMissing.WithLabelValues("cozy-default")); got != 1 { + t.Fatalf("cozystack_backup_default_objects_missing = %v, want 1 while the credentials Secret is absent", got) + } +} + +// TestCheckErrorsCounterMarksTheGaugeStale pins the other half of that +// contract. On an API error the gauge deliberately keeps its last value +// rather than flapping the alert, which means the gauge alone cannot tell +// "healthy" from "not evaluated". The errors counter is what closes that +// gap, and the runbook alerts on both. +func TestCheckErrorsCounterMarksTheGaugeStale(t *testing.T) { + defaultObjectsMissing.Reset() + defaultObjectsCheckErrors.Reset() + // No BackupClass: the check cannot enumerate what must exist. + g, _ := newGate(t, + []client.Object{sourceSecret("bucket-1a2b")}, + helmReleaseObject(), + ) + + if _, _, err := g.Check(context.Background()); err == nil { + t.Fatal("Check succeeded with no BackupClass, want an error") + } + if got := testutil.ToFloat64(defaultObjectsCheckErrors.WithLabelValues("cozy-default")); got != 1 { + t.Fatalf("cozystack_backup_default_objects_check_errors_total = %v, want 1", got) + } + if got := testutil.ToFloat64(defaultObjectsMissing.WithLabelValues("cozy-default")); got != 0 { + t.Fatalf("gauge = %v, want it left untouched by a failed check", got) + } +} + // TestStartDisabledWithoutHelmRelease pins the opt-out: without release // coordinates the runnable must return immediately instead of ticking with // a nil target. diff --git a/packages/system/backupstrategy-controller/templates/deployment.yaml b/packages/system/backupstrategy-controller/templates/deployment.yaml index d56652d0ec..f35e97661a 100644 --- a/packages/system/backupstrategy-controller/templates/deployment.yaml +++ b/packages/system/backupstrategy-controller/templates/deployment.yaml @@ -64,6 +64,30 @@ spec: value: {{ .Release.Name | quote }} - name: BACKUP_DEFAULT_OBJECTS_HELMRELEASE_NAMESPACE value: {{ .Release.Namespace | quote }} + {{- if .Values.backupStorage.provisionBucket }} + # The credentials Secret named by BACKUP_STORAGE_SECRET_NAME is + # itself rendered behind an install-time `lookup` — of the COSI + # Secret, by packages/system/bucket/templates/user-credentials.yaml — + # so it is subject to the same permanent skip, one release earlier in + # the chain. While it is missing the projector has no source, every + # Strategy CR and the Velero BSL stay gated off, and migration 50 + # cannot resolve its snapshot target. These coordinates let the gate + # force the release that renders it. + # + # The name is the Bucket CR name carrying the bucket-rd release + # prefix (packages/system/bucket-rd/cozyrds/bucket.yaml) plus the + # "-system" suffix the parent chart appends + # (packages/apps/bucket/templates/helmrelease.yaml), which is exactly + # BACKUP_STORAGE_SECRET_NAME minus "-credentials". tests/ + # default_objects_gate_test.yaml asserts the two stay in lockstep. + # + # Omitted on provisionBucket=false (external S3): the Secret is + # admin-managed there and no release renders it. + - name: BACKUP_CREDENTIALS_HELMRELEASE_NAME + value: {{ printf "bucket-%s-system" .Values.backupStorage.bucketName | quote }} + - name: BACKUP_CREDENTIALS_HELMRELEASE_NAMESPACE + value: {{ .Values.backupStorage.namespace | quote }} + {{- end }} {{- if .Values.velero.bslEnabled }} - name: BACKUP_DEFAULT_OBJECTS_VELERO_NAMESPACE value: {{ .Values.velero.namespace | quote }} diff --git a/packages/system/backupstrategy-controller/tests/default_objects_gate_test.yaml b/packages/system/backupstrategy-controller/tests/default_objects_gate_test.yaml index 7622822aa8..976fee1667 100644 --- a/packages/system/backupstrategy-controller/tests/default_objects_gate_test.yaml +++ b/packages/system/backupstrategy-controller/tests/default_objects_gate_test.yaml @@ -41,6 +41,71 @@ tests: name: BACKUP_DEFAULT_OBJECTS_VELERO_NAMESPACE value: cozy-velero + # The credentials Secret is rendered one release earlier, by the platform + # bucket's -system release, behind the same kind of install-time + # lookup. Its name is assembled from the bucket-rd release prefix and the + # "-system" suffix the parent bucket chart appends, and it must stay in + # lockstep with systemSecretName: the gate resolves the bucket name from + # that Secret and forces exactly this release when it is absent. Two + # independently written strings, so pin the relationship, not just the value. + - it: "gate targets the bucket release that renders the credentials Secret" + template: templates/deployment.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_CREDENTIALS_HELMRELEASE_NAME + value: bucket-cozy-backups-system + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_CREDENTIALS_HELMRELEASE_NAMESPACE + value: tenant-root + # BACKUP_STORAGE_SECRET_NAME is exactly the release name above with + # "-credentials" appended — that is what makes forcing this release the + # thing that creates that Secret. + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_SECRET_NAME + value: bucket-cozy-backups-system-credentials + + - it: "a renamed bucket moves both the forced release and the Secret together" + template: templates/deployment.yaml + set: + backupStorage.bucketName: platform-backups + backupStorage.systemSecretName: bucket-platform-backups-system-credentials + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_CREDENTIALS_HELMRELEASE_NAME + value: bucket-platform-backups-system + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_SECRET_NAME + value: bucket-platform-backups-system-credentials + + # External S3: the Secret is admin-managed and no release renders it, so + # there is nothing to force. Handing the gate stale coordinates would make + # it patch a HelmRelease that does not exist every MinForceInterval. + - it: "provisionBucket=false: no bucket release to force" + template: templates/deployment.yaml + set: + backupStorage.provisionBucket: false + asserts: + - notContains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_CREDENTIALS_HELMRELEASE_NAME + value: bucket-cozy-backups-system + - notContains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_CREDENTIALS_HELMRELEASE_NAMESPACE + value: tenant-root + - it: "velero.bslEnabled=false: the BSL is not rendered, so the gate must not look for it" template: templates/deployment.yaml set: diff --git a/packages/system/backupstrategy-controller/values.yaml b/packages/system/backupstrategy-controller/values.yaml index 15985983c8..54ba264b17 100644 --- a/packages/system/backupstrategy-controller/values.yaml +++ b/packages/system/backupstrategy-controller/values.yaml @@ -131,16 +131,26 @@ backupStorage: # all; recovery needed a hand-stamped reconcile.fluxcd.io/forceAt + # requestedAt, because a plain reconcile request does not re-render. # - # With this enabled the controller resolves the bucket name from the - # projector's source Secret (no new dependency, no lookup), checks that - # every object cozy-default routes to exists, and forces one Helm upgrade - # when any is missing — throttled, and a no-op in the steady state. It does - # NOT create the objects itself: their bodies are values-driven and Helm - # stays their single owner. + # The systemSecretName Secret above sits one release earlier in the same + # trap: packages/system/bucket/templates/user-credentials.yaml renders it + # behind a `lookup` of the COSI Secret, and skips it just as permanently. + # Without it the projector has no source at all, so nothing downstream can + # resolve — and neither can this gate, which reads the bucket name from it. + # + # With this enabled the controller repairs both, in order: while the + # credentials Secret is missing it forces the bucket's -system + # release; once the bucket name resolves it checks that every object + # cozy-default routes to exists and forces THIS release when any is + # missing. Both are throttled, independently, and a no-op in the steady + # state. A suspended release is skipped rather than re-stamped forever. The + # gate does NOT create the objects itself: their bodies are values-driven + # and Helm stays their single owner. # # Turn this off only if you manage the default Strategy CRs out of band and # accept that a missed install-time render is never repaired. Watch - # cozystack_backup_default_objects_missing if you do. + # cozystack_backup_default_objects_missing (paired with + # cozystack_backup_default_objects_check_errors_total, which marks the + # gauge stale) if you do. reconcileDefaultObjects: true # velero — BSL rendering. The cozy-default BackupStorageLocation is # created in cozy-velero by templates/velero-bsl.yaml so endpoint/bucket/ diff --git a/packages/system/bucket/templates/user-credentials.yaml b/packages/system/bucket/templates/user-credentials.yaml index a0195d3999..6c2a14bd10 100644 --- a/packages/system/bucket/templates/user-credentials.yaml +++ b/packages/system/bucket/templates/user-credentials.yaml @@ -6,35 +6,38 @@ the S3 keys — none of it is knowable at render time, so a `lookup` is unavoidable here. - What is NOT acceptable is silently skipping the Secret when the lookup is - empty. helm-controller does not re-render a release whose chart and values - did not change, and drift detection is off by default, so a skip at - install time is PERMANENT: the Secret is never created, and every consumer - that reads it (the dashboard, and for the platform cozy-backups bucket the - backupstrategy-controller credentials projector — hence every default - BackupClass strategy and Velero itself) stays broken until somebody forces - a real Helm upgrade by hand. + The skip below is NOT self-healing, contrary to what this file used to + imply. helm-controller does not re-render a release whose chart and values + did not change, and drift detection is off on operator-generated + HelmReleases, so a lookup that comes back empty at install time is skipped + PERMANENTLY: the Secret is never created, and every consumer that reads it + stays broken until somebody forces a real Helm upgrade + (reconcile.fluxcd.io/forceAt + requestedAt — a plain `flux reconcile` does + not re-render). - So fail the render instead. This release retries forever - (install/upgrade.remediation.retries: -1 on the -system - HelmRelease, see packages/apps/bucket/templates/helmrelease.yaml) and - every retry re-runs the lookup, so it converges as soon as COSI publishes - the Secret. Failing cannot deadlock its own precondition: the BucketAccess - that produces the COSI Secret is rendered by the PARENT release, not by - this one. + Failing the render instead is tempting and wrong. Helm cannot render a + partial set, so a `fail` inside this range aborts the WHOLE + -system release — the other users' Secrets, and the bucket UI + Deployment/Service/Ingress/HTTPRoute with them. One declared user whose + BucketAccess never provisions (a misconfigured bucketAccessClassName, a + COSI failure) would then take down every other user of that bucket, and on + an already-installed release it would park it in Failed and block every + later upgrade, with no per-bucket way out. - requireUserCredentials: false is the escape hatch for offline renders - (`helm template`, `make show`, CI diffs) where lookup always returns nil - and no apiserver is reachable. + So the repair lives outside the chart, where it is per-object and costs a + throttled annotation patch: backupstrategy-controller's DefaultObjectsGate + (internal/backupcontroller/default_objects_gate.go) forces this release + when the platform bucket's credentials Secret is absent. That covers the + cluster-breaking case — the cozy-backups Secret feeds the credentials + projector, hence every default BackupClass strategy, Velero, and migration + 50. Tenant buckets keep the pre-existing behaviour; generalising the gate + to every bucket needs a controller that owns them and is tracked + separately. */}} {{- range $name, $user := .Values.users }} {{- $secretName := printf "%s-%s" $.Values.bucketName $name }} {{- $existingSecret := lookup "v1" "Secret" $.Release.Namespace $secretName }} -{{- if not $existingSecret }} -{{- if $.Values.requireUserCredentials }} -{{- fail (printf "COSI credentials Secret %s/%s does not exist yet, so %s-credentials cannot be rendered. This is expected while the bucket is being provisioned: the release retries and converges once the objectstorage sidecar has written the Secret for BucketAccess %s. If it never appears, inspect the BucketClaim/BucketAccess in this namespace and the objectstorage-controller logs. Set requireUserCredentials=false to render this chart offline." $.Release.Namespace $secretName $secretName $secretName) }} -{{- end }} -{{- else }} +{{- if $existingSecret }} {{- $bucketInfo := fromJson (b64dec (index $existingSecret.data "BucketInfo")) }} --- apiVersion: v1 diff --git a/packages/system/bucket/tests/user_credentials_test.yaml b/packages/system/bucket/tests/user_credentials_test.yaml index 09e6268f7f..c0c4a16344 100644 --- a/packages/system/bucket/tests/user_credentials_test.yaml +++ b/packages/system/bucket/tests/user_credentials_test.yaml @@ -1,45 +1,44 @@ -suite: per-user credentials Secret — loud failure instead of a permanent silent skip +suite: an unresolvable user must not take the rest of the release down # The --credentials Secret is derived from the COSI Secret the # objectstorage sidecar writes, which is only knowable through a `lookup`. # helm-unittest renders with no live cluster, so lookup always returns nil — # exactly the pre-reconcile install window these cases pin. # -# The old template silently emitted nothing in that window. Because -# helm-controller does not re-render a release whose chart and values did not -# change, that skip was permanent: the Secret was never created, and the -# platform credentials projector (hence every default BackupClass strategy) -# had no source to read. - -templates: - - templates/user-credentials.yaml +# The skip is not self-healing (helm-controller does not re-render an +# unchanged release), and the repair for the platform bucket lives in +# backupstrategy-controller's DefaultObjectsGate, which forces this release. +# +# What these cases guard is the shape of that decision: the render must stay +# partial, never fatal. A `fail` here cannot be scoped to one user — Helm +# renders all or nothing — so it would take the other users' Secrets and the +# bucket UI down with it, and park an installed release in Failed with no +# per-bucket way out. +# +# _namespace / _cluster are set in every case because helm-unittest renders +# the whole chart for each one, not only the templates it asserts on: the +# ingress template dereferences both and would abort the render otherwise. release: name: bucket-cozy-backups namespace: tenant-root -tests: - - it: "fails the render when a declared user's COSI Secret does not exist yet" - set: - bucketName: bucket-cozy-backups - users: - system: - readonly: false - asserts: - - failedTemplate: - errorMessage: >- - COSI credentials Secret tenant-root/bucket-cozy-backups-system does not exist yet, - so bucket-cozy-backups-system-credentials cannot be rendered. This is expected while - the bucket is being provisioned: the release retries and converges once the - objectstorage sidecar has written the Secret for BucketAccess - bucket-cozy-backups-system. If it never appears, inspect the BucketClaim/BucketAccess - in this namespace and the objectstorage-controller logs. Set - requireUserCredentials=false to render this chart offline. +set: &base + bucketName: bucket-cozy-backups + _namespace: + host: example.org + ingress: tenant-root + gateway: "" + _cluster: + issuer-name: letsencrypt-prod + solver: http01 - - it: "requireUserCredentials=false keeps offline renders working (no Secret, no failure)" +tests: + - it: "emits no Secret while a declared user's COSI Secret does not exist yet" + templates: + - templates/user-credentials.yaml set: - bucketName: bucket-cozy-backups - requireUserCredentials: false + <<: *base users: system: readonly: false @@ -47,10 +46,35 @@ tests: - hasDocuments: count: 0 - - it: "no declared users: nothing to render and nothing to fail on" + - it: "no declared users: nothing to render" + templates: + - templates/user-credentials.yaml set: - bucketName: bucket-cozy-backups + <<: *base users: {} asserts: - hasDocuments: count: 0 + + # The regression guard. With every user unresolvable — the multi-user + # partial-failure case — the rest of the release must still render. If a + # `fail` is ever reintroduced in user-credentials.yaml, Helm aborts the + # whole chart and this case fails on the aborted render, not on an + # assertion. + - it: "the bucket UI still renders when no user's COSI Secret resolves" + templates: + - templates/service.yaml + set: + <<: *base + users: + system: + readonly: false + alice: + readonly: true + asserts: + - equal: + path: kind + value: Service + - equal: + path: metadata.name + value: bucket-cozy-backups-ui diff --git a/packages/system/bucket/values.yaml b/packages/system/bucket/values.yaml index d750671322..739c497724 100644 --- a/packages/system/bucket/values.yaml +++ b/packages/system/bucket/values.yaml @@ -1,11 +1,2 @@ bucketName: "cozystack" users: {} -# requireUserCredentials makes the render FAIL while a declared user's COSI -# credentials Secret does not exist yet, instead of silently emitting no -# --credentials Secret. Keep it true on a live cluster: the -# release retries (install/upgrade.remediation.retries: -1) and converges, -# whereas a silent skip is permanent — helm-controller does not re-render an -# unchanged release, so the Secret would never be created at all. -# Set to false only for offline renders (`helm template`, `make show`, CI -# diffs), where `lookup` always returns nil because no apiserver is reachable. -requireUserCredentials: true From dbd9b9ff9ebc4e51aeefd2631f86daf0ca32228c Mon Sep 17 00:00:00 2001 From: Andrey Kolkov Date: Wed, 22 Jul 2026 11:21:53 +0400 Subject: [PATCH 32/51] fix(backups): request S3 checksum only when required for barman-cloud The barman-cloud plugin's sidecar uploads via boto3. Since botocore ~1.36 (early 2025) the default RequestChecksumCalculation is when_supported, which attaches a flexible checksum to every PutObject. Non-AWS S3-compatible backends (Ceph RADOS Gateway, some MinIO / Cloudflare R2 builds) reject the accompanying header with "InvalidArgument: x-amz-content-sha256 must be UNSIGNED-PAYLOAD, STREAMING-AWS4-HMAC-SHA256-PAYLOAD or a valid sha256 value", so every backup/WAL-archive upload fails against them. Pin the sidecar to when_required via the ObjectStore's spec.instanceSidecarConfiguration.env (botocore honors the AWS_REQUEST_CHECKSUM_CALCULATION env var) on both the keycloak system DB and the postgres app (backup + recovery ObjectStores). when_required is also accepted by AWS S3 on a plain PutObject, so it is a safe default everywhere. This mirrors the etcd-operator fix (cozystack/etcd-operator#342, released in v0.5.3), which applied the same WhenRequired policy to its aws-sdk-go-v2 S3 client for the same Ceph RGW backend. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Andrey Kolkov (cherry picked from commit 0c586b9d72d3c3d05c25d4ed74a5039482472589) --- packages/apps/postgres/templates/db.yaml | 14 ++++++++++++++ packages/system/keycloak/templates/db.yaml | 9 +++++++++ 2 files changed, 23 insertions(+) diff --git a/packages/apps/postgres/templates/db.yaml b/packages/apps/postgres/templates/db.yaml index a59b137d15..b9ad00c307 100644 --- a/packages/apps/postgres/templates/db.yaml +++ b/packages/apps/postgres/templates/db.yaml @@ -262,6 +262,15 @@ spec: {{- if .Values.backup.retentionPolicy }} retentionPolicy: {{ .Values.backup.retentionPolicy | quote }} {{- end }} + # Ceph RGW and other non-AWS S3-compatible backends reject botocore's default + # flexible request checksum (the x-amz-content-sha256 that barman-cloud's + # boto3 sends on every upload) with "InvalidArgument: x-amz-content-sha256 + # must be UNSIGNED-PAYLOAD, ...". Compute a checksum only when the operation + # requires one; AWS S3 accepts this too, so it is a safe default everywhere. + instanceSidecarConfiguration: + env: + - name: AWS_REQUEST_CHECKSUM_CALCULATION + value: when_required configuration: destinationPath: {{ $destinationPath }} endpointURL: {{ $endpointURL }} @@ -288,6 +297,11 @@ kind: ObjectStore metadata: name: {{ .Release.Name }}-recovery spec: + # Same non-AWS S3 request-checksum workaround as the backup ObjectStore above. + instanceSidecarConfiguration: + env: + - name: AWS_REQUEST_CHECKSUM_CALCULATION + value: when_required configuration: destinationPath: {{ $destinationPath }} endpointURL: {{ $endpointURL }} diff --git a/packages/system/keycloak/templates/db.yaml b/packages/system/keycloak/templates/db.yaml index 8b8b328234..d337665834 100644 --- a/packages/system/keycloak/templates/db.yaml +++ b/packages/system/keycloak/templates/db.yaml @@ -51,6 +51,15 @@ spec: {{- if .Values.backup.retentionPolicy }} retentionPolicy: {{ .Values.backup.retentionPolicy | quote }} {{- end }} + # Ceph RGW and other non-AWS S3-compatible backends reject botocore's default + # flexible request checksum (the x-amz-content-sha256 that barman-cloud's + # boto3 sends on every upload) with "InvalidArgument: x-amz-content-sha256 + # must be UNSIGNED-PAYLOAD, ...". Compute a checksum only when the operation + # requires one; AWS S3 accepts this too, so it is a safe default everywhere. + instanceSidecarConfiguration: + env: + - name: AWS_REQUEST_CHECKSUM_CALCULATION + value: when_required configuration: destinationPath: {{ .Values.backup.destinationPath | quote }} {{- with .Values.backup.endpointURL }} From 56adea09304df19bc007d3e5be116d2905737aac Mon Sep 17 00:00:00 2001 From: Andrey Kolkov Date: Wed, 22 Jul 2026 15:35:32 +0400 Subject: [PATCH 33/51] fix(backups): cover Go-driven system-bucket ObjectStore + share the checksum helper Address review on the S3 request-checksum fix: - Extend the fix to the platform-managed useSystemBucket=true flow, whose ObjectStore is built in Go by applyClusterPluginBackup (SSA-applied), not chart-rendered. Add InstanceSidecarConfiguration to cnpgtypes.ObjectStoreSpec and set AWS_REQUEST_CHECKSUM_CALCULATION=when_required on it via barmanSidecarConfiguration(). The platform's own default system bucket is SeaweedFS, a non-AWS S3 gateway of the same class as Ceph RGW. Unit test in cnpgstrategy_controller_test.go asserts the applied ObjectStore carries it. - Factor the duplicated instanceSidecarConfiguration block (3x across two charts) into a shared cozy-lib helper (cozy-lib.barman.checksumSidecarConfiguration). Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Andrey Kolkov (cherry picked from commit b574e2176f099305c97e5a8e4a456f463cb7aef1) --- .../cnpgstrategy_controller.go | 23 ++++++++++++++++-- .../cnpgstrategy_controller_test.go | 11 +++++++++ internal/backupcontroller/cnpgtypes/types.go | 17 +++++++++++++ .../cnpgtypes/zz_generated.deepcopy.go | 21 ++++++++++++++++ packages/apps/postgres/templates/db.yaml | 19 ++++----------- .../library/cozy-lib/templates/_barman.tpl | 24 +++++++++++++++++++ packages/system/keycloak/templates/db.yaml | 12 +++------- 7 files changed, 102 insertions(+), 25 deletions(-) create mode 100644 packages/library/cozy-lib/templates/_barman.tpl diff --git a/internal/backupcontroller/cnpgstrategy_controller.go b/internal/backupcontroller/cnpgstrategy_controller.go index f2ee5f5c43..bd5f5981f0 100644 --- a/internal/backupcontroller/cnpgstrategy_controller.go +++ b/internal/backupcontroller/cnpgstrategy_controller.go @@ -436,8 +436,9 @@ func (r *BackupJobReconciler) applyClusterPluginBackup(ctx context.Context, name Controller: &controller, }} objStore.Spec = cnpgtypes.ObjectStoreSpec{ - Configuration: *buildBarmanObjectStore(t.BarmanObjectStore, ""), - RetentionPolicy: t.BarmanObjectStore.RetentionPolicy, + Configuration: *buildBarmanObjectStore(t.BarmanObjectStore, ""), + RetentionPolicy: t.BarmanObjectStore.RetentionPolicy, + InstanceSidecarConfiguration: barmanSidecarConfiguration(), } if err := r.Patch(ctx, objStore, client.Apply, client.FieldOwner(cnpgFieldManager), client.ForceOwnership); err != nil { return "", fmt.Errorf("apply ObjectStore %s/%s: %w", namespace, objStoreName, err) @@ -1391,6 +1392,24 @@ func buildBarmanPlugin(objectStoreName, serverName string) cnpgtypes.PluginConfi } } +// barmanSidecarConfiguration pins the barman-cloud sidecar's boto3 request +// checksum policy to "when_required". Since botocore ~1.36 the default +// (when_supported) attaches a flexible checksum to every PutObject, which +// non-AWS S3-compatible backends (Ceph RGW, and the platform's own default +// SeaweedFS system bucket) reject with "x-amz-content-sha256 must be +// UNSIGNED-PAYLOAD, ...". Compute a checksum only when required; AWS S3 +// accepts that too, so it is a safe default everywhere. This mirrors the same +// env set on the chart-rendered ObjectStores (packages/{apps/postgres, +// system/keycloak}/templates/db.yaml) and the etcd-operator fix (#342). +func barmanSidecarConfiguration() *cnpgtypes.InstanceSidecarConfiguration { + return &cnpgtypes.InstanceSidecarConfiguration{ + Env: []cnpgtypes.EnvVar{{ + Name: "AWS_REQUEST_CHECKSUM_CALCULATION", + Value: "when_required", + }}, + } +} + // buildBarmanObjectStore translates the typed strategy template into the // barman configuration shape shared by the deprecated // spec.backup.barmanObjectStore and the plugin's ObjectStore.spec.configuration. diff --git a/internal/backupcontroller/cnpgstrategy_controller_test.go b/internal/backupcontroller/cnpgstrategy_controller_test.go index 60707473b6..11ac2cf4b0 100644 --- a/internal/backupcontroller/cnpgstrategy_controller_test.go +++ b/internal/backupcontroller/cnpgstrategy_controller_test.go @@ -1464,6 +1464,17 @@ func TestApplyClusterPluginBackup_PatchesExistingCluster(t *testing.T) { if store.Spec.RetentionPolicy != "30d" { t.Errorf("ObjectStore retentionPolicy: got %q", store.Spec.RetentionPolicy) } + // The ObjectStore must pin the barman-cloud sidecar's S3 request-checksum + // policy to when_required, or uploads to non-AWS S3 gateways (Ceph RGW, the + // platform's own SeaweedFS system bucket) fail with an x-amz-content-sha256 + // InvalidArgument. The chart does not render this ObjectStore in the + // platform-managed useSystemBucket=true flow, so this Go path is the only + // place that sets it there. + sc := store.Spec.InstanceSidecarConfiguration + if sc == nil || len(sc.Env) != 1 || + sc.Env[0].Name != "AWS_REQUEST_CHECKSUM_CALCULATION" || sc.Env[0].Value != "when_required" { + t.Errorf("ObjectStore instanceSidecarConfiguration.env: got %+v, want [AWS_REQUEST_CHECKSUM_CALCULATION=when_required]", sc) + } // The ObjectStore must be owner-referenced to the Cluster so Kubernetes GC // removes it when the Cluster is deleted (no orphan in the platform flow, // where the chart does not render this ObjectStore). diff --git a/internal/backupcontroller/cnpgtypes/types.go b/internal/backupcontroller/cnpgtypes/types.go index bf2370ea83..3c03e70b42 100644 --- a/internal/backupcontroller/cnpgtypes/types.go +++ b/internal/backupcontroller/cnpgtypes/types.go @@ -208,6 +208,23 @@ type ObjectStoreSpec struct { // RetentionPolicy is a barman retention expression validated by the plugin // CRD against ^[1-9][0-9]*[dwm]$ (e.g. "30d"). RetentionPolicy string `json:"retentionPolicy,omitempty"` + // InstanceSidecarConfiguration passes settings to the barman-cloud sidecar + // that the plugin injects into the CNPG pods. Only the fields we set are + // modelled (see the upstream ObjectStore CRD for the full type). + InstanceSidecarConfiguration *InstanceSidecarConfiguration `json:"instanceSidecarConfiguration,omitempty"` +} + +// InstanceSidecarConfiguration is a minimal mirror of the barman-cloud +// ObjectStore's spec.instanceSidecarConfiguration (only spec.env). +type InstanceSidecarConfiguration struct { + Env []EnvVar `json:"env,omitempty"` +} + +// EnvVar is a minimal corev1.EnvVar (name/value only) — enough to pass a +// literal environment variable to the barman-cloud sidecar. +type EnvVar struct { + Name string `json:"name"` + Value string `json:"value,omitempty"` } type BackupStatus struct { diff --git a/internal/backupcontroller/cnpgtypes/zz_generated.deepcopy.go b/internal/backupcontroller/cnpgtypes/zz_generated.deepcopy.go index ad6e41864c..f6414fb1b5 100644 --- a/internal/backupcontroller/cnpgtypes/zz_generated.deepcopy.go +++ b/internal/backupcontroller/cnpgtypes/zz_generated.deepcopy.go @@ -239,6 +239,27 @@ func (in *ObjectStoreList) DeepCopyObject() runtime.Object { func (in *ObjectStoreSpec) DeepCopyInto(out *ObjectStoreSpec) { *out = *in in.Configuration.DeepCopyInto(&out.Configuration) + if in.InstanceSidecarConfiguration != nil { + out.InstanceSidecarConfiguration = new(InstanceSidecarConfiguration) + in.InstanceSidecarConfiguration.DeepCopyInto(out.InstanceSidecarConfiguration) + } +} + +func (in *InstanceSidecarConfiguration) DeepCopyInto(out *InstanceSidecarConfiguration) { + *out = *in + if in.Env != nil { + out.Env = make([]EnvVar, len(in.Env)) + copy(out.Env, in.Env) + } +} + +func (in *InstanceSidecarConfiguration) DeepCopy() *InstanceSidecarConfiguration { + if in == nil { + return nil + } + out := new(InstanceSidecarConfiguration) + in.DeepCopyInto(out) + return out } func (in *Backup) DeepCopy() *Backup { diff --git a/packages/apps/postgres/templates/db.yaml b/packages/apps/postgres/templates/db.yaml index b9ad00c307..b03c322f6a 100644 --- a/packages/apps/postgres/templates/db.yaml +++ b/packages/apps/postgres/templates/db.yaml @@ -262,15 +262,9 @@ spec: {{- if .Values.backup.retentionPolicy }} retentionPolicy: {{ .Values.backup.retentionPolicy | quote }} {{- end }} - # Ceph RGW and other non-AWS S3-compatible backends reject botocore's default - # flexible request checksum (the x-amz-content-sha256 that barman-cloud's - # boto3 sends on every upload) with "InvalidArgument: x-amz-content-sha256 - # must be UNSIGNED-PAYLOAD, ...". Compute a checksum only when the operation - # requires one; AWS S3 accepts this too, so it is a safe default everywhere. - instanceSidecarConfiguration: - env: - - name: AWS_REQUEST_CHECKSUM_CALCULATION - value: when_required + # Pin the barman-cloud sidecar's S3 request checksum to when_required + # (rationale in cozy-lib.barman.checksumSidecarConfiguration). + {{- include "cozy-lib.barman.checksumSidecarConfiguration" . | nindent 2 }} configuration: destinationPath: {{ $destinationPath }} endpointURL: {{ $endpointURL }} @@ -297,11 +291,8 @@ kind: ObjectStore metadata: name: {{ .Release.Name }}-recovery spec: - # Same non-AWS S3 request-checksum workaround as the backup ObjectStore above. - instanceSidecarConfiguration: - env: - - name: AWS_REQUEST_CHECKSUM_CALCULATION - value: when_required + # Same S3 request-checksum pin as the backup ObjectStore above. + {{- include "cozy-lib.barman.checksumSidecarConfiguration" . | nindent 2 }} configuration: destinationPath: {{ $destinationPath }} endpointURL: {{ $endpointURL }} diff --git a/packages/library/cozy-lib/templates/_barman.tpl b/packages/library/cozy-lib/templates/_barman.tpl new file mode 100644 index 0000000000..e398598feb --- /dev/null +++ b/packages/library/cozy-lib/templates/_barman.tpl @@ -0,0 +1,24 @@ +{{- /* +cozy-lib.barman.checksumSidecarConfiguration renders a barman-cloud ObjectStore +`spec.instanceSidecarConfiguration` that pins the sidecar's boto3 request-checksum +policy to when_required. + +Since botocore ~1.36 (early 2025) the default RequestChecksumCalculation is +when_supported, which attaches a flexible checksum (the x-amz-content-sha256 +header) to every PutObject. Non-AWS S3-compatible backends (Ceph RADOS Gateway, +the platform's own SeaweedFS system bucket, some MinIO/Cloudflare R2 builds) +reject it with "InvalidArgument: x-amz-content-sha256 must be UNSIGNED-PAYLOAD, +...", so every backup/WAL-archive upload fails against them. when_required +computes a checksum only when the operation mandates one; AWS S3 accepts that on +a plain PutObject too, so it is a safe default everywhere. + +Emit under an ObjectStore `spec:` with `{{- include "cozy-lib.barman.checksumSidecarConfiguration" . | nindent 2 }}`. +The Go-driven platform (useSystemBucket=true) ObjectStore sets the same env in +internal/backupcontroller/cnpgstrategy_controller.go (barmanSidecarConfiguration). +*/ -}} +{{- define "cozy-lib.barman.checksumSidecarConfiguration" -}} +instanceSidecarConfiguration: + env: + - name: AWS_REQUEST_CHECKSUM_CALCULATION + value: when_required +{{- end -}} diff --git a/packages/system/keycloak/templates/db.yaml b/packages/system/keycloak/templates/db.yaml index d337665834..26c25fa360 100644 --- a/packages/system/keycloak/templates/db.yaml +++ b/packages/system/keycloak/templates/db.yaml @@ -51,15 +51,9 @@ spec: {{- if .Values.backup.retentionPolicy }} retentionPolicy: {{ .Values.backup.retentionPolicy | quote }} {{- end }} - # Ceph RGW and other non-AWS S3-compatible backends reject botocore's default - # flexible request checksum (the x-amz-content-sha256 that barman-cloud's - # boto3 sends on every upload) with "InvalidArgument: x-amz-content-sha256 - # must be UNSIGNED-PAYLOAD, ...". Compute a checksum only when the operation - # requires one; AWS S3 accepts this too, so it is a safe default everywhere. - instanceSidecarConfiguration: - env: - - name: AWS_REQUEST_CHECKSUM_CALCULATION - value: when_required + # Pin the barman-cloud sidecar's S3 request checksum to when_required + # (rationale in cozy-lib.barman.checksumSidecarConfiguration). + {{- include "cozy-lib.barman.checksumSidecarConfiguration" . | nindent 2 }} configuration: destinationPath: {{ .Values.backup.destinationPath | quote }} {{- with .Values.backup.endpointURL }} From 4998ae42f5c1a4a5258584a6e5d2e210e9cd6f03 Mon Sep 17 00:00:00 2001 From: Andrey Kolkov Date: Wed, 22 Jul 2026 15:50:25 +0400 Subject: [PATCH 34/51] test(backups): assert the barman-cloud sidecar checksum env on rendered ObjectStores MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit helm-unittest coverage for the S3 request-checksum fix (was mutation-negative before): assert spec.instanceSidecarConfiguration.env pins AWS_REQUEST_CHECKSUM_CALCULATION=when_required on all rendered barman-cloud ObjectStores — postgres backup + bootstrap-recovery (packages/apps/postgres/tests/backup_storage_test.yaml) and the Keycloak DB (new packages/system/keycloak/tests/db_backup_test.yaml). Verified by mutation: reverting the template change now fails these assertions. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Andrey Kolkov (cherry picked from commit 80a03b1e09bac7e34b49f5016c405df913302dbb) --- .../postgres/tests/backup_storage_test.yaml | 31 ++++++++++++++ .../system/keycloak/tests/db_backup_test.yaml | 40 +++++++++++++++++++ 2 files changed, 71 insertions(+) create mode 100644 packages/system/keycloak/tests/db_backup_test.yaml diff --git a/packages/apps/postgres/tests/backup_storage_test.yaml b/packages/apps/postgres/tests/backup_storage_test.yaml index 0a25143138..72274cf4d7 100644 --- a/packages/apps/postgres/tests/backup_storage_test.yaml +++ b/packages/apps/postgres/tests/backup_storage_test.yaml @@ -97,6 +97,22 @@ tests: documentSelector: path: kind value: ObjectStore + # The barman-cloud sidecar's S3 request checksum is pinned to when_required + # (non-AWS S3 gateways reject the botocore default flexible checksum). + - equal: + path: spec.instanceSidecarConfiguration.env[0].name + value: AWS_REQUEST_CHECKSUM_CALCULATION + template: templates/db.yaml + documentSelector: + path: kind + value: ObjectStore + - equal: + path: spec.instanceSidecarConfiguration.env[0].value + value: when_required + template: templates/db.yaml + documentSelector: + path: kind + value: ObjectStore - hasDocuments: count: 1 template: templates/backup-secret.yaml @@ -204,6 +220,21 @@ tests: documentSelector: path: metadata.name value: pg-test-recovery + # The non-AWS S3 request-checksum pin is applied to the recovery ObjectStore too. + - equal: + path: spec.instanceSidecarConfiguration.env[0].name + value: AWS_REQUEST_CHECKSUM_CALCULATION + template: templates/db.yaml + documentSelector: + path: metadata.name + value: pg-test-recovery + - equal: + path: spec.instanceSidecarConfiguration.env[0].value + value: when_required + template: templates/db.yaml + documentSelector: + path: metadata.name + value: pg-test-recovery - it: "bootstrap recovery without serverName: plugin serverName defaults to oldName (native parity)" # Native CNPG defaulted an absent serverName to the externalClusters entry diff --git a/packages/system/keycloak/tests/db_backup_test.yaml b/packages/system/keycloak/tests/db_backup_test.yaml new file mode 100644 index 0000000000..052d34eff6 --- /dev/null +++ b/packages/system/keycloak/tests/db_backup_test.yaml @@ -0,0 +1,40 @@ +suite: keycloak DB backup ObjectStore (barman-cloud plugin) +templates: + - templates/db.yaml +release: + name: keycloak + namespace: cozy-keycloak +tests: + - it: "backup enabled: renders a barman-cloud ObjectStore that pins the sidecar S3 request checksum to when_required" + set: + _cluster: + root-host: example.org + cluster-domain: cozy.local + backup: + enabled: true + destinationPath: s3://aenix/keycloak-db/ + endpointURL: https://s3.frcloud.kz + existingSecretName: keycloak-db-backup-s3-creds + asserts: + # A barman-cloud ObjectStore is rendered for the Keycloak DB. + - equal: + path: metadata.name + value: keycloak-db + documentSelector: + path: kind + value: ObjectStore + # Non-AWS S3 gateways (Ceph RGW, the platform's SeaweedFS system bucket) + # reject botocore's default flexible checksum; the sidecar must request + # one only when required. + - equal: + path: spec.instanceSidecarConfiguration.env[0].name + value: AWS_REQUEST_CHECKSUM_CALCULATION + documentSelector: + path: kind + value: ObjectStore + - equal: + path: spec.instanceSidecarConfiguration.env[0].value + value: when_required + documentSelector: + path: kind + value: ObjectStore From 18a9c9f1a62bfb6b94e85bedd932dabebe4bc676 Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Fri, 14 Aug 2026 13:12:28 +0500 Subject: [PATCH 35/51] fix(api): declare OpenAPIModelName for core and sdn types MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The aggregated apiserver publishes the core.cozystack.io and sdn.cozystack.io models under their Go import path, so a $ref to any of them does not resolve and client-side validation fails on every resource of those groups, not only the one named in the error: error validating data: SchemaError(…core/v1alpha1.Option.spec): unknown model in reference: "github.com~1cozystack~1…v1alpha1.OptionSpec" Since Kubernetes 0.35 the apiserver's DefinitionNamer.GetDefinitionName returns the model name it is handed verbatim; it no longer converts the Go import-path form into the "friendly" reversed-path form. Whatever name the generated openapi map uses therefore becomes both the published definition key and, JSON pointer escaped, the $ref pointing at it. A name containing "/" ships as a definition keyed on the raw path while every reference to it spells each slash "~1", and clients resolve a $ref by trimming "#/definitions/" without unescaping (kube-openapi pkg/util/proto/document.go). The two spellings never meet, the reference dangles, and validation of the whole document fails. Mirror b916d7475, which fixed the same disagreement for apps: add the +k8s:openapi-model-package marker to each doc.go and declare OpenAPIModelName for all 11 core and 9 sdn types, so GetCanonicalTypeName, Scheme.ToOpenAPIDefinitionName and the generated map key all agree on a slash-free dotted name. That takes the number of slash-bearing definition names in the published document from 20 to 0. The methods are hand-written rather than emitted by openapi-gen's --output-model-name-file because that flag also rewrites zz_generated.model_name.go inside the read-only apimachinery and apiextensions module-cache packages the shared gen_openapi helper always passes as inputs, which fails on any consumer including CI that vendors deps from the cache. With the marker in place openapi-gen emits Type{}.OpenAPIModelName() for every type it references, so a future type without a method fails the build instead of silently reintroducing a Go-path name. The generated openapi is the output of the root `make generate`; nothing else in the generated tree moved. kubectl apply --validate has been broken against any cozystack-api built after 2026-07-21 on every resource, and this shipped in v1.6.0 and v1.6.1. In e2e it surfaces as the platform install hanging on cozy-backup-controller/backupstrategy-controller, which takes tenant-root and everything behind cozystack-basics with it. Refs: https://github.com/cozystack/cozystack/issues/3806 Assisted-By: Claude Signed-off-by: Myasnikov Daniil (cherry picked from commit 5f3ad9cd580c9c9de198dc1dc662d23493d3f474) --- pkg/apis/core/v1alpha1/doc.go | 1 + pkg/apis/core/v1alpha1/model_name.go | 100 +++++++ pkg/apis/sdn/v1alpha1/doc.go | 1 + pkg/apis/sdn/v1alpha1/model_name.go | 92 ++++++ pkg/generated/openapi/zz_generated.openapi.go | 272 +++++++++--------- 5 files changed, 331 insertions(+), 135 deletions(-) create mode 100644 pkg/apis/core/v1alpha1/model_name.go create mode 100644 pkg/apis/sdn/v1alpha1/model_name.go diff --git a/pkg/apis/core/v1alpha1/doc.go b/pkg/apis/core/v1alpha1/doc.go index 15b059bed7..c074ab0537 100644 --- a/pkg/apis/core/v1alpha1/doc.go +++ b/pkg/apis/core/v1alpha1/doc.go @@ -15,6 +15,7 @@ limitations under the License. */ // +k8s:openapi-gen=true +// +k8s:openapi-model-package=com.github.cozystack.cozystack.pkg.apis.core.v1alpha1 // +k8s:deepcopy-gen=package // +k8s:conversion-gen=github.com/cozystack/cozystack/pkg/apis/core // +k8s:conversion-gen=k8s.io/apiextensions-apiserver/pkg/apis/apiextensions diff --git a/pkg/apis/core/v1alpha1/model_name.go b/pkg/apis/core/v1alpha1/model_name.go new file mode 100644 index 0000000000..581dca1d2a --- /dev/null +++ b/pkg/apis/core/v1alpha1/model_name.go @@ -0,0 +1,100 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +// OpenAPIModelName pins the OpenAPI model name for the core.cozystack.io types. +// This is hand-written (not generated) on purpose. +// +// Without it, openapi-gen keys the generated definition map on the Go import +// path ("github.com/cozystack/cozystack/pkg/apis/core/v1alpha1.Option"), and +// since Kubernetes 0.35 the apiserver's DefinitionNamer.GetDefinitionName +// returns whatever name it is given verbatim instead of converting it to the +// "friendly" reversed-path form. The published document then keys the definition +// on the raw path while every $ref to it is JSON pointer escaped +// ("#/definitions/github.com~1cozystack~1…v1alpha1.OptionSpec", ~1 being a +// slash), and clients resolve a $ref by trimming "#/definitions/" without +// unescaping. The two spellings never meet, so the reference dangles and +// client-side validation fails on every resource in the group, not only the one +// named in the error: +// +// error validating data: SchemaError(…core/v1alpha1.Option.spec): +// unknown model in reference: "github.com~1cozystack~1…v1alpha1.OptionSpec" +// +// Declaring the dotted name here makes GetCanonicalTypeName, +// Scheme.ToOpenAPIDefinitionName and the generated openapi map key all agree on +// a name with no slash in it, so nothing needs escaping and every $ref resolves. +// The same agreement is what lets DefinitionNamer attach the +// x-kubernetes-group-version-kind extension, which server-side apply needs to +// resolve a kind (see the apps package, where its absence broke SSA instead). +// +// code-generator's openapi-gen could emit these via --output-model-name-file, +// but that also rewrites zz_generated.model_name.go in the read-only apimachinery +// / apiextensions module-cache packages the shared gen_openapi helper always +// passes as inputs, which fails on any consumer (including CI) that vendors deps +// from the module cache. Keeping the methods here sidesteps that. +// +// The +k8s:openapi-model-package marker in doc.go makes openapi-gen emit +// Type{}.OpenAPIModelName() for every type in this package it generates a schema +// or a reference for, so a new core.cozystack.io type without a method here +// fails the build rather than silently reintroducing a Go-path name. The +// returned string must be the dotted form Scheme.ToOpenAPIDefinitionName derives +// from the type's Go package path, which is the marker's value plus the type +// name. See https://github.com/cozystack/cozystack/issues/3806. + +func (in Option) OpenAPIModelName() string { + return "com.github.cozystack.cozystack.pkg.apis.core.v1alpha1.Option" +} + +func (in OptionItem) OpenAPIModelName() string { + return "com.github.cozystack.cozystack.pkg.apis.core.v1alpha1.OptionItem" +} + +func (in OptionList) OpenAPIModelName() string { + return "com.github.cozystack.cozystack.pkg.apis.core.v1alpha1.OptionList" +} + +func (in OptionSpec) OpenAPIModelName() string { + return "com.github.cozystack.cozystack.pkg.apis.core.v1alpha1.OptionSpec" +} + +func (in TenantModule) OpenAPIModelName() string { + return "com.github.cozystack.cozystack.pkg.apis.core.v1alpha1.TenantModule" +} + +func (in TenantModuleList) OpenAPIModelName() string { + return "com.github.cozystack.cozystack.pkg.apis.core.v1alpha1.TenantModuleList" +} + +func (in TenantModuleStatus) OpenAPIModelName() string { + return "com.github.cozystack.cozystack.pkg.apis.core.v1alpha1.TenantModuleStatus" +} + +func (in TenantNamespace) OpenAPIModelName() string { + return "com.github.cozystack.cozystack.pkg.apis.core.v1alpha1.TenantNamespace" +} + +func (in TenantNamespaceList) OpenAPIModelName() string { + return "com.github.cozystack.cozystack.pkg.apis.core.v1alpha1.TenantNamespaceList" +} + +func (in TenantSecret) OpenAPIModelName() string { + return "com.github.cozystack.cozystack.pkg.apis.core.v1alpha1.TenantSecret" +} + +func (in TenantSecretList) OpenAPIModelName() string { + return "com.github.cozystack.cozystack.pkg.apis.core.v1alpha1.TenantSecretList" +} diff --git a/pkg/apis/sdn/v1alpha1/doc.go b/pkg/apis/sdn/v1alpha1/doc.go index 53aa095976..d4f632e15e 100644 --- a/pkg/apis/sdn/v1alpha1/doc.go +++ b/pkg/apis/sdn/v1alpha1/doc.go @@ -15,6 +15,7 @@ limitations under the License. */ // +k8s:openapi-gen=true +// +k8s:openapi-model-package=com.github.cozystack.cozystack.pkg.apis.sdn.v1alpha1 // +k8s:deepcopy-gen=package // +k8s:conversion-gen=github.com/cozystack/cozystack/pkg/apis/sdn // +k8s:conversion-gen=k8s.io/apiextensions-apiserver/pkg/apis/apiextensions diff --git a/pkg/apis/sdn/v1alpha1/model_name.go b/pkg/apis/sdn/v1alpha1/model_name.go new file mode 100644 index 0000000000..8ed9eccc52 --- /dev/null +++ b/pkg/apis/sdn/v1alpha1/model_name.go @@ -0,0 +1,92 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +// OpenAPIModelName pins the OpenAPI model name for the sdn.cozystack.io types. +// This is hand-written (not generated) on purpose. +// +// Without it, openapi-gen keys the generated definition map on the Go import +// path ("github.com/cozystack/cozystack/pkg/apis/sdn/v1alpha1.SecurityGroup"), +// and since Kubernetes 0.35 the apiserver's DefinitionNamer.GetDefinitionName +// returns whatever name it is given verbatim instead of converting it to the +// "friendly" reversed-path form. The published document then keys the definition +// on the raw path while every $ref to it is JSON pointer escaped +// ("#/definitions/github.com~1cozystack~1…v1alpha1.SecurityGroupSpec", ~1 being +// a slash), and clients resolve a $ref by trimming "#/definitions/" without +// unescaping. The two spellings never meet, so the reference dangles and +// client-side validation fails on every resource in the group, not only the one +// named in the error: +// +// error validating data: SchemaError(…sdn/v1alpha1.SecurityGroup.spec): +// unknown model in reference: "github.com~1cozystack~1…v1alpha1.SecurityGroupSpec" +// +// Declaring the dotted name here makes GetCanonicalTypeName, +// Scheme.ToOpenAPIDefinitionName and the generated openapi map key all agree on +// a name with no slash in it, so nothing needs escaping and every $ref resolves. +// The same agreement is what lets DefinitionNamer attach the +// x-kubernetes-group-version-kind extension, which server-side apply needs to +// resolve a kind (see the apps package, where its absence broke SSA instead). +// +// code-generator's openapi-gen could emit these via --output-model-name-file, +// but that also rewrites zz_generated.model_name.go in the read-only apimachinery +// / apiextensions module-cache packages the shared gen_openapi helper always +// passes as inputs, which fails on any consumer (including CI) that vendors deps +// from the module cache. Keeping the methods here sidesteps that. +// +// The +k8s:openapi-model-package marker in doc.go makes openapi-gen emit +// Type{}.OpenAPIModelName() for every type in this package it generates a schema +// or a reference for, so a new sdn.cozystack.io type without a method here fails +// the build rather than silently reintroducing a Go-path name. The returned +// string must be the dotted form Scheme.ToOpenAPIDefinitionName derives from the +// type's Go package path, which is the marker's value plus the type name. See +// https://github.com/cozystack/cozystack/issues/3806. + +func (in ApplicationReference) OpenAPIModelName() string { + return "com.github.cozystack.cozystack.pkg.apis.sdn.v1alpha1.ApplicationReference" +} + +func (in EgressRule) OpenAPIModelName() string { + return "com.github.cozystack.cozystack.pkg.apis.sdn.v1alpha1.EgressRule" +} + +func (in FQDNSelector) OpenAPIModelName() string { + return "com.github.cozystack.cozystack.pkg.apis.sdn.v1alpha1.FQDNSelector" +} + +func (in IngressRule) OpenAPIModelName() string { + return "com.github.cozystack.cozystack.pkg.apis.sdn.v1alpha1.IngressRule" +} + +func (in PortProtocol) OpenAPIModelName() string { + return "com.github.cozystack.cozystack.pkg.apis.sdn.v1alpha1.PortProtocol" +} + +func (in PortRule) OpenAPIModelName() string { + return "com.github.cozystack.cozystack.pkg.apis.sdn.v1alpha1.PortRule" +} + +func (in SecurityGroup) OpenAPIModelName() string { + return "com.github.cozystack.cozystack.pkg.apis.sdn.v1alpha1.SecurityGroup" +} + +func (in SecurityGroupList) OpenAPIModelName() string { + return "com.github.cozystack.cozystack.pkg.apis.sdn.v1alpha1.SecurityGroupList" +} + +func (in SecurityGroupSpec) OpenAPIModelName() string { + return "com.github.cozystack.cozystack.pkg.apis.sdn.v1alpha1.SecurityGroupSpec" +} diff --git a/pkg/generated/openapi/zz_generated.openapi.go b/pkg/generated/openapi/zz_generated.openapi.go index f8d6548197..b50e629be5 100644 --- a/pkg/generated/openapi/zz_generated.openapi.go +++ b/pkg/generated/openapi/zz_generated.openapi.go @@ -23,6 +23,8 @@ package openapi import ( v1alpha1 "github.com/cozystack/cozystack/pkg/apis/apps/v1alpha1" + corev1alpha1 "github.com/cozystack/cozystack/pkg/apis/core/v1alpha1" + sdnv1alpha1 "github.com/cozystack/cozystack/pkg/apis/sdn/v1alpha1" v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" resource "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -34,110 +36,110 @@ import ( func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenAPIDefinition { return map[string]common.OpenAPIDefinition{ - v1alpha1.Application{}.OpenAPIModelName(): schema_pkg_apis_apps_v1alpha1_Application(ref), - v1alpha1.ApplicationList{}.OpenAPIModelName(): schema_pkg_apis_apps_v1alpha1_ApplicationList(ref), - v1alpha1.ApplicationStatus{}.OpenAPIModelName(): schema_pkg_apis_apps_v1alpha1_ApplicationStatus(ref), - "github.com/cozystack/cozystack/pkg/apis/core/v1alpha1.Option": schema_pkg_apis_core_v1alpha1_Option(ref), - "github.com/cozystack/cozystack/pkg/apis/core/v1alpha1.OptionItem": schema_pkg_apis_core_v1alpha1_OptionItem(ref), - "github.com/cozystack/cozystack/pkg/apis/core/v1alpha1.OptionList": schema_pkg_apis_core_v1alpha1_OptionList(ref), - "github.com/cozystack/cozystack/pkg/apis/core/v1alpha1.OptionSpec": schema_pkg_apis_core_v1alpha1_OptionSpec(ref), - "github.com/cozystack/cozystack/pkg/apis/core/v1alpha1.TenantModule": schema_pkg_apis_core_v1alpha1_TenantModule(ref), - "github.com/cozystack/cozystack/pkg/apis/core/v1alpha1.TenantModuleList": schema_pkg_apis_core_v1alpha1_TenantModuleList(ref), - "github.com/cozystack/cozystack/pkg/apis/core/v1alpha1.TenantModuleStatus": schema_pkg_apis_core_v1alpha1_TenantModuleStatus(ref), - "github.com/cozystack/cozystack/pkg/apis/core/v1alpha1.TenantNamespace": schema_pkg_apis_core_v1alpha1_TenantNamespace(ref), - "github.com/cozystack/cozystack/pkg/apis/core/v1alpha1.TenantNamespaceList": schema_pkg_apis_core_v1alpha1_TenantNamespaceList(ref), - "github.com/cozystack/cozystack/pkg/apis/core/v1alpha1.TenantSecret": schema_pkg_apis_core_v1alpha1_TenantSecret(ref), - "github.com/cozystack/cozystack/pkg/apis/core/v1alpha1.TenantSecretList": schema_pkg_apis_core_v1alpha1_TenantSecretList(ref), - "github.com/cozystack/cozystack/pkg/apis/sdn/v1alpha1.ApplicationReference": schema_pkg_apis_sdn_v1alpha1_ApplicationReference(ref), - "github.com/cozystack/cozystack/pkg/apis/sdn/v1alpha1.EgressRule": schema_pkg_apis_sdn_v1alpha1_EgressRule(ref), - "github.com/cozystack/cozystack/pkg/apis/sdn/v1alpha1.FQDNSelector": schema_pkg_apis_sdn_v1alpha1_FQDNSelector(ref), - "github.com/cozystack/cozystack/pkg/apis/sdn/v1alpha1.IngressRule": schema_pkg_apis_sdn_v1alpha1_IngressRule(ref), - "github.com/cozystack/cozystack/pkg/apis/sdn/v1alpha1.PortProtocol": schema_pkg_apis_sdn_v1alpha1_PortProtocol(ref), - "github.com/cozystack/cozystack/pkg/apis/sdn/v1alpha1.PortRule": schema_pkg_apis_sdn_v1alpha1_PortRule(ref), - "github.com/cozystack/cozystack/pkg/apis/sdn/v1alpha1.SecurityGroup": schema_pkg_apis_sdn_v1alpha1_SecurityGroup(ref), - "github.com/cozystack/cozystack/pkg/apis/sdn/v1alpha1.SecurityGroupList": schema_pkg_apis_sdn_v1alpha1_SecurityGroupList(ref), - "github.com/cozystack/cozystack/pkg/apis/sdn/v1alpha1.SecurityGroupSpec": schema_pkg_apis_sdn_v1alpha1_SecurityGroupSpec(ref), - v1.ConversionRequest{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_ConversionRequest(ref), - v1.ConversionResponse{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_ConversionResponse(ref), - v1.ConversionReview{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_ConversionReview(ref), - v1.CustomResourceColumnDefinition{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_CustomResourceColumnDefinition(ref), - v1.CustomResourceConversion{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_CustomResourceConversion(ref), - v1.CustomResourceDefinition{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_CustomResourceDefinition(ref), - v1.CustomResourceDefinitionCondition{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_CustomResourceDefinitionCondition(ref), - v1.CustomResourceDefinitionList{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_CustomResourceDefinitionList(ref), - v1.CustomResourceDefinitionNames{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_CustomResourceDefinitionNames(ref), - v1.CustomResourceDefinitionSpec{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_CustomResourceDefinitionSpec(ref), - v1.CustomResourceDefinitionStatus{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_CustomResourceDefinitionStatus(ref), - v1.CustomResourceDefinitionVersion{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_CustomResourceDefinitionVersion(ref), - v1.CustomResourceSubresourceScale{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_CustomResourceSubresourceScale(ref), - v1.CustomResourceSubresourceStatus{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_CustomResourceSubresourceStatus(ref), - v1.CustomResourceSubresources{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_CustomResourceSubresources(ref), - v1.CustomResourceValidation{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_CustomResourceValidation(ref), - v1.ExternalDocumentation{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_ExternalDocumentation(ref), - v1.JSON{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_JSON(ref), - v1.JSONSchemaProps{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_JSONSchemaProps(ref), - v1.JSONSchemaPropsOrArray{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_JSONSchemaPropsOrArray(ref), - v1.JSONSchemaPropsOrBool{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_JSONSchemaPropsOrBool(ref), - v1.JSONSchemaPropsOrStringArray{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_JSONSchemaPropsOrStringArray(ref), - v1.SelectableField{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_SelectableField(ref), - v1.ServiceReference{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_ServiceReference(ref), - v1.ValidationRule{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_ValidationRule(ref), - v1.WebhookClientConfig{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_WebhookClientConfig(ref), - v1.WebhookConversion{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_WebhookConversion(ref), - resource.Quantity{}.OpenAPIModelName(): schema_apimachinery_pkg_api_resource_Quantity(ref), - metav1.APIGroup{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_APIGroup(ref), - metav1.APIGroupList{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_APIGroupList(ref), - metav1.APIResource{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_APIResource(ref), - metav1.APIResourceList{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_APIResourceList(ref), - metav1.APIVersions{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_APIVersions(ref), - metav1.ApplyOptions{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_ApplyOptions(ref), - metav1.Condition{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_Condition(ref), - metav1.CreateOptions{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_CreateOptions(ref), - metav1.DeleteOptions{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_DeleteOptions(ref), - metav1.Duration{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_Duration(ref), - metav1.FieldSelectorRequirement{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_FieldSelectorRequirement(ref), - metav1.FieldsV1{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_FieldsV1(ref), - metav1.GetOptions{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_GetOptions(ref), - metav1.GroupKind{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_GroupKind(ref), - metav1.GroupResource{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_GroupResource(ref), - metav1.GroupVersion{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_GroupVersion(ref), - metav1.GroupVersionForDiscovery{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_GroupVersionForDiscovery(ref), - metav1.GroupVersionKind{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_GroupVersionKind(ref), - metav1.GroupVersionResource{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_GroupVersionResource(ref), - metav1.InternalEvent{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_InternalEvent(ref), - metav1.LabelSelector{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_LabelSelector(ref), - metav1.LabelSelectorRequirement{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_LabelSelectorRequirement(ref), - metav1.List{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_List(ref), - metav1.ListMeta{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_ListMeta(ref), - metav1.ListOptions{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_ListOptions(ref), - metav1.ManagedFieldsEntry{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_ManagedFieldsEntry(ref), - metav1.MicroTime{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_MicroTime(ref), - metav1.ObjectMeta{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_ObjectMeta(ref), - metav1.OwnerReference{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_OwnerReference(ref), - metav1.PartialObjectMetadata{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_PartialObjectMetadata(ref), - metav1.PartialObjectMetadataList{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_PartialObjectMetadataList(ref), - metav1.Patch{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_Patch(ref), - metav1.PatchOptions{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_PatchOptions(ref), - metav1.Preconditions{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_Preconditions(ref), - metav1.RootPaths{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_RootPaths(ref), - metav1.ServerAddressByClientCIDR{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_ServerAddressByClientCIDR(ref), - metav1.Status{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_Status(ref), - metav1.StatusCause{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_StatusCause(ref), - metav1.StatusDetails{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_StatusDetails(ref), - metav1.Table{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_Table(ref), - metav1.TableColumnDefinition{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_TableColumnDefinition(ref), - metav1.TableOptions{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_TableOptions(ref), - metav1.TableRow{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_TableRow(ref), - metav1.TableRowCondition{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_TableRowCondition(ref), - metav1.Time{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_Time(ref), - metav1.Timestamp{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_Timestamp(ref), - metav1.TypeMeta{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_TypeMeta(ref), - metav1.UpdateOptions{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_UpdateOptions(ref), - metav1.WatchEvent{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_WatchEvent(ref), - runtime.RawExtension{}.OpenAPIModelName(): schema_k8sio_apimachinery_pkg_runtime_RawExtension(ref), - runtime.TypeMeta{}.OpenAPIModelName(): schema_k8sio_apimachinery_pkg_runtime_TypeMeta(ref), - runtime.Unknown{}.OpenAPIModelName(): schema_k8sio_apimachinery_pkg_runtime_Unknown(ref), - version.Info{}.OpenAPIModelName(): schema_k8sio_apimachinery_pkg_version_Info(ref), + v1alpha1.Application{}.OpenAPIModelName(): schema_pkg_apis_apps_v1alpha1_Application(ref), + v1alpha1.ApplicationList{}.OpenAPIModelName(): schema_pkg_apis_apps_v1alpha1_ApplicationList(ref), + v1alpha1.ApplicationStatus{}.OpenAPIModelName(): schema_pkg_apis_apps_v1alpha1_ApplicationStatus(ref), + corev1alpha1.Option{}.OpenAPIModelName(): schema_pkg_apis_core_v1alpha1_Option(ref), + corev1alpha1.OptionItem{}.OpenAPIModelName(): schema_pkg_apis_core_v1alpha1_OptionItem(ref), + corev1alpha1.OptionList{}.OpenAPIModelName(): schema_pkg_apis_core_v1alpha1_OptionList(ref), + corev1alpha1.OptionSpec{}.OpenAPIModelName(): schema_pkg_apis_core_v1alpha1_OptionSpec(ref), + corev1alpha1.TenantModule{}.OpenAPIModelName(): schema_pkg_apis_core_v1alpha1_TenantModule(ref), + corev1alpha1.TenantModuleList{}.OpenAPIModelName(): schema_pkg_apis_core_v1alpha1_TenantModuleList(ref), + corev1alpha1.TenantModuleStatus{}.OpenAPIModelName(): schema_pkg_apis_core_v1alpha1_TenantModuleStatus(ref), + corev1alpha1.TenantNamespace{}.OpenAPIModelName(): schema_pkg_apis_core_v1alpha1_TenantNamespace(ref), + corev1alpha1.TenantNamespaceList{}.OpenAPIModelName(): schema_pkg_apis_core_v1alpha1_TenantNamespaceList(ref), + corev1alpha1.TenantSecret{}.OpenAPIModelName(): schema_pkg_apis_core_v1alpha1_TenantSecret(ref), + corev1alpha1.TenantSecretList{}.OpenAPIModelName(): schema_pkg_apis_core_v1alpha1_TenantSecretList(ref), + sdnv1alpha1.ApplicationReference{}.OpenAPIModelName(): schema_pkg_apis_sdn_v1alpha1_ApplicationReference(ref), + sdnv1alpha1.EgressRule{}.OpenAPIModelName(): schema_pkg_apis_sdn_v1alpha1_EgressRule(ref), + sdnv1alpha1.FQDNSelector{}.OpenAPIModelName(): schema_pkg_apis_sdn_v1alpha1_FQDNSelector(ref), + sdnv1alpha1.IngressRule{}.OpenAPIModelName(): schema_pkg_apis_sdn_v1alpha1_IngressRule(ref), + sdnv1alpha1.PortProtocol{}.OpenAPIModelName(): schema_pkg_apis_sdn_v1alpha1_PortProtocol(ref), + sdnv1alpha1.PortRule{}.OpenAPIModelName(): schema_pkg_apis_sdn_v1alpha1_PortRule(ref), + sdnv1alpha1.SecurityGroup{}.OpenAPIModelName(): schema_pkg_apis_sdn_v1alpha1_SecurityGroup(ref), + sdnv1alpha1.SecurityGroupList{}.OpenAPIModelName(): schema_pkg_apis_sdn_v1alpha1_SecurityGroupList(ref), + sdnv1alpha1.SecurityGroupSpec{}.OpenAPIModelName(): schema_pkg_apis_sdn_v1alpha1_SecurityGroupSpec(ref), + v1.ConversionRequest{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_ConversionRequest(ref), + v1.ConversionResponse{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_ConversionResponse(ref), + v1.ConversionReview{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_ConversionReview(ref), + v1.CustomResourceColumnDefinition{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_CustomResourceColumnDefinition(ref), + v1.CustomResourceConversion{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_CustomResourceConversion(ref), + v1.CustomResourceDefinition{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_CustomResourceDefinition(ref), + v1.CustomResourceDefinitionCondition{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_CustomResourceDefinitionCondition(ref), + v1.CustomResourceDefinitionList{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_CustomResourceDefinitionList(ref), + v1.CustomResourceDefinitionNames{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_CustomResourceDefinitionNames(ref), + v1.CustomResourceDefinitionSpec{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_CustomResourceDefinitionSpec(ref), + v1.CustomResourceDefinitionStatus{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_CustomResourceDefinitionStatus(ref), + v1.CustomResourceDefinitionVersion{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_CustomResourceDefinitionVersion(ref), + v1.CustomResourceSubresourceScale{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_CustomResourceSubresourceScale(ref), + v1.CustomResourceSubresourceStatus{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_CustomResourceSubresourceStatus(ref), + v1.CustomResourceSubresources{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_CustomResourceSubresources(ref), + v1.CustomResourceValidation{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_CustomResourceValidation(ref), + v1.ExternalDocumentation{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_ExternalDocumentation(ref), + v1.JSON{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_JSON(ref), + v1.JSONSchemaProps{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_JSONSchemaProps(ref), + v1.JSONSchemaPropsOrArray{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_JSONSchemaPropsOrArray(ref), + v1.JSONSchemaPropsOrBool{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_JSONSchemaPropsOrBool(ref), + v1.JSONSchemaPropsOrStringArray{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_JSONSchemaPropsOrStringArray(ref), + v1.SelectableField{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_SelectableField(ref), + v1.ServiceReference{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_ServiceReference(ref), + v1.ValidationRule{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_ValidationRule(ref), + v1.WebhookClientConfig{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_WebhookClientConfig(ref), + v1.WebhookConversion{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_WebhookConversion(ref), + resource.Quantity{}.OpenAPIModelName(): schema_apimachinery_pkg_api_resource_Quantity(ref), + metav1.APIGroup{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_APIGroup(ref), + metav1.APIGroupList{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_APIGroupList(ref), + metav1.APIResource{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_APIResource(ref), + metav1.APIResourceList{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_APIResourceList(ref), + metav1.APIVersions{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_APIVersions(ref), + metav1.ApplyOptions{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_ApplyOptions(ref), + metav1.Condition{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_Condition(ref), + metav1.CreateOptions{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_CreateOptions(ref), + metav1.DeleteOptions{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_DeleteOptions(ref), + metav1.Duration{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_Duration(ref), + metav1.FieldSelectorRequirement{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_FieldSelectorRequirement(ref), + metav1.FieldsV1{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_FieldsV1(ref), + metav1.GetOptions{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_GetOptions(ref), + metav1.GroupKind{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_GroupKind(ref), + metav1.GroupResource{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_GroupResource(ref), + metav1.GroupVersion{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_GroupVersion(ref), + metav1.GroupVersionForDiscovery{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_GroupVersionForDiscovery(ref), + metav1.GroupVersionKind{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_GroupVersionKind(ref), + metav1.GroupVersionResource{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_GroupVersionResource(ref), + metav1.InternalEvent{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_InternalEvent(ref), + metav1.LabelSelector{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_LabelSelector(ref), + metav1.LabelSelectorRequirement{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_LabelSelectorRequirement(ref), + metav1.List{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_List(ref), + metav1.ListMeta{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_ListMeta(ref), + metav1.ListOptions{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_ListOptions(ref), + metav1.ManagedFieldsEntry{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_ManagedFieldsEntry(ref), + metav1.MicroTime{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_MicroTime(ref), + metav1.ObjectMeta{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_ObjectMeta(ref), + metav1.OwnerReference{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_OwnerReference(ref), + metav1.PartialObjectMetadata{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_PartialObjectMetadata(ref), + metav1.PartialObjectMetadataList{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_PartialObjectMetadataList(ref), + metav1.Patch{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_Patch(ref), + metav1.PatchOptions{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_PatchOptions(ref), + metav1.Preconditions{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_Preconditions(ref), + metav1.RootPaths{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_RootPaths(ref), + metav1.ServerAddressByClientCIDR{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_ServerAddressByClientCIDR(ref), + metav1.Status{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_Status(ref), + metav1.StatusCause{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_StatusCause(ref), + metav1.StatusDetails{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_StatusDetails(ref), + metav1.Table{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_Table(ref), + metav1.TableColumnDefinition{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_TableColumnDefinition(ref), + metav1.TableOptions{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_TableOptions(ref), + metav1.TableRow{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_TableRow(ref), + metav1.TableRowCondition{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_TableRowCondition(ref), + metav1.Time{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_Time(ref), + metav1.Timestamp{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_Timestamp(ref), + metav1.TypeMeta{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_TypeMeta(ref), + metav1.UpdateOptions{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_UpdateOptions(ref), + metav1.WatchEvent{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_WatchEvent(ref), + runtime.RawExtension{}.OpenAPIModelName(): schema_k8sio_apimachinery_pkg_runtime_RawExtension(ref), + runtime.TypeMeta{}.OpenAPIModelName(): schema_k8sio_apimachinery_pkg_runtime_TypeMeta(ref), + runtime.Unknown{}.OpenAPIModelName(): schema_k8sio_apimachinery_pkg_runtime_Unknown(ref), + version.Info{}.OpenAPIModelName(): schema_k8sio_apimachinery_pkg_version_Info(ref), } } @@ -321,14 +323,14 @@ func schema_pkg_apis_core_v1alpha1_Option(ref common.ReferenceCallback) common.O "spec": { SchemaProps: spec.SchemaProps{ Default: map[string]interface{}{}, - Ref: ref("github.com/cozystack/cozystack/pkg/apis/core/v1alpha1.OptionSpec"), + Ref: ref(corev1alpha1.OptionSpec{}.OpenAPIModelName()), }, }, }, }, }, Dependencies: []string{ - "github.com/cozystack/cozystack/pkg/apis/core/v1alpha1.OptionSpec", metav1.ObjectMeta{}.OpenAPIModelName()}, + corev1alpha1.OptionSpec{}.OpenAPIModelName(), metav1.ObjectMeta{}.OpenAPIModelName()}, } } @@ -408,7 +410,7 @@ func schema_pkg_apis_core_v1alpha1_OptionList(ref common.ReferenceCallback) comm Schema: &spec.Schema{ SchemaProps: spec.SchemaProps{ Default: map[string]interface{}{}, - Ref: ref("github.com/cozystack/cozystack/pkg/apis/core/v1alpha1.Option"), + Ref: ref(corev1alpha1.Option{}.OpenAPIModelName()), }, }, }, @@ -419,7 +421,7 @@ func schema_pkg_apis_core_v1alpha1_OptionList(ref common.ReferenceCallback) comm }, }, Dependencies: []string{ - "github.com/cozystack/cozystack/pkg/apis/core/v1alpha1.Option", metav1.ListMeta{}.OpenAPIModelName()}, + corev1alpha1.Option{}.OpenAPIModelName(), metav1.ListMeta{}.OpenAPIModelName()}, } } @@ -437,7 +439,7 @@ func schema_pkg_apis_core_v1alpha1_OptionSpec(ref common.ReferenceCallback) comm Schema: &spec.Schema{ SchemaProps: spec.SchemaProps{ Default: map[string]interface{}{}, - Ref: ref("github.com/cozystack/cozystack/pkg/apis/core/v1alpha1.OptionItem"), + Ref: ref(corev1alpha1.OptionItem{}.OpenAPIModelName()), }, }, }, @@ -447,7 +449,7 @@ func schema_pkg_apis_core_v1alpha1_OptionSpec(ref common.ReferenceCallback) comm }, }, Dependencies: []string{ - "github.com/cozystack/cozystack/pkg/apis/core/v1alpha1.OptionItem"}, + corev1alpha1.OptionItem{}.OpenAPIModelName()}, } } @@ -489,14 +491,14 @@ func schema_pkg_apis_core_v1alpha1_TenantModule(ref common.ReferenceCallback) co SchemaProps: spec.SchemaProps{ Description: "Status contains the module status", Default: map[string]interface{}{}, - Ref: ref("github.com/cozystack/cozystack/pkg/apis/core/v1alpha1.TenantModuleStatus"), + Ref: ref(corev1alpha1.TenantModuleStatus{}.OpenAPIModelName()), }, }, }, }, }, Dependencies: []string{ - "github.com/cozystack/cozystack/pkg/apis/core/v1alpha1.TenantModuleStatus", metav1.ObjectMeta{}.OpenAPIModelName()}, + corev1alpha1.TenantModuleStatus{}.OpenAPIModelName(), metav1.ObjectMeta{}.OpenAPIModelName()}, } } @@ -534,7 +536,7 @@ func schema_pkg_apis_core_v1alpha1_TenantModuleList(ref common.ReferenceCallback Schema: &spec.Schema{ SchemaProps: spec.SchemaProps{ Default: map[string]interface{}{}, - Ref: ref("github.com/cozystack/cozystack/pkg/apis/core/v1alpha1.TenantModule"), + Ref: ref(corev1alpha1.TenantModule{}.OpenAPIModelName()), }, }, }, @@ -545,7 +547,7 @@ func schema_pkg_apis_core_v1alpha1_TenantModuleList(ref common.ReferenceCallback }, }, Dependencies: []string{ - "github.com/cozystack/cozystack/pkg/apis/core/v1alpha1.TenantModule", metav1.ListMeta{}.OpenAPIModelName()}, + corev1alpha1.TenantModule{}.OpenAPIModelName(), metav1.ListMeta{}.OpenAPIModelName()}, } } @@ -654,7 +656,7 @@ func schema_pkg_apis_core_v1alpha1_TenantNamespaceList(ref common.ReferenceCallb Schema: &spec.Schema{ SchemaProps: spec.SchemaProps{ Default: map[string]interface{}{}, - Ref: ref("github.com/cozystack/cozystack/pkg/apis/core/v1alpha1.TenantNamespace"), + Ref: ref(corev1alpha1.TenantNamespace{}.OpenAPIModelName()), }, }, }, @@ -665,7 +667,7 @@ func schema_pkg_apis_core_v1alpha1_TenantNamespaceList(ref common.ReferenceCallb }, }, Dependencies: []string{ - "github.com/cozystack/cozystack/pkg/apis/core/v1alpha1.TenantNamespace", metav1.ListMeta{}.OpenAPIModelName()}, + corev1alpha1.TenantNamespace{}.OpenAPIModelName(), metav1.ListMeta{}.OpenAPIModelName()}, } } @@ -772,7 +774,7 @@ func schema_pkg_apis_core_v1alpha1_TenantSecretList(ref common.ReferenceCallback Schema: &spec.Schema{ SchemaProps: spec.SchemaProps{ Default: map[string]interface{}{}, - Ref: ref("github.com/cozystack/cozystack/pkg/apis/core/v1alpha1.TenantSecret"), + Ref: ref(corev1alpha1.TenantSecret{}.OpenAPIModelName()), }, }, }, @@ -783,7 +785,7 @@ func schema_pkg_apis_core_v1alpha1_TenantSecretList(ref common.ReferenceCallback }, }, Dependencies: []string{ - "github.com/cozystack/cozystack/pkg/apis/core/v1alpha1.TenantSecret", metav1.ListMeta{}.OpenAPIModelName()}, + corev1alpha1.TenantSecret{}.OpenAPIModelName(), metav1.ListMeta{}.OpenAPIModelName()}, } } @@ -839,7 +841,7 @@ func schema_pkg_apis_sdn_v1alpha1_EgressRule(ref common.ReferenceCallback) commo Schema: &spec.Schema{ SchemaProps: spec.SchemaProps{ Default: map[string]interface{}{}, - Ref: ref("github.com/cozystack/cozystack/pkg/apis/sdn/v1alpha1.ApplicationReference"), + Ref: ref(sdnv1alpha1.ApplicationReference{}.OpenAPIModelName()), }, }, }, @@ -883,7 +885,7 @@ func schema_pkg_apis_sdn_v1alpha1_EgressRule(ref common.ReferenceCallback) commo Schema: &spec.Schema{ SchemaProps: spec.SchemaProps{ Default: map[string]interface{}{}, - Ref: ref("github.com/cozystack/cozystack/pkg/apis/sdn/v1alpha1.FQDNSelector"), + Ref: ref(sdnv1alpha1.FQDNSelector{}.OpenAPIModelName()), }, }, }, @@ -897,7 +899,7 @@ func schema_pkg_apis_sdn_v1alpha1_EgressRule(ref common.ReferenceCallback) commo Schema: &spec.Schema{ SchemaProps: spec.SchemaProps{ Default: map[string]interface{}{}, - Ref: ref("github.com/cozystack/cozystack/pkg/apis/sdn/v1alpha1.PortRule"), + Ref: ref(sdnv1alpha1.PortRule{}.OpenAPIModelName()), }, }, }, @@ -907,7 +909,7 @@ func schema_pkg_apis_sdn_v1alpha1_EgressRule(ref common.ReferenceCallback) commo }, }, Dependencies: []string{ - "github.com/cozystack/cozystack/pkg/apis/sdn/v1alpha1.ApplicationReference", "github.com/cozystack/cozystack/pkg/apis/sdn/v1alpha1.FQDNSelector", "github.com/cozystack/cozystack/pkg/apis/sdn/v1alpha1.PortRule"}, + sdnv1alpha1.ApplicationReference{}.OpenAPIModelName(), sdnv1alpha1.FQDNSelector{}.OpenAPIModelName(), sdnv1alpha1.PortRule{}.OpenAPIModelName()}, } } @@ -953,7 +955,7 @@ func schema_pkg_apis_sdn_v1alpha1_IngressRule(ref common.ReferenceCallback) comm Schema: &spec.Schema{ SchemaProps: spec.SchemaProps{ Default: map[string]interface{}{}, - Ref: ref("github.com/cozystack/cozystack/pkg/apis/sdn/v1alpha1.ApplicationReference"), + Ref: ref(sdnv1alpha1.ApplicationReference{}.OpenAPIModelName()), }, }, }, @@ -997,7 +999,7 @@ func schema_pkg_apis_sdn_v1alpha1_IngressRule(ref common.ReferenceCallback) comm Schema: &spec.Schema{ SchemaProps: spec.SchemaProps{ Default: map[string]interface{}{}, - Ref: ref("github.com/cozystack/cozystack/pkg/apis/sdn/v1alpha1.PortRule"), + Ref: ref(sdnv1alpha1.PortRule{}.OpenAPIModelName()), }, }, }, @@ -1007,7 +1009,7 @@ func schema_pkg_apis_sdn_v1alpha1_IngressRule(ref common.ReferenceCallback) comm }, }, Dependencies: []string{ - "github.com/cozystack/cozystack/pkg/apis/sdn/v1alpha1.ApplicationReference", "github.com/cozystack/cozystack/pkg/apis/sdn/v1alpha1.PortRule"}, + sdnv1alpha1.ApplicationReference{}.OpenAPIModelName(), sdnv1alpha1.PortRule{}.OpenAPIModelName()}, } } @@ -1053,7 +1055,7 @@ func schema_pkg_apis_sdn_v1alpha1_PortRule(ref common.ReferenceCallback) common. Schema: &spec.Schema{ SchemaProps: spec.SchemaProps{ Default: map[string]interface{}{}, - Ref: ref("github.com/cozystack/cozystack/pkg/apis/sdn/v1alpha1.PortProtocol"), + Ref: ref(sdnv1alpha1.PortProtocol{}.OpenAPIModelName()), }, }, }, @@ -1063,7 +1065,7 @@ func schema_pkg_apis_sdn_v1alpha1_PortRule(ref common.ReferenceCallback) common. }, }, Dependencies: []string{ - "github.com/cozystack/cozystack/pkg/apis/sdn/v1alpha1.PortProtocol"}, + sdnv1alpha1.PortProtocol{}.OpenAPIModelName()}, } } @@ -1098,14 +1100,14 @@ func schema_pkg_apis_sdn_v1alpha1_SecurityGroup(ref common.ReferenceCallback) co SchemaProps: spec.SchemaProps{ Description: "Spec describes the applications this SecurityGroup attaches to and the traffic it allows.", Default: map[string]interface{}{}, - Ref: ref("github.com/cozystack/cozystack/pkg/apis/sdn/v1alpha1.SecurityGroupSpec"), + Ref: ref(sdnv1alpha1.SecurityGroupSpec{}.OpenAPIModelName()), }, }, }, }, }, Dependencies: []string{ - "github.com/cozystack/cozystack/pkg/apis/sdn/v1alpha1.SecurityGroupSpec", metav1.ObjectMeta{}.OpenAPIModelName()}, + sdnv1alpha1.SecurityGroupSpec{}.OpenAPIModelName(), metav1.ObjectMeta{}.OpenAPIModelName()}, } } @@ -1143,7 +1145,7 @@ func schema_pkg_apis_sdn_v1alpha1_SecurityGroupList(ref common.ReferenceCallback Schema: &spec.Schema{ SchemaProps: spec.SchemaProps{ Default: map[string]interface{}{}, - Ref: ref("github.com/cozystack/cozystack/pkg/apis/sdn/v1alpha1.SecurityGroup"), + Ref: ref(sdnv1alpha1.SecurityGroup{}.OpenAPIModelName()), }, }, }, @@ -1154,7 +1156,7 @@ func schema_pkg_apis_sdn_v1alpha1_SecurityGroupList(ref common.ReferenceCallback }, }, Dependencies: []string{ - "github.com/cozystack/cozystack/pkg/apis/sdn/v1alpha1.SecurityGroup", metav1.ListMeta{}.OpenAPIModelName()}, + sdnv1alpha1.SecurityGroup{}.OpenAPIModelName(), metav1.ListMeta{}.OpenAPIModelName()}, } } @@ -1173,7 +1175,7 @@ func schema_pkg_apis_sdn_v1alpha1_SecurityGroupSpec(ref common.ReferenceCallback Schema: &spec.Schema{ SchemaProps: spec.SchemaProps{ Default: map[string]interface{}{}, - Ref: ref("github.com/cozystack/cozystack/pkg/apis/sdn/v1alpha1.ApplicationReference"), + Ref: ref(sdnv1alpha1.ApplicationReference{}.OpenAPIModelName()), }, }, }, @@ -1187,7 +1189,7 @@ func schema_pkg_apis_sdn_v1alpha1_SecurityGroupSpec(ref common.ReferenceCallback Schema: &spec.Schema{ SchemaProps: spec.SchemaProps{ Default: map[string]interface{}{}, - Ref: ref("github.com/cozystack/cozystack/pkg/apis/sdn/v1alpha1.IngressRule"), + Ref: ref(sdnv1alpha1.IngressRule{}.OpenAPIModelName()), }, }, }, @@ -1201,7 +1203,7 @@ func schema_pkg_apis_sdn_v1alpha1_SecurityGroupSpec(ref common.ReferenceCallback Schema: &spec.Schema{ SchemaProps: spec.SchemaProps{ Default: map[string]interface{}{}, - Ref: ref("github.com/cozystack/cozystack/pkg/apis/sdn/v1alpha1.EgressRule"), + Ref: ref(sdnv1alpha1.EgressRule{}.OpenAPIModelName()), }, }, }, @@ -1211,7 +1213,7 @@ func schema_pkg_apis_sdn_v1alpha1_SecurityGroupSpec(ref common.ReferenceCallback }, }, Dependencies: []string{ - "github.com/cozystack/cozystack/pkg/apis/sdn/v1alpha1.ApplicationReference", "github.com/cozystack/cozystack/pkg/apis/sdn/v1alpha1.EgressRule", "github.com/cozystack/cozystack/pkg/apis/sdn/v1alpha1.IngressRule"}, + sdnv1alpha1.ApplicationReference{}.OpenAPIModelName(), sdnv1alpha1.EgressRule{}.OpenAPIModelName(), sdnv1alpha1.IngressRule{}.OpenAPIModelName()}, } } From 0e4b096a6f67d41ef0645c032907b0c02872d576 Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Fri, 14 Aug 2026 13:12:39 +0500 Subject: [PATCH 36/51] test(api): guard published openapi definition names and refs Nothing asserted that a published definition name contains no slash, which is why the core and sdn groups shipped as Go import paths in v1.6.0 and v1.6.1 without anyone noticing. The same omission on a future group would be just as invisible. TestDefinitionNamesAreDottedModelNames fails on any definition name containing "/", names the offender and points at the fix (the +k8s:openapi-model-package marker plus an OpenAPIModelName method). TestDefinitionRefsResolve is the failure mode itself: it builds each $ref the way kube-openapi's builder does, JSON pointer escaped, then resolves it the way a client does, trimming "#/definitions/" with no unescaping, and reports any that dangles. It also checks the declared Dependencies against the published names, which catches a group whose types disagree with each other rather than uniformly. Both assert the invariant instead of listing today's 20 names, so a new group that omits the marker is caught rather than a changed count, and both guard against going vacuous if the definition map is ever emptied or stops covering cozystack's own types. Both fail on the pre-fix tree: the first names all 20 offending definitions, and the second reproduces the reported error verbatim, down to "github.com~1cozystack~1cozystack~1pkg~1apis~1core~1v1alpha1.OptionSpec". Refs: https://github.com/cozystack/cozystack/issues/3806 Assisted-By: Claude Signed-off-by: Myasnikov Daniil (cherry picked from commit cea1703a6d7b68e190addfd7a28488bafcf0de65) --- pkg/generated/openapi/definitions_test.go | 157 ++++++++++++++++++++++ 1 file changed, 157 insertions(+) create mode 100644 pkg/generated/openapi/definitions_test.go diff --git a/pkg/generated/openapi/definitions_test.go b/pkg/generated/openapi/definitions_test.go new file mode 100644 index 0000000000..9440a7283a --- /dev/null +++ b/pkg/generated/openapi/definitions_test.go @@ -0,0 +1,157 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package openapi + +import ( + "sort" + "strings" + "testing" + + "k8s.io/kube-openapi/pkg/common" + "k8s.io/kube-openapi/pkg/validation/spec" +) + +// The two tests below guard the published OpenAPI document against the class of +// defect in https://github.com/cozystack/cozystack/issues/3806: a definition +// name that is a Go import path rather than a dotted Kubernetes model name. +// +// Since Kubernetes 0.35 the apiserver's DefinitionNamer.GetDefinitionName +// returns the model name it was given verbatim — it no longer converts the Go +// import-path form into the "friendly" reversed-path form — so whatever name +// this generated map uses becomes both the published definition key and, JSON +// pointer escaped, the $ref that points at it. A name containing "/" therefore +// ships as a definition keyed on the raw path while every reference to it is +// spelled with "~1" in place of each slash, and clients resolve a $ref by +// trimming "#/definitions/" without unescaping (kube-openapi +// pkg/util/proto/document.go). The two spellings never meet, so the reference +// dangles and client-side validation fails on every resource of the group: +// +// error validating data: SchemaError(…core/v1alpha1.Option.spec): +// unknown model in reference: "github.com~1cozystack~1…v1alpha1.OptionSpec" +// +// The fix for a group is the +k8s:openapi-model-package marker in its doc.go +// plus an OpenAPIModelName method per type (see each package's model_name.go). +// Nothing asserted that, which is why the core and sdn groups shipped broken in +// v1.6.0 and v1.6.1. These tests assert the invariant rather than today's set of +// names, so a future group that omits the marker fails here instead of in a +// cluster. + +// buildDefinitions builds the generated definition map, recording every model +// name the generated code passes to the reference callback. Refs are built +// exactly as kube-openapi's builder builds them, so the recorded name and the +// emitted $ref differ in the same way they do in the served document. +func buildDefinitions() (defs map[string]common.OpenAPIDefinition, refNames []string) { + defs = GetOpenAPIDefinitions(func(path string) spec.Ref { + refNames = append(refNames, path) + return spec.MustCreateRef("#/definitions/" + common.EscapeJsonPointer(path)) + }) + return defs, refNames +} + +// sortedKeys returns the definition names in a stable order so failures are +// reproducible rather than map-iteration ordered. +func sortedKeys(defs map[string]common.OpenAPIDefinition) []string { + out := make([]string, 0, len(defs)) + for name := range defs { + out = append(out, name) + } + sort.Strings(out) + return out +} + +// TestDefinitionNamesAreDottedModelNames asserts that no published definition +// name contains a slash. A slash-bearing name is a Go import path that escaped +// as-is, which means the group it belongs to is missing its +// +k8s:openapi-model-package marker and its OpenAPIModelName methods. +func TestDefinitionNamesAreDottedModelNames(t *testing.T) { + defs, _ := buildDefinitions() + + // Guard against the assertion going vacuous: the map must be populated and + // must actually cover cozystack's own types, not only the vendored + // apimachinery and apiextensions models that already declare their names. + if len(defs) == 0 { + t.Fatal("GetOpenAPIDefinitions returned no definitions; this test is asserting nothing") + } + own := 0 + for _, name := range sortedKeys(defs) { + if strings.Contains(name, "cozystack") { + own++ + } + } + if own == 0 { + t.Fatal("no cozystack-owned definitions found; this test is asserting nothing") + } + + for _, name := range sortedKeys(defs) { + if strings.Contains(name, "/") { + t.Errorf("definition %q is a Go import path, not a dotted model name: "+ + "every $ref to it is escaped with ~1 and will not resolve. Add "+ + "+k8s:openapi-model-package= to that package's doc.go and an "+ + "OpenAPIModelName method for the type in its model_name.go, then re-run make generate", + name) + } + } +} + +// TestDefinitionRefsResolve asserts that every $ref the generated document +// emits resolves to a published definition, resolving it the way a client does: +// trim "#/definitions/" and look the remainder up verbatim, with no +// unescaping. This is the failure the user sees, so it is checked directly +// rather than only through the slash-freedom proxy above; it also catches a +// group whose types disagree with each other (some named, some not) and a ref +// to a model that is not published at all. +func TestDefinitionRefsResolve(t *testing.T) { + defs, refNames := buildDefinitions() + + if len(refNames) == 0 { + t.Fatal("generated definitions emitted no $ref; this test is asserting nothing") + } + + // Index the published definitions under the key a client sees, which is the + // definition name exactly as published. + published := make(map[string]struct{}, len(defs)) + for name := range defs { + published[name] = struct{}{} + } + + seen := make(map[string]struct{}, len(refNames)) + for _, modelName := range refNames { + // The served $ref, built the way kube-openapi's builder builds it. + ref := "#/definitions/" + common.EscapeJsonPointer(modelName) + // The lookup a client performs on it. + resolved := strings.TrimPrefix(ref, "#/definitions/") + if _, ok := published[resolved]; ok { + continue + } + if _, dup := seen[resolved]; dup { + continue + } + seen[resolved] = struct{}{} + t.Errorf("$ref %q does not resolve: no definition is published under %q (model name %q)", + ref, resolved, modelName) + } + + // Dependencies must agree with the refs: the generated code lists them from + // the same set, and consumers walk them to build the transitive closure. + for _, name := range sortedKeys(defs) { + for _, dep := range defs[name].Dependencies { + if _, ok := published[dep]; !ok { + t.Errorf("definition %q declares dependency %q, which is not published", name, dep) + } + } + } +} From f47685b48768d0c9f691549b408989413b2f192e Mon Sep 17 00:00:00 2001 From: IvanHunters Date: Tue, 4 Aug 2026 23:07:10 +0300 Subject: [PATCH 37/51] fix(flux-shard-operator): strip proxy env from cloned helm-controller A sharded helm-controller inherits HTTP_PROXY/HTTPS_PROXY/NO_PROXY from the flux-aio helm-controller container. A standalone shard needs no external egress (source-controller fetches artifacts), so behind an unreachable corporate proxy it makes a blocking startup HTTPS call that never completes, the manager never serves /healthz, and the liveness probe crashloops the pod forever, freezing every HelmRelease carrying a sharding key. flux-aio itself survives only because it starts once and never restarts. Drop the proxy env (and the now-pointless NO_PROXY) when cloning, alongside the existing localhost/KubePrism sanitisation, so the shard talks to the in-cluster apiserver directly. Signed-off-by: IvanHunters (cherry picked from commit c475faae0fbe1b72961f8aa62149531024e0fe10) --- internal/fluxshardoperator/provisioner.go | 11 +++++++++++ internal/fluxshardoperator/provisioner_test.go | 14 ++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/internal/fluxshardoperator/provisioner.go b/internal/fluxshardoperator/provisioner.go index a0acb9400c..fdbe24e864 100644 --- a/internal/fluxshardoperator/provisioner.go +++ b/internal/fluxshardoperator/provisioner.go @@ -316,6 +316,17 @@ func BuildShardDeployment(flux *appsv1.Deployment, idx int, cfg *Config) (*appsv // is not hostNetwork, so it must fall back to the in-cluster defaults. case "KUBERNETES_SERVICE_HOST", "KUBERNETES_SERVICE_PORT": continue + // Corporate-proxy env inherited from flux-aio. A standalone shard needs + // no external egress (source-controller fetches artifacts), so the proxy + // only harms it: a startup HTTPS call that stalls through an unreachable + // proxy blocks the manager before it serves /healthz, and the liveness + // probe then crashloops the pod indefinitely. flux-aio survives only + // because it starts once and never restarts. Drop the proxy env (and the + // now-pointless NO_PROXY) so the shard talks to the in-cluster apiserver + // directly. + case "HTTP_PROXY", "HTTPS_PROXY", "NO_PROXY", + "http_proxy", "https_proxy", "no_proxy": + continue } env = append(env, e) } diff --git a/internal/fluxshardoperator/provisioner_test.go b/internal/fluxshardoperator/provisioner_test.go index 678f93833c..96f742a08e 100644 --- a/internal/fluxshardoperator/provisioner_test.go +++ b/internal/fluxshardoperator/provisioner_test.go @@ -92,6 +92,15 @@ func fluxAIODeployment() *appsv1.Deployment { FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.namespace"}, }}, {Name: "TUF_ROOT", Value: "/tmp/.sigstore"}, + // Corporate-proxy env the installer puts on + // flux-aio; harmless there but hangs a + // standalone shard at startup. + {Name: "HTTP_PROXY", Value: "http://proxy.example:3128"}, + {Name: "HTTPS_PROXY", Value: "http://proxy.example:3128"}, + {Name: "NO_PROXY", Value: ".svc"}, + {Name: "http_proxy", Value: "http://proxy.example:3128"}, + {Name: "https_proxy", Value: "http://proxy.example:3128"}, + {Name: "no_proxy", Value: ".svc"}, }, VolumeMounts: []corev1.VolumeMount{{Name: "tmp", MountPath: "/tmp"}}, }, @@ -188,6 +197,11 @@ func TestBuildShardDeployment(t *testing.T) { // A non-hostNetwork pod dialing the node-local KubePrism endpoint // crashloops on "dial tcp [::1]:7445: connect: connection refused". t.Fatalf("node-local apiserver endpoint env leaked through: %s", e.Name) + case "HTTP_PROXY", "HTTPS_PROXY", "NO_PROXY", + "http_proxy", "https_proxy", "no_proxy": + // A standalone shard behind an unreachable proxy hangs at startup + // and crashloops before it ever serves /healthz. + t.Fatalf("corporate-proxy env leaked through: %s", e.Name) } } envNames := make([]string, 0, len(hc.Env)) From 86c3af61fa09d5853ec02b9c1453ef1fb2b7df3b Mon Sep 17 00:00:00 2001 From: IvanHunters Date: Tue, 4 Aug 2026 23:08:39 +0300 Subject: [PATCH 38/51] fix(flux-shard-operator): add startupProbe to cloned helm-controller The cloned shard Deployment inherits only a liveness probe (~30s window, no startupProbe), so a controller still syncing caches on startup is killed before it serves /healthz and never recovers. Derive a startupProbe from the liveness handler with a generous budget so a slow start is not fatal, while liveness still catches a wedged running pod. Signed-off-by: IvanHunters (cherry picked from commit d44c87d9394652b688d2776c320db82f49ccc71f) --- internal/fluxshardoperator/provisioner.go | 16 ++++++++++++++ .../fluxshardoperator/provisioner_test.go | 21 +++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/internal/fluxshardoperator/provisioner.go b/internal/fluxshardoperator/provisioner.go index fdbe24e864..d6ad8aa426 100644 --- a/internal/fluxshardoperator/provisioner.go +++ b/internal/fluxshardoperator/provisioner.go @@ -338,6 +338,22 @@ func BuildShardDeployment(flux *appsv1.Deployment, idx int, cfg *Config) (*appsv mergeResourceList(&hc.Resources.Requests, cfg.ShardResources.Requests) mergeResourceList(&hc.Resources.Limits, cfg.ShardResources.Limits) + // Guard startup with a startupProbe derived from the liveness handler. + // Without it the inherited ~30s liveness window kills a controller that is + // still syncing caches (a slow start on a large cluster, or a transient + // dependency), turning any slow start into an unrecoverable crashloop. The + // generous startup budget defers liveness until the manager is serving, + // then liveness still catches a wedged running pod. + if hc.LivenessProbe != nil && hc.StartupProbe == nil { + sp := hc.LivenessProbe.DeepCopy() + sp.InitialDelaySeconds = 0 + sp.PeriodSeconds = 10 + sp.TimeoutSeconds = 1 + sp.SuccessThreshold = 1 + sp.FailureThreshold = 30 + hc.StartupProbe = sp + } + mounted := map[string]bool{} for _, m := range hc.VolumeMounts { mounted[m.Name] = true diff --git a/internal/fluxshardoperator/provisioner_test.go b/internal/fluxshardoperator/provisioner_test.go index 96f742a08e..9f4a0e1cf9 100644 --- a/internal/fluxshardoperator/provisioner_test.go +++ b/internal/fluxshardoperator/provisioner_test.go @@ -8,6 +8,7 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" ) // fluxAIODeployment models the relevant shape of the flux-aio "flux" @@ -103,6 +104,16 @@ func fluxAIODeployment() *appsv1.Deployment { {Name: "no_proxy", Value: ".svc"}, }, VolumeMounts: []corev1.VolumeMount{{Name: "tmp", MountPath: "/tmp"}}, + LivenessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/healthz", + Port: intstr.FromString("healthz-hc"), + }, + }, + PeriodSeconds: 10, + FailureThreshold: 3, + }, }, {Name: "notification-controller", Image: "ghcr.io/fluxcd/notification-controller:v1.8.0"}, }, @@ -215,6 +226,16 @@ func TestBuildShardDeployment(t *testing.T) { if hc.Resources.Limits.Memory().String() != "1Gi" { t.Fatalf("resources override not applied: %v", hc.Resources) } + + if hc.StartupProbe == nil { + t.Fatal("startupProbe must be added so a slow start is not liveness-killed into a crashloop") + } + if hc.StartupProbe.HTTPGet == nil || hc.StartupProbe.HTTPGet.Path != "/healthz" { + t.Fatalf("startupProbe must reuse the liveness /healthz handler: %+v", hc.StartupProbe) + } + if hc.StartupProbe.FailureThreshold < 10 { + t.Fatalf("startupProbe budget too small to cover a slow cache sync: %d", hc.StartupProbe.FailureThreshold) + } } func TestBuildShardDeploymentInheritsResourcesWhenUnset(t *testing.T) { From 1cf292c551700d0ef4f5c9b93e0f11a462d94201 Mon Sep 17 00:00:00 2001 From: IvanHunters Date: Tue, 4 Aug 2026 23:22:59 +0300 Subject: [PATCH 39/51] fix(flux-shard-operator): inherit probe timeout, correct comments Review follow-ups: - Do not force the cloned startupProbe TimeoutSeconds to 1: keep the value inherited from the liveness handler, otherwise a future flux-aio with a larger liveness timeout would get a stricter startup probe and could recreate the crashloop. Covered by a regression guard (liveness timeout 5s must be inherited). - Soften the proxy-strip rationale to the A/B-confirmed mechanism (blocking external HTTPS call at startup) and note the remote spec.kubeConfig exception that does not apply to cozystack in-cluster guest apiservers. - Correct the test fixture comment: the proxy env is operator/patch-level, not installer-injected (the installer injects only KUBERNETES_SERVICE_*). Signed-off-by: IvanHunters (cherry picked from commit affdd70e9d38f4560613e8f41003e1b6f2426757) --- internal/fluxshardoperator/provisioner.go | 25 ++++++++++++------- .../fluxshardoperator/provisioner_test.go | 12 ++++++--- 2 files changed, 25 insertions(+), 12 deletions(-) diff --git a/internal/fluxshardoperator/provisioner.go b/internal/fluxshardoperator/provisioner.go index d6ad8aa426..3d6a0bde59 100644 --- a/internal/fluxshardoperator/provisioner.go +++ b/internal/fluxshardoperator/provisioner.go @@ -317,13 +317,17 @@ func BuildShardDeployment(flux *appsv1.Deployment, idx int, cfg *Config) (*appsv case "KUBERNETES_SERVICE_HOST", "KUBERNETES_SERVICE_PORT": continue // Corporate-proxy env inherited from flux-aio. A standalone shard needs - // no external egress (source-controller fetches artifacts), so the proxy - // only harms it: a startup HTTPS call that stalls through an unreachable - // proxy blocks the manager before it serves /healthz, and the liveness - // probe then crashloops the pod indefinitely. flux-aio survives only - // because it starts once and never restarts. Drop the proxy env (and the - // now-pointless NO_PROXY) so the shard talks to the in-cluster apiserver - // directly. + // no external egress (source-controller does artifact fetching and + // cosign/TUF verification), so the proxy only harms it: a blocking + // external HTTPS call at startup stalls through an unreachable proxy, + // the manager never serves /healthz, and the liveness probe then + // crashloops the pod. (flux-aio itself is unaffected in practice: it is + // long-lived and does not restart, so it never re-hits this startup + // path.) Drop the proxy env (and the now-pointless NO_PROXY) so the + // shard reaches the in-cluster apiserver directly. A HelmRelease + // targeting a remote cluster via spec.kubeConfig reachable only through + // the proxy is the one theoretical exception; cozystack guest-cluster + // apiservers are in-cluster, so this does not apply here. case "HTTP_PROXY", "HTTPS_PROXY", "NO_PROXY", "http_proxy", "https_proxy", "no_proxy": continue @@ -343,12 +347,15 @@ func BuildShardDeployment(flux *appsv1.Deployment, idx int, cfg *Config) (*appsv // still syncing caches (a slow start on a large cluster, or a transient // dependency), turning any slow start into an unrecoverable crashloop. The // generous startup budget defers liveness until the manager is serving, - // then liveness still catches a wedged running pod. + // then liveness still catches a wedged running pod. Only the budget fields + // are normalised; the liveness handler and its TimeoutSeconds are inherited, + // so a controller whose /healthz is slow under load keeps the same tolerance + // at startup as at runtime (overriding TimeoutSeconds down could make the + // startup probe stricter than liveness and recreate the crashloop). if hc.LivenessProbe != nil && hc.StartupProbe == nil { sp := hc.LivenessProbe.DeepCopy() sp.InitialDelaySeconds = 0 sp.PeriodSeconds = 10 - sp.TimeoutSeconds = 1 sp.SuccessThreshold = 1 sp.FailureThreshold = 30 hc.StartupProbe = sp diff --git a/internal/fluxshardoperator/provisioner_test.go b/internal/fluxshardoperator/provisioner_test.go index 9f4a0e1cf9..0058941116 100644 --- a/internal/fluxshardoperator/provisioner_test.go +++ b/internal/fluxshardoperator/provisioner_test.go @@ -93,9 +93,10 @@ func fluxAIODeployment() *appsv1.Deployment { FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.namespace"}, }}, {Name: "TUF_ROOT", Value: "/tmp/.sigstore"}, - // Corporate-proxy env the installer puts on - // flux-aio; harmless there but hangs a - // standalone shard at startup. + // Corporate-proxy env present on flux-aio in + // proxied installs (operator/patch level, not the + // installer, which injects only KUBERNETES_SERVICE_*); + // harmless there, hangs a standalone shard at startup. {Name: "HTTP_PROXY", Value: "http://proxy.example:3128"}, {Name: "HTTPS_PROXY", Value: "http://proxy.example:3128"}, {Name: "NO_PROXY", Value: ".svc"}, @@ -112,6 +113,7 @@ func fluxAIODeployment() *appsv1.Deployment { }, }, PeriodSeconds: 10, + TimeoutSeconds: 5, FailureThreshold: 3, }, }, @@ -236,6 +238,10 @@ func TestBuildShardDeployment(t *testing.T) { if hc.StartupProbe.FailureThreshold < 10 { t.Fatalf("startupProbe budget too small to cover a slow cache sync: %d", hc.StartupProbe.FailureThreshold) } + if hc.StartupProbe.TimeoutSeconds != 5 { + t.Fatalf("startupProbe must inherit the liveness TimeoutSeconds (5), not force it stricter: got %d", + hc.StartupProbe.TimeoutSeconds) + } } func TestBuildShardDeploymentInheritsResourcesWhenUnset(t *testing.T) { From 55de6c011dfa08caae7d39c996756dea63f5f4e3 Mon Sep 17 00:00:00 2001 From: IvanHunters Date: Fri, 7 Aug 2026 16:33:13 +0300 Subject: [PATCH 40/51] docs(flux-shard-operator): record proxy-env drop and startupProbe in sanitisation list The BuildShardDeployment doc comment and the operator README are the only record of how far the cloned shard container has diverged from flux-aio. Add the two steps this fix introduced (dropping the inherited corporate-proxy env and deriving a startupProbe) so the list stays the authoritative account and does not invite re-adding what was deliberately removed. Signed-off-by: IvanHunters (cherry picked from commit 036a31b550e080553da3fef1c6365ed9b579fe45) --- internal/fluxshardoperator/provisioner.go | 9 ++++++++- packages/system/flux-shard-operator/README.md | 2 +- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/internal/fluxshardoperator/provisioner.go b/internal/fluxshardoperator/provisioner.go index 3d6a0bde59..f430ba8947 100644 --- a/internal/fluxshardoperator/provisioner.go +++ b/internal/fluxshardoperator/provisioner.go @@ -263,7 +263,14 @@ func mergeResourceList(dst *corev1.ResourceList, overrides corev1.ResourceList) // - the required podAntiAffinity cloned from flux-aio (which keeps its own // replicas off one node) is dropped, since it targets // app.kubernetes.io/name=flux and would otherwise leave every shard -// Pending on a single-node cluster. +// Pending on a single-node cluster; +// - the corporate-proxy env (HTTP_PROXY/HTTPS_PROXY/NO_PROXY) inherited from +// flux-aio is dropped: a standalone shard needs no external egress, and an +// unreachable proxy stalls a blocking startup HTTPS call, so the manager +// never serves /healthz and the pod crashloops; +// - a startupProbe is derived from the liveness handler (generous failure +// budget, liveness handler and TimeoutSeconds inherited) so a slow but +// progressing start is not killed by the short inherited liveness window. func BuildShardDeployment(flux *appsv1.Deployment, idx int, cfg *Config) (*appsv1.Deployment, error) { var src *corev1.Container for i := range flux.Spec.Template.Spec.Containers { diff --git a/packages/system/flux-shard-operator/README.md b/packages/system/flux-shard-operator/README.md index 660a6542b6..9f06cb9b6a 100644 --- a/packages/system/flux-shard-operator/README.md +++ b/packages/system/flux-shard-operator/README.md @@ -6,7 +6,7 @@ Spreads tenant HelmReleases across multiple helm-controller shards so a noisy te The operator has three parts, all served by one Deployment (leader-elected controllers, webhook on every replica): -1. **Shard runtime.** Reconciles `shardCount` helm-controller Deployments (`helm-controller-shard`, `--watch-label-selector=sharding.fluxcd.io/key=shard`) in the flux namespace, cloned from the flux-aio `flux` Deployment's helm-controller container and sanitised (no host networking, no localhost cross-container wiring). The helm-controller image and feature-gates are inherited from flux-aio automatically. Deployments beyond `shardCount` are pruned once they drain, and the legacy hand-rolled `flux-tenants` Deployment is retired once no HelmRelease carries `sharding.fluxcd.io/key=tenants`. +1. **Shard runtime.** Reconciles `shardCount` helm-controller Deployments (`helm-controller-shard`, `--watch-label-selector=sharding.fluxcd.io/key=shard`) in the flux namespace, cloned from the flux-aio `flux` Deployment's helm-controller container and sanitised (no host networking, no localhost cross-container wiring, no inherited corporate-proxy env, and a startupProbe guarding a slow start). The helm-controller image and feature-gates are inherited from flux-aio automatically. Deployments beyond `shardCount` are pruned once they drain, and the legacy hand-rolled `flux-tenants` Deployment is retired once no HelmRelease carries `sharding.fluxcd.io/key=tenants`. 2. **Placement controller.** Owns the tenant→shard assignment. The unit of placement is the tenant: all HelmReleases of one tenant (parent `tenant-` plus everything in namespace `tenant-`) carry the same shard label, so a noisy tenant's blast radius is bounded to its shard's co-residents. Tenants are distributed greedy least-loaded, weighted by HelmRelease count (N tenants over N shards land exactly 1 per shard). The assignment is recorded as the `internal.cozystack.io/flux-shard` label on the tenant namespace; HelmRelease labels remain the source of truth on restarts. Moves are paced and deleting tenants are never moved. Watches are metadata-only, so the controller does not decode the helm-controller status-patch firehose. From daf44098f566ebded3938cf3839eb104c19156fb Mon Sep 17 00:00:00 2001 From: IvanHunters Date: Fri, 7 Aug 2026 16:33:21 +0300 Subject: [PATCH 41/51] test(flux-shard-operator): assert full startupProbe contract and reject alias regression Model the source liveness probe as flux-aio actually ships it (a bare httpGet with no timing fields) and tighten the startupProbe assertions: - assert the complete normalised budget (delay/period/success/failure) instead of only FailureThreshold, so a later PeriodSeconds regression cannot silently restore a short crashloop window; - assert TimeoutSeconds is inherited from the source probe rather than a hardcoded value, so forcing it stricter is caught; - assert the startupProbe is a DeepCopy of liveness, not an alias, which would otherwise stamp the 30-failure startup budget onto liveness and produce a never-failing liveness probe. The alias assertion is non-vacuous: it fails when the DeepCopy is regressed to a plain alias. Signed-off-by: IvanHunters (cherry picked from commit a10fcb3e4a6b8074c25eec46dce68bc4db4384b4) --- .../fluxshardoperator/provisioner_test.go | 55 +++++++++++++++---- 1 file changed, 45 insertions(+), 10 deletions(-) diff --git a/internal/fluxshardoperator/provisioner_test.go b/internal/fluxshardoperator/provisioner_test.go index 0058941116..96ad34ff00 100644 --- a/internal/fluxshardoperator/provisioner_test.go +++ b/internal/fluxshardoperator/provisioner_test.go @@ -105,6 +105,11 @@ func fluxAIODeployment() *appsv1.Deployment { {Name: "no_proxy", Value: ".svc"}, }, VolumeMounts: []corev1.VolumeMount{{Name: "tmp", MountPath: "/tmp"}}, + // flux-aio ships helm-controller with a bare httpGet + // liveness probe: no timing fields at all, so the + // kube-apiserver defaults (~30s window) apply. The + // startupProbe must inherit this handler and its unset + // TimeoutSeconds and only normalise the startup budget. LivenessProbe: &corev1.Probe{ ProbeHandler: corev1.ProbeHandler{ HTTPGet: &corev1.HTTPGetAction{ @@ -112,9 +117,6 @@ func fluxAIODeployment() *appsv1.Deployment { Port: intstr.FromString("healthz-hc"), }, }, - PeriodSeconds: 10, - TimeoutSeconds: 5, - FailureThreshold: 3, }, }, {Name: "notification-controller", Image: "ghcr.io/fluxcd/notification-controller:v1.8.0"}, @@ -136,7 +138,18 @@ func TestBuildShardDeployment(t *testing.T) { }, } - dep, err := BuildShardDeployment(fluxAIODeployment(), 2, cfg) + flux := fluxAIODeployment() + // Capture the source liveness probe verbatim so the startupProbe assertions + // can check inheritance against what flux-aio actually ships, rather than + // against a hardcoded value baked into the test. + var srcLiveness *corev1.Probe + for i := range flux.Spec.Template.Spec.Containers { + if flux.Spec.Template.Spec.Containers[i].Name == "helm-controller" { + srcLiveness = flux.Spec.Template.Spec.Containers[i].LivenessProbe.DeepCopy() + } + } + + dep, err := BuildShardDeployment(flux, 2, cfg) if err != nil { t.Fatal(err) } @@ -232,15 +245,37 @@ func TestBuildShardDeployment(t *testing.T) { if hc.StartupProbe == nil { t.Fatal("startupProbe must be added so a slow start is not liveness-killed into a crashloop") } + // Handler inherited from liveness verbatim. if hc.StartupProbe.HTTPGet == nil || hc.StartupProbe.HTTPGet.Path != "/healthz" { t.Fatalf("startupProbe must reuse the liveness /healthz handler: %+v", hc.StartupProbe) } - if hc.StartupProbe.FailureThreshold < 10 { - t.Fatalf("startupProbe budget too small to cover a slow cache sync: %d", hc.StartupProbe.FailureThreshold) - } - if hc.StartupProbe.TimeoutSeconds != 5 { - t.Fatalf("startupProbe must inherit the liveness TimeoutSeconds (5), not force it stricter: got %d", - hc.StartupProbe.TimeoutSeconds) + // Assert the complete normalised startup budget, not just FailureThreshold: + // a threshold check alone still passes if PeriodSeconds later regresses to 1, + // which would silently restore a short crashloop window. + if hc.StartupProbe.InitialDelaySeconds != 0 || + hc.StartupProbe.PeriodSeconds != 10 || + hc.StartupProbe.SuccessThreshold != 1 || + hc.StartupProbe.FailureThreshold != 30 { + t.Fatalf("startupProbe budget not normalised to the expected contract "+ + "(delay=0 period=10 success=1 failure=30): %+v", hc.StartupProbe) + } + // TimeoutSeconds is inherited from the source liveness probe, never forced. + // flux-aio ships a bare probe (TimeoutSeconds 0), so a stray + // sp.TimeoutSeconds = N would diverge from the source and fail here. + if hc.StartupProbe.TimeoutSeconds != srcLiveness.TimeoutSeconds { + t.Fatalf("startupProbe must inherit the liveness TimeoutSeconds (%d), got %d", + srcLiveness.TimeoutSeconds, hc.StartupProbe.TimeoutSeconds) + } + // The startupProbe must be a DeepCopy of the liveness probe, never an alias. + // If it aliased, normalising the startup FailureThreshold to 30 would also + // stamp 30 onto liveness, producing exactly the never-failing liveness probe + // this change exists to avoid. + if hc.LivenessProbe == hc.StartupProbe { + t.Fatal("startupProbe must be a DeepCopy of liveness, not an alias sharing its backing probe") + } + if hc.LivenessProbe.FailureThreshold != srcLiveness.FailureThreshold { + t.Fatalf("liveness FailureThreshold was clobbered by the startup budget "+ + "(alias regression): source=%d live=%d", srcLiveness.FailureThreshold, hc.LivenessProbe.FailureThreshold) } } From 2107d80eb187bee9faad0252c009603c067135e7 Mon Sep 17 00:00:00 2001 From: IvanHunters Date: Mon, 10 Aug 2026 12:56:42 +0300 Subject: [PATCH 42/51] docs(flux-shard-operator): drop unproven mechanism from proxy comments Name all six proxy env spellings in the sanitisation list, and replace the unestablished 'external HTTPS call' / 'flux-aio does not restart' claims with the grounded in-cluster mechanism: after the KUBERNETES_SERVICE_HOST drop above, the shard talks to the management apiserver over the kubelet-injected ClusterIP, which NO_PROXY=.svc does not match, so that startup call is what stalls behind an unreachable proxy. Comment-only; no behaviour change. Signed-off-by: IvanHunters (cherry picked from commit 66df3b20e05f0b64a321ccb8866a80b4ac93b68a) --- internal/fluxshardoperator/provisioner.go | 34 +++++++++++++---------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/internal/fluxshardoperator/provisioner.go b/internal/fluxshardoperator/provisioner.go index f430ba8947..1a0066d59d 100644 --- a/internal/fluxshardoperator/provisioner.go +++ b/internal/fluxshardoperator/provisioner.go @@ -264,10 +264,11 @@ func mergeResourceList(dst *corev1.ResourceList, overrides corev1.ResourceList) // replicas off one node) is dropped, since it targets // app.kubernetes.io/name=flux and would otherwise leave every shard // Pending on a single-node cluster; -// - the corporate-proxy env (HTTP_PROXY/HTTPS_PROXY/NO_PROXY) inherited from -// flux-aio is dropped: a standalone shard needs no external egress, and an -// unreachable proxy stalls a blocking startup HTTPS call, so the manager -// never serves /healthz and the pod crashloops; +// - the corporate-proxy env inherited from flux-aio is dropped, in both the +// upper- and lower-case spellings (HTTP_PROXY/HTTPS_PROXY/NO_PROXY): a +// standalone shard needs no external egress, and behind an unreachable +// proxy a stalled startup call leaves the manager never serving /healthz, +// so the liveness probe crashloops the pod; // - a startupProbe is derived from the liveness handler (generous failure // budget, liveness handler and TimeoutSeconds inherited) so a slow but // progressing start is not killed by the short inherited liveness window. @@ -324,17 +325,20 @@ func BuildShardDeployment(flux *appsv1.Deployment, idx int, cfg *Config) (*appsv case "KUBERNETES_SERVICE_HOST", "KUBERNETES_SERVICE_PORT": continue // Corporate-proxy env inherited from flux-aio. A standalone shard needs - // no external egress (source-controller does artifact fetching and - // cosign/TUF verification), so the proxy only harms it: a blocking - // external HTTPS call at startup stalls through an unreachable proxy, - // the manager never serves /healthz, and the liveness probe then - // crashloops the pod. (flux-aio itself is unaffected in practice: it is - // long-lived and does not restart, so it never re-hits this startup - // path.) Drop the proxy env (and the now-pointless NO_PROXY) so the - // shard reaches the in-cluster apiserver directly. A HelmRelease - // targeting a remote cluster via spec.kubeConfig reachable only through - // the proxy is the one theoretical exception; cozystack guest-cluster - // apiservers are in-cluster, so this does not apply here. + // no external egress: source-controller does artifact fetching and + // cosign/TUF verification, and every hop the shard makes is in-cluster + // (artifacts advertise as flux.$(RUNTIME_NAMESPACE).svc, guest-cluster + // apiservers are reached over .svc kubeconfigs). NO_PROXY=.svc covers + // those, but not the management apiserver: the KUBERNETES_SERVICE_HOST + // case above makes the shard fall back to the kubelet-injected + // ClusterIP, a bare IP that no .svc suffix matches, so that startup + // call is the one that stalls through an unreachable proxy and leaves + // the manager never serving /healthz until the liveness probe + // crashloops the pod. Dropping the proxy env (and the then-pointless + // NO_PROXY) keeps every hop direct. A HelmRelease targeting a remote + // cluster via spec.kubeConfig reachable only through the proxy is the + // one theoretical exception; cozystack guest-cluster apiservers are + // in-cluster, so this does not apply here. case "HTTP_PROXY", "HTTPS_PROXY", "NO_PROXY", "http_proxy", "https_proxy", "no_proxy": continue From e85482baccf5ce0129d9e5036cd0b7e4da6d075d Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Mon, 17 Aug 2026 14:52:28 +0500 Subject: [PATCH 43/51] ci(release): complete the candidate-aware promotion pipeline on release-1.6 Promote RC refuses to promote a release whose target base cannot verify the packages candidate. It checks five markers on the base branch before its first registry write and fails closed with "Target base 'release-1.6' lacks ''". This branch satisfied one of the five (hack/lib/image-refs.sh), so the next patch release off release-1.6 could not be dispatched at all. Add the two halves the base has to carry: * hack/verify-promoted-packages.sh and hack/lib/promoted-packages.sh, byte-identical to main. The verifier pulls the candidate by its committed digest and proves it matches the merge tree apart from the artifact's impossible self-reference, that promotion rewrote every version-line image tag, and that the container repo@digest set is unchanged from the rc artifact. * the two jobs that run it. verify-release-candidate in pull-requests.yaml checks GitHub's prospective merge tree so a packages/ edit pushed to the promote PR shows up as a failed check, and a Verify stable packages candidate step in the finalize job of pull-requests-release.yaml repeats it against the real merge commit. Position is the whole point of the finalize half: it runs before the write-once stable tag, the draft-release publish and the retag, so a bad candidate is refused while nothing stable-named exists yet. The former "Set up toolchain (skopeo, yq, helm)" and "Login to registry (GHCR)" steps moved up with it and are no longer duplicated later in the job; the toolchain now also installs flux and pins yq by version and sha256, because an irreversible step must not depend on whatever releases/latest/download resolves to today. verify-release-candidate is keyed on the promote PR's author and its head-branch shape rather than on the `release` label main matches. This branch's on.pull_request.types has no `labeled` event, and the label is applied about a second after the PR opens (a separate call in gh pr create), so the `opened` payload carries no labels and a label-keyed guard would never fire here. .release-tooling is gitignored, matching main, so the trusted-base checkout can never be staged. Signed-off-by: Myasnikov Daniil --- .github/workflows/pull-requests-release.yaml | 108 +++++++-- .github/workflows/pull-requests.yaml | 100 ++++++++ .gitignore | 5 + hack/lib/promoted-packages.sh | 25 ++ hack/verify-promoted-packages.sh | 230 +++++++++++++++++++ 5 files changed, 450 insertions(+), 18 deletions(-) create mode 100644 hack/lib/promoted-packages.sh create mode 100755 hack/verify-promoted-packages.sh diff --git a/.github/workflows/pull-requests-release.yaml b/.github/workflows/pull-requests-release.yaml index fc1608a343..5e61bac423 100644 --- a/.github/workflows/pull-requests-release.yaml +++ b/.github/workflows/pull-requests-release.yaml @@ -91,6 +91,90 @@ jobs: # app token authenticates, matching how promote-rc.yaml already pushes. persist-credentials: false + # Use the trusted pre-merge base version of the verifier, while its root + # argument remains the merged packages tree checked out above. This keeps + # the final guard independent of the release PR's executable contents. + - name: Checkout release tooling from base + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + ref: ${{ github.event.pull_request.base.sha }} + path: .release-tooling + sparse-checkout: hack + persist-credentials: false + + # Resolve and verify the candidate before creating any write-once stable + # git or registry name. The candidate was serialized at promote time from + # the tag-rewritten tree; this guard pulls it by the committed digest and + # proves it still matches the merge commit (apart from its impossible + # self-reference) and still pins the rc-tested container digests. + # + # This step block runs HERE, ahead of the tag creation, the draft-release + # publish and the retag, rather than in its old position after the publish: + # a guard that runs after the write-once names exist cannot refuse them. + # It absorbs the former "Set up toolchain (skopeo, yq, helm)" and "Login to + # registry (GHCR)" steps, which is why neither appears later in this job + # any more — the tools and the logins are set up once, before the first + # irreversible action, and stay in scope for the retag and chart publish. + # + # flux and yq are version-pinned: they are what this job verifies the + # candidate with and what stamps the stable chart, so `latest/download` + # would make an irreversible step depend on whatever mikefarah shipped + # this morning. The version tracks + # packages/core/testing/images/e2e-sandbox/Dockerfile. yq is additionally + # pinned by content — it stamps the chart's platformVersion, and this step + # is the last one before write-once names exist. The Flux installer is not, + # because it checksums the binary it fetches and the mutable part is its + # own bootstrap script. + - name: Set up promotion toolchain (flux, skopeo, yq, helm) + if: ${{ !contains(steps.get_tag.outputs.tag, '-') }} + env: + FLUX_VERSION: "2.8.6" + YQ_VERSION: "4.53.3" + YQ_SHA256: "fa52a4e758c63d38299163fbdd1edfb4c4963247918bf9c1c5d31d84789eded4" + run: | + if ! flux version --client 2>/dev/null | grep -qx "flux: v${FLUX_VERSION}"; then + install_script="$(mktemp)" + curl -fsSL https://fluxcd.io/install.sh -o "$install_script" + sudo env "FLUX_VERSION=$FLUX_VERSION" bash "$install_script" + rm -f "$install_script" + fi + if ! yq --version 2>/dev/null | grep -q "mikefarah.* version v${YQ_VERSION}\$"; then + yq_bin="$(mktemp)" + curl -fsSL -o "$yq_bin" \ + "https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_linux_amd64" + printf '%s %s\n' "$YQ_SHA256" "$yq_bin" | sha256sum -c - + sudo install -m 0755 "$yq_bin" /usr/local/bin/yq + rm -f "$yq_bin" + fi + command -v skopeo >/dev/null \ + || { sudo apt-get update -qq && sudo apt-get install -y -qq skopeo; } + command -v helm >/dev/null \ + || curl -fsSL https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash + # Not installed here, only required: `flux pull artifact` reads its + # registry credentials from ~/.docker/config.json, which is why the + # login step below runs `docker login` as well as skopeo's and helm's. + # This job's runner class is not one of the docker-using build ones, so + # say what is missing here rather than letting the login step fail with + # a bare "command not found" in the middle of the release path. + command -v docker >/dev/null \ + || { echo "::error::docker is required on this runner: flux reads registry credentials from ~/.docker/config.json"; exit 1; } + + - name: Login to registry (GHCR) + if: ${{ !contains(steps.get_tag.outputs.tag, '-') }} + env: + GHCR_USER: ${{ github.actor }} + GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + echo "$GHCR_TOKEN" | docker login "${REGISTRY%%/*}" -u "$GHCR_USER" --password-stdin + echo "$GHCR_TOKEN" | skopeo login "${REGISTRY%%/*}" -u "$GHCR_USER" --password-stdin + echo "$GHCR_TOKEN" | helm registry login "${REGISTRY%%/*}" -u "$GHCR_USER" --password-stdin + + - name: Verify stable packages candidate + if: ${{ !contains(steps.get_tag.outputs.tag, '-') }} + env: + TAG: ${{ steps.get_tag.outputs.tag }} + run: .release-tooling/hack/verify-promoted-packages.sh "${TAG#v}" + # Create the release tag at the merge commit — write-once. # # The tag is attached to the PR merge commit, which did not exist before @@ -393,24 +477,12 @@ jobs: # The checked-out merge commit is the promoted stable tree: the rc's # digests with the tag string rewritten to stable. skopeo retags those # digests to the stable image tag; helm packages+pushes the stable chart. - - name: Set up toolchain (skopeo, yq, helm) - if: ${{ !contains(steps.get_tag.outputs.tag, '-') }} - run: | - command -v yq >/dev/null \ - || { sudo curl -sSL -o /usr/local/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 && sudo chmod +x /usr/local/bin/yq; } - command -v skopeo >/dev/null \ - || { sudo apt-get update -qq && sudo apt-get install -y -qq skopeo; } - command -v helm >/dev/null \ - || curl -fsSL https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash - - - name: Login to registry (GHCR) - if: ${{ !contains(steps.get_tag.outputs.tag, '-') }} - env: - GHCR_USER: ${{ github.actor }} - GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - echo "$GHCR_TOKEN" | skopeo login "${REGISTRY%%/*}" -u "$GHCR_USER" --password-stdin - echo "$GHCR_TOKEN" | helm registry login "${REGISTRY%%/*}" -u "$GHCR_USER" --password-stdin + # + # The toolchain and the GHCR logins these two steps need are established + # earlier in this job, by "Set up promotion toolchain (flux, skopeo, yq, + # helm)" and "Login to registry (GHCR)" — both moved ahead of the tag + # creation so the candidate verification can refuse a release before any + # write-once name exists. Same job, so both are still in scope here. # Retag every rc image/digest to the stable tag. No rebuild — bit-for-bit # the e2e-tested rc. :latest moves only when this stable is the newest diff --git a/.github/workflows/pull-requests.yaml b/.github/workflows/pull-requests.yaml index a5056ca00e..cdc3435b15 100644 --- a/.github/workflows/pull-requests.yaml +++ b/.github/workflows/pull-requests.yaml @@ -83,6 +83,106 @@ jobs: echo "any=true" >> "$GITHUB_OUTPUT" fi + # Verify GitHub's prospective merge tree here, so a `packages/` edit pushed to + # this PR fails this check before it can merge. + # + # Checking the MERGE tree rather than the head is what makes base drift + # visible when this job runs — but nothing re-runs it when the base moves: + # GitHub does not fire `pull_request` on the base branch advancing, it only + # recomputes refs/pull/N/merge. So a `packages/` change landing on the base + # after this went green is never re-verified here, and the PR stays mergeable + # on the stale result. Finalize repeats the check against the real merge + # commit before creating the write-once stable tag, which is where that case + # is actually caught — loudly, and before any stable name exists. + # + # The trigger predicate is the promote PR's author plus its head-branch shape, + # NOT the `release` label main keys on. This branch's `on.pull_request.types` + # has no `labeled` event, and `gh pr create --label release` applies the label + # in a separate call one second after the PR opens (observed on the v1.6.1 + # promote PR), so the `opened` payload carries no labels at all and a + # label-keyed guard would never fire here. Every `cozystack-ci[bot]` PR whose + # head starts with `release-` is a promote PR, so this is if anything the + # narrower predicate — and it re-runs on `synchronize`, which is what makes a + # `packages/` edit pushed to the PR visible. A prerelease staging head + # (release-X.Y.Z-rc.N) never gets a PR under the checks-at-rc flow; if one ever + # did, the verifier's own X.Y.Z assertion fails it closed rather than passing. + verify-release-candidate: + name: Verify release packages candidate + runs-on: ubuntu-latest + # Matching `checks` and `finalize`. Without a ceiling the job inherits the + # 6-hour default, and a stalled `flux pull artifact` would hold this check + # pending for all of it. + timeout-minutes: 30 + if: | + github.event.pull_request.user.login == 'cozystack-ci[bot]' + && startsWith(github.head_ref, 'release-') + permissions: + contents: read + packages: read + steps: + # The default pull_request checkout is GitHub's prospective merge commit, + # not just the head branch. That makes base-branch package drift visible. + - name: Checkout prospective merge + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + fetch-depth: 0 + + # Execute the verifier from the trusted base SHA. An older rc-derived head + # may not contain it, and a PR must not be able to weaken its own guard. + - name: Checkout release tooling from base + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + ref: ${{ github.event.pull_request.base.sha }} + path: .release-tooling + sparse-checkout: hack + persist-credentials: false + + # Both tools are version-pinned. This job decides whether a stable release + # may be created, so what it runs has to be a fixed input: `latest/download` + # resolves to whatever mikefarah shipped this morning, and a yq that changed + # how it emits or compares a value turns a release gate into a moving one. + # The version tracks packages/core/testing/images/e2e-sandbox/Dockerfile. + # yq is additionally pinned by content; the Flux installer is not, because + # it checksums the binary it fetches and the mutable part is its own + # bootstrap script. + - name: Set up promotion toolchain (flux, yq) + env: + FLUX_VERSION: "2.8.6" + YQ_VERSION: "4.53.3" + YQ_SHA256: "fa52a4e758c63d38299163fbdd1edfb4c4963247918bf9c1c5d31d84789eded4" + run: | + if ! flux version --client 2>/dev/null | grep -qx "flux: v${FLUX_VERSION}"; then + install_script="$(mktemp)" + curl -fsSL https://fluxcd.io/install.sh -o "$install_script" + sudo env "FLUX_VERSION=$FLUX_VERSION" bash "$install_script" + rm -f "$install_script" + fi + if ! yq --version 2>/dev/null | grep -q "mikefarah.* version v${YQ_VERSION}\$"; then + yq_bin="$(mktemp)" + curl -fsSL -o "$yq_bin" \ + "https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_linux_amd64" + printf '%s %s\n' "$YQ_SHA256" "$yq_bin" | sha256sum -c - + sudo install -m 0755 "$yq_bin" /usr/local/bin/yq + rm -f "$yq_bin" + fi + + - name: Login to registry (GHCR) + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3 + with: + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + registry: ghcr.io + env: + DOCKER_CONFIG: ${{ runner.temp }}/.docker + + - name: Verify stable packages candidate + env: + DOCKER_CONFIG: ${{ runner.temp }}/.docker + HEAD_BRANCH: ${{ github.head_ref }} + run: | + STABLE_VERSION="${HEAD_BRANCH#release-}" + .release-tooling/hack/verify-promoted-packages.sh "$STABLE_VERSION" + # Helm-unittest + Go controller tests. Decoupled from the image build and from # e2e so they give fast, independent signal; they gate the PR as their own # required check rather than blocking e2e from starting. diff --git a/.gitignore b/.gitignore index 4d15bd0bd2..ae05d04195 100644 --- a/.gitignore +++ b/.gitignore @@ -104,3 +104,8 @@ examples/backups/*/.bucket-info.env examples/backups/*/.sentinel.env examples/backups/*/.backup-name.env packages/core/platform/images/migrations/etcd-operator-crds/ + +# CI scratch: the candidate-verification jobs check the trusted base ref here so +# release tooling (the packages verifier and its library) is the pre-merge +# version even when the checked-out tree predates it. Never committed. +.release-tooling diff --git a/hack/lib/promoted-packages.sh b/hack/lib/promoted-packages.sh new file mode 100644 index 0000000000..e962eef761 --- /dev/null +++ b/hack/lib/promoted-packages.sh @@ -0,0 +1,25 @@ +# shellcheck shell=sh +# Shared definitions for the stable-candidate packages artifact. +# +# Sourced by hack/promote-packages-artifact.sh, which publishes the candidate, +# and hack/verify-promoted-packages.sh, which proves it. The publisher's whole +# reason to inspect the pin at all is to make the verifier's refusal land at +# dispatch time, before a registry write, two commits and a draft release exist +# — so the two have to be testing the same thing. Copied constants would drift +# apart silently and turn that dispatch-time refusal back into the late failure +# it was added to remove; they live here instead. +# +# Not a general library: promotion is the only caller, and both halves run from +# a `hack`-only sparse checkout of a trusted ref, so this file has to stay +# inside hack/ next to them. + +# The one OCI repository a packages candidate may ever be published to or read +# back from. Overridable only so the behavioural suites can point both halves +# at a fixture registry; production never sets it. +EXPECTED_PACKAGES_REPOSITORY="${EXPECTED_PACKAGES_REPOSITORY:-oci://ghcr.io/cozystack/cozystack/cozystack-packages}" + +# An immutable Flux OCI source pin, `digest=sha256:<64 lowercase hex>`. The +# tag form Flux also accepts is exactly what promotion must neither publish nor +# accept: a stable installer pinned by tag resolves to whatever that tag points +# at later, which is how a stable install ends up serving rc content (#3477). +PACKAGES_DIGEST_REF_PATTERN='^digest=sha256:[0-9a-f]{64}$' diff --git a/hack/verify-promoted-packages.sh b/hack/verify-promoted-packages.sh new file mode 100755 index 0000000000..bbd625ff3d --- /dev/null +++ b/hack/verify-promoted-packages.sh @@ -0,0 +1,230 @@ +#!/bin/sh +# Verify that the packages artifact pinned by a stable promotion is exactly the +# packages tree in the merge commit, apart from the artifact's impossible +# self-reference. This runs before finalize creates any stable git/release tag. +# +# Usage: hack/verify-promoted-packages.sh [root] +# X.Y.Z (without a leading v) +# [root] release packages tree; defaults to packages +# +# Environment: +# EXPECTED_PACKAGES_REPOSITORY exact trusted OCI repository; defaults to +# the public Cozystack release repository +# VERIFY_PACKAGES_WORKDIR caller-owned work directory; when set, the +# verifier neither creates nor removes it +set -eu + +STABLE_VERSION="${1:?usage: verify-promoted-packages.sh [root]}" +ROOT="${2:-packages}" + +# EXPECTED_PACKAGES_REPOSITORY and PACKAGES_DIGEST_REF_PATTERN, shared with the +# publisher so its dispatch-time preflight cannot drift from this guard. +# shellcheck source=hack/lib/promoted-packages.sh +. "$(dirname "$0")/lib/promoted-packages.sh" + +printf '%s\n' "$STABLE_VERSION" | grep -Eq '^[0-9]+\.[0-9]+\.[0-9]+$' \ + || { echo "stable-version '$STABLE_VERSION' must match X.Y.Z" >&2; exit 1; } +[ -d "$ROOT" ] || { echo "root '$ROOT' is not a directory" >&2; exit 1; } +VALUES="$ROOT/core/installer/values.yaml" +[ -f "$VALUES" ] || { echo "root '$ROOT' has no core/installer/values.yaml" >&2; exit 1; } + +command -v flux >/dev/null || { echo "flux is required" >&2; exit 1; } +command -v yq >/dev/null || { echo "yq (mikefarah) is required" >&2; exit 1; } +yq --version 2>&1 | grep -q mikefarah \ + || { echo "yq (mikefarah) is required" >&2; exit 1; } + +source_url="$(yq -e -r '.cozystackOperator.platformSourceUrl' "$VALUES")" +source_ref="$(yq -e -r '.cozystackOperator.platformSourceRef' "$VALUES")" +[ "$source_url" = "$EXPECTED_PACKAGES_REPOSITORY" ] \ + || { echo "platformSourceUrl '$source_url' must equal trusted repository '$EXPECTED_PACKAGES_REPOSITORY'" >&2; exit 1; } +printf '%s\n' "$source_ref" | grep -Eq "$PACKAGES_DIGEST_REF_PATTERN" \ + || { echo "platformSourceRef '$source_ref' is not an immutable digest" >&2; exit 1; } + +if [ -n "${VERIFY_PACKAGES_WORKDIR:-}" ]; then + tmp="$VERIFY_PACKAGES_WORKDIR" + [ ! -e "$tmp" ] || { echo "VERIFY_PACKAGES_WORKDIR '$tmp' already exists" >&2; exit 1; } + mkdir -p "$tmp" +else + tmp="$(mktemp -d)" + cleanup() { + case "$tmp" in + "${TMPDIR:-/tmp}"/*) rm -r -- "$tmp" ;; + *) echo "refusing to clean unexpected work directory '$tmp'" >&2 ;; + esac + } + trap cleanup EXIT HUP INT TERM +fi +artifact="$tmp/artifact" +mkdir -p "$artifact" +flux pull artifact "${source_url}@${source_ref#digest=}" --output "$artifact" + +ARTIFACT_VALUES="$artifact/core/installer/values.yaml" +[ -f "$ARTIFACT_VALUES" ] \ + || { echo "promoted artifact has no core/installer/values.yaml" >&2; exit 1; } + +# The candidate was pushed before its own digest was written into installer +# values, so its embedded platformSourceRef is the immutable rc artifact it was +# derived from. Pull that baseline now; comparing its normalized refs below is +# what proves promotion changed tag strings rather than container bytes. +rc_source_url="$(yq -e -r '.cozystackOperator.platformSourceUrl' "$ARTIFACT_VALUES")" +rc_source_ref="$(yq -e -r '.cozystackOperator.platformSourceRef' "$ARTIFACT_VALUES")" +[ "$rc_source_url" = "$EXPECTED_PACKAGES_REPOSITORY" ] \ + || { echo "candidate's original platformSourceUrl '$rc_source_url' must equal trusted repository '$EXPECTED_PACKAGES_REPOSITORY'" >&2; exit 1; } +printf '%s\n' "$rc_source_ref" | grep -Eq "$PACKAGES_DIGEST_REF_PATTERN" \ + || { echo "candidate's original platformSourceRef '$rc_source_ref' is not an immutable digest" >&2; exit 1; } + +# Promotion must have rewritten every version-line image tag in the artifact, +# not merely in git. Keep the match scoped to image-reference positions so an +# unrelated dependency version or historical prose cannot abort a release. +stable_esc="$(printf '%s' "$STABLE_VERSION" | sed 's/\./\\./g')" +scan_err="$tmp/rc-scan-err" +leftovers="$(find "$artifact" -type f ! -path '*/charts/*' ! -name '*.md' \ + -exec grep -lE -- \ + "(image|repository|tag)[\"']?:[^#]*${stable_esc}-rc\.[0-9]+|${stable_esc}-rc\.[0-9]+@sha256:" \ + {} + 2>"$scan_err" || true)" +# An unreadable file is NOT "a file with no match" — the silent-skip shape +# hack/promote-rewrite-tags.sh refuses by name, and the `|| true` here has to +# stay because `-exec … +` reports the legitimate "nothing matched" as failure +# too. That leaves grep's own diagnostics as the only signal that a file went +# unscanned, so treat any of them as a failed scan. Defence in depth: the +# per-file `cmp` below would catch a leftover rc string anyway, since the +# release tree has none. Cheap enough to not depend on that. +if [ -s "$scan_err" ]; then + echo "::error::could not scan the promoted artifact for rc image references:" >&2 + cat "$scan_err" >&2 + exit 1 +fi +if [ -n "$leftovers" ]; then + echo "::error::promoted packages artifact still carries rc image references:" >&2 + printf '%s\n' "$leftovers" >&2 + exit 1 +fi + +# Flux's default archive excludes these file classes and omits symlinks rather +# than following or storing them. Match that established RC artifact view here; +# changing it would be a separate packages-format migration. .build-revision is +# an ignored build nonce generated only by the rc image-packages target; it is +# not consumed at runtime. The installer values are compared separately below +# after normalizing the one impossible self-reference. +# +# This list is a RESTATEMENT of flux's built-in `excludeOCI` set, not a +# derivation of it — the CLI does not expose it — so it is coupled to a flux +# version. That version is pinned as FLUX_VERSION (2.8.6) in all three workflow +# steps that install the toolchain, and the promote-packages contract test pins +# that they agree. If flux ever changes the set, the symptom is a "contain +# different files" failure naming the newly included or excluded class, which +# is loud and lands before any stable name exists; the fix is to update this +# list and FLUX_VERSION together, in that order. +artifact_entries() { + ( + cd "$1" + find . -type f \ + ! -name '.gitignore' \ + ! -name '.gitmodules' \ + ! -name '.gitattributes' \ + ! -name '*.jpg' \ + ! -name '*.jpeg' \ + ! -name '*.gif' \ + ! -name '*.png' \ + ! -name '*.wmv' \ + ! -name '*.flv' \ + ! -name '*.tar.gz' \ + ! -name '*.zip' \ + ! -name '.build-revision' \ + ! -path './core/installer/values.yaml' \ + | LC_ALL=C sort + ) +} + +artifact_entries "$artifact" > "$tmp/artifact-files" +artifact_entries "$ROOT" > "$tmp/release-files" +if ! cmp -s "$tmp/artifact-files" "$tmp/release-files"; then + echo "::error::promoted artifact and release tree contain different files:" >&2 + diff -u "$tmp/artifact-files" "$tmp/release-files" >&2 || true + exit 1 +fi + +while IFS= read -r rel; do + rel="${rel#./}" + artifact_file="$artifact/$rel" + release_file="$ROOT/$rel" + if ! cmp -s "$artifact_file" "$release_file"; then + echo "::error::promoted artifact file differs from release tree: $rel" >&2 + exit 1 + fi + # Resolve each side to a value first. Written as one `&&`/`||` chain this + # reads like a symmetric difference and is not one: POSIX gives the two + # operators equal precedence and left associativity, so `A && B || C && D` + # groups as `((A && B) || C) && D` and the candidate-executable direction — + # the one that makes content executable inside the artifact the operator + # installs — is accepted silently. + if [ -x "$artifact_file" ]; then artifact_exec=1; else artifact_exec=0; fi + if [ -x "$release_file" ]; then release_exec=1; else release_exec=0; fi + if [ "$artifact_exec" -ne "$release_exec" ]; then + echo "::error::promoted artifact executable bit differs from release tree: $rel" >&2 + exit 1 + fi +done < "$tmp/artifact-files" + +normalize_installer() { + sed -E 's|^([[:space:]]*platformSourceRef:[[:space:]]*).*$|\1digest=sha256:SELF|' "$1" +} +normalize_installer "$ARTIFACT_VALUES" > "$tmp/artifact-installer" +normalize_installer "$VALUES" > "$tmp/release-installer" +if ! cmp -s "$tmp/artifact-installer" "$tmp/release-installer"; then + echo "::error::promoted artifact installer values differ beyond their self-reference" >&2 + diff -u "$tmp/artifact-installer" "$tmp/release-installer" >&2 || true + exit 1 +fi + +# Compare normalized repo@digest sets against the rc artifact as an explicit +# proof that promotion did not change any container bytes. The packages +# artifact's own digest is omitted because that is the intentional declarative +# content change this verification is proving. +# shellcheck source=hack/lib/image-refs.sh +. "$(dirname "$0")/lib/image-refs.sh" +normalized_refs() { + # Collect into a file rather than piping straight into the loop. As the head + # of a pipeline the collector's exit status is thrown away — the pipeline + # reports `sort -u`'s, POSIX sh has no `pipefail` and this script has to stay + # POSIX — so `set -e` never sees it and a failed or truncated collection + # becomes a smaller set that the comparison below happily matches. Same + # fail-open class as the rc-reference scan above, on the one guard that is + # supposed to prove the container bytes did not move across promotion. + _nr_raw="$tmp/refs-raw" + collect_image_refs "$1" > "$_nr_raw" + # A collection that comes back empty is that same failure wearing a zero exit + # status: two empty sets compare equal, so the proof passes having examined + # nothing at all. Every packages tree carries image references — none means + # the collector understood nothing it was given, not that there was nothing + # to find. + [ -s "$_nr_raw" ] \ + || { echo "::error::collected no image references from '$1'; cannot prove the container digests are unchanged" >&2; exit 1; } + while IFS= read -r raw; do + [ -n "$raw" ] || continue + without_digest="${raw%@*}" + digest="${raw##*@}" + image="${without_digest##*/}" + if [ "$without_digest" = "$image" ]; then + repo="${image%:*}" + else + repo="${without_digest%/*}/${image%:*}" + fi + case "$repo" in + */cozystack-packages) continue ;; + esac + printf '%s@%s\n' "$repo" "$digest" + done < "$_nr_raw" | LC_ALL=C sort -u +} +rc_artifact="$tmp/rc-artifact" +mkdir -p "$rc_artifact" +flux pull artifact "${rc_source_url}@${rc_source_ref#digest=}" --output "$rc_artifact" +normalized_refs "$rc_artifact" > "$tmp/rc-refs" +normalized_refs "$artifact" > "$tmp/artifact-refs" +if ! cmp -s "$tmp/rc-refs" "$tmp/artifact-refs"; then + echo "::error::promotion changed the container repository/digest set" >&2 + diff -u "$tmp/rc-refs" "$tmp/artifact-refs" >&2 || true + exit 1 +fi + +echo "Verified ${source_url}@${source_ref#digest=}: stable tags, identical package tree, and the rc artifact's container digests." From 7df35017b550df1b148860d54f79e2faa3bd1d3a Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Mon, 17 Aug 2026 14:54:01 +0500 Subject: [PATCH 44/51] ci(release): run the full e2e suite on rc tag pushes from release-1.6 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Workflows run from the ref they fire on, so a tag pushed on this branch executes THIS tags.yaml, not main's. main grew an rc-e2e job on 2026-07-24; release-1.6 never did. The v1.6.1-rc.1 push therefore produced only Prepare Release, a skipped changelog and skipped website docs — no E2E Release Candidate job at all. Promote RC looks for its evidence by exact job name, found none, and v1.6.1 was promoted with the gate bypassed. That is the hole this closes. Add e2e-tag.yaml byte-identical to main and call it from a new rc-e2e job. The suite step works as written here because this branch already carries hack/e2e-chainsaw/ and a test-chainsaw target, and the sandbox image already installs chainsaw 0.2.15. Two things make the wiring load-bearing rather than cosmetic: * the called job's name must stay "E2E ${{ inputs.tag }} (full suite)". Promote RC composes the string it searches for from the rc tag, and GitHub renders a called job as " / ", which the gate's suffix matcher accepts. Rename either side and the gate fails closed. * the caller must grant contents: read AND checks: write. A caller's permissions are the ceiling for every job in the called workflow and GitHub validates that ceiling statically, at run creation, before any `if:` is evaluated. e2e-tag.yaml's e2e job declares checks: write for a dispatch-only breakpoint workflow_call can never reach, so omitting the grant would fail the entire tags.yaml run for EVERY tag push, rc and stable alike, building nothing. prepare-release already publishes the rc as a non-draft prerelease with the nocloud-amd64.raw.xz asset and pushes the release-X.Y.Z-rc.N staging branch before rc-e2e can start, which is exactly what e2e-tag.yaml's resolve job requires. Signed-off-by: Myasnikov Daniil --- .github/workflows/e2e-tag.yaml | 374 +++++++++++++++++++++++++++++++++ .github/workflows/tags.yaml | 29 +++ 2 files changed, 403 insertions(+) create mode 100644 .github/workflows/e2e-tag.yaml diff --git a/.github/workflows/e2e-tag.yaml b/.github/workflows/e2e-tag.yaml new file mode 100644 index 0000000000..d70b62be5e --- /dev/null +++ b/.github/workflows/e2e-tag.yaml @@ -0,0 +1,374 @@ +name: E2E Release Tag + +# Validate the installable release closure for an existing version tag. +# +# Every RC must pass this lane after tags.yaml has published the prerelease and +# pushed its release-X.Y.Z-rc.N digest-pinned staging branch — tags.yaml's +# rc-e2e job calls this workflow as that mandatory post-cut check. The +# workflow_dispatch trigger is the manual validation button for published +# alpha, beta, RC, and stable releases. +# +# This lane validates published tags only; draft and promote-time validation are +# explicitly out of scope because release checks run at RC publication time. +# Prerelease tags use the matching digest-pinned staging branch, while stable +# tags use their pinned merge tree directly. In both cases the nocloud disk +# comes from the published release identified by the same tag. + +on: + workflow_call: + inputs: + tag: + description: "Release tag to validate, e.g. v1.7.0-rc.1" + required: true + type: string + workflow_dispatch: + inputs: + tag: + description: "Release tag to validate, e.g. v1.7.0-rc.1" + required: true + type: string + debug: + description: "Open a maintainer SSH breakpoint if E2E fails" + required: false + default: false + type: boolean + +concurrency: + group: e2e-tag-${{ inputs.tag }} + cancel-in-progress: false + +# Listing published releases, reading Git refs, downloading a release asset, and +# checking out the pinned tree require only repository contents read access. +# Draft releases are not part of this lane's contract. The public GHCR images +# referenced by that tree need no package token. +# +# CALLERS MUST GRANT `contents: read` AND `checks: write`. A caller's permissions +# are the ceiling for every job here, and GitHub validates that ceiling when it +# creates the run, before any `if:` is evaluated — so the `checks: write` the e2e +# job below declares for its dispatch-only breakpoint has to be granted even by a +# caller whose invocation can never reach that step. Granting less does not +# degrade the breakpoint; it fails the caller's entire workflow run at startup. +permissions: + contents: read + +jobs: + resolve: + name: Resolve release assets and tree + runs-on: ubuntu-latest + timeout-minutes: 10 + outputs: + disk_id: ${{ steps.resolve.outputs.disk_id }} + tree_ref: ${{ steps.resolve.outputs.tree_ref }} + steps: + - name: Validate tag and resolve release + id: resolve + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7 + env: + TAG: ${{ inputs.tag }} + with: + script: | + const tag = process.env.TAG; + const pattern = /^v[0-9]+\.[0-9]+\.[0-9]+(-(alpha|beta|rc)\.[0-9]+)?$/; + if (!pattern.test(tag)) { + core.setFailed(`Tag '${tag}' must match vX.Y.Z, vX.Y.Z-alpha.N, vX.Y.Z-beta.N, or vX.Y.Z-rc.N`); + return; + } + + const isPrerelease = /-(alpha|beta|rc)\.[0-9]+$/.test(tag); + const releases = await github.paginate(github.rest.repos.listReleases, { + owner: context.repo.owner, + repo: context.repo.repo, + per_page: 100 + }); + const release = releases.find(candidate => candidate.tag_name === tag); + if (!release) { + core.setFailed(`GitHub release '${tag}' not found`); + return; + } + + if (release.draft) { + core.setFailed(`Release for tag '${tag}' is still a draft — this lane validates published tags only; RC validation runs after tags.yaml publishes the prerelease`); + return; + } else if (isPrerelease && !release.prerelease) { + core.setFailed(`Release '${tag}' must be published as a prerelease`); + return; + } else if (!isPrerelease && release.prerelease) { + core.setFailed(`Release '${tag}' must be published as a stable release`); + return; + } + + const disk = release.assets.find(asset => asset.name === 'nocloud-amd64.raw.xz'); + if (!disk) { + core.setFailed(`Required asset 'nocloud-amd64.raw.xz' is missing from release '${tag}'`); + return; + } + + const treeRef = isPrerelease ? `release-${tag.slice(1)}` : tag; + const gitRef = isPrerelease ? `heads/${treeRef}` : `tags/${treeRef}`; + try { + await github.rest.git.getRef({ + owner: context.repo.owner, + repo: context.repo.repo, + ref: gitRef + }); + } catch (error) { + if (error.status === 404 && isPrerelease) { + core.setFailed(`Expected staging branch '${treeRef}' for prerelease tag '${tag}' was not found`); + return; + } + if (error.status === 404) { + core.setFailed(`Expected stable tag ref '${tag}' was not found`); + return; + } + throw error; + } + + core.setOutput('disk_id', String(disk.id)); + core.setOutput('tree_ref', treeRef); + core.info(`Resolved ${tag}: tree '${treeRef}', disk asset ${disk.id}`); + + e2e: + name: E2E ${{ inputs.tag }} (full suite) + needs: resolve + # The sandbox boots three 8-vCPU / 24-GiB QEMU guests. This runner leaves + # enough host headroom for QEMU, networking, containerd, and the runner + # agent while the full platform installs across all three guests. + runs-on: oracle-vm-32cpu-128gb-x86-64 + timeout-minutes: 180 + permissions: + contents: read + checks: write # Breakpoint action updates its dedicated "Breakpoint Open" Check Run. + steps: + # The resolved ref is the digest-pinned staging branch for a prerelease, + # or the stable tag's pinned merge tree. + - name: Checkout resolved release tree + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + ref: ${{ needs.resolve.outputs.tree_ref }} + fetch-depth: 1 + persist-credentials: false + + - name: Download disk asset from release + env: + DISK_ID: ${{ needs.resolve.outputs.disk_id }} + GH_TOKEN: ${{ github.token }} + run: | + mkdir -p _out/assets + curl --fail-with-body --silent --show-error --location \ + -H "Authorization: Bearer ${GH_TOKEN}" \ + -H "Accept: application/octet-stream" \ + -o _out/assets/nocloud-amd64.raw.xz \ + "https://api.github.com/repos/${GITHUB_REPOSITORY}/releases/assets/${DISK_ID}" + + - name: Set sandbox ID + env: + TAG: ${{ inputs.tag }} + run: echo "SANDBOX_NAME=cozy-e2e-sandbox-$(echo "${GITHUB_REPOSITORY}:${GITHUB_WORKFLOW}:${TAG}" | sha256sum | cut -c1-10)" >> "$GITHUB_ENV" + + - name: Prepare workspace + run: | + rm -rf "/tmp/$SANDBOX_NAME" + cp -r "${{ github.workspace }}" "/tmp/$SANDBOX_NAME" + + # This is the only retried stage: Talos image unpacking, QEMU startup, and + # sandbox networking are pure runner infrastructure, not product logic. + - name: Prepare environment + run: | + cd "/tmp/$SANDBOX_NAME" + attempt=0 + until make SANDBOX_NAME=$SANDBOX_NAME prepare-env; do + attempt=$((attempt + 1)) + if [ $attempt -ge 3 ]; then + echo "Attempt $attempt failed, exiting..." + exit 1 + fi + echo "Attempt $attempt failed, retrying..." + done + echo "Prepare environment completed after $((attempt + 1)) attempts" + + - name: Install Cozystack into sandbox + run: | + cd "/tmp/$SANDBOX_NAME" + if ! make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME install-cozystack; then + echo "❌ Install failed (no retry — see diagnostics below)" + echo "::group::Diagnostics: HelmRelease status" + docker exec "$SANDBOX_NAME" sh -c 'kubectl get hr -A -o wide 2>&1 | tail -100' || true + echo "::endgroup::" + echo "::group::Diagnostics: not-Ready HelmReleases (describe)" + docker exec "$SANDBOX_NAME" sh -c 'kubectl get hr -A --no-headers 2>/dev/null | awk "\$4 != \"True\" {print \$1\" \"\$2}"' \ + | while read -r ns name; do + echo "--- $ns/$name ---" + docker exec "$SANDBOX_NAME" sh -c "kubectl describe hr '$name' -n '$ns' 2>&1 | tail -50" || true + done + echo "::endgroup::" + echo "::group::Diagnostics: recent events" + docker exec "$SANDBOX_NAME" sh -c 'kubectl get events -A --sort-by=.lastTimestamp 2>&1 | tail -50' || true + echo "::endgroup::" + exit 1 + fi + + - name: Run OpenAPI tests + run: | + cd "/tmp/$SANDBOX_NAME" + make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME test-openapi + + # A tag has no PR diff to scope with Test Impact Analysis. Empty + # CHAINSAW_SUITES is the testing Makefile's full-suite mode. + - name: Run E2E tests (full suite) + id: e2e_tests + run: | + cd "/tmp/$SANDBOX_NAME" + if ! make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME test-chainsaw CHAINSAW_SUITES=""; then + echo "❌ Chainsaw E2E failed (see the assertion diffs above and diagnostics below)" + echo "::group::Diagnostics: HelmRelease status" + docker exec "$SANDBOX_NAME" sh -c 'kubectl get hr -A -o wide 2>&1 | tail -50' || true + echo "::endgroup::" + echo "::group::Diagnostics: recent events" + docker exec "$SANDBOX_NAME" sh -c 'kubectl get events -A --sort-by=.lastTimestamp 2>&1 | tail -30' || true + echo "::endgroup::" + # COSI CRDs have no printer columns, so capture readiness fields at + # failure time before tests and report collection can move state. + echo "::group::Diagnostics: COSI bucket state" + docker exec "$SANDBOX_NAME" sh -c ' + kubectl get bucketclaims.objectstorage.k8s.io -A -o custom-columns="NAMESPACE:.metadata.namespace,NAME:.metadata.name,READY:.status.bucketReady,BUCKET:.status.bucketName" 2>&1 + kubectl get buckets.objectstorage.k8s.io -o custom-columns="NAME:.metadata.name,READY:.status.bucketReady,ID:.status.bucketID,CLAIMNS:.spec.bucketClaim.namespace" 2>&1 + kubectl get bucketaccesses.objectstorage.k8s.io -A -o custom-columns="NAMESPACE:.metadata.namespace,NAME:.metadata.name,GRANTED:.status.accessGranted,CLAIM:.spec.bucketClaimName" 2>&1 + ' || true + echo "::endgroup::" + exit 1 + fi + echo "✅ All E2E tests passed" + + - name: Collect chainsaw report + if: always() + run: | + cd "/tmp/$SANDBOX_NAME" + mkdir -p _out + docker cp "$SANDBOX_NAME":/workspace/hack/e2e-chainsaw/chainsaw-report.xml \ + _out/chainsaw-report.xml || true + + - name: Upload chainsaw report + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: chainsaw-report-${{ inputs.tag }} + path: /tmp/${{ env.SANDBOX_NAME }}/_out/chainsaw-report.xml + if-no-files-found: ignore + + # collect-report packages cozyreport plus per-failed-test crust-gather + # snapshots captured before Chainsaw cleanup. + # + # A ceiling of its own, not the job's 180 minutes. `Upload cozyreport.tgz` + # is the next step and the collector writes its tarball at the very end of + # its run, so a read that never returns spends what is left of the job + # budget, the job is cancelled, and the upload never runs -- the artifact + # is lost rather than truncated. 30 minutes is measured against the step: + # it takes 3-42s across sampled runs, failed e2e jobs included. + # `continue-on-error` because the ceiling must not change what the job + # reports: `|| true` says collecting diagnostics is never fatal, and a step + # killed at its ceiling never reaches it. + - name: Collect report + id: collect_report + if: always() + timeout-minutes: 30 + continue-on-error: true + run: | + cd "/tmp/$SANDBOX_NAME" + make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME collect-report || true + + # `continue-on-error` keeps the overrun out of the job's result, which also + # keeps it out of everyone's view: a step that ended at its ceiling renders + # green and the only trace is in the raw log. This branch's own standard is + # that a bound which fires is recorded, and that has to hold for the outer + # bound too. + # + # The wording says what `outcome` establishes and no more. A step can end + # non-zero without any ceiling firing -- in e2e-tag.yaml the body opens with + # a `cd` into a sandbox tree an earlier step may never have created, and the + # default shell is `bash -e` -- so naming the timeout as the cause would be + # the same false-mechanism claim the collector refuses to make for a 137. + - name: Report collection overran its ceiling + if: always() && steps.collect_report.outcome == 'failure' + run: | + echo "::warning title=cozyreport::Collect report ended non-zero -- its timeout-minutes fired, or the step failed before the collector ran; cozyreport.tgz is missing or partial for this run" + + - name: Upload cozyreport.tgz + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: cozyreport-${{ inputs.tag }} + path: /tmp/${{ env.SANDBOX_NAME }}/_out/cozyreport.tgz + + - name: Collect images list + if: always() + run: | + cd "/tmp/$SANDBOX_NAME" + make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME collect-images || true + + - name: Upload image list + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: image-list-${{ inputs.tag }} + path: /tmp/${{ env.SANDBOX_NAME }}/_out/images.txt + + # ▸ Open an SSH breakpoint to the failing sandbox so maintainers can attach, + # inspect Talos/Cozystack state and resume with `breakpoint resume`. + # + # Gated by the workflow_dispatch `debug` boolean and the configured + # BREAKPOINT_ENDPOINT repository variable. workflow_call runs never get + # a breakpoint. authorized-users is kept as a second defense layer at + # the breakpoint level. + # + # Uses cozystack/breakpoint-action (fork of namespacelabs/breakpoint-action) + # pinned by SHA. The fork adds: pause-idle mode (initial grace period for + # the first SSH connection, idle-aware exit afterwards), endpoint output + # and ::notice:: annotation, and a dedicated Check Run "Breakpoint Open" + # that carries the SSH endpoint in output.summary while the breakpoint is + # paused (conclusion=failure → standard ✗ in checks; updated to + # conclusion=success when the breakpoint exits). + - name: Breakpoint on E2E failure + if: | + failure() && + github.event_name == 'workflow_dispatch' && + inputs.debug == true && + vars.BREAKPOINT_ENDPOINT != '' + # cozystack/breakpoint-action v2-cozy.1 + # mode: pause-idle defaults: grace-period=20m, idle-timeout=10m + uses: cozystack/breakpoint-action@a6f3a6f87be398ad63b6577351e3398e53f578e4 + with: + mode: pause-idle + endpoint: ${{ vars.BREAKPOINT_ENDPOINT }} + authorized-users: androndo, Arsolitt, IvanHunters, kvaps, lexfrei, lllamnyp, mattia-eleuteri, matthieu-robin, myasnikovdaniil, sircthulhu, tym83 + check-run-name: "Breakpoint Open" + github-token: ${{ github.token }} + check-run-summary-template: | + ## 🔴 SSH breakpoint open — paused for debug + + ``` + {endpoint} + ``` + + Enter the e2e sandbox after SSH: + ``` + docker exec -ti $(docker ps --filter name=cozy-e2e-sandbox -q | head -1) bash + export KUBECONFIG=/workspace/kubeconfig + ``` + + Resume from inside: `breakpoint resume`. Otherwise the breakpoint + exits 10 minutes after the last SSH session disconnects. + + - name: Tear down sandbox + if: always() + run: make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME delete || true + + - name: Remove workspace + if: always() + run: rm -rf "/tmp/$SANDBOX_NAME" + + - name: Summarize outcome + if: always() + env: + OUTCOME: ${{ job.status }} + TAG: ${{ inputs.tag }} + run: printf '| E2E tag `%s` | %s |\n' "$TAG" "$OUTCOME" >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/tags.yaml b/.github/workflows/tags.yaml index 7b1062e254..517d5b0013 100644 --- a/.github/workflows/tags.yaml +++ b/.github/workflows/tags.yaml @@ -311,6 +311,35 @@ jobs: # not exists" step was gated on `is_stable && release_exists==false` — # unreachable now that a hand-pushed stable tag is rejected above. + # Mandatory only for vX.Y.Z-rc.N tags. Alpha/beta tags deliberately skip this + # job and use e2e-tag.yaml's manual dispatch when validation is wanted. + # + # Deliberately not gated on release_exists: validating an already-published rc + # again is an idempotent re-tag-run check, while prepare-release still provides + # the ordering guarantee for a fresh rc. Its "Publish rc release" and "Create + # release branch" steps finish before this reusable workflow can start. + rc-e2e: + name: E2E Release Candidate + needs: prepare-release + if: needs.prepare-release.result == 'success' && contains(github.ref_name, '-rc.') + # A caller's permissions are the ceiling for every job in the called + # workflow, and GitHub validates that ceiling STATICALLY, when it creates the + # run — before this job's `if:` is evaluated, let alone before a step runs. + # e2e-tag.yaml's e2e job declares `checks: write` for its + # workflow_dispatch-only breakpoint, so even though workflow_call can never + # take that branch, omitting the grant here fails the whole tags.yaml run with + # "requesting 'checks: write', but is only allowed 'checks: none'". Not just + # rc cuts: the `if:` cannot gate a startup failure, so EVERY tag push — rc and + # stable, including the one finalize pushes to publish a release — would build + # nothing at all. hack/promote-gate-contract.bats pins the pair, because no CI + # lane exercises this file (it runs only on tag pushes). + permissions: + contents: read + checks: write + uses: ./.github/workflows/e2e-tag.yaml + with: + tag: ${{ github.ref_name }} + generate-changelog: name: Generate Changelog # git + node + copilot CLI only — no docker, no repo toolchain: GitHub-hosted. From 5673249d475b91801264be42cb0588610a47920c Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Mon, 17 Aug 2026 14:55:12 +0500 Subject: [PATCH 45/51] ci(release): validate the tag-time changelog and port it from the tag MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since the promote PR is based on release-1.6, the changelog it carries merges onto the maintenance line and never reaches main. The backstop job in tags.yaml only asks whether the file is on origin/main, so it sees "absent", spends an AI run, and produces text nobody reviewed — which update-releasenotes.yaml then pushes into the published release body, overwriting the reviewed notes the release actually shipped with. Ask the second question too. `at_tag` is whether the changelog is present in this checkout, which is the tag commit and therefore the promote merge commit. When it is, port that reviewed file to main verbatim and skip generation entirely; the AI now runs only when the changelog is genuinely absent on both sides. Add hack/validate-changelog.sh, byte-identical to main, and gate the commit on it. A ported file came from the tag and a generated one came from a step that is deliberately allowed to fail, so neither had been checked: the old guard only asked whether the file was non-empty, which a truncated Copilot stream satisfies. The validator asserts the H1, the release-link comment, the trailing compare link and at least one section, all carrying the expected version, so a fragment or a changelog generated against the wrong tag cannot reach a release body. Verified against this branch's own shipped changelogs: v1.6.1, v1.5.2 and v1.5.1 all pass, and v1.6.1 checked against 1.6.2 fails as it should. This takes effect for tags cut after it lands — tags.yaml and the script it calls both come from the tag's own tree. Signed-off-by: Myasnikov Daniil --- .github/workflows/tags.yaml | 57 +++++++++++++++++++++++++++------- hack/validate-changelog.sh | 62 +++++++++++++++++++++++++++++++++++++ 2 files changed, 108 insertions(+), 11 deletions(-) create mode 100755 hack/validate-changelog.sh diff --git a/.github/workflows/tags.yaml b/.github/workflows/tags.yaml index 517d5b0013..7f0d78e8c6 100644 --- a/.github/workflows/tags.yaml +++ b/.github/workflows/tags.yaml @@ -409,11 +409,22 @@ jobs: fetch-tags: true token: ${{ steps.app-token.outputs.token }} - # Check whether the changelog already exists on origin/main. - # The previous step checked out the tag commit, which never has the - # changelog (the changelog PR merges to main AFTER the tag is cut). - # Looking at origin/main is what the gate is supposed to do — skip - # generation on workflow reruns when the changelog has already merged. + # Two independent questions, and the answers select between three paths. + # + # `exists` — is the changelog already on origin/main? Then this whole job + # is a no-op. On a maintenance line that is the EXCEPTION: the promote PR + # merges into release-1.6, so the file it carried never touched main, and + # this is true only if something else put it there. + # + # `at_tag` — is it present in THIS checkout (the tag commit)? Under the + # promote flow it normally is, because the promote PR carried it onto + # release-1.6 and the stable tag sits on that merge commit. So this is the + # usual state on arrival: `exists` false, `at_tag` true, a reviewed and + # already-published changelog sitting right here — port it rather than + # regenerate. + # + # Only when both are false is there genuinely no changelog to publish, and + # only then does the AI run. - name: Check if changelog already exists id: check_changelog env: @@ -428,19 +439,36 @@ jobs: echo "exists=false" >> $GITHUB_OUTPUT echo "Changelog file $CHANGELOG_FILE does not exist on origin/main" fi + # Is the changelog present at the TAG (this checkout)? Under the promote + # flow it normally is: promote-rc.yaml commits it onto release-X.Y.Z and + # it reaches the tag via the merge commit. That PR targets the maintenance + # line, so the file lands on release-1.6 and never reaches main — + # `exists` above is false even though a reviewed, already-PUBLISHED + # changelog exists right here. + # + # Regenerating in that case would spend a second AI run and, once the + # backstop PR merged, update-releasenotes.yaml would overwrite the + # published release body with text nobody approved. Port the reviewed + # file verbatim instead; the AI is only for the genuinely-absent case. + if [ -s "$CHANGELOG_FILE" ]; then + echo "at_tag=true" >> $GITHUB_OUTPUT + echo "Changelog present at the tag — will port it to main verbatim, no regeneration." + else + echo "at_tag=false" >> $GITHUB_OUTPUT + fi - name: Setup Node.js - if: steps.check_changelog.outputs.exists == 'false' + if: steps.check_changelog.outputs.exists == 'false' && steps.check_changelog.outputs.at_tag == 'false' uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 with: node-version: 22 - name: Install GitHub Copilot CLI - if: steps.check_changelog.outputs.exists == 'false' + if: steps.check_changelog.outputs.exists == 'false' && steps.check_changelog.outputs.at_tag == 'false' run: npm i -g @github/copilot - name: Generate changelog using AI - if: steps.check_changelog.outputs.exists == 'false' + if: steps.check_changelog.outputs.exists == 'false' && steps.check_changelog.outputs.at_tag == 'false' timeout-minutes: 30 env: COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} @@ -462,12 +490,19 @@ jobs: CHANGELOG_FILE="docs/changelogs/v${VERSION}.md" CHANGELOG_BRANCH="changelog-v${VERSION}" + # Two ways the file can be here: ported verbatim from the tag commit + # (at_tag=true — the reviewed, already-published changelog reaching main + # for the first time), or freshly generated above. Either is valid. if [ ! -f "$CHANGELOG_FILE" ]; then - echo "::error::Changelog file $CHANGELOG_FILE was not produced by the Generate changelog using AI step" + echo "::error::Changelog file $CHANGELOG_FILE is absent — it was neither present at the tag nor produced by the Generate changelog using AI step" exit 1 fi - if [ ! -s "$CHANGELOG_FILE" ]; then - echo "::error::Changelog file $CHANGELOG_FILE is empty" + # A ported file came from the tag and a generated one from a step allowed + # to fail, so neither has been checked here. update-releasenotes.yaml + # will push whatever this PR merges into the published release body, so a + # fragment must not get that far. + if ! REASON="$(hack/validate-changelog.sh "$CHANGELOG_FILE" "${VERSION}" 2>&1)"; then + echo "::error::Changelog file $CHANGELOG_FILE is not usable: ${REASON}" exit 1 fi diff --git a/hack/validate-changelog.sh b/hack/validate-changelog.sh new file mode 100755 index 0000000000..117391be1c --- /dev/null +++ b/hack/validate-changelog.sh @@ -0,0 +1,62 @@ +#!/bin/sh +# Usage: hack/validate-changelog.sh +# +# Decides whether a changelog file is publishable as a GitHub Release body. +# +# promote-rc.yaml lets the AI generation step fail (a Copilot outage must never +# block a release), which means a run can die mid-stream and leave a fragment +# behind. Without this check that fragment would be committed, advertised as a +# valid changelog in the promote PR body, and published verbatim by finalize as +# the release notes — worse than having no changelog, because nobody is told. +# +# The assertions are deliberately narrow: only invariants every shipped changelog +# in docs/changelogs/ actually satisfies. Two forms exist and both are valid — +# minor releases use `# Cozystack vX.Y.Z`, patch releases use `# vX.Y.Z (date)` — +# so the header check accepts either. There is deliberately NO line-count floor: +# v1.5.1 is a complete, shipped, 19-line patch changelog, and a floor set above +# it would reject good output and mislabel it as a generation failure. +# +# The version is checked into the header, the release-link comment and the +# compare link, so a changelog generated for the wrong version (an rc string, or +# a stale version carried over from a retry) is caught rather than published. +# +# Exit 0 = publishable. Exit 1 = not publishable, with the reason on stderr. + +set -eu + +CL="${1:?usage: validate-changelog.sh }" +VERSION="${2:?usage: validate-changelog.sh }" + +fail() { + echo "$1" >&2 + exit 1 +} + +[ -f "$CL" ] || fail "file absent: $CL" +[ -s "$CL" ] || fail "file empty: $CL" +grep -q '[^[:space:]]' "$CL" || fail "file is only whitespace: $CL" + +# Both header conventions. \b is not portable in POSIX grep -E, so anchor on +# "end of line or a non-version character" instead: that keeps v1.5.1 from +# satisfying a check for v1.5.11. +grep -qE "^# (Cozystack )?v${VERSION}([^0-9.]|\$)" "$CL" \ + || fail "missing an H1 for v${VERSION}: expected '# Cozystack v${VERSION}' (minor) or '# v${VERSION} ()' (patch)" + +# Leading HTML comment pointing at the release. Present in every shipped +# changelog and the first thing a truncated write would still have, so on its +# own it is weak — it is the version check here that earns its keep. +grep -qF "releases/tag/v${VERSION}" "$CL" \ + || fail "missing the release-link comment for v${VERSION}" + +# A truncated stream loses the tail, so assert something that only exists at the +# end. The compare link must also name this version, catching a changelog +# generated against the rc tag or a stale version. +grep -qE "compare/.*\.\.\.v${VERSION}([^0-9.]|\$)" "$CL" \ + || fail "missing the 'Full Changelog' compare link ending in v${VERSION} (likely truncated, or generated for the wrong version)" + +# At least one section. Distinguishes a real changelog from a header plus a +# stub line, without assuming which sections a given release happens to have. +grep -qE '^## ' "$CL" \ + || fail "no '## ' sections — not a complete changelog" + +echo "OK: $CL is publishable as the v${VERSION} release body ($(wc -l < "$CL" | tr -d ' ') lines)" From 672a32b9dfd463d5e33eab9dcd53d5ede1d7d3cd Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Mon, 17 Aug 2026 14:58:38 +0500 Subject: [PATCH 46/51] test(release): pin the promotion contract this branch has to satisfy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit hack/verify-promoted-packages_test.bats is byte-identical to main: 13 tests driving the verifier against a mocked `flux`, so they need no registry. They cover the fail-open shapes that matter most — an unreadable file during the rc-reference scan, a collector that returns nothing, a digest that moved while candidate and merge tree still match. hack/promote-gate-contract.bats is NOT main's copy, and the difference is deliberate. main's version pins a `parse` job, a `skip_e2e_gate` input, promote-time changelog and website-docs jobs, and the `labeled` pull_request trigger they depend on — none of which exist here, because `Promote RC` is a workflow_dispatch and runs from the ref it is dispatched from, which is main. Run verbatim against this branch it fails on its first test and the runner aborts the suite. So this pins the half release-1.6 actually owns — the producer side: * an rc tag push calls e2e-tag.yaml with the pushed ref, only for -rc. tags, and grants the static permission ceiling the called workflow declares; * the called job's name is the exact literal main's gate composes and searches for, asserted with grep -F because `${{` as a basic regular expression matches nothing and would make the pin pass by finding its own subject absent; * the suite it runs is the full one (empty CHAINSAW_SUITES), with the Makefile target and the sandbox chainsaw binary it needs; * all five files promote-rc.yaml's preflight reads off the target base are present, plus the two libraries the verifier sources by relative path; * verify-release-candidate is keyed on author and head branch and carries NO label condition, so restoring main's from muscle memory switches the job off here and fails this test instead; * the candidate verification precedes the stable tag, the submodule tag, the release publish, the retag and the chart publish, and follows the toolchain, the login and the tooling checkout it needs; * the tag-time changelog is validated and ported, not regenerated. Every pin was mutation-checked: breaking the thing it describes turns it red. Signed-off-by: Myasnikov Daniil --- hack/promote-gate-contract.bats | 269 +++++++++++++++++ hack/verify-promoted-packages_test.bats | 367 ++++++++++++++++++++++++ 2 files changed, 636 insertions(+) create mode 100644 hack/promote-gate-contract.bats create mode 100644 hack/verify-promoted-packages_test.bats diff --git a/hack/promote-gate-contract.bats b/hack/promote-gate-contract.bats new file mode 100644 index 0000000000..c7e8b47454 --- /dev/null +++ b/hack/promote-gate-contract.bats @@ -0,0 +1,269 @@ +#!/usr/bin/env bats + +# Contract for the halves of the rc-E2E promotion gate that live on THIS branch. +# These tests intentionally pin executable/structural workflow lines, not prose: +# a commented-out gate, job, or step must never satisfy the contract. +# +# Scope, and why it is narrower than main's copy of this file. +# +# The gate itself is not here. `Promote RC` is a workflow_dispatch, so it runs +# from the ref the maintainer dispatches — main — and it is main's promote-rc.yaml +# that composes the expected job name, walks the Actions API for evidence, and +# refuses to promote without it. This branch's promote-rc.yaml predates all of +# that and is deliberately left alone: porting it would drag in a `parse` job, a +# `skip_e2e_gate` input, promote-time changelog and website-docs jobs, and the +# `labeled` pull_request trigger those depend on. main's copy of this file pins +# exactly those things, which is why it cannot be used verbatim here. +# +# What release-1.6 owns is the PRODUCER side of the same contract, and that is +# what these tests pin: +# +# * a tag push must actually run the full suite, under a job name whose +# rendered form is what the gate on main searches for. Workflows run from the +# ref they fire on, so only this file's tags.yaml can do that for an rc cut +# on this line; +# * the target base must carry the five files promote-rc.yaml's preflight +# demands before its first registry write, or the dispatch fails closed with +# "Target base 'release-1.6' lacks ''"; +# * the candidate verification in finalize must run before anything +# irreversible, because a guard that runs after the write-once stable tag +# exists cannot refuse it. +# +# No CI lane exercises tags.yaml (it runs only on tag pushes) and no lane +# exercises finalize's ordering, which is why these are pinned mechanically here. + +REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)" +PULL_REQUESTS="$REPO_ROOT/.github/workflows/pull-requests.yaml" +TAGS="$REPO_ROOT/.github/workflows/tags.yaml" +FINALIZE="$REPO_ROOT/.github/workflows/pull-requests-release.yaml" +E2E_TAG="$REPO_ROOT/.github/workflows/e2e-tag.yaml" + +job_block() { + awk -v job=" $1:" ' + $0 == job { inside = 1; next } + /^ [a-z0-9_-]+:$/ { inside = 0 } + inside' "$2" +} + +# Comment-stripping filter for the pins below. POSIX `grep` only: the unit-test +# runner has no ripgrep, and a missing filter used to be swallowed by `|| true`, +# silently reducing every pin to "0 matches" instead of failing on the real +# cause. grep exits 1 when nothing is selected (legitimate for an empty block) +# and 2 on an actual error. Do not read the `rc` check below as what catches +# that: code_lines is never the last stage of a pipeline in this file, so +# ordinary (non-pipefail) pipe semantics discard whatever it itself returns, +# and `hack/cozytest.sh`'s translator appends `return 0` to every line that is +# exactly `}`, code_lines' closing brace included, making its own exit status +# even less trustworthy as a signal. What catches a real grep error is the +# OUTPUT, and only for one class of pin: the error empties code_lines' stream, +# so a pin that requires something to be present in that stream fails on it. A +# pin demanding an absence stays blind, because an exact count of zero is what +# an emptied stream produces anyway, and what the pin was written to assert. +code_lines() { + local rc=0 + grep -v '^[[:space:]]*#' || rc=$? + [ "$rc" -le 1 ] +} + +# First line number of a named step inside an already-extracted job block, in +# code_lines' comment-stripped view. Top-level rather than nested inside a test +# so its closing brace is the only one the runner's translator has to rewrite. +step_line() { + printf '%s\n' "$2" | code_lines | grep -nF " - name: $1" \ + | awk -F: 'NR == 1 { print $1 }' +} + +# ── the producer half of the rc-E2E gate ───────────────────────────────────── + +@test "an rc tag push calls the full-suite e2e workflow" { + [ -f "$E2E_TAG" ] + + rc_e2e="$(job_block rc-e2e "$TAGS")" + [ -n "$rc_e2e" ] + + printf '%s\n' "$rc_e2e" | code_lines | grep -qF 'uses: ./.github/workflows/e2e-tag.yaml' + + # The tag under test must be the pushed ref. The gate correlates evidence by a + # job name that embeds the exact tag, so passing anything else here would + # produce a green run that no promotion can ever match to its rc. + printf '%s\n' "$rc_e2e" | code_lines | grep -qF 'tag: ${{ github.ref_name }}' + + # rc tags only. A stable tag push must not re-run the suite, and alpha/beta use + # e2e-tag.yaml's manual dispatch. + printf '%s\n' "$rc_e2e" | code_lines | grep -qF "contains(github.ref_name, '-rc.')" +} + +@test "rc-e2e grants the ceiling e2e-tag.yaml's jobs declare" { + rc_e2e="$(job_block rc-e2e "$TAGS")" + [ -n "$rc_e2e" ] + + # Every permission any job in the called workflow declares must be granted by + # the caller. GitHub validates that ceiling STATICALLY, when it creates the run, + # before this job's `if:` is evaluated — so a missing grant does not skip the + # rc lane, it fails the whole tags.yaml run for EVERY tag push, rc and stable + # alike, building nothing at all. Assert the pair in both files, so removing + # either side surfaces here instead of at the next release. + e2e_job="$(job_block e2e "$E2E_TAG")" + [ -n "$e2e_job" ] + printf '%s\n' "$e2e_job" | code_lines | grep -qF 'checks: write' + + printf '%s\n' "$rc_e2e" | code_lines | grep -qF 'contents: read' + printf '%s\n' "$rc_e2e" | code_lines | grep -qF 'checks: write' +} + +@test "the e2e job name is the exact string the promote gate searches for" { + # main's promote-rc.yaml builds `E2E ${rcTag} (full suite)` and matches a job + # whose name either equals it or ends in " / " plus it (the second form is how + # GitHub renders a called workflow's job: " / "). This + # branch supplies the called half, so the literal below is the contract — a + # rename on either side makes the gate fail closed and blocks promotion until + # someone notices. Cheap to pin, expensive to debug. + # + # -F is not optional: the pattern contains `${{`, and as a basic regular + # expression that silently matches nothing rather than erroring, which would + # turn this pin into a test that passes by finding its own subject absent. + count="$(code_lines < "$E2E_TAG" | grep -cF 'name: E2E ${{ inputs.tag }} (full suite)' || true)" + [ "${count:-0}" -eq 1 ] +} + +@test "the tag-push e2e lane runs the full chainsaw suite it claims to" { + e2e_job="$(job_block e2e "$E2E_TAG")" + [ -n "$e2e_job" ] + + # Empty CHAINSAW_SUITES is the testing Makefile's full-suite mode. A tag has no + # PR diff to scope with Test Impact Analysis, so a narrowed suite here would + # make the gate's evidence weaker than its name advertises. + printf '%s\n' "$e2e_job" | code_lines | grep -qF 'test-chainsaw CHAINSAW_SUITES=""' + + # The target and the sandbox binary the step above depends on. + grep -qE '^test-chainsaw' "$REPO_ROOT/packages/core/testing/Makefile" + grep -qF 'CHAINSAW_VERSION' "$REPO_ROOT/packages/core/testing/images/e2e-sandbox/Dockerfile" +} + +# ── the candidate-aware promotion pipeline this base must carry ────────────── + +@test "the base carries every file promote-rc.yaml's preflight demands" { + # promote-rc.yaml resolves the target base (release-X.Y when it exists) and + # reads each path below off it, refusing to promote unless the content contains + # the paired marker. It fails closed BEFORE the first registry write, so a base + # missing any of these cannot be promoted at all — which is exactly the state + # this branch was in. Keep the list in lockstep with the requiredBaseFiles array + # in main's promote-rc.yaml. + grep -qF 'verify-release-candidate:' "$PULL_REQUESTS" + grep -qF 'Verify stable packages candidate' "$FINALIZE" + grep -qF 'EXPECTED_PACKAGES_REPOSITORY' "$REPO_ROOT/hack/verify-promoted-packages.sh" + grep -qF 'collect_image_refs()' "$REPO_ROOT/hack/lib/image-refs.sh" + grep -qF 'PACKAGES_DIGEST_REF_PATTERN' "$REPO_ROOT/hack/lib/promoted-packages.sh" + + # The verifier sources both libraries by path relative to itself. A base + # carrying the script without them fails at the PR gate and again at finalize, + # after the candidate, the staging branch and the draft release already exist. + [ -x "$REPO_ROOT/hack/verify-promoted-packages.sh" ] + [ -f "$REPO_ROOT/hack/lib/promoted-packages.sh" ] + [ -f "$REPO_ROOT/hack/lib/image-refs.sh" ] +} + +@test "verify-release-candidate fires on the promote PR without needing a label" { + block="$(job_block verify-release-candidate "$PULL_REQUESTS")" + [ -n "$block" ] + + # This branch's on.pull_request.types has no `labeled` event, and the promote + # PR's `release` label is applied a moment AFTER the PR opens, so the `opened` + # payload carries no labels at all. A label-keyed guard would therefore never + # fire here. Pin the author + head-branch predicate that does, and pin the + # absence of the label dependency so nobody "restores" it from main and + # silently switches the job off. + printf '%s\n' "$block" | code_lines | grep -qF "github.event.pull_request.user.login == 'cozystack-ci[bot]'" + printf '%s\n' "$block" | code_lines | grep -qF "startsWith(github.head_ref, 'release-')" + + count="$(printf '%s\n' "$block" | code_lines | grep -cF "contains(github.event.pull_request.labels.*.name, 'release')" || true)" + [ "${count:-0}" -eq 0 ] + + # The verifier must come from the trusted base, not from the PR's own tree: a + # promote PR must not be able to weaken the guard that judges it. + printf '%s\n' "$block" | code_lines | grep -qF 'ref: ${{ github.event.pull_request.base.sha }}' + printf '%s\n' "$block" | code_lines | grep -qF '.release-tooling/hack/verify-promoted-packages.sh' +} + +@test "the candidate verification runs before every irreversible finalize step" { + block="$(job_block finalize "$FINALIZE")" + [ -n "$block" ] + + verify="$(step_line 'Verify stable packages candidate' "$block")" + [ -n "$verify" ] + + # Each of these creates or moves a name that cannot be taken back: the + # write-once stable git tag, the API submodule tag, the published release, the + # stable image tags (and :latest), and the stable installer chart. The + # verification is only a gate if it precedes all of them. + tag_step="$(step_line 'Create tag on merge commit (write-once)' "$block")" + [ -n "$tag_step" ] + [ "$verify" -lt "$tag_step" ] + + submodule_step="$(step_line 'Tag API submodule (write-once)' "$block")" + [ -n "$submodule_step" ] + [ "$verify" -lt "$submodule_step" ] + + publish_step="$(step_line 'Publish draft release' "$block")" + [ -n "$publish_step" ] + [ "$verify" -lt "$publish_step" ] + + retag_step="$(step_line 'Retag rc images to stable' "$block")" + [ -n "$retag_step" ] + [ "$verify" -lt "$retag_step" ] + + chart_step="$(step_line 'Publish stable cozy-installer chart' "$block")" + [ -n "$chart_step" ] + [ "$verify" -lt "$chart_step" ] + + # The toolchain and the registry login the verification needs must precede it + # too. They used to sit after "Publish draft release"; moving the verification + # up without them would make it fail on a missing `flux` instead of verifying. + toolchain_step="$(step_line 'Set up promotion toolchain (flux, skopeo, yq, helm)' "$block")" + [ -n "$toolchain_step" ] + [ "$toolchain_step" -lt "$verify" ] + + login_step="$(step_line 'Login to registry (GHCR)' "$block")" + [ -n "$login_step" ] + [ "$login_step" -lt "$verify" ] + + tooling_step="$(step_line 'Checkout release tooling from base' "$block")" + [ -n "$tooling_step" ] + [ "$tooling_step" -lt "$verify" ] +} + +@test "finalize checkout does not persist credentials so the app-token tag push triggers tags.yaml" { + block="$(job_block finalize "$FINALIZE")" + [ -n "$block" ] + checkout="$(printf '%s\n' "$block" | awk ' + /^ - name: Checkout repo$/ { inside = 1; next } + /^ - name: / { inside = 0 } + inside')" + [ -n "$checkout" ] + + # The one-line root-cause fix. Without persist-credentials:false the checkout + # persists GITHUB_TOKEN as http.extraheader, which silently defeats the app token + # each later `git remote set-url` injects onto the tag pushes — and a + # GITHUB_TOKEN-authenticated push creates no workflow run (anti-recursion), so + # tags.yaml's stable-tag backstops never fire (v1.6.0's tag never triggered it). + count="$(printf '%s\n' "$checkout" | code_lines | grep -cF 'persist-credentials: false' || true)" + [ "${count:-0}" -eq 1 ] +} + +# ── the tag-time changelog backstop ────────────────────────────────────────── + +@test "the tag-time changelog is validated and ported rather than regenerated" { + block="$(job_block generate-changelog "$TAGS")" + [ -n "$block" ] + + # The promote PR merges into release-1.6, so its reviewed changelog never + # reaches main and `exists` alone reads as "absent". at_tag is what stops a + # second AI run from overwriting an already-published release body. + printf '%s\n' "$block" | code_lines | grep -qF 'at_tag=true' + printf '%s\n' "$block" | code_lines | grep -qF "steps.check_changelog.outputs.at_tag == 'false'" + + # Neither a ported nor a generated file has been checked at this point, and the + # generating step is allowed to fail, so a truncated fragment must not pass. + printf '%s\n' "$block" | code_lines | grep -qF 'hack/validate-changelog.sh' + [ -x "$REPO_ROOT/hack/validate-changelog.sh" ] +} diff --git a/hack/verify-promoted-packages_test.bats b/hack/verify-promoted-packages_test.bats new file mode 100644 index 0000000000..1d75f89dbb --- /dev/null +++ b/hack/verify-promoted-packages_test.bats @@ -0,0 +1,367 @@ +#!/usr/bin/env bats +# Behavioural tests for the pre-publication packages artifact verification. +# Run with: hack/cozytest.sh hack/verify-promoted-packages_test.bats + +_test_workspace() { + if [ -n "${BATS_TEST_TMPDIR:-}" ]; then + printf '%s\n' "$BATS_TEST_TMPDIR" + else + printf '%s\n' "${tmp:?cozytest workspace is missing}" + fi +} + +_make_verify_fixture() { + t="$1" + mkdir -p "$t/bin" "$t/release/core/installer" "$t/release/system/dashboard" + mkdir -p "$t/artifact/core/installer" "$t/artifact/system/dashboard" + mkdir -p "$t/rc-artifact/core/installer" "$t/rc-artifact/system/dashboard" + cat > "$t/bin/flux" <<'EOF' +#!/bin/sh +set -eu +[ "$1" = "pull" ] +[ "$2" = "artifact" ] +shift 2 +ref="$1" +printf '%s\n' "$@" >> "$MOCK_FLUX_LOG" +out="" +while [ "$#" -gt 0 ]; do + if [ "$1" = "--output" ]; then + out="$2" + shift 2 + else + shift + fi +done +[ -n "$out" ] +case "$ref" in + *@sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb) + src="$MOCK_ARTIFACT" + ;; + *@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa) + src="$MOCK_RC_ARTIFACT" + ;; + *) + echo "unexpected artifact ref: $ref" >&2 + exit 1 + ;; +esac +cp -R "$src"/. "$out"/ +EOF + chmod +x "$t/bin/flux" + cat > "$t/release/core/installer/values.yaml" <<'EOF' +cozystackOperator: + platformSourceUrl: oci://ghcr.io/cozystack/cozystack/cozystack-packages + platformSourceRef: digest=sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb +EOF + cat > "$t/artifact/core/installer/values.yaml" <<'EOF' +cozystackOperator: + platformSourceUrl: oci://ghcr.io/cozystack/cozystack/cozystack-packages + platformSourceRef: digest=sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +EOF + image='ghcr.io/cozystack/cozystack/cozystack-ui:v9.9.9@sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc' + printf 'console:\n image: %s\n' "$image" > "$t/release/system/dashboard/values.yaml" + printf 'console:\n image: %s\n' "$image" > "$t/artifact/system/dashboard/values.yaml" + cp "$t/artifact/core/installer/values.yaml" "$t/rc-artifact/core/installer/values.yaml" + printf 'console:\n image: %s\n' \ + 'ghcr.io/cozystack/cozystack/cozystack-ui:v9.9.9-rc.3@sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc' \ + > "$t/rc-artifact/system/dashboard/values.yaml" + # Flux's established packages archive omits symlinks. The verifier must + # compare against that archive view rather than demanding this node back. + ln -s values.yaml "$t/release/system/dashboard/values-link.yaml" +} + +@test "accepts an identical candidate with only the self-reference changed" { + tmp="$(_test_workspace)/fixture" + _make_verify_fixture "$tmp" + + rc=0 + MOCK_ARTIFACT="$tmp/artifact" MOCK_RC_ARTIFACT="$tmp/rc-artifact" MOCK_FLUX_LOG="$tmp/flux.log" \ + VERIFY_PACKAGES_WORKDIR="$tmp/work" \ + PATH="$tmp/bin:$PATH" \ + hack/verify-promoted-packages.sh 9.9.9 "$tmp/release" \ + > "$tmp/out" 2> "$tmp/err" || rc=$? + if [ "$rc" -ne 0 ]; then + echo "verification exited $rc" >&2 + cat "$tmp/err" >&2 + return "$rc" + fi + + grep -qx 'oci://ghcr.io/cozystack/cozystack/cozystack-packages@sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb' "$tmp/flux.log" + grep -q "stable tags, identical package tree, and the rc artifact's container digests" "$tmp/out" +} + +@test "rejects an rc image reference inside the candidate" { + tmp="$(_test_workspace)/fixture" + _make_verify_fixture "$tmp" + sed 's/:v9.9.9@/:v9.9.9-rc.3@/' "$tmp/artifact/system/dashboard/values.yaml" \ + > "$tmp/rc-values.yaml" + mv "$tmp/rc-values.yaml" "$tmp/artifact/system/dashboard/values.yaml" + + rc=0 + MOCK_ARTIFACT="$tmp/artifact" MOCK_RC_ARTIFACT="$tmp/rc-artifact" MOCK_FLUX_LOG="$tmp/flux.log" \ + VERIFY_PACKAGES_WORKDIR="$tmp/work" \ + PATH="$tmp/bin:$PATH" \ + hack/verify-promoted-packages.sh 9.9.9 "$tmp/release" \ + > "$tmp/out" 2> "$tmp/err" || rc=$? + + [ "$rc" -ne 0 ] + grep -q 'still carries rc image references' "$tmp/err" +} + +# The scan's `|| true` is not optional — `find … -exec … +` reports the +# legitimate "nothing matched" as a failure too — which leaves grep's own +# diagnostics as the only remaining signal that a file went unscanned. Produce +# that signal the way an unreadable file produces it, without depending on the +# runner's uid: root reads a chmod 000 fixture anyway, so a permissions-based +# test would quietly stop testing anything wherever the suite runs as root. +# A grep on PATH answering the scan's invocation the way the real one answers a +# file it cannot open is the same injection idiom as _tooling_with_collector +# below, one dependency further out. +_grep_failing_the_rc_scan() { + real_grep="$(command -v grep)" + cat > "$1/grep" < . Every +# other grep in the verifier, and in the collector it sources, must behave. +if [ "\$1" = "-lE" ]; then + for _a in "\$@"; do _last="\$_a"; done + echo "grep: \$_last: Permission denied" >&2 + exit 2 +fi +exec $real_grep "\$@" +EOF + chmod +x "$1/grep" +} + +@test "refuses when the rc-reference scan could not read a file" { + tmp="$(_test_workspace)/fixture" + _make_verify_fixture "$tmp" + _grep_failing_the_rc_scan "$tmp/bin" + + rc=0 + MOCK_ARTIFACT="$tmp/artifact" MOCK_RC_ARTIFACT="$tmp/rc-artifact" MOCK_FLUX_LOG="$tmp/flux.log" \ + VERIFY_PACKAGES_WORKDIR="$tmp/work" \ + PATH="$tmp/bin:$PATH" \ + hack/verify-promoted-packages.sh 9.9.9 "$tmp/release" \ + > "$tmp/out" 2> "$tmp/err" || rc=$? + + [ "$rc" -ne 0 ] + grep -q 'could not scan the promoted artifact for rc image references' "$tmp/err" + # The diagnostic itself has to reach the log, or the failure names no file and + # nobody can tell which one went unscanned. + grep -q 'Permission denied' "$tmp/err" + # …and the verifier did not go on to report a proof it skipped part of. + # Counted rather than negated with `!`, which suppresses errexit and would + # pass regardless. + count="$(grep -c 'identical package tree' "$tmp/out" || true)" + [ "${count:-0}" -eq 0 ] +} + +# The threat this whole job exists for: someone edits packages/ on the release +# PR after promotion published the candidate, so the artifact the installer +# resolves is no longer the tree that was reviewed and tagged. A file present on +# BOTH sides with different bytes is the shape that takes, and only the per-file +# comparison catches it — the file-set compare sees the same names, the installer +# compare reads one file, and the rc-refs compare never looks at the release tree +# at all. The edit here is deliberately not an image reference, so no other leg +# can fire and claim the credit. +@test "rejects a file whose content changed on only one side" { + tmp="$(_test_workspace)/fixture" + _make_verify_fixture "$tmp" + printf ' replicas: 3\n' >> "$tmp/release/system/dashboard/values.yaml" + + rc=0 + MOCK_ARTIFACT="$tmp/artifact" MOCK_RC_ARTIFACT="$tmp/rc-artifact" MOCK_FLUX_LOG="$tmp/flux.log" \ + VERIFY_PACKAGES_WORKDIR="$tmp/work" \ + PATH="$tmp/bin:$PATH" \ + hack/verify-promoted-packages.sh 9.9.9 "$tmp/release" \ + > "$tmp/out" 2> "$tmp/err" || rc=$? + + [ "$rc" -ne 0 ] + grep -q 'promoted artifact file differs from release tree: system/dashboard/values.yaml' "$tmp/err" +} + +@test "rejects package content changed after the candidate was published" { + tmp="$(_test_workspace)/fixture" + _make_verify_fixture "$tmp" + printf 'changed: true\n' > "$tmp/release/system/dashboard/extra.yaml" + + rc=0 + MOCK_ARTIFACT="$tmp/artifact" MOCK_RC_ARTIFACT="$tmp/rc-artifact" MOCK_FLUX_LOG="$tmp/flux.log" \ + VERIFY_PACKAGES_WORKDIR="$tmp/work" \ + PATH="$tmp/bin:$PATH" \ + hack/verify-promoted-packages.sh 9.9.9 "$tmp/release" \ + > "$tmp/out" 2> "$tmp/err" || rc=$? + + [ "$rc" -ne 0 ] + grep -q 'contain different files' "$tmp/err" +} + +@test "rejects a candidate outside the trusted release repository" { + tmp="$(_test_workspace)/fixture" + _make_verify_fixture "$tmp" + sed 's#oci://ghcr.io/cozystack/cozystack/cozystack-packages#oci://example.invalid/cozystack-packages#' \ + "$tmp/release/core/installer/values.yaml" > "$tmp/untrusted-values.yaml" + mv "$tmp/untrusted-values.yaml" "$tmp/release/core/installer/values.yaml" + + rc=0 + MOCK_ARTIFACT="$tmp/artifact" MOCK_RC_ARTIFACT="$tmp/rc-artifact" MOCK_FLUX_LOG="$tmp/flux.log" \ + VERIFY_PACKAGES_WORKDIR="$tmp/work" \ + PATH="$tmp/bin:$PATH" \ + hack/verify-promoted-packages.sh 9.9.9 "$tmp/release" \ + > "$tmp/out" 2> "$tmp/err" || rc=$? + + [ "$rc" -ne 0 ] + grep -q 'must equal trusted repository' "$tmp/err" + [ ! -s "$tmp/flux.log" ] +} + +@test "rejects an embedded rc baseline outside the trusted release repository" { + tmp="$(_test_workspace)/fixture" + _make_verify_fixture "$tmp" + sed 's#oci://ghcr.io/cozystack/cozystack/cozystack-packages#oci://example.invalid/cozystack-packages#' \ + "$tmp/artifact/core/installer/values.yaml" > "$tmp/untrusted-values.yaml" + mv "$tmp/untrusted-values.yaml" "$tmp/artifact/core/installer/values.yaml" + + rc=0 + MOCK_ARTIFACT="$tmp/artifact" MOCK_RC_ARTIFACT="$tmp/rc-artifact" MOCK_FLUX_LOG="$tmp/flux.log" \ + VERIFY_PACKAGES_WORKDIR="$tmp/work" \ + PATH="$tmp/bin:$PATH" \ + hack/verify-promoted-packages.sh 9.9.9 "$tmp/release" \ + > "$tmp/out" 2> "$tmp/err" || rc=$? + + [ "$rc" -ne 0 ] + grep -q "candidate's original platformSourceUrl.*must equal trusted repository" "$tmp/err" +} + +# The mode check has to catch BOTH directions, and only one of them is the +# dangerous one. A candidate whose file is executable where the release tree's +# is not is content the merge reviewer never saw made runnable inside the +# artifact the operator installs — cmp(1) compares bytes and says nothing about +# it. The mirror case is the benign one, and it was the only one an earlier +# `&&`/`||` chain actually rejected: POSIX gives the two operators equal +# precedence and left associativity, so the guard grouped as +# `((A && B) || C) && D` and quietly accepted a candidate-only executable bit. +@test "rejects an executable bit set only in the candidate" { + tmp="$(_test_workspace)/fixture" + _make_verify_fixture "$tmp" + chmod +x "$tmp/artifact/system/dashboard/values.yaml" + + rc=0 + MOCK_ARTIFACT="$tmp/artifact" MOCK_RC_ARTIFACT="$tmp/rc-artifact" MOCK_FLUX_LOG="$tmp/flux.log" \ + VERIFY_PACKAGES_WORKDIR="$tmp/work" \ + PATH="$tmp/bin:$PATH" \ + hack/verify-promoted-packages.sh 9.9.9 "$tmp/release" \ + > "$tmp/out" 2> "$tmp/err" || rc=$? + + [ "$rc" -ne 0 ] + grep -q 'executable bit differs from release tree' "$tmp/err" +} + +@test "rejects an executable bit set only in the release tree" { + tmp="$(_test_workspace)/fixture" + _make_verify_fixture "$tmp" + chmod +x "$tmp/release/system/dashboard/values.yaml" + + rc=0 + MOCK_ARTIFACT="$tmp/artifact" MOCK_RC_ARTIFACT="$tmp/rc-artifact" MOCK_FLUX_LOG="$tmp/flux.log" \ + VERIFY_PACKAGES_WORKDIR="$tmp/work" \ + PATH="$tmp/bin:$PATH" \ + hack/verify-promoted-packages.sh 9.9.9 "$tmp/release" \ + > "$tmp/out" 2> "$tmp/err" || rc=$? + + [ "$rc" -ne 0 ] + grep -q 'executable bit differs from release tree' "$tmp/err" +} + +@test "rejects installer drift beyond the self-reference" { + tmp="$(_test_workspace)/fixture" + _make_verify_fixture "$tmp" + printf ' platformSourceSecret: changed-after-push\n' \ + >> "$tmp/release/core/installer/values.yaml" + + rc=0 + MOCK_ARTIFACT="$tmp/artifact" MOCK_RC_ARTIFACT="$tmp/rc-artifact" MOCK_FLUX_LOG="$tmp/flux.log" \ + VERIFY_PACKAGES_WORKDIR="$tmp/work" \ + PATH="$tmp/bin:$PATH" \ + hack/verify-promoted-packages.sh 9.9.9 "$tmp/release" \ + > "$tmp/out" 2> "$tmp/err" || rc=$? + + [ "$rc" -ne 0 ] + grep -q 'installer values differ beyond their self-reference' "$tmp/err" +} + +# Run the verifier from a copy whose sibling lib/ is ours. That is already how +# the workflows invoke it — `.release-tooling/hack/verify-promoted-packages.sh`, +# with its libraries resolved next to the script — so replacing one library is +# the faithful way to reach a failure inside it rather than a contrived one. +_tooling_with_collector() { + mkdir -p "$1/lib" + cp hack/verify-promoted-packages.sh "$1/" + cp hack/lib/promoted-packages.sh "$1/lib/" + cat > "$1/lib/image-refs.sh" <&2 + return 3' + + rc=0 + MOCK_ARTIFACT="$tmp/artifact" MOCK_RC_ARTIFACT="$tmp/rc-artifact" MOCK_FLUX_LOG="$tmp/flux.log" \ + VERIFY_PACKAGES_WORKDIR="$tmp/work" \ + PATH="$tmp/bin:$PATH" \ + "$tmp/tooling/verify-promoted-packages.sh" 9.9.9 "$tmp/release" \ + > "$tmp/out" 2> "$tmp/err" || rc=$? + + [ "$rc" -ne 0 ] + grep -q 'collector exploded' "$tmp/err" + # …and it did not also claim the proof it never completed. Counted rather + # than negated with `!`, which suppresses errexit and would pass regardless. + count="$(grep -c 'identical package tree' "$tmp/out" || true)" + [ "${count:-0}" -eq 0 ] +} + +@test "refuses when the collector reports no image references at all" { + tmp="$(_test_workspace)/fixture" + _make_verify_fixture "$tmp" + _tooling_with_collector "$tmp/tooling" ' return 0' + + rc=0 + MOCK_ARTIFACT="$tmp/artifact" MOCK_RC_ARTIFACT="$tmp/rc-artifact" MOCK_FLUX_LOG="$tmp/flux.log" \ + VERIFY_PACKAGES_WORKDIR="$tmp/work" \ + PATH="$tmp/bin:$PATH" \ + "$tmp/tooling/verify-promoted-packages.sh" 9.9.9 "$tmp/release" \ + > "$tmp/out" 2> "$tmp/err" || rc=$? + + [ "$rc" -ne 0 ] + grep -q 'collected no image references' "$tmp/err" +} + +@test "rejects a changed container digest even when candidate and merge tree match" { + tmp="$(_test_workspace)/fixture" + _make_verify_fixture "$tmp" + sed 's/cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc/dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd/' \ + "$tmp/artifact/system/dashboard/values.yaml" > "$tmp/changed-values.yaml" + mv "$tmp/changed-values.yaml" "$tmp/artifact/system/dashboard/values.yaml" + cp "$tmp/artifact/system/dashboard/values.yaml" "$tmp/release/system/dashboard/values.yaml" + + rc=0 + MOCK_ARTIFACT="$tmp/artifact" MOCK_RC_ARTIFACT="$tmp/rc-artifact" MOCK_FLUX_LOG="$tmp/flux.log" \ + VERIFY_PACKAGES_WORKDIR="$tmp/work" \ + PATH="$tmp/bin:$PATH" \ + hack/verify-promoted-packages.sh 9.9.9 "$tmp/release" \ + > "$tmp/out" 2> "$tmp/err" || rc=$? + + [ "$rc" -ne 0 ] + grep -q 'promotion changed the container repository/digest set' "$tmp/err" +} From 19e158fbabd54228ccd4ad521c65581806f3d069 Mon Sep 17 00:00:00 2001 From: "cozystack-ci[bot]" <274107086+cozystack-ci[bot]@users.noreply.github.com> Date: Tue, 18 Aug 2026 08:01:26 +0000 Subject: [PATCH 47/51] Prepare release v1.6.2-rc.1 Signed-off-by: cozystack-ci[bot] <274107086+cozystack-ci[bot]@users.noreply.github.com> --- .../apps/clickhouse/images/altinity-clickhouse-backup.tag | 2 +- packages/apps/clickhouse/images/clickhouse-backup.tag | 2 +- packages/apps/http-cache/images/nginx-cache.tag | 2 +- packages/apps/kubernetes/images/cluster-autoscaler.tag | 2 +- packages/apps/kubernetes/images/kubevirt-cloud-provider.tag | 2 +- packages/apps/kubernetes/images/kubevirt-csi-driver.tag | 2 +- packages/apps/kubernetes/images/talos-csr-signer.tag | 2 +- packages/apps/mariadb/images/mariadb-backup.tag | 2 +- packages/core/installer/values.yaml | 4 ++-- packages/core/platform/values.yaml | 2 +- packages/core/testing/values.yaml | 2 +- packages/extra/bootbox/images/matchbox.tag | 2 +- packages/extra/seaweedfs/images/objectstorage-sidecar.tag | 2 +- packages/system/backup-controller/values.yaml | 2 +- packages/system/backupstrategy-controller/values.yaml | 4 ++-- packages/system/bucket/images/s3manager.tag | 2 +- packages/system/cilium/values.yaml | 2 +- packages/system/cozystack-api/values.yaml | 2 +- packages/system/cozystack-controller/values.yaml | 2 +- packages/system/dashboard/values.yaml | 4 ++-- packages/system/flux-shard-operator/values.yaml | 2 +- .../system/grafana-operator/images/grafana-dashboards.tag | 2 +- packages/system/kamaji/values.yaml | 4 ++-- packages/system/kubeovn-plunger/values.yaml | 2 +- packages/system/kubeovn-webhook/values.yaml | 2 +- packages/system/kubevirt-csi-node/values.yaml | 2 +- packages/system/lineage-controller-webhook/values.yaml | 2 +- packages/system/linstor-gui/values.yaml | 2 +- packages/system/linstor/values.yaml | 4 ++-- packages/system/metallb/values.yaml | 4 ++-- packages/system/monitoring/images/grafana.tag | 2 +- packages/system/multus/templates/multus-daemonset-thick.yml | 4 ++-- packages/system/objectstorage-controller/values.yaml | 2 +- packages/system/seaweedfs/values.yaml | 2 +- packages/system/securitygroup-controller/values.yaml | 2 +- 35 files changed, 42 insertions(+), 42 deletions(-) diff --git a/packages/apps/clickhouse/images/altinity-clickhouse-backup.tag b/packages/apps/clickhouse/images/altinity-clickhouse-backup.tag index 6ee6e2547a..66957e3e50 100644 --- a/packages/apps/clickhouse/images/altinity-clickhouse-backup.tag +++ b/packages/apps/clickhouse/images/altinity-clickhouse-backup.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/altinity-clickhouse-backup:v1.6.1@sha256:e890d78cd83968bdff9a36c9ca4312afc05e138af486e476da0a935f81f9b7a7 +ghcr.io/cozystack/cozystack/altinity-clickhouse-backup:v1.6.2-rc.1@sha256:e890d78cd83968bdff9a36c9ca4312afc05e138af486e476da0a935f81f9b7a7 diff --git a/packages/apps/clickhouse/images/clickhouse-backup.tag b/packages/apps/clickhouse/images/clickhouse-backup.tag index e709b9c75b..f6dfcd865d 100644 --- a/packages/apps/clickhouse/images/clickhouse-backup.tag +++ b/packages/apps/clickhouse/images/clickhouse-backup.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/clickhouse-backup:v1.6.1@sha256:8efef62d59fb82d39544823f81a797c2b9ac130a50241fee48e701161ce2670d +ghcr.io/cozystack/cozystack/clickhouse-backup:v1.6.2-rc.1@sha256:8efef62d59fb82d39544823f81a797c2b9ac130a50241fee48e701161ce2670d diff --git a/packages/apps/http-cache/images/nginx-cache.tag b/packages/apps/http-cache/images/nginx-cache.tag index 94a6a92cab..58d5fa00ab 100644 --- a/packages/apps/http-cache/images/nginx-cache.tag +++ b/packages/apps/http-cache/images/nginx-cache.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/nginx-cache:v1.6.1@sha256:bd042a8a9789e8b21b12cb266e390484a3a48f4af6ca6e2d2864a8a83c74bd25 +ghcr.io/cozystack/cozystack/nginx-cache:v1.6.2-rc.1@sha256:bd042a8a9789e8b21b12cb266e390484a3a48f4af6ca6e2d2864a8a83c74bd25 diff --git a/packages/apps/kubernetes/images/cluster-autoscaler.tag b/packages/apps/kubernetes/images/cluster-autoscaler.tag index 4bff49114f..027c22ca47 100644 --- a/packages/apps/kubernetes/images/cluster-autoscaler.tag +++ b/packages/apps/kubernetes/images/cluster-autoscaler.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/cluster-autoscaler:v1.6.1@sha256:e77ff02b328e4e119439efd1edd3aa71e9325fabf10a4a84b03a87d12d90b308 +ghcr.io/cozystack/cozystack/cluster-autoscaler:v1.6.2-rc.1@sha256:e77ff02b328e4e119439efd1edd3aa71e9325fabf10a4a84b03a87d12d90b308 diff --git a/packages/apps/kubernetes/images/kubevirt-cloud-provider.tag b/packages/apps/kubernetes/images/kubevirt-cloud-provider.tag index 35cbec44e4..84cccc5312 100644 --- a/packages/apps/kubernetes/images/kubevirt-cloud-provider.tag +++ b/packages/apps/kubernetes/images/kubevirt-cloud-provider.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/kubevirt-cloud-provider:v1.6.1@sha256:f005b99041d191eed9dde98f9f6691ba2d0cbfa1b3b0b2fe4b32cc0fdba74acf +ghcr.io/cozystack/cozystack/kubevirt-cloud-provider:v1.6.2-rc.1@sha256:f005b99041d191eed9dde98f9f6691ba2d0cbfa1b3b0b2fe4b32cc0fdba74acf diff --git a/packages/apps/kubernetes/images/kubevirt-csi-driver.tag b/packages/apps/kubernetes/images/kubevirt-csi-driver.tag index 97c2e3ab13..77c6197d4e 100644 --- a/packages/apps/kubernetes/images/kubevirt-csi-driver.tag +++ b/packages/apps/kubernetes/images/kubevirt-csi-driver.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/kubevirt-csi-driver:v1.6.1@sha256:7a47bb5aa46fccfe33dc0e0d857a6ec2ce2c6284a213e37beff5edcf6f37c673 +ghcr.io/cozystack/cozystack/kubevirt-csi-driver:v1.6.2-rc.1@sha256:0fe6808c5857493a0d7e98344a2bcdfc3fea34917386fc654c03328fc11afe2e diff --git a/packages/apps/kubernetes/images/talos-csr-signer.tag b/packages/apps/kubernetes/images/talos-csr-signer.tag index b4a6f53f08..01099bac83 100644 --- a/packages/apps/kubernetes/images/talos-csr-signer.tag +++ b/packages/apps/kubernetes/images/talos-csr-signer.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/talos-csr-signer:v1.6.1@sha256:7051e90245cb446ce4deb857d4337e2fd69ce84436243ba60939d374defc2a07 +ghcr.io/cozystack/cozystack/talos-csr-signer:v1.6.2-rc.1@sha256:7051e90245cb446ce4deb857d4337e2fd69ce84436243ba60939d374defc2a07 diff --git a/packages/apps/mariadb/images/mariadb-backup.tag b/packages/apps/mariadb/images/mariadb-backup.tag index 77268f30d2..04043407b1 100644 --- a/packages/apps/mariadb/images/mariadb-backup.tag +++ b/packages/apps/mariadb/images/mariadb-backup.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/mariadb-backup:v1.6.1@sha256:1de944b5c4fbeef94004b8620b0865aa7886a692dacb23d50c8f9b31fbeece07 +ghcr.io/cozystack/cozystack/mariadb-backup:v1.6.2-rc.1@sha256:1de944b5c4fbeef94004b8620b0865aa7886a692dacb23d50c8f9b31fbeece07 diff --git a/packages/core/installer/values.yaml b/packages/core/installer/values.yaml index 7c9ea260ae..816b2ed48a 100644 --- a/packages/core/installer/values.yaml +++ b/packages/core/installer/values.yaml @@ -14,9 +14,9 @@ cozystackOperator: # to the value baked at build time. The Makefile sets this for the # kubectl-apply release artifacts (_out/assets/cozystack-operator-*.yaml). platformVersion: "" - image: ghcr.io/cozystack/cozystack/cozystack-operator:v1.6.1@sha256:ab2c0c8f87dc70f675bf36ce55f8615f810bbe3d0840d3a7d1a412f38abdf78a + image: ghcr.io/cozystack/cozystack/cozystack-operator:v1.6.2-rc.1@sha256:41ea5ca9c6a7d471105920b09fc8ed68279b8ca0dbffe7f839dccd530b42e4c7 platformSourceUrl: 'oci://ghcr.io/cozystack/cozystack/cozystack-packages' - platformSourceRef: 'digest=sha256:2d7e07e4cf3d49a97f5b9c62ad732cfb93586483723788eed489b32643a1a8df' + platformSourceRef: 'digest=sha256:02bc4a989b12ba6bce4cae79a2feb1092dbb094028eaea5959b1d833de641201' # When non-empty, overrides the operator's --helmrelease-interval flag # (operator default: 5m). E2E sets this to 30s; production should leave empty. helmReleaseInterval: "" diff --git a/packages/core/platform/values.yaml b/packages/core/platform/values.yaml index 278665c641..f03472f8da 100644 --- a/packages/core/platform/values.yaml +++ b/packages/core/platform/values.yaml @@ -13,7 +13,7 @@ migrations: # below therefore lags targetVersion in-tree by design; run-migrations.sh # refuses to advance past a migration file missing from the image (exit 1), so # a stale pin fails loudly rather than silently skipping the etcd adoption. - image: ghcr.io/cozystack/cozystack/platform-migrations:v1.6.1@sha256:a34e7156a52dc55b27662917e5f9d5bbea671188d8db233296d0a98c800dc6c6 + image: ghcr.io/cozystack/cozystack/platform-migrations:v1.6.2-rc.1@sha256:a34e7156a52dc55b27662917e5f9d5bbea671188d8db233296d0a98c800dc6c6 targetVersion: 54 # Adopt legacy etcd clusters onto the v1alpha2 operator WITHOUT the mandatory # pre-adoption safety snapshot (migration 50). Leave false unless migration 50 diff --git a/packages/core/testing/values.yaml b/packages/core/testing/values.yaml index 4f3beb63cb..f01eb0b5ce 100644 --- a/packages/core/testing/values.yaml +++ b/packages/core/testing/values.yaml @@ -1,2 +1,2 @@ e2e: - image: ghcr.io/cozystack/cozystack/e2e-sandbox:v1.6.1@sha256:a9980e8c48d6e50ed2b5d8245a4e0aae00970777b6e2255577fdb66035a44c68 + image: ghcr.io/cozystack/cozystack/e2e-sandbox:v1.6.2-rc.1@sha256:6f9f059305b057dce4dc0c68e82b86fe5573ecae40795f997b70db2f7e16f82d diff --git a/packages/extra/bootbox/images/matchbox.tag b/packages/extra/bootbox/images/matchbox.tag index 7a3ea6c975..3aa407d25f 100644 --- a/packages/extra/bootbox/images/matchbox.tag +++ b/packages/extra/bootbox/images/matchbox.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/matchbox:v1.6.1@sha256:3f492d2bd2d14de94c88dc995e3092911e2ca8ed7e17d305398e7fd49135d351 +ghcr.io/cozystack/cozystack/matchbox:v1.6.2-rc.1@sha256:d3db5e7b5eb146447c3be22845bc49e7322641a617d60dcd7803f502f0e93bb7 diff --git a/packages/extra/seaweedfs/images/objectstorage-sidecar.tag b/packages/extra/seaweedfs/images/objectstorage-sidecar.tag index 4f903c420a..d8f79d7a35 100644 --- a/packages/extra/seaweedfs/images/objectstorage-sidecar.tag +++ b/packages/extra/seaweedfs/images/objectstorage-sidecar.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/objectstorage-sidecar:v1.6.1@sha256:ef3c154c1a6dd1ac9fa1b0763a2bd23707d68f2f16f16a97ebf331f3646b53ce +ghcr.io/cozystack/cozystack/objectstorage-sidecar:v1.6.2-rc.1@sha256:ef3c154c1a6dd1ac9fa1b0763a2bd23707d68f2f16f16a97ebf331f3646b53ce diff --git a/packages/system/backup-controller/values.yaml b/packages/system/backup-controller/values.yaml index 1ddce6ae32..452cad8111 100644 --- a/packages/system/backup-controller/values.yaml +++ b/packages/system/backup-controller/values.yaml @@ -1,5 +1,5 @@ backupController: - image: "ghcr.io/cozystack/cozystack/backup-controller:v1.6.1@sha256:ef3c45ac4b23d6231a4166b4331a622374960caeeef3e4a901b0ee1dabcb6922" + image: "ghcr.io/cozystack/cozystack/backup-controller:v1.6.2-rc.1@sha256:0774d068befce98dd067cb6239e5d37cb0b03b3d17a429e80caa46787d7f3726" replicas: 2 debug: false metrics: diff --git a/packages/system/backupstrategy-controller/values.yaml b/packages/system/backupstrategy-controller/values.yaml index 54ba264b17..ce736253a2 100644 --- a/packages/system/backupstrategy-controller/values.yaml +++ b/packages/system/backupstrategy-controller/values.yaml @@ -1,5 +1,5 @@ backupStrategyController: - image: "ghcr.io/cozystack/cozystack/backupstrategy-controller:v1.6.1@sha256:7cb5af7bd8fae2ef5d6e58d1eb6eb7d9579bff8df228ac3d936c6e14ae5e063f" + image: "ghcr.io/cozystack/cozystack/backupstrategy-controller:v1.6.2-rc.1@sha256:390caeb8cabf80dfe9bcd9f38b4ed97f0e71c22af36496fb744cdc50bb9501fc" # chBackupClientImage is the image rendered into the Altinity strategy # Pod: it drives clickhouse-backup's HTTP API via curl + jq. # @@ -35,7 +35,7 @@ backupStrategyController: # retag both to the same stable tag, and the write-once guard in # hack/promote-retag.sh fails the release. hack/image-pin-consistency.bats # asserts no repository is pinned at more than one digest. - chBackupClientImage: "ghcr.io/cozystack/cozystack/platform-migrations:v1.6.1@sha256:a34e7156a52dc55b27662917e5f9d5bbea671188d8db233296d0a98c800dc6c6" + chBackupClientImage: "ghcr.io/cozystack/cozystack/platform-migrations:v1.6.2-rc.1@sha256:a34e7156a52dc55b27662917e5f9d5bbea671188d8db233296d0a98c800dc6c6" replicas: 2 debug: false metrics: diff --git a/packages/system/bucket/images/s3manager.tag b/packages/system/bucket/images/s3manager.tag index 83fec5eec0..554b14bbe3 100644 --- a/packages/system/bucket/images/s3manager.tag +++ b/packages/system/bucket/images/s3manager.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/s3manager:v1.6.1@sha256:73665aaf7f5406b4a1e456d14060fe874e0ae2a436ba7fd3058fa73abff8a927 +ghcr.io/cozystack/cozystack/s3manager:v1.6.2-rc.1@sha256:73665aaf7f5406b4a1e456d14060fe874e0ae2a436ba7fd3058fa73abff8a927 diff --git a/packages/system/cilium/values.yaml b/packages/system/cilium/values.yaml index 9700f73a79..2f27b497d3 100644 --- a/packages/system/cilium/values.yaml +++ b/packages/system/cilium/values.yaml @@ -23,7 +23,7 @@ cilium: mode: "kubernetes" image: repository: ghcr.io/cozystack/cozystack/cilium - tag: v1.6.1 + tag: v1.6.2-rc.1 digest: "sha256:136a7dfff4d6adcd448424c2354144160028a8e96eb6558b917fd3ee6d7b39fe" envoy: enabled: true diff --git a/packages/system/cozystack-api/values.yaml b/packages/system/cozystack-api/values.yaml index 31e9e253e7..d3ed4f8c30 100644 --- a/packages/system/cozystack-api/values.yaml +++ b/packages/system/cozystack-api/values.yaml @@ -1,3 +1,3 @@ cozystackAPI: - image: ghcr.io/cozystack/cozystack/cozystack-api:v1.6.1@sha256:6d1db0a4fc1f21282f7825fed13807f80e40998f6703a087f372f19cc4197173 + image: ghcr.io/cozystack/cozystack/cozystack-api:v1.6.2-rc.1@sha256:8f412e6332e96370c678c776f663ff81b813bd84ed5b4303ecbd6a47dbba9a01 replicas: 2 diff --git a/packages/system/cozystack-controller/values.yaml b/packages/system/cozystack-controller/values.yaml index a3444ceaef..16d52d8ac6 100644 --- a/packages/system/cozystack-controller/values.yaml +++ b/packages/system/cozystack-controller/values.yaml @@ -1,4 +1,4 @@ cozystackController: - image: ghcr.io/cozystack/cozystack/cozystack-controller:v1.6.1@sha256:1b60f18b37c140fa9e50baacd62a8fe9ed8d549c26746b081d5b460c7ca98a04 + image: ghcr.io/cozystack/cozystack/cozystack-controller:v1.6.2-rc.1@sha256:fd0cc8e40293f52ea12ac8a7933dc0104a828045f9d7a73b3a38095da86c40f7 debug: false disableTelemetry: false diff --git a/packages/system/dashboard/values.yaml b/packages/system/dashboard/values.yaml index 1b232d9e98..4846584c03 100644 --- a/packages/system/dashboard/values.yaml +++ b/packages/system/dashboard/values.yaml @@ -1,4 +1,4 @@ console: - image: ghcr.io/cozystack/cozystack/cozystack-ui:v1.6.1@sha256:e8a7f00cf8f970c2d8e598570ca4ed98c5448e2362489f4470a2a89543f760ed + image: ghcr.io/cozystack/cozystack/cozystack-ui:v1.6.2-rc.1@sha256:051282dbc216ed4f7b237ee93f172a5c8f9302e549805b70373c849c7dea6c7b tokenProxy: - image: ghcr.io/cozystack/cozystack/token-proxy:v1.6.1@sha256:97124c012246c33cdcac60e8f884097b48c0081624716fe8161efc2b3ceeb2a8 + image: ghcr.io/cozystack/cozystack/token-proxy:v1.6.2-rc.1@sha256:97124c012246c33cdcac60e8f884097b48c0081624716fe8161efc2b3ceeb2a8 diff --git a/packages/system/flux-shard-operator/values.yaml b/packages/system/flux-shard-operator/values.yaml index 1a6a201ec4..1308115429 100644 --- a/packages/system/flux-shard-operator/values.yaml +++ b/packages/system/flux-shard-operator/values.yaml @@ -1,5 +1,5 @@ fluxShardOperator: - image: ghcr.io/cozystack/cozystack/flux-shard-operator:v1.6.1@sha256:5774b606d182ebc70319f1adf6641c52afe417e968bdd64540eb32fb8f28605f + image: ghcr.io/cozystack/cozystack/flux-shard-operator:v1.6.2-rc.1@sha256:b59ec5d228575a0aee678b95b352270acf6020fb80caf9636ed690225b7858d7 debug: false replicas: 2 ## Number of helm-controller shards to provision and distribute tenants diff --git a/packages/system/grafana-operator/images/grafana-dashboards.tag b/packages/system/grafana-operator/images/grafana-dashboards.tag index 49327da8d4..ef1ebcfdb1 100644 --- a/packages/system/grafana-operator/images/grafana-dashboards.tag +++ b/packages/system/grafana-operator/images/grafana-dashboards.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/grafana-dashboards:v1.6.1@sha256:fbbcfed72e9d23f223d990b9cd38a8c49ea3f985d7da65defc79eb730c0861e3 +ghcr.io/cozystack/cozystack/grafana-dashboards:v1.6.2-rc.1@sha256:cf8880fc5615e8c5309b31ad2624af206158958cf839d7f51dbf043ff7ceef21 diff --git a/packages/system/kamaji/values.yaml b/packages/system/kamaji/values.yaml index 858b9e8d1f..367f3f5600 100644 --- a/packages/system/kamaji/values.yaml +++ b/packages/system/kamaji/values.yaml @@ -30,7 +30,7 @@ kamaji: topologyKey: kubernetes.io/hostname image: pullPolicy: IfNotPresent - tag: v1.6.1@sha256:dd81f5b76208e11665f2d2eed00396653c32182079ee53f44169074ac35d62ed + tag: v1.6.2-rc.1@sha256:dd81f5b76208e11665f2d2eed00396653c32182079ee53f44169074ac35d62ed repository: ghcr.io/cozystack/cozystack/kamaji resources: limits: @@ -63,4 +63,4 @@ kamaji: periodSeconds: 10 timeoutSeconds: 1 extraArgs: - - --migrate-image=ghcr.io/cozystack/cozystack/kamaji:v1.6.1@sha256:dd81f5b76208e11665f2d2eed00396653c32182079ee53f44169074ac35d62ed + - --migrate-image=ghcr.io/cozystack/cozystack/kamaji:v1.6.2-rc.1@sha256:dd81f5b76208e11665f2d2eed00396653c32182079ee53f44169074ac35d62ed diff --git a/packages/system/kubeovn-plunger/values.yaml b/packages/system/kubeovn-plunger/values.yaml index ed38af48a6..e83594e8e0 100644 --- a/packages/system/kubeovn-plunger/values.yaml +++ b/packages/system/kubeovn-plunger/values.yaml @@ -1,4 +1,4 @@ portSecurity: true routes: "" -image: ghcr.io/cozystack/cozystack/kubeovn-plunger:v1.6.1@sha256:6f38f72c16f86a5937630ba58ee42afca2fd8f14dfe45bb79670f9a4a8447b60 +image: ghcr.io/cozystack/cozystack/kubeovn-plunger:v1.6.2-rc.1@sha256:6f38f72c16f86a5937630ba58ee42afca2fd8f14dfe45bb79670f9a4a8447b60 ovnCentralName: ovn-central diff --git a/packages/system/kubeovn-webhook/values.yaml b/packages/system/kubeovn-webhook/values.yaml index c356efd936..a388d69620 100644 --- a/packages/system/kubeovn-webhook/values.yaml +++ b/packages/system/kubeovn-webhook/values.yaml @@ -1,3 +1,3 @@ portSecurity: true routes: "" -image: ghcr.io/cozystack/cozystack/kubeovn-webhook:v1.6.1@sha256:593c324a38db59495497c5b68b90cad25dfe6753c73b7a287b3280e26f70a15f +image: ghcr.io/cozystack/cozystack/kubeovn-webhook:v1.6.2-rc.1@sha256:8abc089c4469b6572d7eb6cba1bd87528dc1298101a2250d4b351d6ba0ca0620 diff --git a/packages/system/kubevirt-csi-node/values.yaml b/packages/system/kubevirt-csi-node/values.yaml index 98c80c4d78..39ebb70556 100644 --- a/packages/system/kubevirt-csi-node/values.yaml +++ b/packages/system/kubevirt-csi-node/values.yaml @@ -9,4 +9,4 @@ storageClasses: infraStorageClass: replicated default: false csiDriver: - image: ghcr.io/cozystack/cozystack/kubevirt-csi-driver:v1.6.1@sha256:7a47bb5aa46fccfe33dc0e0d857a6ec2ce2c6284a213e37beff5edcf6f37c673 + image: ghcr.io/cozystack/cozystack/kubevirt-csi-driver:v1.6.2-rc.1@sha256:0fe6808c5857493a0d7e98344a2bcdfc3fea34917386fc654c03328fc11afe2e diff --git a/packages/system/lineage-controller-webhook/values.yaml b/packages/system/lineage-controller-webhook/values.yaml index e535d4ca8a..4d8d536e9e 100644 --- a/packages/system/lineage-controller-webhook/values.yaml +++ b/packages/system/lineage-controller-webhook/values.yaml @@ -1,5 +1,5 @@ lineageControllerWebhook: - image: ghcr.io/cozystack/cozystack/lineage-controller-webhook:v1.6.1@sha256:b8bfe1573ef75f74e32eff6fe3c57d16413860836299d2f64a8a28fd482a65ab + image: ghcr.io/cozystack/cozystack/lineage-controller-webhook:v1.6.2-rc.1@sha256:454e3da9468e578bf627210ea1ff664cf58b1f618edeaa158f52c9a930a44794 debug: false replicas: 2 # DEPRECATED. Injects KUBERNETES_SERVICE_HOST=status.hostIP and diff --git a/packages/system/linstor-gui/values.yaml b/packages/system/linstor-gui/values.yaml index b690912057..c3fbc008c0 100644 --- a/packages/system/linstor-gui/values.yaml +++ b/packages/system/linstor-gui/values.yaml @@ -3,7 +3,7 @@ ## @param image.tag LINSTOR GUI container image tag (digest recommended) image: repository: ghcr.io/cozystack/cozystack/linstor-gui - tag: v1.6.1@sha256:6e11829de86709f5e21636cdd1a1ada1239a7dcc1741f91e9d95d9f6ade603a2 + tag: v1.6.2-rc.1@sha256:6e11829de86709f5e21636cdd1a1ada1239a7dcc1741f91e9d95d9f6ade603a2 ## @section Deployment ## @param replicas Number of linstor-gui replicas replicas: 1 diff --git a/packages/system/linstor/values.yaml b/packages/system/linstor/values.yaml index be08bf00fc..39df945066 100644 --- a/packages/system/linstor/values.yaml +++ b/packages/system/linstor/values.yaml @@ -1,7 +1,7 @@ piraeusServer: image: repository: ghcr.io/cozystack/cozystack/piraeus-server - tag: v1.6.1@sha256:82d2234181ac8d5942e57286101b242a08fbcc49a9bf37d5df529921624fdca9 + tag: v1.6.2-rc.1@sha256:82d2234181ac8d5942e57286101b242a08fbcc49a9bf37d5df529921624fdca9 # Talos-specific workarounds (disable for generic Linux like Ubuntu/Debian) talos: enabled: true @@ -13,7 +13,7 @@ linstor: linstorCSI: image: repository: ghcr.io/cozystack/cozystack/linstor-csi - tag: v1.6.1@sha256:3228487ae861ddf02cac984289f8c377fc9e66a71fb3c387e9af9f72c1dbc78b + tag: v1.6.2-rc.1@sha256:3228487ae861ddf02cac984289f8c377fc9e66a71fb3c387e9af9f72c1dbc78b controller: # Resource requests for the linstor-controller JVM. Requests (not limits) are # set on purpose: a CPU request guarantees CPU shares so the JVM can answer its diff --git a/packages/system/metallb/values.yaml b/packages/system/metallb/values.yaml index 9404c13e60..df6b04e1d8 100644 --- a/packages/system/metallb/values.yaml +++ b/packages/system/metallb/values.yaml @@ -37,11 +37,11 @@ metallb: controller: image: repository: ghcr.io/cozystack/cozystack/metallb-controller - tag: v1.6.1@sha256:9d8ba76cdb9c7c6221334ad05d706dee22b138b3e90c1fe8fc884925b7480c02 + tag: v1.6.2-rc.1@sha256:9d8ba76cdb9c7c6221334ad05d706dee22b138b3e90c1fe8fc884925b7480c02 speaker: image: repository: ghcr.io/cozystack/cozystack/metallb-speaker - tag: v1.6.1@sha256:87df3c82d0b6ea223b26fd5d6fbba6e940c13e56418dfc1cd90863d145795be9 + tag: v1.6.2-rc.1@sha256:87df3c82d0b6ea223b26fd5d6fbba6e940c13e56418dfc1cd90863d145795be9 # The vendored metallb chart's values.yaml leaves `frr-k8s.prometheus:` as # a YAML block containing only commented examples — it parses to null. # Helm-controller's deep-merge then writes that null over the frr-k8s diff --git a/packages/system/monitoring/images/grafana.tag b/packages/system/monitoring/images/grafana.tag index 7221f07092..03450722be 100644 --- a/packages/system/monitoring/images/grafana.tag +++ b/packages/system/monitoring/images/grafana.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/grafana:v1.6.1@sha256:d29c2306b96dfc47aab05e6afd1bce581c552f02d022084c11737632d8839029 +ghcr.io/cozystack/cozystack/grafana:v1.6.2-rc.1@sha256:63c90beb8bf31f49820f529c74ebeb9e8be10acb02c45ac517844aecdfd1a4f2 diff --git a/packages/system/multus/templates/multus-daemonset-thick.yml b/packages/system/multus/templates/multus-daemonset-thick.yml index da69f773b8..9ba9f5a8c0 100644 --- a/packages/system/multus/templates/multus-daemonset-thick.yml +++ b/packages/system/multus/templates/multus-daemonset-thick.yml @@ -156,7 +156,7 @@ spec: serviceAccountName: multus containers: - name: kube-multus - image: ghcr.io/cozystack/cozystack/multus-cni:v1.6.1@sha256:f94fdd68d674a1f8c8a2b45fa8fd4b478fc632e66fa147aea73a6b60b721920c + image: ghcr.io/cozystack/cozystack/multus-cni:v1.6.2-rc.1@sha256:f94fdd68d674a1f8c8a2b45fa8fd4b478fc632e66fa147aea73a6b60b721920c command: [ "/usr/src/multus-cni/bin/multus-daemon" ] resources: requests: @@ -202,7 +202,7 @@ spec: fieldPath: spec.nodeName initContainers: - name: install-multus-binary - image: ghcr.io/cozystack/cozystack/multus-cni:v1.6.1@sha256:f94fdd68d674a1f8c8a2b45fa8fd4b478fc632e66fa147aea73a6b60b721920c + image: ghcr.io/cozystack/cozystack/multus-cni:v1.6.2-rc.1@sha256:f94fdd68d674a1f8c8a2b45fa8fd4b478fc632e66fa147aea73a6b60b721920c command: - "/usr/src/multus-cni/bin/install_multus" - "-d" diff --git a/packages/system/objectstorage-controller/values.yaml b/packages/system/objectstorage-controller/values.yaml index c1dc68cc36..35e3b89d41 100644 --- a/packages/system/objectstorage-controller/values.yaml +++ b/packages/system/objectstorage-controller/values.yaml @@ -1,3 +1,3 @@ objectstorage: controller: - image: "ghcr.io/cozystack/cozystack/objectstorage-controller:v1.6.1@sha256:0d9ff6c2a3453fdc7e72894bbfdd3e5a292be78342c7c5352f655c976c3b7f0b" + image: "ghcr.io/cozystack/cozystack/objectstorage-controller:v1.6.2-rc.1@sha256:0d9ff6c2a3453fdc7e72894bbfdd3e5a292be78342c7c5352f655c976c3b7f0b" diff --git a/packages/system/seaweedfs/values.yaml b/packages/system/seaweedfs/values.yaml index 253dae4f81..1c4016a685 100644 --- a/packages/system/seaweedfs/values.yaml +++ b/packages/system/seaweedfs/values.yaml @@ -259,7 +259,7 @@ seaweedfs: bucketClassName: "seaweedfs" region: "" sidecar: - image: "ghcr.io/cozystack/cozystack/objectstorage-sidecar:v1.6.1@sha256:ef3c154c1a6dd1ac9fa1b0763a2bd23707d68f2f16f16a97ebf331f3646b53ce" + image: "ghcr.io/cozystack/cozystack/objectstorage-sidecar:v1.6.2-rc.1@sha256:ef3c154c1a6dd1ac9fa1b0763a2bd23707d68f2f16f16a97ebf331f3646b53ce" certificates: commonName: "SeaweedFS CA" ipAddresses: [] diff --git a/packages/system/securitygroup-controller/values.yaml b/packages/system/securitygroup-controller/values.yaml index 987e944e98..83a58ca29c 100644 --- a/packages/system/securitygroup-controller/values.yaml +++ b/packages/system/securitygroup-controller/values.yaml @@ -1,5 +1,5 @@ securityGroupController: - image: "ghcr.io/cozystack/cozystack/securitygroup-controller:v1.6.1@sha256:1eaaed61b595ed79460709a410d1da830031e82926b0a2fd8cfd917e0e120844" + image: "ghcr.io/cozystack/cozystack/securitygroup-controller:v1.6.2-rc.1@sha256:b1c7153a93344318ec96363c6802e20a93c559ac0586d4ea6eabd8d95baa324b" replicas: 2 debug: false resources: From 4fcc111142a9883b01dc738a850ffbd5d775b346 Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Wed, 19 Aug 2026 11:13:22 +0500 Subject: [PATCH 48/51] fix(release): compare host-less image refs on digest alone hack/verify-promoted-packages.sh reported "promotion changed the container repository/digest set" for a v1.6.2 promotion that moved no container bytes at all: all 48 real repo@digest refs and all 48 digests were byte-identical between the rc artifact and the promotion candidate. The six entries it flagged as changed repositories were not repositories. They were bare tag strings -- v1.6.2-rc.1 on the rc side, v1.6.2 on the candidate side -- pairing one-to-one by identical digest. hack/lib/image-refs.sh emits two entries per shape-3 split map: the correctly joined repository@digest, plus shape 1's recursive scrape of the raw .tag scalar, which carries no repository. The library documents that degeneracy and leaves host-less refs to its callers; promote-retag.sh drops them, the verifier had only a */cozystack-packages exclusion. So `${image%:*}` -- a no-op on a string holding no colon -- handed the tag over as the repository, and promotion rewrites exactly those tags. Set an empty repository for a ref with no `/` ahead of its digest, so it compares on the digest, which is the whole container identity such a ref carries and the only part promotion must not move. Dropping these refs instead would be wrong rather than blunter: packages/system/kuberture carries an `image:` map with a `tag:` and no `repository:`, so shape 1 is the only rule that sees its digest, and skipping it would silently stop proving that digest unchanged. Reproduced over the release-1.6 packages tree with the shape-3 tags rewritten rc-to-stable: before, the six pairs above diverge; after, both sides normalize to 60 identical entries, kuberture's digest included and kube-ovn's real repository still visible through its shape-4 join. Signed-off-by: Myasnikov Daniil --- hack/verify-promoted-packages.sh | 39 +++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/hack/verify-promoted-packages.sh b/hack/verify-promoted-packages.sh index bbd625ff3d..26089e4188 100755 --- a/hack/verify-promoted-packages.sh +++ b/hack/verify-promoted-packages.sh @@ -206,7 +206,44 @@ normalized_refs() { digest="${raw##*@}" image="${without_digest##*/}" if [ "$without_digest" = "$image" ]; then - repo="${image%:*}" + # No `/` anywhere ahead of the digest, so nothing in this ref names a + # registry host or a repository path — it is a bare scalar sitting in + # front of a digest, and the collector emits those by design. Shape 3 + # (`repository:` on one key, `tag: @sha256:` on another) is + # the dominant shape in the tree, and shape 1's recursive descent sees + # that `tag` scalar on its own, so every shape-3 map arrives here twice: + # once correctly joined, once as the bare tag. hack/lib/image-refs.sh + # documents that degeneracy and leaves host-less refs to its callers — + # promote-retag.sh drops them through its ownership filter, and this + # function had no equivalent. `${image%:*}` is then a no-op (a tag holds + # no colon) and the TAG arrives as the repository. Promotion rewrites + # exactly those tags, which is how a promotion that moved no container + # bytes at all reported six changed "repositories" whose digests were + # identical on both sides. + # + # Compare such a ref on its digest alone. The digest is the whole of the + # container identity a host-less ref carries, and it is the only part + # promotion must not move; the empty repository yields a leading `@`, + # which no real repository name can produce, so these cannot collide with + # a host-bearing entry. Where a repository name does exist behind one of + # them — kube-ovn's `global.images.kubeovn`, whose host lives in + # `global.registry.address` — the same image also arrives host-bearing + # from shape 4, so a repository change stays visible through that entry. + # + # Dropping host-less refs outright would be wrong rather than merely + # blunter: packages/system/kuberture/values.yaml carries an `image:` map + # with a `tag:` and no `repository:`, so shape 1 is the only rule that + # ever sees its digest. Skipping it would silently stop proving that + # digest is unchanged — a hole in the very guard this branch repairs. + # + # Do NOT reach for promote-retag.sh's `${REGISTRY}/` ownership filter to + # do this job. The verify job's REGISTRY names the private build registry + # while both artifacts under comparison live on the public one, so that + # filter drops every ref; and the emptiness guard above tests the RAW + # collection rather than the filtered set, so the proof would then pass + # by comparing two empty sets — failing open on the one invariant it + # exists to establish. + repo="" else repo="${without_digest%/*}/${image%:*}" fi From 2cb8c743cb0f5371fd6271928bde26ef60ca0146 Mon Sep 17 00:00:00 2001 From: Myasnikov Daniil Date: Wed, 19 Aug 2026 11:16:29 +0500 Subject: [PATCH 49/51] test(release): cover the split-map and repository-less ref shapes No fixture in verify-promoted-packages_test.bats built a shape-3 split map -- `repository:` on one key, `tag: @sha256:` on another -- which hack/lib/image-refs.sh itself calls the dominant shape in the tree. The existing fixture's single-string ref carries a registry host, so its rc-to-stable rewrite normalizes through the other branch and passes whether or not host-less refs are handled, which is why the suite looked covered while a clean promotion could still be rejected. Add two cases. The first promotes a split map whose tag moves from rc to stable with the repository and digest held constant; reverting the normalization fix reddens it on "promotion changed the container repository/digest set", diffing v9.9.9-rc.3@sha256:eee... against v9.9.9@sha256:eee... -- the same tag-as-repository pair, with identical digests, that CI reported. The second pins the property a blunter fix would break. A digest-only `image:` map with no `repository:` (the kuberture shape) is seen by no rule but shape 1, so it must still contribute its digest: the case moves that digest and nothing else, and asserts the rejection comes from the digest set rather than a neighbouring leg. Replacing the fix with an outright `continue` over host-less refs makes the verifier exit 0 on that changed digest and reddens the case at its exit-status assertion. Negative assertions are counted rather than `!`-negated: under hack/cozytest.sh a `! grep -q` cannot fail, so it would be a comment shaped like an assertion. Signed-off-by: Myasnikov Daniil --- hack/verify-promoted-packages_test.bats | 105 ++++++++++++++++++++++++ 1 file changed, 105 insertions(+) diff --git a/hack/verify-promoted-packages_test.bats b/hack/verify-promoted-packages_test.bats index 1d75f89dbb..1f6bad367d 100644 --- a/hack/verify-promoted-packages_test.bats +++ b/hack/verify-promoted-packages_test.bats @@ -365,3 +365,108 @@ EOF [ "$rc" -ne 0 ] grep -q 'promotion changed the container repository/digest set' "$tmp/err" } + +# Shape 3 — `repository:` on one key, `tag: @sha256:` on another — +# is the shape hack/lib/image-refs.sh itself calls the dominant one, and the one +# no fixture above builds. The collector emits TWO entries for such a map: the +# correctly joined repository@digest, plus shape 1's recursive scrape of the +# bare `tag` scalar, which names no repository at all. Promotion rewrites +# exactly that scalar, so normalization keeping it — treating the tag as the +# repository — makes every clean promotion look as though it moved container +# bytes. The single-string ref in _make_verify_fixture cannot reach that path: +# it carries a registry host, so it normalizes through the other branch and its +# own rc-to-stable rewrite passes either way. That is why the suite looked +# covered while the dominant shape went untested. +_write_split_map() { + cat > "$1" < "$tmp/out" 2> "$tmp/err" || rc=$? + if [ "$rc" -ne 0 ]; then + echo "verification exited $rc" >&2 + cat "$tmp/err" >&2 + return "$rc" + fi + + grep -q "stable tags, identical package tree, and the rc artifact's container digests" "$tmp/out" +} + +# The property a blunter fix would break. packages/system/kuberture/values.yaml +# carries an `image:` map with a `tag:` and no `repository:` — shape 2 wants a +# `digest` key and shape 3 a `repository` key, so shape 1's scrape of the bare +# tag scalar is the ONLY rule that ever sees that image's digest. Discarding +# host-less refs rather than comparing them on their digest would silently stop +# proving it unchanged, which is a hole in the same guard being repaired here. +# Asserted from the rejecting direction, because that is the direction able to +# go red: the digest moves and nothing else does, so the digest-set comparison +# is the only leg that can fire — artifact and release tree stay byte-identical, +# no rc string appears in the candidate, and the installer values match. +_write_repository_less_map() { + cat > "$1" < "$tmp/out" 2> "$tmp/err" || rc=$? + + [ "$rc" -ne 0 ] + grep -q 'promotion changed the container repository/digest set' "$tmp/err" + # The rejection has to come from the digest set and not from a neighbouring leg + # rejecting a tree this test never perturbed, or it would still pass with the + # digest dropped from the comparison entirely. Counted rather than negated with + # `!`, which suppresses errexit and would pass regardless. + for _other in 'contain different files' 'differs from release tree' \ + 'still carries rc image references' 'installer values differ'; do + count="$(grep -c "$_other" "$tmp/err" || true)" + [ "${count:-0}" -eq 0 ] + done +} From 269d339c0c23998a3555c86cc5fb912b918692f1 Mon Sep 17 00:00:00 2001 From: "cozystack-ci[bot]" <274107086+cozystack-ci[bot]@users.noreply.github.com> Date: Wed, 19 Aug 2026 07:31:23 +0000 Subject: [PATCH 50/51] Prepare release v1.6.2 (promoted from v1.6.2-rc.1) Signed-off-by: cozystack-ci[bot] <274107086+cozystack-ci[bot]@users.noreply.github.com> --- docs/changelogs/v1.6.2.md | 50 +++++++++++++++++++ .../images/altinity-clickhouse-backup.tag | 2 +- .../clickhouse/images/clickhouse-backup.tag | 2 +- .../apps/http-cache/images/nginx-cache.tag | 2 +- .../kubernetes/images/cluster-autoscaler.tag | 2 +- .../images/kubevirt-cloud-provider.tag | 2 +- .../kubernetes/images/kubevirt-csi-driver.tag | 2 +- .../kubernetes/images/talos-csr-signer.tag | 2 +- .../apps/mariadb/images/mariadb-backup.tag | 2 +- packages/core/installer/values.yaml | 2 +- packages/core/platform/values.yaml | 2 +- packages/core/testing/values.yaml | 2 +- packages/extra/bootbox/images/matchbox.tag | 2 +- .../images/objectstorage-sidecar.tag | 2 +- packages/system/backup-controller/values.yaml | 2 +- .../backupstrategy-controller/values.yaml | 4 +- packages/system/bucket/images/s3manager.tag | 2 +- packages/system/cilium/values.yaml | 2 +- packages/system/cozystack-api/values.yaml | 2 +- .../system/cozystack-controller/values.yaml | 2 +- packages/system/dashboard/values.yaml | 4 +- .../system/flux-shard-operator/values.yaml | 2 +- .../images/grafana-dashboards.tag | 2 +- packages/system/kamaji/values.yaml | 4 +- packages/system/kubeovn-plunger/values.yaml | 2 +- packages/system/kubeovn-webhook/values.yaml | 2 +- packages/system/kubevirt-csi-node/values.yaml | 2 +- .../lineage-controller-webhook/values.yaml | 2 +- packages/system/linstor-gui/values.yaml | 2 +- packages/system/linstor/values.yaml | 4 +- packages/system/metallb/values.yaml | 4 +- packages/system/monitoring/images/grafana.tag | 2 +- .../templates/multus-daemonset-thick.yml | 4 +- .../objectstorage-controller/values.yaml | 2 +- packages/system/seaweedfs/values.yaml | 2 +- .../securitygroup-controller/values.yaml | 2 +- 36 files changed, 91 insertions(+), 41 deletions(-) create mode 100644 docs/changelogs/v1.6.2.md diff --git a/docs/changelogs/v1.6.2.md b/docs/changelogs/v1.6.2.md new file mode 100644 index 0000000000..a6f0a8c845 --- /dev/null +++ b/docs/changelogs/v1.6.2.md @@ -0,0 +1,50 @@ + + +# v1.6.2 (2026-08-19) + +A patch release with six fixes covering the backup-strategy controller, kube-ovn's webhook certificate, Velero CRD upgrades, CNPG barman-cloud backups, `flux-shard-operator`, and the published OpenAPI definitions, plus a release-pipeline reliability fix. + +## Fixes + +* **fix(backupstrategy-controller): repair lookup-gated backup objects**: The default backup `Strategy` CRs and the Velero `BackupStorageLocation` are gated on a Helm `lookup` performed while the referenced object is still being created; when that lookup came back empty the objects were skipped permanently, since helm-controller does not re-render a release whose chart and values are unchanged. The gate now resolves the default bucket credentials Secret through the RESTMapper, bounds each check, and tolerates an absent Secret instead of looping, so the default backup objects are created reliably instead of silently vanishing for months ([**@mattia-eleuteri**](https://github.com/mattia-eleuteri) in #3524, backport #3731). + +* **fix(kube-ovn): reload kubeovn-webhook serving certificate on cert-manager renewal**: `kube-ovn-webhook` loaded its TLS serving certificate once at startup and never re-read it; once cert-manager renewed the backing Secret and the old certificate expired, the apiserver's calls to the webhook failed verification and, because the `MutatingWebhookConfiguration` uses `failurePolicy: Fail`, every pod creation in tenant namespaces was rejected — including `virt-launcher` pods, blocking VMI startup. The webhook now serves its certificate through a reloading callback that re-reads the key pair when the mounted files change and widens `renewBefore` to 720h, so cert-manager renewals are honored without a pod restart ([**@IvanHunters**](https://github.com/IvanHunters) in #3557, backport #3730). + +* **fix(velero): apply CRD updates on upgrade via CreateReplace**: Velero's CRDs stayed frozen at whatever version was first installed, since Helm never touches a chart's `crds/` directory on upgrade; when the Velero image moved to a version that added new backup phases, the apiserver rejected phase transitions against the stale CRDs and backups silently stopped while the HelmRelease stayed green. The Velero package now opts into `upgradeCRDs: CreateReplace`, so CRDs are kept current on upgrade and backups keep working ([**@lexfrei**](https://github.com/lexfrei) in #3727, backport #3728). + +* **fix(backups): request S3 checksum only when required for barman-cloud (non-AWS S3 / Ceph RGW)**: CNPG's barman-cloud plugin sidecar defaulted to computing a flexible checksum on every upload, which several S3-compatible backends (Ceph RGW, some MinIO / Cloudflare R2 builds) reject outright, so every backup and WAL-archive upload to those backends failed and `ScheduledBackup`s never stored anything. Every barman-cloud `ObjectStore` Cozystack creates — Keycloak's system DB, the postgres app's backup and recovery stores, and the platform-managed system-bucket store — now sets `AWS_REQUEST_CHECKSUM_CALCULATION=when_required`, a safe default accepted by both AWS S3 and the affected backends ([**@androndo**](https://github.com/androndo) in #3417, backport #3767). + +* **fix(flux-shard-operator): repair sharded helm-controller crashloop behind an HTTP proxy**: The cloned `helm-controller-shard` Deployment inherited `HTTP_PROXY`/`HTTPS_PROXY`/`NO_PROXY` from the flux-aio all-in-one wiring even though a standalone shard needs no external egress; behind an unreachable proxy the controller's blocking startup HTTPS call never completed, the manager never served `/healthz`, and every HelmRelease sharded to that controller was frozen. The sanitisation now also drops the inherited proxy env and adds a `startupProbe` derived from the liveness handler, so sharded HelmReleases keep reconciling in proxied environments instead of crashlooping forever ([**@IvanHunters**](https://github.com/IvanHunters) in #3546, backport #3818). + +* **fix(api): declare OpenAPIModelName for core and sdn types**: The `core` and `sdn` API groups did not declare `OpenAPIModelName` the way the `apps` group already did, so their published OpenAPI definition names were Go import paths while every `$ref` pointing at them escaped each slash — the two spellings never matched, the reference dangled, and `kubectl apply --validate` failed on any resource against a `cozystack-api` built after the underlying Kubernetes 0.35 change. Declaring `OpenAPIModelName` for `core` and `sdn` too makes every published definition name the dotted Kubernetes model name, so client-side validation against the published OpenAPI works again ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #3808, backport #3812). + +## Development, Testing, and CI/CD + +* **ci(release): complete the candidate-aware promotion pipeline on release-1.6**: `release-1.6` was missing the e2e and packages-verification jobs that `Promote RC` requires on its target base, so `v1.6.1` was promoted with the rc e2e gate bypassed and the next patch release could not even be dispatched. Adds the `rc-e2e` job, the `verify-release-candidate` checks, `hack/verify-promoted-packages.sh`, `hack/validate-changelog.sh` and regression tests pinning the pipeline's contract, so future patch releases off `release-1.6` run the same e2e and package-verification gates as `main` before promoting, and the tag-time changelog is validated and ported from the tag rather than regenerated ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #3893). + +## Documentation + +* **[website] docs: import the operator guides that lived in the cozystack repo**: Moves the operator-facing guides that used to live in the `cozystack` repo over to the documentation site, so operators find them alongside the rest of the docs instead of scattered across two repositories ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in cozystack/website#648). + +* **[website] docs(oidc): document private CA and staging trust**: Documents how to configure tenant OIDC to trust a private certificate authority and staging certificates, closing a gap for operators running their own CA or testing with a staging issuer ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in cozystack/website#650). + +* **[website] feat(community): add a Community page and link it from the main menu**: Adds a Community page linked from the site's main menu, giving visitors a single place to find how to get in touch with and contribute to the Cozystack community ([**@tym83**](https://github.com/tym83) in cozystack/website#637). + +* **[website] chore(telemetry): publish July 2026 and explain how the figures are derived**: Publishes the July 2026 telemetry figures and documents how those figures are derived, giving the community visibility into adoption trends and how the numbers are calculated ([**@tym83**](https://github.com/tym83) in cozystack/website#644). + +* **[website] feat(blog): new Blockstor banner**: Adds a new banner promoting Blockstor to the blog, improving the visibility of the storage control plane's announcement ([**@tym83**](https://github.com/tym83) in cozystack/website#646). + +## Contributors + +Thanks to everyone who contributed to this patch release: + +* [**@IvanHunters**](https://github.com/IvanHunters) +* [**@androndo**](https://github.com/androndo) +* [**@lexfrei**](https://github.com/lexfrei) +* [**@mattia-eleuteri**](https://github.com/mattia-eleuteri) +* [**@myasnikovdaniil**](https://github.com/myasnikovdaniil) +* [**@tym83**](https://github.com/tym83) + +**Full Changelog**: https://github.com/cozystack/cozystack/compare/v1.6.1...v1.6.2 diff --git a/packages/apps/clickhouse/images/altinity-clickhouse-backup.tag b/packages/apps/clickhouse/images/altinity-clickhouse-backup.tag index 66957e3e50..0a693fdc87 100644 --- a/packages/apps/clickhouse/images/altinity-clickhouse-backup.tag +++ b/packages/apps/clickhouse/images/altinity-clickhouse-backup.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/altinity-clickhouse-backup:v1.6.2-rc.1@sha256:e890d78cd83968bdff9a36c9ca4312afc05e138af486e476da0a935f81f9b7a7 +ghcr.io/cozystack/cozystack/altinity-clickhouse-backup:v1.6.2@sha256:e890d78cd83968bdff9a36c9ca4312afc05e138af486e476da0a935f81f9b7a7 diff --git a/packages/apps/clickhouse/images/clickhouse-backup.tag b/packages/apps/clickhouse/images/clickhouse-backup.tag index f6dfcd865d..216a99c319 100644 --- a/packages/apps/clickhouse/images/clickhouse-backup.tag +++ b/packages/apps/clickhouse/images/clickhouse-backup.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/clickhouse-backup:v1.6.2-rc.1@sha256:8efef62d59fb82d39544823f81a797c2b9ac130a50241fee48e701161ce2670d +ghcr.io/cozystack/cozystack/clickhouse-backup:v1.6.2@sha256:8efef62d59fb82d39544823f81a797c2b9ac130a50241fee48e701161ce2670d diff --git a/packages/apps/http-cache/images/nginx-cache.tag b/packages/apps/http-cache/images/nginx-cache.tag index 58d5fa00ab..d25e1e3445 100644 --- a/packages/apps/http-cache/images/nginx-cache.tag +++ b/packages/apps/http-cache/images/nginx-cache.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/nginx-cache:v1.6.2-rc.1@sha256:bd042a8a9789e8b21b12cb266e390484a3a48f4af6ca6e2d2864a8a83c74bd25 +ghcr.io/cozystack/cozystack/nginx-cache:v1.6.2@sha256:bd042a8a9789e8b21b12cb266e390484a3a48f4af6ca6e2d2864a8a83c74bd25 diff --git a/packages/apps/kubernetes/images/cluster-autoscaler.tag b/packages/apps/kubernetes/images/cluster-autoscaler.tag index 027c22ca47..83233f03dc 100644 --- a/packages/apps/kubernetes/images/cluster-autoscaler.tag +++ b/packages/apps/kubernetes/images/cluster-autoscaler.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/cluster-autoscaler:v1.6.2-rc.1@sha256:e77ff02b328e4e119439efd1edd3aa71e9325fabf10a4a84b03a87d12d90b308 +ghcr.io/cozystack/cozystack/cluster-autoscaler:v1.6.2@sha256:e77ff02b328e4e119439efd1edd3aa71e9325fabf10a4a84b03a87d12d90b308 diff --git a/packages/apps/kubernetes/images/kubevirt-cloud-provider.tag b/packages/apps/kubernetes/images/kubevirt-cloud-provider.tag index 84cccc5312..4629ad3a7f 100644 --- a/packages/apps/kubernetes/images/kubevirt-cloud-provider.tag +++ b/packages/apps/kubernetes/images/kubevirt-cloud-provider.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/kubevirt-cloud-provider:v1.6.2-rc.1@sha256:f005b99041d191eed9dde98f9f6691ba2d0cbfa1b3b0b2fe4b32cc0fdba74acf +ghcr.io/cozystack/cozystack/kubevirt-cloud-provider:v1.6.2@sha256:f005b99041d191eed9dde98f9f6691ba2d0cbfa1b3b0b2fe4b32cc0fdba74acf diff --git a/packages/apps/kubernetes/images/kubevirt-csi-driver.tag b/packages/apps/kubernetes/images/kubevirt-csi-driver.tag index 77c6197d4e..22fea6235c 100644 --- a/packages/apps/kubernetes/images/kubevirt-csi-driver.tag +++ b/packages/apps/kubernetes/images/kubevirt-csi-driver.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/kubevirt-csi-driver:v1.6.2-rc.1@sha256:0fe6808c5857493a0d7e98344a2bcdfc3fea34917386fc654c03328fc11afe2e +ghcr.io/cozystack/cozystack/kubevirt-csi-driver:v1.6.2@sha256:0fe6808c5857493a0d7e98344a2bcdfc3fea34917386fc654c03328fc11afe2e diff --git a/packages/apps/kubernetes/images/talos-csr-signer.tag b/packages/apps/kubernetes/images/talos-csr-signer.tag index 01099bac83..0e6f67d82d 100644 --- a/packages/apps/kubernetes/images/talos-csr-signer.tag +++ b/packages/apps/kubernetes/images/talos-csr-signer.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/talos-csr-signer:v1.6.2-rc.1@sha256:7051e90245cb446ce4deb857d4337e2fd69ce84436243ba60939d374defc2a07 +ghcr.io/cozystack/cozystack/talos-csr-signer:v1.6.2@sha256:7051e90245cb446ce4deb857d4337e2fd69ce84436243ba60939d374defc2a07 diff --git a/packages/apps/mariadb/images/mariadb-backup.tag b/packages/apps/mariadb/images/mariadb-backup.tag index 04043407b1..cc10075498 100644 --- a/packages/apps/mariadb/images/mariadb-backup.tag +++ b/packages/apps/mariadb/images/mariadb-backup.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/mariadb-backup:v1.6.2-rc.1@sha256:1de944b5c4fbeef94004b8620b0865aa7886a692dacb23d50c8f9b31fbeece07 +ghcr.io/cozystack/cozystack/mariadb-backup:v1.6.2@sha256:1de944b5c4fbeef94004b8620b0865aa7886a692dacb23d50c8f9b31fbeece07 diff --git a/packages/core/installer/values.yaml b/packages/core/installer/values.yaml index 816b2ed48a..21790feca0 100644 --- a/packages/core/installer/values.yaml +++ b/packages/core/installer/values.yaml @@ -14,7 +14,7 @@ cozystackOperator: # to the value baked at build time. The Makefile sets this for the # kubectl-apply release artifacts (_out/assets/cozystack-operator-*.yaml). platformVersion: "" - image: ghcr.io/cozystack/cozystack/cozystack-operator:v1.6.2-rc.1@sha256:41ea5ca9c6a7d471105920b09fc8ed68279b8ca0dbffe7f839dccd530b42e4c7 + image: ghcr.io/cozystack/cozystack/cozystack-operator:v1.6.2@sha256:41ea5ca9c6a7d471105920b09fc8ed68279b8ca0dbffe7f839dccd530b42e4c7 platformSourceUrl: 'oci://ghcr.io/cozystack/cozystack/cozystack-packages' platformSourceRef: 'digest=sha256:02bc4a989b12ba6bce4cae79a2feb1092dbb094028eaea5959b1d833de641201' # When non-empty, overrides the operator's --helmrelease-interval flag diff --git a/packages/core/platform/values.yaml b/packages/core/platform/values.yaml index f03472f8da..4c01da3403 100644 --- a/packages/core/platform/values.yaml +++ b/packages/core/platform/values.yaml @@ -13,7 +13,7 @@ migrations: # below therefore lags targetVersion in-tree by design; run-migrations.sh # refuses to advance past a migration file missing from the image (exit 1), so # a stale pin fails loudly rather than silently skipping the etcd adoption. - image: ghcr.io/cozystack/cozystack/platform-migrations:v1.6.2-rc.1@sha256:a34e7156a52dc55b27662917e5f9d5bbea671188d8db233296d0a98c800dc6c6 + image: ghcr.io/cozystack/cozystack/platform-migrations:v1.6.2@sha256:a34e7156a52dc55b27662917e5f9d5bbea671188d8db233296d0a98c800dc6c6 targetVersion: 54 # Adopt legacy etcd clusters onto the v1alpha2 operator WITHOUT the mandatory # pre-adoption safety snapshot (migration 50). Leave false unless migration 50 diff --git a/packages/core/testing/values.yaml b/packages/core/testing/values.yaml index f01eb0b5ce..ffc3cb2818 100644 --- a/packages/core/testing/values.yaml +++ b/packages/core/testing/values.yaml @@ -1,2 +1,2 @@ e2e: - image: ghcr.io/cozystack/cozystack/e2e-sandbox:v1.6.2-rc.1@sha256:6f9f059305b057dce4dc0c68e82b86fe5573ecae40795f997b70db2f7e16f82d + image: ghcr.io/cozystack/cozystack/e2e-sandbox:v1.6.2@sha256:6f9f059305b057dce4dc0c68e82b86fe5573ecae40795f997b70db2f7e16f82d diff --git a/packages/extra/bootbox/images/matchbox.tag b/packages/extra/bootbox/images/matchbox.tag index 3aa407d25f..b7f6101120 100644 --- a/packages/extra/bootbox/images/matchbox.tag +++ b/packages/extra/bootbox/images/matchbox.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/matchbox:v1.6.2-rc.1@sha256:d3db5e7b5eb146447c3be22845bc49e7322641a617d60dcd7803f502f0e93bb7 +ghcr.io/cozystack/cozystack/matchbox:v1.6.2@sha256:d3db5e7b5eb146447c3be22845bc49e7322641a617d60dcd7803f502f0e93bb7 diff --git a/packages/extra/seaweedfs/images/objectstorage-sidecar.tag b/packages/extra/seaweedfs/images/objectstorage-sidecar.tag index d8f79d7a35..c9e15b1c33 100644 --- a/packages/extra/seaweedfs/images/objectstorage-sidecar.tag +++ b/packages/extra/seaweedfs/images/objectstorage-sidecar.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/objectstorage-sidecar:v1.6.2-rc.1@sha256:ef3c154c1a6dd1ac9fa1b0763a2bd23707d68f2f16f16a97ebf331f3646b53ce +ghcr.io/cozystack/cozystack/objectstorage-sidecar:v1.6.2@sha256:ef3c154c1a6dd1ac9fa1b0763a2bd23707d68f2f16f16a97ebf331f3646b53ce diff --git a/packages/system/backup-controller/values.yaml b/packages/system/backup-controller/values.yaml index 452cad8111..483d9eb5e5 100644 --- a/packages/system/backup-controller/values.yaml +++ b/packages/system/backup-controller/values.yaml @@ -1,5 +1,5 @@ backupController: - image: "ghcr.io/cozystack/cozystack/backup-controller:v1.6.2-rc.1@sha256:0774d068befce98dd067cb6239e5d37cb0b03b3d17a429e80caa46787d7f3726" + image: "ghcr.io/cozystack/cozystack/backup-controller:v1.6.2@sha256:0774d068befce98dd067cb6239e5d37cb0b03b3d17a429e80caa46787d7f3726" replicas: 2 debug: false metrics: diff --git a/packages/system/backupstrategy-controller/values.yaml b/packages/system/backupstrategy-controller/values.yaml index ce736253a2..de005bac46 100644 --- a/packages/system/backupstrategy-controller/values.yaml +++ b/packages/system/backupstrategy-controller/values.yaml @@ -1,5 +1,5 @@ backupStrategyController: - image: "ghcr.io/cozystack/cozystack/backupstrategy-controller:v1.6.2-rc.1@sha256:390caeb8cabf80dfe9bcd9f38b4ed97f0e71c22af36496fb744cdc50bb9501fc" + image: "ghcr.io/cozystack/cozystack/backupstrategy-controller:v1.6.2@sha256:390caeb8cabf80dfe9bcd9f38b4ed97f0e71c22af36496fb744cdc50bb9501fc" # chBackupClientImage is the image rendered into the Altinity strategy # Pod: it drives clickhouse-backup's HTTP API via curl + jq. # @@ -35,7 +35,7 @@ backupStrategyController: # retag both to the same stable tag, and the write-once guard in # hack/promote-retag.sh fails the release. hack/image-pin-consistency.bats # asserts no repository is pinned at more than one digest. - chBackupClientImage: "ghcr.io/cozystack/cozystack/platform-migrations:v1.6.2-rc.1@sha256:a34e7156a52dc55b27662917e5f9d5bbea671188d8db233296d0a98c800dc6c6" + chBackupClientImage: "ghcr.io/cozystack/cozystack/platform-migrations:v1.6.2@sha256:a34e7156a52dc55b27662917e5f9d5bbea671188d8db233296d0a98c800dc6c6" replicas: 2 debug: false metrics: diff --git a/packages/system/bucket/images/s3manager.tag b/packages/system/bucket/images/s3manager.tag index 554b14bbe3..0fb589ef4d 100644 --- a/packages/system/bucket/images/s3manager.tag +++ b/packages/system/bucket/images/s3manager.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/s3manager:v1.6.2-rc.1@sha256:73665aaf7f5406b4a1e456d14060fe874e0ae2a436ba7fd3058fa73abff8a927 +ghcr.io/cozystack/cozystack/s3manager:v1.6.2@sha256:73665aaf7f5406b4a1e456d14060fe874e0ae2a436ba7fd3058fa73abff8a927 diff --git a/packages/system/cilium/values.yaml b/packages/system/cilium/values.yaml index 2f27b497d3..dd60e1a88f 100644 --- a/packages/system/cilium/values.yaml +++ b/packages/system/cilium/values.yaml @@ -23,7 +23,7 @@ cilium: mode: "kubernetes" image: repository: ghcr.io/cozystack/cozystack/cilium - tag: v1.6.2-rc.1 + tag: v1.6.2 digest: "sha256:136a7dfff4d6adcd448424c2354144160028a8e96eb6558b917fd3ee6d7b39fe" envoy: enabled: true diff --git a/packages/system/cozystack-api/values.yaml b/packages/system/cozystack-api/values.yaml index d3ed4f8c30..99bba399b4 100644 --- a/packages/system/cozystack-api/values.yaml +++ b/packages/system/cozystack-api/values.yaml @@ -1,3 +1,3 @@ cozystackAPI: - image: ghcr.io/cozystack/cozystack/cozystack-api:v1.6.2-rc.1@sha256:8f412e6332e96370c678c776f663ff81b813bd84ed5b4303ecbd6a47dbba9a01 + image: ghcr.io/cozystack/cozystack/cozystack-api:v1.6.2@sha256:8f412e6332e96370c678c776f663ff81b813bd84ed5b4303ecbd6a47dbba9a01 replicas: 2 diff --git a/packages/system/cozystack-controller/values.yaml b/packages/system/cozystack-controller/values.yaml index 16d52d8ac6..5219a55d1a 100644 --- a/packages/system/cozystack-controller/values.yaml +++ b/packages/system/cozystack-controller/values.yaml @@ -1,4 +1,4 @@ cozystackController: - image: ghcr.io/cozystack/cozystack/cozystack-controller:v1.6.2-rc.1@sha256:fd0cc8e40293f52ea12ac8a7933dc0104a828045f9d7a73b3a38095da86c40f7 + image: ghcr.io/cozystack/cozystack/cozystack-controller:v1.6.2@sha256:fd0cc8e40293f52ea12ac8a7933dc0104a828045f9d7a73b3a38095da86c40f7 debug: false disableTelemetry: false diff --git a/packages/system/dashboard/values.yaml b/packages/system/dashboard/values.yaml index 4846584c03..427df90411 100644 --- a/packages/system/dashboard/values.yaml +++ b/packages/system/dashboard/values.yaml @@ -1,4 +1,4 @@ console: - image: ghcr.io/cozystack/cozystack/cozystack-ui:v1.6.2-rc.1@sha256:051282dbc216ed4f7b237ee93f172a5c8f9302e549805b70373c849c7dea6c7b + image: ghcr.io/cozystack/cozystack/cozystack-ui:v1.6.2@sha256:051282dbc216ed4f7b237ee93f172a5c8f9302e549805b70373c849c7dea6c7b tokenProxy: - image: ghcr.io/cozystack/cozystack/token-proxy:v1.6.2-rc.1@sha256:97124c012246c33cdcac60e8f884097b48c0081624716fe8161efc2b3ceeb2a8 + image: ghcr.io/cozystack/cozystack/token-proxy:v1.6.2@sha256:97124c012246c33cdcac60e8f884097b48c0081624716fe8161efc2b3ceeb2a8 diff --git a/packages/system/flux-shard-operator/values.yaml b/packages/system/flux-shard-operator/values.yaml index 1308115429..cf89e440bd 100644 --- a/packages/system/flux-shard-operator/values.yaml +++ b/packages/system/flux-shard-operator/values.yaml @@ -1,5 +1,5 @@ fluxShardOperator: - image: ghcr.io/cozystack/cozystack/flux-shard-operator:v1.6.2-rc.1@sha256:b59ec5d228575a0aee678b95b352270acf6020fb80caf9636ed690225b7858d7 + image: ghcr.io/cozystack/cozystack/flux-shard-operator:v1.6.2@sha256:b59ec5d228575a0aee678b95b352270acf6020fb80caf9636ed690225b7858d7 debug: false replicas: 2 ## Number of helm-controller shards to provision and distribute tenants diff --git a/packages/system/grafana-operator/images/grafana-dashboards.tag b/packages/system/grafana-operator/images/grafana-dashboards.tag index ef1ebcfdb1..337aa6b017 100644 --- a/packages/system/grafana-operator/images/grafana-dashboards.tag +++ b/packages/system/grafana-operator/images/grafana-dashboards.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/grafana-dashboards:v1.6.2-rc.1@sha256:cf8880fc5615e8c5309b31ad2624af206158958cf839d7f51dbf043ff7ceef21 +ghcr.io/cozystack/cozystack/grafana-dashboards:v1.6.2@sha256:cf8880fc5615e8c5309b31ad2624af206158958cf839d7f51dbf043ff7ceef21 diff --git a/packages/system/kamaji/values.yaml b/packages/system/kamaji/values.yaml index 367f3f5600..be97fa5c0a 100644 --- a/packages/system/kamaji/values.yaml +++ b/packages/system/kamaji/values.yaml @@ -30,7 +30,7 @@ kamaji: topologyKey: kubernetes.io/hostname image: pullPolicy: IfNotPresent - tag: v1.6.2-rc.1@sha256:dd81f5b76208e11665f2d2eed00396653c32182079ee53f44169074ac35d62ed + tag: v1.6.2@sha256:dd81f5b76208e11665f2d2eed00396653c32182079ee53f44169074ac35d62ed repository: ghcr.io/cozystack/cozystack/kamaji resources: limits: @@ -63,4 +63,4 @@ kamaji: periodSeconds: 10 timeoutSeconds: 1 extraArgs: - - --migrate-image=ghcr.io/cozystack/cozystack/kamaji:v1.6.2-rc.1@sha256:dd81f5b76208e11665f2d2eed00396653c32182079ee53f44169074ac35d62ed + - --migrate-image=ghcr.io/cozystack/cozystack/kamaji:v1.6.2@sha256:dd81f5b76208e11665f2d2eed00396653c32182079ee53f44169074ac35d62ed diff --git a/packages/system/kubeovn-plunger/values.yaml b/packages/system/kubeovn-plunger/values.yaml index e83594e8e0..8e5242c743 100644 --- a/packages/system/kubeovn-plunger/values.yaml +++ b/packages/system/kubeovn-plunger/values.yaml @@ -1,4 +1,4 @@ portSecurity: true routes: "" -image: ghcr.io/cozystack/cozystack/kubeovn-plunger:v1.6.2-rc.1@sha256:6f38f72c16f86a5937630ba58ee42afca2fd8f14dfe45bb79670f9a4a8447b60 +image: ghcr.io/cozystack/cozystack/kubeovn-plunger:v1.6.2@sha256:6f38f72c16f86a5937630ba58ee42afca2fd8f14dfe45bb79670f9a4a8447b60 ovnCentralName: ovn-central diff --git a/packages/system/kubeovn-webhook/values.yaml b/packages/system/kubeovn-webhook/values.yaml index a388d69620..134fed945d 100644 --- a/packages/system/kubeovn-webhook/values.yaml +++ b/packages/system/kubeovn-webhook/values.yaml @@ -1,3 +1,3 @@ portSecurity: true routes: "" -image: ghcr.io/cozystack/cozystack/kubeovn-webhook:v1.6.2-rc.1@sha256:8abc089c4469b6572d7eb6cba1bd87528dc1298101a2250d4b351d6ba0ca0620 +image: ghcr.io/cozystack/cozystack/kubeovn-webhook:v1.6.2@sha256:8abc089c4469b6572d7eb6cba1bd87528dc1298101a2250d4b351d6ba0ca0620 diff --git a/packages/system/kubevirt-csi-node/values.yaml b/packages/system/kubevirt-csi-node/values.yaml index 39ebb70556..563fd45d8a 100644 --- a/packages/system/kubevirt-csi-node/values.yaml +++ b/packages/system/kubevirt-csi-node/values.yaml @@ -9,4 +9,4 @@ storageClasses: infraStorageClass: replicated default: false csiDriver: - image: ghcr.io/cozystack/cozystack/kubevirt-csi-driver:v1.6.2-rc.1@sha256:0fe6808c5857493a0d7e98344a2bcdfc3fea34917386fc654c03328fc11afe2e + image: ghcr.io/cozystack/cozystack/kubevirt-csi-driver:v1.6.2@sha256:0fe6808c5857493a0d7e98344a2bcdfc3fea34917386fc654c03328fc11afe2e diff --git a/packages/system/lineage-controller-webhook/values.yaml b/packages/system/lineage-controller-webhook/values.yaml index 4d8d536e9e..7c56993732 100644 --- a/packages/system/lineage-controller-webhook/values.yaml +++ b/packages/system/lineage-controller-webhook/values.yaml @@ -1,5 +1,5 @@ lineageControllerWebhook: - image: ghcr.io/cozystack/cozystack/lineage-controller-webhook:v1.6.2-rc.1@sha256:454e3da9468e578bf627210ea1ff664cf58b1f618edeaa158f52c9a930a44794 + image: ghcr.io/cozystack/cozystack/lineage-controller-webhook:v1.6.2@sha256:454e3da9468e578bf627210ea1ff664cf58b1f618edeaa158f52c9a930a44794 debug: false replicas: 2 # DEPRECATED. Injects KUBERNETES_SERVICE_HOST=status.hostIP and diff --git a/packages/system/linstor-gui/values.yaml b/packages/system/linstor-gui/values.yaml index c3fbc008c0..8f9b888933 100644 --- a/packages/system/linstor-gui/values.yaml +++ b/packages/system/linstor-gui/values.yaml @@ -3,7 +3,7 @@ ## @param image.tag LINSTOR GUI container image tag (digest recommended) image: repository: ghcr.io/cozystack/cozystack/linstor-gui - tag: v1.6.2-rc.1@sha256:6e11829de86709f5e21636cdd1a1ada1239a7dcc1741f91e9d95d9f6ade603a2 + tag: v1.6.2@sha256:6e11829de86709f5e21636cdd1a1ada1239a7dcc1741f91e9d95d9f6ade603a2 ## @section Deployment ## @param replicas Number of linstor-gui replicas replicas: 1 diff --git a/packages/system/linstor/values.yaml b/packages/system/linstor/values.yaml index 39df945066..ee9884e2b9 100644 --- a/packages/system/linstor/values.yaml +++ b/packages/system/linstor/values.yaml @@ -1,7 +1,7 @@ piraeusServer: image: repository: ghcr.io/cozystack/cozystack/piraeus-server - tag: v1.6.2-rc.1@sha256:82d2234181ac8d5942e57286101b242a08fbcc49a9bf37d5df529921624fdca9 + tag: v1.6.2@sha256:82d2234181ac8d5942e57286101b242a08fbcc49a9bf37d5df529921624fdca9 # Talos-specific workarounds (disable for generic Linux like Ubuntu/Debian) talos: enabled: true @@ -13,7 +13,7 @@ linstor: linstorCSI: image: repository: ghcr.io/cozystack/cozystack/linstor-csi - tag: v1.6.2-rc.1@sha256:3228487ae861ddf02cac984289f8c377fc9e66a71fb3c387e9af9f72c1dbc78b + tag: v1.6.2@sha256:3228487ae861ddf02cac984289f8c377fc9e66a71fb3c387e9af9f72c1dbc78b controller: # Resource requests for the linstor-controller JVM. Requests (not limits) are # set on purpose: a CPU request guarantees CPU shares so the JVM can answer its diff --git a/packages/system/metallb/values.yaml b/packages/system/metallb/values.yaml index df6b04e1d8..1ad3a0b295 100644 --- a/packages/system/metallb/values.yaml +++ b/packages/system/metallb/values.yaml @@ -37,11 +37,11 @@ metallb: controller: image: repository: ghcr.io/cozystack/cozystack/metallb-controller - tag: v1.6.2-rc.1@sha256:9d8ba76cdb9c7c6221334ad05d706dee22b138b3e90c1fe8fc884925b7480c02 + tag: v1.6.2@sha256:9d8ba76cdb9c7c6221334ad05d706dee22b138b3e90c1fe8fc884925b7480c02 speaker: image: repository: ghcr.io/cozystack/cozystack/metallb-speaker - tag: v1.6.2-rc.1@sha256:87df3c82d0b6ea223b26fd5d6fbba6e940c13e56418dfc1cd90863d145795be9 + tag: v1.6.2@sha256:87df3c82d0b6ea223b26fd5d6fbba6e940c13e56418dfc1cd90863d145795be9 # The vendored metallb chart's values.yaml leaves `frr-k8s.prometheus:` as # a YAML block containing only commented examples — it parses to null. # Helm-controller's deep-merge then writes that null over the frr-k8s diff --git a/packages/system/monitoring/images/grafana.tag b/packages/system/monitoring/images/grafana.tag index 03450722be..b36fb89286 100644 --- a/packages/system/monitoring/images/grafana.tag +++ b/packages/system/monitoring/images/grafana.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/grafana:v1.6.2-rc.1@sha256:63c90beb8bf31f49820f529c74ebeb9e8be10acb02c45ac517844aecdfd1a4f2 +ghcr.io/cozystack/cozystack/grafana:v1.6.2@sha256:63c90beb8bf31f49820f529c74ebeb9e8be10acb02c45ac517844aecdfd1a4f2 diff --git a/packages/system/multus/templates/multus-daemonset-thick.yml b/packages/system/multus/templates/multus-daemonset-thick.yml index 9ba9f5a8c0..56684711c3 100644 --- a/packages/system/multus/templates/multus-daemonset-thick.yml +++ b/packages/system/multus/templates/multus-daemonset-thick.yml @@ -156,7 +156,7 @@ spec: serviceAccountName: multus containers: - name: kube-multus - image: ghcr.io/cozystack/cozystack/multus-cni:v1.6.2-rc.1@sha256:f94fdd68d674a1f8c8a2b45fa8fd4b478fc632e66fa147aea73a6b60b721920c + image: ghcr.io/cozystack/cozystack/multus-cni:v1.6.2@sha256:f94fdd68d674a1f8c8a2b45fa8fd4b478fc632e66fa147aea73a6b60b721920c command: [ "/usr/src/multus-cni/bin/multus-daemon" ] resources: requests: @@ -202,7 +202,7 @@ spec: fieldPath: spec.nodeName initContainers: - name: install-multus-binary - image: ghcr.io/cozystack/cozystack/multus-cni:v1.6.2-rc.1@sha256:f94fdd68d674a1f8c8a2b45fa8fd4b478fc632e66fa147aea73a6b60b721920c + image: ghcr.io/cozystack/cozystack/multus-cni:v1.6.2@sha256:f94fdd68d674a1f8c8a2b45fa8fd4b478fc632e66fa147aea73a6b60b721920c command: - "/usr/src/multus-cni/bin/install_multus" - "-d" diff --git a/packages/system/objectstorage-controller/values.yaml b/packages/system/objectstorage-controller/values.yaml index 35e3b89d41..42e116a8c8 100644 --- a/packages/system/objectstorage-controller/values.yaml +++ b/packages/system/objectstorage-controller/values.yaml @@ -1,3 +1,3 @@ objectstorage: controller: - image: "ghcr.io/cozystack/cozystack/objectstorage-controller:v1.6.2-rc.1@sha256:0d9ff6c2a3453fdc7e72894bbfdd3e5a292be78342c7c5352f655c976c3b7f0b" + image: "ghcr.io/cozystack/cozystack/objectstorage-controller:v1.6.2@sha256:0d9ff6c2a3453fdc7e72894bbfdd3e5a292be78342c7c5352f655c976c3b7f0b" diff --git a/packages/system/seaweedfs/values.yaml b/packages/system/seaweedfs/values.yaml index 1c4016a685..7462cc5794 100644 --- a/packages/system/seaweedfs/values.yaml +++ b/packages/system/seaweedfs/values.yaml @@ -259,7 +259,7 @@ seaweedfs: bucketClassName: "seaweedfs" region: "" sidecar: - image: "ghcr.io/cozystack/cozystack/objectstorage-sidecar:v1.6.2-rc.1@sha256:ef3c154c1a6dd1ac9fa1b0763a2bd23707d68f2f16f16a97ebf331f3646b53ce" + image: "ghcr.io/cozystack/cozystack/objectstorage-sidecar:v1.6.2@sha256:ef3c154c1a6dd1ac9fa1b0763a2bd23707d68f2f16f16a97ebf331f3646b53ce" certificates: commonName: "SeaweedFS CA" ipAddresses: [] diff --git a/packages/system/securitygroup-controller/values.yaml b/packages/system/securitygroup-controller/values.yaml index 83a58ca29c..3cb5dcfe89 100644 --- a/packages/system/securitygroup-controller/values.yaml +++ b/packages/system/securitygroup-controller/values.yaml @@ -1,5 +1,5 @@ securityGroupController: - image: "ghcr.io/cozystack/cozystack/securitygroup-controller:v1.6.2-rc.1@sha256:b1c7153a93344318ec96363c6802e20a93c559ac0586d4ea6eabd8d95baa324b" + image: "ghcr.io/cozystack/cozystack/securitygroup-controller:v1.6.2@sha256:b1c7153a93344318ec96363c6802e20a93c559ac0586d4ea6eabd8d95baa324b" replicas: 2 debug: false resources: From 00dd9499224f497b49dcdb74fb5bbbd15d857f8c Mon Sep 17 00:00:00 2001 From: "cozystack-ci[bot]" <274107086+cozystack-ci[bot]@users.noreply.github.com> Date: Wed, 19 Aug 2026 07:31:25 +0000 Subject: [PATCH 51/51] Pin promoted packages artifact for v1.6.2 Signed-off-by: cozystack-ci[bot] <274107086+cozystack-ci[bot]@users.noreply.github.com> --- packages/core/installer/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/core/installer/values.yaml b/packages/core/installer/values.yaml index 21790feca0..52a1cbb237 100644 --- a/packages/core/installer/values.yaml +++ b/packages/core/installer/values.yaml @@ -16,7 +16,7 @@ cozystackOperator: platformVersion: "" image: ghcr.io/cozystack/cozystack/cozystack-operator:v1.6.2@sha256:41ea5ca9c6a7d471105920b09fc8ed68279b8ca0dbffe7f839dccd530b42e4c7 platformSourceUrl: 'oci://ghcr.io/cozystack/cozystack/cozystack-packages' - platformSourceRef: 'digest=sha256:02bc4a989b12ba6bce4cae79a2feb1092dbb094028eaea5959b1d833de641201' + platformSourceRef: 'digest=sha256:251626e80efad3e87561b138e03301d8b63c15a42895359716aa18802b04fed5' # When non-empty, overrides the operator's --helmrelease-interval flag # (operator default: 5m). E2E sets this to 30s; production should leave empty. helmReleaseInterval: ""