diff --git a/.github/workflows/build-release.yaml b/.github/workflows/build-release.yaml new file mode 100644 index 0000000000..c70aa92d2e --- /dev/null +++ b/.github/workflows/build-release.yaml @@ -0,0 +1,116 @@ +name: Build release line + +# Publishes the per-line equivalent of what build-main.yaml publishes for main: +# every image tagged with the line's branch name, and the whole packages tree as +# the `cozystack-packages:` OCI artifact with each image reference +# digest-pinned to the images this run just pushed (`make build` -> +# packages/core/installer image-packages does the `flux push artifact`). +# +# Why it exists: pull-requests.yaml overlays image refs for every package a PR +# did NOT rebuild, so that e2e and the installer do not test last-release images +# for everything outside the PR's build matrix. That overlay had only +# `cozystack-packages:main` to read, which meant a release-line PR was handed +# MAIN's binaries to run against its own line's charts — a cross-generation mix +# that failed deterministically (see #3437: main's cozystack-controller served an +# aggregated OpenAPI release-1.6's charts could not validate against). With this +# artifact published per line, the overlay reads the PR's own base branch and the +# generations match. +# +# Cost: one full `make build` per push to a maintained line, i.e. per merged +# backport. That is the price of release-line PRs testing their line's tip rather +# than its last release. + +env: + # Same per-CI registry as pull-requests.yaml and build-main.yaml. + REGISTRY: iad.ocir.io/idyksih5sir9/cozystack + +on: + push: + # Maintained LINE branches only: `release-.`. The `+` quantifies + # the preceding character class, and `.` is literal, so this matches + # release-1.6 but NOT the per-release staging branches promote-rc.yaml creates + # (release-1.6.1) nor rc staging branches (release-1.6.0-rc.4) — building + # those would be pure waste, since their images are built by tags.yaml. + branches: ['release-[0-9]+.[0-9]+'] + paths-ignore: + - 'docs/**' + +# Per line: a newer push to the same line supersedes an in-flight build, but +# different lines build independently. +concurrency: + group: build-release-${{ github.ref_name }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + build-line: + name: Build ${{ github.ref_name }} images and packages artifact + runs-on: oracle-vm-24cpu-96gb-x86-64 + timeout-minutes: 120 + permissions: + contents: read + packages: write + + steps: + - name: Checkout code + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + fetch-depth: 0 + fetch-tags: true + # This workflow holds packages:write and an OCIR push token; it never + # pushes git, so don't leave the checkout's GITHUB_TOKEN in .git/config + # where a later build step could read it. + persist-credentials: false + + # Ephemeral runners lack the flux CLI the installer's image-packages step + # shells out to; install if absent (idempotent). + - name: Set up build toolchain + run: | + command -v flux >/dev/null \ + || curl -fsSL https://fluxcd.io/install.sh | sudo bash + + - name: Set up Docker config + run: | + if [ -d ~/.docker ]; then + cp -r ~/.docker "${{ runner.temp }}/.docker" + fi + + - name: Login to OCIR + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3 + with: + username: ${{ secrets.OCIR_USER }} + password: ${{ secrets.OCIR_TOKEN }} + registry: iad.ocir.io + env: + DOCKER_CONFIG: ${{ runner.temp }}/.docker + + # Same isolated buildkit as build-main.yaml, so a line build never contends + # with PR builds on the shared embedded builder. + - name: Set up Buildx (docker-container driver) + id: buildx + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 + with: + driver: docker-container + env: + DOCKER_CONFIG: ${{ runner.temp }}/.docker + + - name: Build line images and publish the packages artifact + run: make build + env: + DOCKER_CONFIG: ${{ runner.temp }}/.docker + BUILDER: ${{ steps.buildx.outputs.name }} + # READ the shared mode=max cache, never write it. CACHE_REGISTRY/: + # buildcache is a single ref per image and build-main.yaml is + # deliberately its only writer, serialized, precisely so concurrent + # builds cannot race on the cache manifest (the 409 collision class + # #2711 fixed for image tags). A line build can overlap a main build, so + # writing here would reintroduce exactly that race. + WRITE_CACHE: '0' + # Floating per-line handle: images and the packages artifact both land + # under the branch name (e.g. :release-1.6). Versioned and :latest tags + # belong to releases (tags.yaml / finalize), never to this workflow. + IMAGE_TAG: ${{ github.ref_name }} + PUBLISH_VERSIONED: '0' + PUBLISH_FLOATING: '0' diff --git a/.github/workflows/e2e-tag.yaml b/.github/workflows/e2e-tag.yaml new file mode 100644 index 0000000000..d70b62be5e --- /dev/null +++ b/.github/workflows/e2e-tag.yaml @@ -0,0 +1,374 @@ +name: E2E Release Tag + +# Validate the installable release closure for an existing version tag. +# +# Every RC must pass this lane after tags.yaml has published the prerelease and +# pushed its release-X.Y.Z-rc.N digest-pinned staging branch — tags.yaml's +# rc-e2e job calls this workflow as that mandatory post-cut check. The +# workflow_dispatch trigger is the manual validation button for published +# alpha, beta, RC, and stable releases. +# +# This lane validates published tags only; draft and promote-time validation are +# explicitly out of scope because release checks run at RC publication time. +# Prerelease tags use the matching digest-pinned staging branch, while stable +# tags use their pinned merge tree directly. In both cases the nocloud disk +# comes from the published release identified by the same tag. + +on: + workflow_call: + inputs: + tag: + description: "Release tag to validate, e.g. v1.7.0-rc.1" + required: true + type: string + workflow_dispatch: + inputs: + tag: + description: "Release tag to validate, e.g. v1.7.0-rc.1" + required: true + type: string + debug: + description: "Open a maintainer SSH breakpoint if E2E fails" + required: false + default: false + type: boolean + +concurrency: + group: e2e-tag-${{ inputs.tag }} + cancel-in-progress: false + +# Listing published releases, reading Git refs, downloading a release asset, and +# checking out the pinned tree require only repository contents read access. +# Draft releases are not part of this lane's contract. The public GHCR images +# referenced by that tree need no package token. +# +# CALLERS MUST GRANT `contents: read` AND `checks: write`. A caller's permissions +# are the ceiling for every job here, and GitHub validates that ceiling when it +# creates the run, before any `if:` is evaluated — so the `checks: write` the e2e +# job below declares for its dispatch-only breakpoint has to be granted even by a +# caller whose invocation can never reach that step. Granting less does not +# degrade the breakpoint; it fails the caller's entire workflow run at startup. +permissions: + contents: read + +jobs: + resolve: + name: Resolve release assets and tree + runs-on: ubuntu-latest + timeout-minutes: 10 + outputs: + disk_id: ${{ steps.resolve.outputs.disk_id }} + tree_ref: ${{ steps.resolve.outputs.tree_ref }} + steps: + - name: Validate tag and resolve release + id: resolve + uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7 + env: + TAG: ${{ inputs.tag }} + with: + script: | + const tag = process.env.TAG; + const pattern = /^v[0-9]+\.[0-9]+\.[0-9]+(-(alpha|beta|rc)\.[0-9]+)?$/; + if (!pattern.test(tag)) { + core.setFailed(`Tag '${tag}' must match vX.Y.Z, vX.Y.Z-alpha.N, vX.Y.Z-beta.N, or vX.Y.Z-rc.N`); + return; + } + + const isPrerelease = /-(alpha|beta|rc)\.[0-9]+$/.test(tag); + const releases = await github.paginate(github.rest.repos.listReleases, { + owner: context.repo.owner, + repo: context.repo.repo, + per_page: 100 + }); + const release = releases.find(candidate => candidate.tag_name === tag); + if (!release) { + core.setFailed(`GitHub release '${tag}' not found`); + return; + } + + if (release.draft) { + core.setFailed(`Release for tag '${tag}' is still a draft — this lane validates published tags only; RC validation runs after tags.yaml publishes the prerelease`); + return; + } else if (isPrerelease && !release.prerelease) { + core.setFailed(`Release '${tag}' must be published as a prerelease`); + return; + } else if (!isPrerelease && release.prerelease) { + core.setFailed(`Release '${tag}' must be published as a stable release`); + return; + } + + const disk = release.assets.find(asset => asset.name === 'nocloud-amd64.raw.xz'); + if (!disk) { + core.setFailed(`Required asset 'nocloud-amd64.raw.xz' is missing from release '${tag}'`); + return; + } + + const treeRef = isPrerelease ? `release-${tag.slice(1)}` : tag; + const gitRef = isPrerelease ? `heads/${treeRef}` : `tags/${treeRef}`; + try { + await github.rest.git.getRef({ + owner: context.repo.owner, + repo: context.repo.repo, + ref: gitRef + }); + } catch (error) { + if (error.status === 404 && isPrerelease) { + core.setFailed(`Expected staging branch '${treeRef}' for prerelease tag '${tag}' was not found`); + return; + } + if (error.status === 404) { + core.setFailed(`Expected stable tag ref '${tag}' was not found`); + return; + } + throw error; + } + + core.setOutput('disk_id', String(disk.id)); + core.setOutput('tree_ref', treeRef); + core.info(`Resolved ${tag}: tree '${treeRef}', disk asset ${disk.id}`); + + e2e: + name: E2E ${{ inputs.tag }} (full suite) + needs: resolve + # The sandbox boots three 8-vCPU / 24-GiB QEMU guests. This runner leaves + # enough host headroom for QEMU, networking, containerd, and the runner + # agent while the full platform installs across all three guests. + runs-on: oracle-vm-32cpu-128gb-x86-64 + timeout-minutes: 180 + permissions: + contents: read + checks: write # Breakpoint action updates its dedicated "Breakpoint Open" Check Run. + steps: + # The resolved ref is the digest-pinned staging branch for a prerelease, + # or the stable tag's pinned merge tree. + - name: Checkout resolved release tree + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + ref: ${{ needs.resolve.outputs.tree_ref }} + fetch-depth: 1 + persist-credentials: false + + - name: Download disk asset from release + env: + DISK_ID: ${{ needs.resolve.outputs.disk_id }} + GH_TOKEN: ${{ github.token }} + run: | + mkdir -p _out/assets + curl --fail-with-body --silent --show-error --location \ + -H "Authorization: Bearer ${GH_TOKEN}" \ + -H "Accept: application/octet-stream" \ + -o _out/assets/nocloud-amd64.raw.xz \ + "https://api.github.com/repos/${GITHUB_REPOSITORY}/releases/assets/${DISK_ID}" + + - name: Set sandbox ID + env: + TAG: ${{ inputs.tag }} + run: echo "SANDBOX_NAME=cozy-e2e-sandbox-$(echo "${GITHUB_REPOSITORY}:${GITHUB_WORKFLOW}:${TAG}" | sha256sum | cut -c1-10)" >> "$GITHUB_ENV" + + - name: Prepare workspace + run: | + rm -rf "/tmp/$SANDBOX_NAME" + cp -r "${{ github.workspace }}" "/tmp/$SANDBOX_NAME" + + # This is the only retried stage: Talos image unpacking, QEMU startup, and + # sandbox networking are pure runner infrastructure, not product logic. + - name: Prepare environment + run: | + cd "/tmp/$SANDBOX_NAME" + attempt=0 + until make SANDBOX_NAME=$SANDBOX_NAME prepare-env; do + attempt=$((attempt + 1)) + if [ $attempt -ge 3 ]; then + echo "Attempt $attempt failed, exiting..." + exit 1 + fi + echo "Attempt $attempt failed, retrying..." + done + echo "Prepare environment completed after $((attempt + 1)) attempts" + + - name: Install Cozystack into sandbox + run: | + cd "/tmp/$SANDBOX_NAME" + if ! make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME install-cozystack; then + echo "❌ Install failed (no retry — see diagnostics below)" + echo "::group::Diagnostics: HelmRelease status" + docker exec "$SANDBOX_NAME" sh -c 'kubectl get hr -A -o wide 2>&1 | tail -100' || true + echo "::endgroup::" + echo "::group::Diagnostics: not-Ready HelmReleases (describe)" + docker exec "$SANDBOX_NAME" sh -c 'kubectl get hr -A --no-headers 2>/dev/null | awk "\$4 != \"True\" {print \$1\" \"\$2}"' \ + | while read -r ns name; do + echo "--- $ns/$name ---" + docker exec "$SANDBOX_NAME" sh -c "kubectl describe hr '$name' -n '$ns' 2>&1 | tail -50" || true + done + echo "::endgroup::" + echo "::group::Diagnostics: recent events" + docker exec "$SANDBOX_NAME" sh -c 'kubectl get events -A --sort-by=.lastTimestamp 2>&1 | tail -50' || true + echo "::endgroup::" + exit 1 + fi + + - name: Run OpenAPI tests + run: | + cd "/tmp/$SANDBOX_NAME" + make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME test-openapi + + # A tag has no PR diff to scope with Test Impact Analysis. Empty + # CHAINSAW_SUITES is the testing Makefile's full-suite mode. + - name: Run E2E tests (full suite) + id: e2e_tests + run: | + cd "/tmp/$SANDBOX_NAME" + if ! make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME test-chainsaw CHAINSAW_SUITES=""; then + echo "❌ Chainsaw E2E failed (see the assertion diffs above and diagnostics below)" + echo "::group::Diagnostics: HelmRelease status" + docker exec "$SANDBOX_NAME" sh -c 'kubectl get hr -A -o wide 2>&1 | tail -50' || true + echo "::endgroup::" + echo "::group::Diagnostics: recent events" + docker exec "$SANDBOX_NAME" sh -c 'kubectl get events -A --sort-by=.lastTimestamp 2>&1 | tail -30' || true + echo "::endgroup::" + # COSI CRDs have no printer columns, so capture readiness fields at + # failure time before tests and report collection can move state. + echo "::group::Diagnostics: COSI bucket state" + docker exec "$SANDBOX_NAME" sh -c ' + kubectl get bucketclaims.objectstorage.k8s.io -A -o custom-columns="NAMESPACE:.metadata.namespace,NAME:.metadata.name,READY:.status.bucketReady,BUCKET:.status.bucketName" 2>&1 + kubectl get buckets.objectstorage.k8s.io -o custom-columns="NAME:.metadata.name,READY:.status.bucketReady,ID:.status.bucketID,CLAIMNS:.spec.bucketClaim.namespace" 2>&1 + kubectl get bucketaccesses.objectstorage.k8s.io -A -o custom-columns="NAMESPACE:.metadata.namespace,NAME:.metadata.name,GRANTED:.status.accessGranted,CLAIM:.spec.bucketClaimName" 2>&1 + ' || true + echo "::endgroup::" + exit 1 + fi + echo "✅ All E2E tests passed" + + - name: Collect chainsaw report + if: always() + run: | + cd "/tmp/$SANDBOX_NAME" + mkdir -p _out + docker cp "$SANDBOX_NAME":/workspace/hack/e2e-chainsaw/chainsaw-report.xml \ + _out/chainsaw-report.xml || true + + - name: Upload chainsaw report + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: chainsaw-report-${{ inputs.tag }} + path: /tmp/${{ env.SANDBOX_NAME }}/_out/chainsaw-report.xml + if-no-files-found: ignore + + # collect-report packages cozyreport plus per-failed-test crust-gather + # snapshots captured before Chainsaw cleanup. + # + # A ceiling of its own, not the job's 180 minutes. `Upload cozyreport.tgz` + # is the next step and the collector writes its tarball at the very end of + # its run, so a read that never returns spends what is left of the job + # budget, the job is cancelled, and the upload never runs -- the artifact + # is lost rather than truncated. 30 minutes is measured against the step: + # it takes 3-42s across sampled runs, failed e2e jobs included. + # `continue-on-error` because the ceiling must not change what the job + # reports: `|| true` says collecting diagnostics is never fatal, and a step + # killed at its ceiling never reaches it. + - name: Collect report + id: collect_report + if: always() + timeout-minutes: 30 + continue-on-error: true + run: | + cd "/tmp/$SANDBOX_NAME" + make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME collect-report || true + + # `continue-on-error` keeps the overrun out of the job's result, which also + # keeps it out of everyone's view: a step that ended at its ceiling renders + # green and the only trace is in the raw log. This branch's own standard is + # that a bound which fires is recorded, and that has to hold for the outer + # bound too. + # + # The wording says what `outcome` establishes and no more. A step can end + # non-zero without any ceiling firing -- in e2e-tag.yaml the body opens with + # a `cd` into a sandbox tree an earlier step may never have created, and the + # default shell is `bash -e` -- so naming the timeout as the cause would be + # the same false-mechanism claim the collector refuses to make for a 137. + - name: Report collection overran its ceiling + if: always() && steps.collect_report.outcome == 'failure' + run: | + echo "::warning title=cozyreport::Collect report ended non-zero -- its timeout-minutes fired, or the step failed before the collector ran; cozyreport.tgz is missing or partial for this run" + + - name: Upload cozyreport.tgz + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: cozyreport-${{ inputs.tag }} + path: /tmp/${{ env.SANDBOX_NAME }}/_out/cozyreport.tgz + + - name: Collect images list + if: always() + run: | + cd "/tmp/$SANDBOX_NAME" + make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME collect-images || true + + - name: Upload image list + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: image-list-${{ inputs.tag }} + path: /tmp/${{ env.SANDBOX_NAME }}/_out/images.txt + + # ▸ Open an SSH breakpoint to the failing sandbox so maintainers can attach, + # inspect Talos/Cozystack state and resume with `breakpoint resume`. + # + # Gated by the workflow_dispatch `debug` boolean and the configured + # BREAKPOINT_ENDPOINT repository variable. workflow_call runs never get + # a breakpoint. authorized-users is kept as a second defense layer at + # the breakpoint level. + # + # Uses cozystack/breakpoint-action (fork of namespacelabs/breakpoint-action) + # pinned by SHA. The fork adds: pause-idle mode (initial grace period for + # the first SSH connection, idle-aware exit afterwards), endpoint output + # and ::notice:: annotation, and a dedicated Check Run "Breakpoint Open" + # that carries the SSH endpoint in output.summary while the breakpoint is + # paused (conclusion=failure → standard ✗ in checks; updated to + # conclusion=success when the breakpoint exits). + - name: Breakpoint on E2E failure + if: | + failure() && + github.event_name == 'workflow_dispatch' && + inputs.debug == true && + vars.BREAKPOINT_ENDPOINT != '' + # cozystack/breakpoint-action v2-cozy.1 + # mode: pause-idle defaults: grace-period=20m, idle-timeout=10m + uses: cozystack/breakpoint-action@a6f3a6f87be398ad63b6577351e3398e53f578e4 + with: + mode: pause-idle + endpoint: ${{ vars.BREAKPOINT_ENDPOINT }} + authorized-users: androndo, Arsolitt, IvanHunters, kvaps, lexfrei, lllamnyp, mattia-eleuteri, matthieu-robin, myasnikovdaniil, sircthulhu, tym83 + check-run-name: "Breakpoint Open" + github-token: ${{ github.token }} + check-run-summary-template: | + ## 🔴 SSH breakpoint open — paused for debug + + ``` + {endpoint} + ``` + + Enter the e2e sandbox after SSH: + ``` + docker exec -ti $(docker ps --filter name=cozy-e2e-sandbox -q | head -1) bash + export KUBECONFIG=/workspace/kubeconfig + ``` + + Resume from inside: `breakpoint resume`. Otherwise the breakpoint + exits 10 minutes after the last SSH session disconnects. + + - name: Tear down sandbox + if: always() + run: make -C packages/core/testing SANDBOX_NAME=$SANDBOX_NAME delete || true + + - name: Remove workspace + if: always() + run: rm -rf "/tmp/$SANDBOX_NAME" + + - name: Summarize outcome + if: always() + env: + OUTCOME: ${{ job.status }} + TAG: ${{ inputs.tag }} + run: printf '| E2E tag `%s` | %s |\n' "$TAG" "$OUTCOME" >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/pull-requests-release.yaml b/.github/workflows/pull-requests-release.yaml index 5abdd20ca1..5e61bac423 100644 --- a/.github/workflows/pull-requests-release.yaml +++ b/.github/workflows/pull-requests-release.yaml @@ -3,8 +3,15 @@ name: "Releasing PR" on: pull_request: types: [closed] - paths-ignore: - - 'docs/**/*' + # No paths filter. A `paths-ignore: docs/**/*` used to live here as an + # optimization, but the promote PR now carries docs/changelogs/vX.Y.Z.md — + # and `Prepare stable branch` explicitly tolerates producing no tag-string + # changes ("digests already stable"). That combination can yield a docs-only + # promote PR, which the filter would drop: no finalize run, so no tag, no + # release, and no error anywhere. Filtering is left entirely to the `if:` + # gate below (merged + `release` label + release-bot author), which is what + # actually decides whether this workflow should do anything. The cost is a + # skipped job on unrelated closed PRs. # Cancel in‑flight runs for the same PR when a new push arrives. concurrency: @@ -74,6 +81,99 @@ jobs: uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: fetch-depth: 0 + # Do NOT persist GITHUB_TOKEN as http.extraheader. Every tag/branch push + # below re-injects the app token via `git remote set-url`, but a persisted + # extraheader silently wins over the URL credential — so those pushes + # would authenticate as GITHUB_TOKEN, and a GITHUB_TOKEN push creates no + # workflow run (GitHub anti-recursion). That is exactly why v1.6.0's + # stable tag triggered no tags.yaml run (the generate-changelog and + # update-website-docs backstops never fired). With this false, the set-url + # app token authenticates, matching how promote-rc.yaml already pushes. + persist-credentials: false + + # Use the trusted pre-merge base version of the verifier, while its root + # argument remains the merged packages tree checked out above. This keeps + # the final guard independent of the release PR's executable contents. + - name: Checkout release tooling from base + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + ref: ${{ github.event.pull_request.base.sha }} + path: .release-tooling + sparse-checkout: hack + persist-credentials: false + + # Resolve and verify the candidate before creating any write-once stable + # git or registry name. The candidate was serialized at promote time from + # the tag-rewritten tree; this guard pulls it by the committed digest and + # proves it still matches the merge commit (apart from its impossible + # self-reference) and still pins the rc-tested container digests. + # + # This step block runs HERE, ahead of the tag creation, the draft-release + # publish and the retag, rather than in its old position after the publish: + # a guard that runs after the write-once names exist cannot refuse them. + # It absorbs the former "Set up toolchain (skopeo, yq, helm)" and "Login to + # registry (GHCR)" steps, which is why neither appears later in this job + # any more — the tools and the logins are set up once, before the first + # irreversible action, and stay in scope for the retag and chart publish. + # + # flux and yq are version-pinned: they are what this job verifies the + # candidate with and what stamps the stable chart, so `latest/download` + # would make an irreversible step depend on whatever mikefarah shipped + # this morning. The version tracks + # packages/core/testing/images/e2e-sandbox/Dockerfile. yq is additionally + # pinned by content — it stamps the chart's platformVersion, and this step + # is the last one before write-once names exist. The Flux installer is not, + # because it checksums the binary it fetches and the mutable part is its + # own bootstrap script. + - name: Set up promotion toolchain (flux, skopeo, yq, helm) + if: ${{ !contains(steps.get_tag.outputs.tag, '-') }} + env: + FLUX_VERSION: "2.8.6" + YQ_VERSION: "4.53.3" + YQ_SHA256: "fa52a4e758c63d38299163fbdd1edfb4c4963247918bf9c1c5d31d84789eded4" + run: | + if ! flux version --client 2>/dev/null | grep -qx "flux: v${FLUX_VERSION}"; then + install_script="$(mktemp)" + curl -fsSL https://fluxcd.io/install.sh -o "$install_script" + sudo env "FLUX_VERSION=$FLUX_VERSION" bash "$install_script" + rm -f "$install_script" + fi + if ! yq --version 2>/dev/null | grep -q "mikefarah.* version v${YQ_VERSION}\$"; then + yq_bin="$(mktemp)" + curl -fsSL -o "$yq_bin" \ + "https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_linux_amd64" + printf '%s %s\n' "$YQ_SHA256" "$yq_bin" | sha256sum -c - + sudo install -m 0755 "$yq_bin" /usr/local/bin/yq + rm -f "$yq_bin" + fi + command -v skopeo >/dev/null \ + || { sudo apt-get update -qq && sudo apt-get install -y -qq skopeo; } + command -v helm >/dev/null \ + || curl -fsSL https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash + # Not installed here, only required: `flux pull artifact` reads its + # registry credentials from ~/.docker/config.json, which is why the + # login step below runs `docker login` as well as skopeo's and helm's. + # This job's runner class is not one of the docker-using build ones, so + # say what is missing here rather than letting the login step fail with + # a bare "command not found" in the middle of the release path. + command -v docker >/dev/null \ + || { echo "::error::docker is required on this runner: flux reads registry credentials from ~/.docker/config.json"; exit 1; } + + - name: Login to registry (GHCR) + if: ${{ !contains(steps.get_tag.outputs.tag, '-') }} + env: + GHCR_USER: ${{ github.actor }} + GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + echo "$GHCR_TOKEN" | docker login "${REGISTRY%%/*}" -u "$GHCR_USER" --password-stdin + echo "$GHCR_TOKEN" | skopeo login "${REGISTRY%%/*}" -u "$GHCR_USER" --password-stdin + echo "$GHCR_TOKEN" | helm registry login "${REGISTRY%%/*}" -u "$GHCR_USER" --password-stdin + + - name: Verify stable packages candidate + if: ${{ !contains(steps.get_tag.outputs.tag, '-') }} + env: + TAG: ${{ steps.get_tag.outputs.tag }} + run: .release-tooling/hack/verify-promoted-packages.sh "${TAG#v}" # Create the release tag at the merge commit — write-once. # @@ -312,6 +412,39 @@ jobs: console.log(`🏷️ ${tag} is the highest version, make_latest: true`); } + // Release body comes from the changelog that merged with THIS PR + // (promote-rc.yaml commits it onto the release-X.Y.Z branch), read + // from the merge commit already checked out here. + // + // Deliberately not left to update-releasenotes.yaml: that workflow + // triggers on pushes to main touching docs/changelogs/v*.md, so it + // would race this job on the very same push — and if it won, it + // would find no published release yet, skip, and never fire again, + // leaving the release body permanently empty. It also only watches + // main, so a patch release whose promote PR targets release-X.Y + // would never be synced at all. Reading the file here is both + // race-free and branch-agnostic; update-releasenotes.yaml remains + // for later maintainer edits and manual re-syncs. + const fs = require('fs'); + const changelogPath = `docs/changelogs/${tag}.md`; + let body; + if (fs.existsSync(changelogPath)) { + const raw = fs.readFileSync(changelogPath, 'utf8'); + // Last line of defence before the body becomes public and + // immutable-by-convention. promote-rc.yaml validates the changelog + // structurally, but it can also arrive by hand-commit, and an + // empty or whitespace-only file here would blank the release notes + // more thoroughly than publishing the draft's body would. + if (raw.trim().length === 0) { + console.log(`::warning::${changelogPath} is empty or whitespace-only — refusing to use it as the release body for ${tag}. Publishing with the draft's existing body instead.`); + } else { + body = raw; + console.log(`📝 Release body from ${changelogPath} (${body.length} bytes)`); + } + } else { + console.log(`::warning::${changelogPath} not found in the merge commit — publishing ${tag} with the draft's existing body. Add the changelog and re-run 'Update Release Notes' to backfill it.`); + } + // Publish the release await github.rest.repos.updateRelease({ owner: context.repo.owner, @@ -319,7 +452,8 @@ jobs: release_id: draft.id, draft: false, prerelease: isRc, - make_latest: makeLatest + make_latest: makeLatest, + ...(body ? { body } : {}) }); console.log(`🚀 Published release ${tag}`); @@ -343,24 +477,12 @@ jobs: # The checked-out merge commit is the promoted stable tree: the rc's # digests with the tag string rewritten to stable. skopeo retags those # digests to the stable image tag; helm packages+pushes the stable chart. - - name: Set up toolchain (skopeo, yq, helm) - if: ${{ !contains(steps.get_tag.outputs.tag, '-') }} - run: | - command -v yq >/dev/null \ - || { sudo curl -sSL -o /usr/local/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 && sudo chmod +x /usr/local/bin/yq; } - command -v skopeo >/dev/null \ - || { sudo apt-get update -qq && sudo apt-get install -y -qq skopeo; } - command -v helm >/dev/null \ - || curl -fsSL https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash - - - name: Login to registry (GHCR) - if: ${{ !contains(steps.get_tag.outputs.tag, '-') }} - env: - GHCR_USER: ${{ github.actor }} - GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - echo "$GHCR_TOKEN" | skopeo login "${REGISTRY%%/*}" -u "$GHCR_USER" --password-stdin - echo "$GHCR_TOKEN" | helm registry login "${REGISTRY%%/*}" -u "$GHCR_USER" --password-stdin + # + # The toolchain and the GHCR logins these two steps need are established + # earlier in this job, by "Set up promotion toolchain (flux, skopeo, yq, + # helm)" and "Login to registry (GHCR)" — both moved ahead of the tag + # creation so the candidate verification can refuse a release before any + # write-once name exists. Same job, so both are still in scope here. # Retag every rc image/digest to the stable tag. No rebuild — bit-for-bit # the e2e-tested rc. :latest moves only when this stable is the newest diff --git a/.github/workflows/pull-requests.yaml b/.github/workflows/pull-requests.yaml index 7bb448248d..cdc3435b15 100644 --- a/.github/workflows/pull-requests.yaml +++ b/.github/workflows/pull-requests.yaml @@ -83,6 +83,106 @@ jobs: echo "any=true" >> "$GITHUB_OUTPUT" fi + # Verify GitHub's prospective merge tree here, so a `packages/` edit pushed to + # this PR fails this check before it can merge. + # + # Checking the MERGE tree rather than the head is what makes base drift + # visible when this job runs — but nothing re-runs it when the base moves: + # GitHub does not fire `pull_request` on the base branch advancing, it only + # recomputes refs/pull/N/merge. So a `packages/` change landing on the base + # after this went green is never re-verified here, and the PR stays mergeable + # on the stale result. Finalize repeats the check against the real merge + # commit before creating the write-once stable tag, which is where that case + # is actually caught — loudly, and before any stable name exists. + # + # The trigger predicate is the promote PR's author plus its head-branch shape, + # NOT the `release` label main keys on. This branch's `on.pull_request.types` + # has no `labeled` event, and `gh pr create --label release` applies the label + # in a separate call one second after the PR opens (observed on the v1.6.1 + # promote PR), so the `opened` payload carries no labels at all and a + # label-keyed guard would never fire here. Every `cozystack-ci[bot]` PR whose + # head starts with `release-` is a promote PR, so this is if anything the + # narrower predicate — and it re-runs on `synchronize`, which is what makes a + # `packages/` edit pushed to the PR visible. A prerelease staging head + # (release-X.Y.Z-rc.N) never gets a PR under the checks-at-rc flow; if one ever + # did, the verifier's own X.Y.Z assertion fails it closed rather than passing. + verify-release-candidate: + name: Verify release packages candidate + runs-on: ubuntu-latest + # Matching `checks` and `finalize`. Without a ceiling the job inherits the + # 6-hour default, and a stalled `flux pull artifact` would hold this check + # pending for all of it. + timeout-minutes: 30 + if: | + github.event.pull_request.user.login == 'cozystack-ci[bot]' + && startsWith(github.head_ref, 'release-') + permissions: + contents: read + packages: read + steps: + # The default pull_request checkout is GitHub's prospective merge commit, + # not just the head branch. That makes base-branch package drift visible. + - name: Checkout prospective merge + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + fetch-depth: 0 + + # Execute the verifier from the trusted base SHA. An older rc-derived head + # may not contain it, and a PR must not be able to weaken its own guard. + - name: Checkout release tooling from base + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + ref: ${{ github.event.pull_request.base.sha }} + path: .release-tooling + sparse-checkout: hack + persist-credentials: false + + # Both tools are version-pinned. This job decides whether a stable release + # may be created, so what it runs has to be a fixed input: `latest/download` + # resolves to whatever mikefarah shipped this morning, and a yq that changed + # how it emits or compares a value turns a release gate into a moving one. + # The version tracks packages/core/testing/images/e2e-sandbox/Dockerfile. + # yq is additionally pinned by content; the Flux installer is not, because + # it checksums the binary it fetches and the mutable part is its own + # bootstrap script. + - name: Set up promotion toolchain (flux, yq) + env: + FLUX_VERSION: "2.8.6" + YQ_VERSION: "4.53.3" + YQ_SHA256: "fa52a4e758c63d38299163fbdd1edfb4c4963247918bf9c1c5d31d84789eded4" + run: | + if ! flux version --client 2>/dev/null | grep -qx "flux: v${FLUX_VERSION}"; then + install_script="$(mktemp)" + curl -fsSL https://fluxcd.io/install.sh -o "$install_script" + sudo env "FLUX_VERSION=$FLUX_VERSION" bash "$install_script" + rm -f "$install_script" + fi + if ! yq --version 2>/dev/null | grep -q "mikefarah.* version v${YQ_VERSION}\$"; then + yq_bin="$(mktemp)" + curl -fsSL -o "$yq_bin" \ + "https://github.com/mikefarah/yq/releases/download/v${YQ_VERSION}/yq_linux_amd64" + printf '%s %s\n' "$YQ_SHA256" "$yq_bin" | sha256sum -c - + sudo install -m 0755 "$yq_bin" /usr/local/bin/yq + rm -f "$yq_bin" + fi + + - name: Login to registry (GHCR) + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3 + with: + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + registry: ghcr.io + env: + DOCKER_CONFIG: ${{ runner.temp }}/.docker + + - name: Verify stable packages candidate + env: + DOCKER_CONFIG: ${{ runner.temp }}/.docker + HEAD_BRANCH: ${{ github.head_ref }} + run: | + STABLE_VERSION="${HEAD_BRANCH#release-}" + .release-tooling/hack/verify-promoted-packages.sh "$STABLE_VERSION" + # Helm-unittest + Go controller tests. Decoupled from the image build and from # e2e so they give fast, independent signal; they gate the PR as their own # required check rather than blocking e2e from starting. @@ -377,20 +477,50 @@ jobs: # pr.patch e2e applies — already reflect the overlay. A missing artifact # (build-main not yet green) is a no-op: packages keep their committed # refs (prior behaviour). See hack/overlay-main-images.sh. - - name: Pull current-main packages tree + # + # Read the artifact for the PR's OWN base branch, not always main. Overlaying + # main's images onto a release-line PR mixes generations: main's binaries + # against that line's charts. #3437 (a one-line change on release-1.6) failed + # install deterministically that way, because main's cozystack-controller + # served an aggregated OpenAPI release-1.6's charts could not validate + # against (`unknown model in reference: …v1alpha1.OptionSpec`) — a broken + # lane reading as a broken PR. build-release.yaml publishes + # cozystack-packages: for every maintained release-. + # branch, so each base branch has its own artifact to read. + # + # A missing artifact stays a no-op (packages keep their committed refs), which + # also covers a line whose first build-release run has not landed yet. + - name: Pull base-branch packages tree run: | rm -rf _out/mainpkgs mkdir -p _out/mainpkgs - flux pull artifact "oci://${REGISTRY}/cozystack-packages:main" \ - --output _out/mainpkgs \ - || { echo "cozystack-packages:main pull failed — keeping committed refs"; rm -rf _out/mainpkgs; } + if ! flux pull artifact "oci://${REGISTRY}/cozystack-packages:${BASE_REF}" \ + --output _out/mainpkgs; then + rm -rf _out/mainpkgs + # Degrading to committed refs is safe, but SAY so: on a release line a + # missing artifact means build-release.yaml has not published this line + # yet (first build pending, or its branch filter does not match this + # branch), and the PR is then testing the line's last release instead of + # its tip. Silent would look identical to a working overlay. + if [ "${BASE_REF}" = "main" ]; then + echo "cozystack-packages:${BASE_REF} pull failed — keeping committed refs" + else + echo "::warning title=No packages artifact for ${BASE_REF}::Keeping committed refs, so unbuilt packages use this line's last release rather than its tip. Check that build-release.yaml has run for ${BASE_REF}." + fi + fi env: DOCKER_CONFIG: ${{ runner.temp }}/.docker - - name: Overlay current-main refs for unbuilt packages + # Env-passed, not interpolated into the script: base_ref is repo-controlled + # here (a branch name in this repo), but the workflow-injection hygiene + # applies uniformly. + BASE_REF: ${{ github.base_ref }} + - name: Overlay base-branch refs for unbuilt packages # Best-effort: this only improves WHICH images e2e and the installer use, # so an unexpected failure must degrade to the committed refs (prior # behaviour), never block the PR. overlay-main-images.sh already handles - # the expected paths (missing artifact, drift, absent unit) internally. + # the expected paths (missing artifact, drift, absent unit) internally — + # including the empty directory the step above leaves when a line has no + # artifact yet. run: | hack/overlay-main-images.sh _out/mainpkgs "$BUILD_MATRIX" "$PR_TOUCHED" \ || echo "::warning::overlay-main-images failed; unbuilt packages keep committed refs" diff --git a/.github/workflows/tags.yaml b/.github/workflows/tags.yaml index 7b1062e254..7f0d78e8c6 100644 --- a/.github/workflows/tags.yaml +++ b/.github/workflows/tags.yaml @@ -311,6 +311,35 @@ jobs: # not exists" step was gated on `is_stable && release_exists==false` — # unreachable now that a hand-pushed stable tag is rejected above. + # Mandatory only for vX.Y.Z-rc.N tags. Alpha/beta tags deliberately skip this + # job and use e2e-tag.yaml's manual dispatch when validation is wanted. + # + # Deliberately not gated on release_exists: validating an already-published rc + # again is an idempotent re-tag-run check, while prepare-release still provides + # the ordering guarantee for a fresh rc. Its "Publish rc release" and "Create + # release branch" steps finish before this reusable workflow can start. + rc-e2e: + name: E2E Release Candidate + needs: prepare-release + if: needs.prepare-release.result == 'success' && contains(github.ref_name, '-rc.') + # A caller's permissions are the ceiling for every job in the called + # workflow, and GitHub validates that ceiling STATICALLY, when it creates the + # run — before this job's `if:` is evaluated, let alone before a step runs. + # e2e-tag.yaml's e2e job declares `checks: write` for its + # workflow_dispatch-only breakpoint, so even though workflow_call can never + # take that branch, omitting the grant here fails the whole tags.yaml run with + # "requesting 'checks: write', but is only allowed 'checks: none'". Not just + # rc cuts: the `if:` cannot gate a startup failure, so EVERY tag push — rc and + # stable, including the one finalize pushes to publish a release — would build + # nothing at all. hack/promote-gate-contract.bats pins the pair, because no CI + # lane exercises this file (it runs only on tag pushes). + permissions: + contents: read + checks: write + uses: ./.github/workflows/e2e-tag.yaml + with: + tag: ${{ github.ref_name }} + generate-changelog: name: Generate Changelog # git + node + copilot CLI only — no docker, no repo toolchain: GitHub-hosted. @@ -380,11 +409,22 @@ jobs: fetch-tags: true token: ${{ steps.app-token.outputs.token }} - # Check whether the changelog already exists on origin/main. - # The previous step checked out the tag commit, which never has the - # changelog (the changelog PR merges to main AFTER the tag is cut). - # Looking at origin/main is what the gate is supposed to do — skip - # generation on workflow reruns when the changelog has already merged. + # Two independent questions, and the answers select between three paths. + # + # `exists` — is the changelog already on origin/main? Then this whole job + # is a no-op. On a maintenance line that is the EXCEPTION: the promote PR + # merges into release-1.6, so the file it carried never touched main, and + # this is true only if something else put it there. + # + # `at_tag` — is it present in THIS checkout (the tag commit)? Under the + # promote flow it normally is, because the promote PR carried it onto + # release-1.6 and the stable tag sits on that merge commit. So this is the + # usual state on arrival: `exists` false, `at_tag` true, a reviewed and + # already-published changelog sitting right here — port it rather than + # regenerate. + # + # Only when both are false is there genuinely no changelog to publish, and + # only then does the AI run. - name: Check if changelog already exists id: check_changelog env: @@ -399,19 +439,36 @@ jobs: echo "exists=false" >> $GITHUB_OUTPUT echo "Changelog file $CHANGELOG_FILE does not exist on origin/main" fi + # Is the changelog present at the TAG (this checkout)? Under the promote + # flow it normally is: promote-rc.yaml commits it onto release-X.Y.Z and + # it reaches the tag via the merge commit. That PR targets the maintenance + # line, so the file lands on release-1.6 and never reaches main — + # `exists` above is false even though a reviewed, already-PUBLISHED + # changelog exists right here. + # + # Regenerating in that case would spend a second AI run and, once the + # backstop PR merged, update-releasenotes.yaml would overwrite the + # published release body with text nobody approved. Port the reviewed + # file verbatim instead; the AI is only for the genuinely-absent case. + if [ -s "$CHANGELOG_FILE" ]; then + echo "at_tag=true" >> $GITHUB_OUTPUT + echo "Changelog present at the tag — will port it to main verbatim, no regeneration." + else + echo "at_tag=false" >> $GITHUB_OUTPUT + fi - name: Setup Node.js - if: steps.check_changelog.outputs.exists == 'false' + if: steps.check_changelog.outputs.exists == 'false' && steps.check_changelog.outputs.at_tag == 'false' uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 with: node-version: 22 - name: Install GitHub Copilot CLI - if: steps.check_changelog.outputs.exists == 'false' + if: steps.check_changelog.outputs.exists == 'false' && steps.check_changelog.outputs.at_tag == 'false' run: npm i -g @github/copilot - name: Generate changelog using AI - if: steps.check_changelog.outputs.exists == 'false' + if: steps.check_changelog.outputs.exists == 'false' && steps.check_changelog.outputs.at_tag == 'false' timeout-minutes: 30 env: COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} @@ -433,12 +490,19 @@ jobs: CHANGELOG_FILE="docs/changelogs/v${VERSION}.md" CHANGELOG_BRANCH="changelog-v${VERSION}" + # Two ways the file can be here: ported verbatim from the tag commit + # (at_tag=true — the reviewed, already-published changelog reaching main + # for the first time), or freshly generated above. Either is valid. if [ ! -f "$CHANGELOG_FILE" ]; then - echo "::error::Changelog file $CHANGELOG_FILE was not produced by the Generate changelog using AI step" + echo "::error::Changelog file $CHANGELOG_FILE is absent — it was neither present at the tag nor produced by the Generate changelog using AI step" exit 1 fi - if [ ! -s "$CHANGELOG_FILE" ]; then - echo "::error::Changelog file $CHANGELOG_FILE is empty" + # A ported file came from the tag and a generated one from a step allowed + # to fail, so neither has been checked here. update-releasenotes.yaml + # will push whatever this PR merges into the published release body, so a + # fragment must not get that far. + if ! REASON="$(hack/validate-changelog.sh "$CHANGELOG_FILE" "${VERSION}" 2>&1)"; then + echo "::error::Changelog file $CHANGELOG_FILE is not usable: ${REASON}" exit 1 fi diff --git a/.gitignore b/.gitignore index 4d15bd0bd2..ae05d04195 100644 --- a/.gitignore +++ b/.gitignore @@ -104,3 +104,8 @@ examples/backups/*/.bucket-info.env examples/backups/*/.sentinel.env examples/backups/*/.backup-name.env packages/core/platform/images/migrations/etcd-operator-crds/ + +# CI scratch: the candidate-verification jobs check the trusted base ref here so +# release tooling (the packages verifier and its library) is the pre-merge +# version even when the checked-out tree predates it. Never committed. +.release-tooling diff --git a/cmd/backupstrategy-controller/main.go b/cmd/backupstrategy-controller/main.go index d5995af04d..eeec8bf010 100644 --- a/cmd/backupstrategy-controller/main.go +++ b/cmd/backupstrategy-controller/main.go @@ -27,6 +27,7 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" @@ -203,6 +204,39 @@ func main() { } } + // The default Strategy CRs and the Velero BSL are Helm-templated behind + // a lookup of the BucketClaim the same chart creates, so a fresh install + // renders them empty — permanently, because helm-controller does not + // re-render an unchanged, successful release. The gate detects that and + // forces the one real Helm upgrade that materialises them. Disabled when + // the HelmRelease coordinates are not plumbed (e.g. a chart-less local run). + if hrName := os.Getenv("BACKUP_DEFAULT_OBJECTS_HELMRELEASE_NAME"); hrName != "" { + gate := &backupcontroller.DefaultObjectsGate{ + Client: mgr.GetClient(), + Config: credentialsConfig, + BackupClassName: os.Getenv("BACKUP_DEFAULT_OBJECTS_BACKUPCLASS"), + HelmRelease: types.NamespacedName{ + Namespace: os.Getenv("BACKUP_DEFAULT_OBJECTS_HELMRELEASE_NAMESPACE"), + Name: hrName, + }, + VeleroNamespace: os.Getenv("BACKUP_DEFAULT_OBJECTS_VELERO_NAMESPACE"), + // The platform bucket's -system release renders the + // credentials Secret the projector reads, behind the same kind + // of install-time lookup — and while that Secret is missing + // nothing downstream can resolve. Empty when the bucket is not + // provisioned by Cozystack (external S3), where the Secret is + // admin-managed and no release renders it. + CredentialsHelmRelease: types.NamespacedName{ + Namespace: os.Getenv("BACKUP_CREDENTIALS_HELMRELEASE_NAMESPACE"), + Name: os.Getenv("BACKUP_CREDENTIALS_HELMRELEASE_NAME"), + }, + } + if err := gate.SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to add DefaultObjectsGate runnable") + os.Exit(1) + } + } + if err = (&backupcontroller.BackupJobReconciler{ Client: mgr.GetClient(), Scheme: mgr.GetScheme(), diff --git a/docs/changelogs/v1.6.1.md b/docs/changelogs/v1.6.1.md new file mode 100644 index 0000000000..94f690f505 --- /dev/null +++ b/docs/changelogs/v1.6.1.md @@ -0,0 +1,72 @@ + + +# v1.6.1 (2026-08-05) + +A patch release with seven fixes covering PostgreSQL, etcd, managed Kubernetes, Keycloak, `cozystack-basics`, and SeaweedFS, plus release-pipeline reliability fixes and a `talm` update adding declarative Talos preset knobs. + +## Fixes + +* **fix(postgres-operator): align CNPG operator and CRDs to 1.28.2 for PVC resize-deadlock fix**: A simultaneous `resources` + `size` change on a single-instance PostgreSQL cluster could make the CloudNativePG operator delete the sole primary Pod, classify the PVC as `resizing`, and never recreate the Pod — wedging the cluster with zero instances and leaving the filesystem resize incomplete. Bumping the operator image and CRDs together to 1.28.2 (which carries upstream's fix, cloudnative-pg#9980 / cloudnative-pg#9981) resolves the deadlock ([**@scooby87**](https://github.com/scooby87) in #3510, backport #3542). + +* **chore(etcd-operator): bump etcd-operator to v0.5.4**: Rolls up four upstream controller bug fixes: the operator no longer exempts the bootstrap seed from crash-loop self-heal, self-heal now also covers memory-backed etcd members, `--initial-cluster-state` is derived from cluster phase instead of the seed, and each EtcdCluster's PodDisruptionBudget switches from `maxUnavailable` to `minAvailable` (existing clusters are reconciled onto the new field automatically on upgrade). Tenant etcd clusters recover more reliably from member crashes and are less likely to have their PDB block a node drain ([**@androndo**](https://github.com/androndo) in #3529, backport #3538). + +* **fix(kubernetes): render the talos-reconcile Job for the default md0 group**: On a managed Kubernetes cluster left with the default (undeclared) `nodeGroups`, the Job that creates each worker's `TalosConfigTemplate` and patches the control-plane's cert SANs only iterated the user-supplied node-group map, so it silently skipped the implicit `md0` group. Nothing failed at install time, since `md0` defaults to zero replicas, but the first scale-up (for example, autoscaler-driven growth after enabling ingress-nginx) left new Machines permanently blocked with no matching `TalosConfigTemplate`. The Job now iterates the same helper that produces the `MachineDeployment`, so `md0` gets its reconcile Job like any explicitly declared group ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #3535, backport #3536). + +* **fix(keycloak-configure): patch HelmRelease in release namespace on teardown**: The `keycloak-configure` pre-delete Job cleared the Flux `HelmRelease` finalizer in a hardcoded namespace that did not match where the release actually installs (`cozy-keycloak`), so its `ServiceAccount` was forbidden to patch it, the Job retried forever, and the `HelmRelease` stuck in `Terminating` — blocking any uninstall or reinstall of Keycloak. The teardown Job now templates both the release name and namespace from the Helm release itself, so teardown completes correctly ([**@lexfrei**](https://github.com/lexfrei) in #3372, backport #3478). + +* **fix(cozystack-basics): gate the hostname VAP policies on the VAP API**: The hostname `ValidatingAdmissionPolicy` templates rendered unconditionally, so a first install on a cluster where the `ValidatingAdmissionPolicy` API is unavailable dropped the policies permanently — a later cluster upgrade that gains the API would not bring them back. The templates are now gated on `.Capabilities.APIVersions.Has`, so they render only where the API exists and are picked up automatically once it becomes available ([**@lexfrei**](https://github.com/lexfrei) in #3409, backport #3442). + +* **fix(seaweedfs): make naming audit fail closed on kubectl and payload errors**: `hack/seaweedfs-naming-audit.sh`, used by operators to classify SeaweedFS instances before the naming-migration cleanup, was fail-open — any `kubectl` failure or unreadable Helm release payload produced an empty result table indistinguishable from a genuinely clean fleet. Since the runbook uses this script as the gate before deleting old PVCs, a transient API error could have green-lit destroying live data. Every query now fails loudly on error instead of silently reporting "nothing found," and incomplete evidence now falls back to a conservative "direction cannot be established" verdict rather than a wrong deletion candidate ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #3436, backport #3474). + +* **chore(release): don't activate `kubernetes-nodes` on the release-1.6 line**: The `kubernetes-nodes` app package was not ready to ship on the 1.6 line, so its single include is removed from the `iaas` platform bundle — the platform stops activating the package on this line while its code, API types, and `kubernetes-nodes-rd` system package are all kept intact for when it is ready ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #3437). + +## Development, Testing, and CI/CD + +* **ci(release): carry the finalize fixes onto the 1.6 line**: `release-1.6` was cut before three release-pipeline fixes landed on `main`: dropping `persist-credentials` on checkout (which had let a stale `GITHUB_TOKEN` silently win over the app token, so the stable tag push created no workflow run and the automated changelog/docs backstops never fired, as happened for v1.6.0), publishing the GitHub release with the merged changelog as its body instead of a placeholder, and dropping a `paths-ignore` filter that could drop a changelog-only promotion PR. All three are backported so v1.6.1 releases correctly with real release notes ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #3530). + +* **fix(ci): overlay images from the PR base branch, and publish per-line artifacts**: PR validation always overlaid unbuilt packages from `cozystack-packages:main`, so a `release-1.6` PR was tested against `main`'s controller binaries against its own line's charts — which is exactly what made #3437 above fail install deterministically with a schema-validation error. Each maintained `release-X.Y` branch now builds and publishes its own packages artifact, and PR validation overlays from the artifact matching the PR's own base branch ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #3471, backport #3514). + +* **fix(release): make promote-retag digest verification media-type-agnostic**: The v1.6.0 finalize run aborted partway through promoting rc images to stable tags because its post-copy digest check used `skopeo inspect --format '{{.Digest}}'`, which prints nothing for OCI artifacts like `cozystack-packages`, leaving most repositories without a stable tag and skipping the installer publish. The digest is now computed as the sha256 of the raw manifest, which works identically for container images and OCI artifacts, so promotion can no longer abort mid-way on this class of artifact ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #3435, backport #3473). + +## Other repositories + +### talm v0.34.0 + +* **[talm] feat(charts): add preset value knobs**: Exposes `timeServers`, control-plane component `extraArgs`, `registryMirrors`, per-host `registryTLS`, multiple Layer2 `vips`, `network.preserveExisting`, and `network.extraLinks` (bonds, VLANs, extra addresses and routes) as values on the cozystack, generic, and talm presets, so a node's Talos machine config can be described declaratively instead of via a template fork. Every knob defaults empty and a stock render stays byte-identical; each input Talos would reject fails fast at render time with a hinted error ([**@lexfrei**](https://github.com/lexfrei) in cozystack/talm#232). + +* **[talm] chore(deps): migrate to Helm 4 and drop the cozystack/talos fork**: Moves talm's vendored Helm template engine from v3 to v4 and drops the `cozystack/talos` fork (carried solely for a `--skip-verify` flag, now reimplemented locally), tracking stock upstream Talos v1.13.7. Golden render snapshots confirm the generated machine config is unchanged for users ([**@lexfrei**](https://github.com/lexfrei) in cozystack/talm#231). + +## Documentation + +* **[website] feat(blog): add Cozystack 1.6 release and Blockstor announcement**: Publishes the v1.6.0 release-announcement blog post — covering Talos Linux tenant workers, tenant-controlled OIDC, the `SecurityGroup` API, hierarchical quotas, and in-place etcd-operator adoption — alongside a companion post announcing the open-sourcing of Blockstor, the LINSTOR-compatible storage control plane ([**@tym83**](https://github.com/tym83) in cozystack/website#641). + +* **[website] chore(blog): repair front matter, links and bundle names**: Follow-up cleanup on the two new blog posts, fixing front matter, internal links, and page bundle names ([**@tym83**](https://github.com/tym83) in cozystack/website@4b5d1ad). + +* **[website] chore: upgrade Hugo to 0.164.0 and convert HTML content to markdown**: Upgrades the site generator to Hugo 0.164.0, requiring Node 22 under its new node permission model, and converts remaining raw-HTML content to markdown along the way, keeping the site buildable on current tooling ([**@tym83**](https://github.com/tym83) in cozystack/website#636). + +* **[website] chore(blog): make the two taxonomy axes disjoint**: Cleans up the blog's tagging so its two taxonomy axes no longer overlap, making blog post categorization and filtering more consistent ([**@tym83**](https://github.com/tym83) in cozystack/website#635). + +* **[website] docs(talm): describe the preset value knobs for network and registries**: Documents the new `talm` preset value knobs (cozystack/talm#232) for time servers, control-plane extra args, registry mirrors and TLS, VIPs, and network links, so operators can find the declarative equivalents to a template fork ([**@lexfrei**](https://github.com/lexfrei) in cozystack/website#633). + +* **[website] feat(hack): generate docs from a pre-tag ref and fail loudly on fetch errors**: Hardens the docs-generation tooling to build from a pinned pre-tag ref and to fail loudly instead of silently on a fetch error, reducing the chance of publishing docs generated from the wrong ref ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in cozystack/website#631). + +* **[website] docs(backups): document PostgreSQL point-in-time recovery (PITR)**: Adds documentation for PostgreSQL point-in-time recovery, covering how to configure and perform a PITR restore for managed PostgreSQL clusters ([**@androndo**](https://github.com/androndo) in cozystack/website#629). + +* **[website] docs: update managed apps reference for v1.6.0**: Refreshes the managed applications reference pages to match what shipped in v1.6.0 ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in cozystack/website#628). + +* **[website] docs(monitoring): add OIDC authentication guide for Grafana**: Adds a guide for configuring OIDC single sign-on authentication for per-instance Grafana, complementing v1.6.0's tenant OIDC support ([**@IvanHunters**](https://github.com/IvanHunters) in cozystack/website#597). + +## Contributors + +Thanks to everyone who contributed to this patch release: + +* [**@IvanHunters**](https://github.com/IvanHunters) +* [**@androndo**](https://github.com/androndo) +* [**@lexfrei**](https://github.com/lexfrei) +* [**@myasnikovdaniil**](https://github.com/myasnikovdaniil) +* [**@scooby87**](https://github.com/scooby87) +* [**@tym83**](https://github.com/tym83) + +**Full Changelog**: https://github.com/cozystack/cozystack/compare/v1.6.0...v1.6.1 diff --git a/docs/changelogs/v1.6.2.md b/docs/changelogs/v1.6.2.md new file mode 100644 index 0000000000..a6f0a8c845 --- /dev/null +++ b/docs/changelogs/v1.6.2.md @@ -0,0 +1,50 @@ + + +# v1.6.2 (2026-08-19) + +A patch release with six fixes covering the backup-strategy controller, kube-ovn's webhook certificate, Velero CRD upgrades, CNPG barman-cloud backups, `flux-shard-operator`, and the published OpenAPI definitions, plus a release-pipeline reliability fix. + +## Fixes + +* **fix(backupstrategy-controller): repair lookup-gated backup objects**: The default backup `Strategy` CRs and the Velero `BackupStorageLocation` are gated on a Helm `lookup` performed while the referenced object is still being created; when that lookup came back empty the objects were skipped permanently, since helm-controller does not re-render a release whose chart and values are unchanged. The gate now resolves the default bucket credentials Secret through the RESTMapper, bounds each check, and tolerates an absent Secret instead of looping, so the default backup objects are created reliably instead of silently vanishing for months ([**@mattia-eleuteri**](https://github.com/mattia-eleuteri) in #3524, backport #3731). + +* **fix(kube-ovn): reload kubeovn-webhook serving certificate on cert-manager renewal**: `kube-ovn-webhook` loaded its TLS serving certificate once at startup and never re-read it; once cert-manager renewed the backing Secret and the old certificate expired, the apiserver's calls to the webhook failed verification and, because the `MutatingWebhookConfiguration` uses `failurePolicy: Fail`, every pod creation in tenant namespaces was rejected — including `virt-launcher` pods, blocking VMI startup. The webhook now serves its certificate through a reloading callback that re-reads the key pair when the mounted files change and widens `renewBefore` to 720h, so cert-manager renewals are honored without a pod restart ([**@IvanHunters**](https://github.com/IvanHunters) in #3557, backport #3730). + +* **fix(velero): apply CRD updates on upgrade via CreateReplace**: Velero's CRDs stayed frozen at whatever version was first installed, since Helm never touches a chart's `crds/` directory on upgrade; when the Velero image moved to a version that added new backup phases, the apiserver rejected phase transitions against the stale CRDs and backups silently stopped while the HelmRelease stayed green. The Velero package now opts into `upgradeCRDs: CreateReplace`, so CRDs are kept current on upgrade and backups keep working ([**@lexfrei**](https://github.com/lexfrei) in #3727, backport #3728). + +* **fix(backups): request S3 checksum only when required for barman-cloud (non-AWS S3 / Ceph RGW)**: CNPG's barman-cloud plugin sidecar defaulted to computing a flexible checksum on every upload, which several S3-compatible backends (Ceph RGW, some MinIO / Cloudflare R2 builds) reject outright, so every backup and WAL-archive upload to those backends failed and `ScheduledBackup`s never stored anything. Every barman-cloud `ObjectStore` Cozystack creates — Keycloak's system DB, the postgres app's backup and recovery stores, and the platform-managed system-bucket store — now sets `AWS_REQUEST_CHECKSUM_CALCULATION=when_required`, a safe default accepted by both AWS S3 and the affected backends ([**@androndo**](https://github.com/androndo) in #3417, backport #3767). + +* **fix(flux-shard-operator): repair sharded helm-controller crashloop behind an HTTP proxy**: The cloned `helm-controller-shard` Deployment inherited `HTTP_PROXY`/`HTTPS_PROXY`/`NO_PROXY` from the flux-aio all-in-one wiring even though a standalone shard needs no external egress; behind an unreachable proxy the controller's blocking startup HTTPS call never completed, the manager never served `/healthz`, and every HelmRelease sharded to that controller was frozen. The sanitisation now also drops the inherited proxy env and adds a `startupProbe` derived from the liveness handler, so sharded HelmReleases keep reconciling in proxied environments instead of crashlooping forever ([**@IvanHunters**](https://github.com/IvanHunters) in #3546, backport #3818). + +* **fix(api): declare OpenAPIModelName for core and sdn types**: The `core` and `sdn` API groups did not declare `OpenAPIModelName` the way the `apps` group already did, so their published OpenAPI definition names were Go import paths while every `$ref` pointing at them escaped each slash — the two spellings never matched, the reference dangled, and `kubectl apply --validate` failed on any resource against a `cozystack-api` built after the underlying Kubernetes 0.35 change. Declaring `OpenAPIModelName` for `core` and `sdn` too makes every published definition name the dotted Kubernetes model name, so client-side validation against the published OpenAPI works again ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #3808, backport #3812). + +## Development, Testing, and CI/CD + +* **ci(release): complete the candidate-aware promotion pipeline on release-1.6**: `release-1.6` was missing the e2e and packages-verification jobs that `Promote RC` requires on its target base, so `v1.6.1` was promoted with the rc e2e gate bypassed and the next patch release could not even be dispatched. Adds the `rc-e2e` job, the `verify-release-candidate` checks, `hack/verify-promoted-packages.sh`, `hack/validate-changelog.sh` and regression tests pinning the pipeline's contract, so future patch releases off `release-1.6` run the same e2e and package-verification gates as `main` before promoting, and the tag-time changelog is validated and ported from the tag rather than regenerated ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in #3893). + +## Documentation + +* **[website] docs: import the operator guides that lived in the cozystack repo**: Moves the operator-facing guides that used to live in the `cozystack` repo over to the documentation site, so operators find them alongside the rest of the docs instead of scattered across two repositories ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in cozystack/website#648). + +* **[website] docs(oidc): document private CA and staging trust**: Documents how to configure tenant OIDC to trust a private certificate authority and staging certificates, closing a gap for operators running their own CA or testing with a staging issuer ([**@myasnikovdaniil**](https://github.com/myasnikovdaniil) in cozystack/website#650). + +* **[website] feat(community): add a Community page and link it from the main menu**: Adds a Community page linked from the site's main menu, giving visitors a single place to find how to get in touch with and contribute to the Cozystack community ([**@tym83**](https://github.com/tym83) in cozystack/website#637). + +* **[website] chore(telemetry): publish July 2026 and explain how the figures are derived**: Publishes the July 2026 telemetry figures and documents how those figures are derived, giving the community visibility into adoption trends and how the numbers are calculated ([**@tym83**](https://github.com/tym83) in cozystack/website#644). + +* **[website] feat(blog): new Blockstor banner**: Adds a new banner promoting Blockstor to the blog, improving the visibility of the storage control plane's announcement ([**@tym83**](https://github.com/tym83) in cozystack/website#646). + +## Contributors + +Thanks to everyone who contributed to this patch release: + +* [**@IvanHunters**](https://github.com/IvanHunters) +* [**@androndo**](https://github.com/androndo) +* [**@lexfrei**](https://github.com/lexfrei) +* [**@mattia-eleuteri**](https://github.com/mattia-eleuteri) +* [**@myasnikovdaniil**](https://github.com/myasnikovdaniil) +* [**@tym83**](https://github.com/tym83) + +**Full Changelog**: https://github.com/cozystack/cozystack/compare/v1.6.1...v1.6.2 diff --git a/docs/operations/backup-classes.md b/docs/operations/backup-classes.md index e96b406f2a..de9fb63332 100644 --- a/docs/operations/backup-classes.md +++ b/docs/operations/backup-classes.md @@ -93,9 +93,52 @@ On a fresh-cluster install, the Velero `BackupStorageLocation` `cozy-default` is ### Cozy-default Bucket bootstrap -`cozy-default` ships an `apps.cozystack.io/Bucket cozy-backups` CR in `tenant-root`, which the bucket-application chart turns into a `BucketClaim`; the COSI driver then assigns the real S3 bucket name and writes it to the BucketClaim's `.status.bucketName`. The strategy templates and the Velero BSL all read that real bucket name (Helm `lookup` against the BucketClaim). On a fresh install the BucketClaim takes a short reconcile cycle to populate its status — until it does, the strategy templates render empty and only the `Bucket` CR + `BackupClass` are present in the cluster. The HelmRelease re-reconciles on its interval (5 minutes by default — set by the cozystack operator's `helmrelease-interval` flag, not a Flux default), at which point the populated BucketClaim status causes the missing strategy templates to materialise. +`cozy-default` ships an `apps.cozystack.io/Bucket cozy-backups` CR in `tenant-root`, which the bucket-application chart turns into a `BucketClaim`; the COSI driver then assigns the real S3 bucket name (`bucket-`, so it cannot be computed in advance) and writes it to the BucketClaim's `.status.bucketName`. The strategy templates and the Velero BSL all read that real bucket name (Helm `lookup` against the BucketClaim). On a fresh install the BucketClaim takes a reconcile cycle to populate its status — until it does, the strategy templates render empty and only the `Bucket` CR + `BackupClass` are present in the cluster. -If you need the BackupClass functional immediately (e.g. an e2e), trigger a Flux reconcile (`flux reconcile helmrelease backupstrategy-controller -n cozy-backup-controller`) once you see `kubectl get bucketclaim -n tenant-root bucket-cozy-backups -o jsonpath='{.status.bucketName}'` non-empty. +**That skip does not repair itself on a reconcile.** helm-controller re-renders a release only when its chart or values change; the interval reconcile is a no-op for a healthy release, and drift detection is off on operator-generated HelmReleases. A cluster that lost the race at install time therefore keeps `BackupClass cozy-default` with **no** `Strategy` CRs and **no** Velero BSL indefinitely — which also fail-closes the pre-adoption snapshot in the v1.6.0 etcd migration. + +The same trap sits one release earlier, on the Secret everything else depends on. `bucket-cozy-backups-system-credentials` is rendered by the `bucket-cozy-backups-system` release behind a `lookup` of the COSI Secret, and is skipped just as permanently when that lookup is empty. Without it the credentials projector has no source at all, so no strategy, no Velero and no v1.6.0 etcd migration can resolve — and the gate cannot even determine the bucket name, which it reads from that Secret. + +Convergence is driven instead by the controller's default-objects gate (`backupStorage.reconcileDefaultObjects`, on by default), which repairs both, in order. While the credentials Secret is absent or carries no bucket name, the gate stamps `reconcile.fluxcd.io/forceAt` + `requestedAt` on the **bucket's** `bucket--system` HelmRelease. Once the bucket name resolves it checks that every object `cozy-default` routes to exists and stamps the same pair on the **`backupstrategy-controller`** HelmRelease, forcing the real Helm upgrade that re-runs the lookups. The two are throttled independently (one force per 5 minutes each), so the second is not delayed by the first. Expect the objects within a minute or two of the bucket becoming ready. + +The gate does not force a suspended HelmRelease — helm-controller ignores both annotations while `spec.suspend` is true, so it logs the skip and leaves the force counter alone. A release left suspended (`cozyhr suspend`) is therefore never repaired until it is resumed. + +Watch it with: + +```bash +kubectl -n cozy-backup-controller logs deploy/backupstrategy-controller | grep default-objects-gate +``` + +#### Manual recovery on an affected cluster + +Only needed on a cluster running a version without the gate (or with `reconcileDefaultObjects: false`). A plain `flux reconcile helmrelease` does **nothing** here — it does not re-render. You need a forced upgrade, and **both** annotations: `forceAt` is what makes helm-controller run a real Helm upgrade, and it is only honoured together with `requestedAt`. + +First confirm the bucket name is actually resolvable — forcing before that just re-runs the same empty lookup: + +```bash +kubectl -n tenant-root get bucketclaim bucket-cozy-backups -o jsonpath='{.status.bucketName}' +``` + +Then force the two releases, **in this order**: + +```bash +# 1. The credentials Secret. It is rendered by the -system release, not by +# bucket-cozy-backups — easy to miss, and the projector (hence every +# strategy and Velero) has no source without it. +ts=$(date -u +%Y-%m-%dT%H:%M:%SZ) +kubectl -n tenant-root annotate helmrelease bucket-cozy-backups-system \ + reconcile.fluxcd.io/forceAt="$ts" reconcile.fluxcd.io/requestedAt="$ts" --overwrite +kubectl -n tenant-root get secret bucket-cozy-backups-system-credentials + +# 2. The Strategy CRs and the Velero BSL. +ts=$(date -u +%Y-%m-%dT%H:%M:%SZ) +kubectl -n cozy-backup-controller annotate helmrelease backupstrategy-controller \ + reconcile.fluxcd.io/forceAt="$ts" reconcile.fluxcd.io/requestedAt="$ts" --overwrite +kubectl get $(kubectl get crd -o name | grep strategy.backups.cozystack.io) 2>/dev/null +kubectl -n cozy-velero get backupstoragelocation cozy-default +``` + +The race is nested: step 2's `lookup` resolves from the BucketClaim status, but the projector — and the v1.6.0 etcd migration — read the Secret from step 1, so a cluster missing both needs both. ### Observability @@ -106,6 +149,12 @@ The credentials projector emits two Prometheus counters labelled by `namespace` Alert on `rate(cozystack_backup_credentials_projection_failures_total[5m]) > 0` or `absent_over_time(cozystack_backup_credentials_projection_successes_total[10m])` to catch a stale BSL credential or a malformed source Secret without log scraping. +The default-objects gate emits three more: + +- `cozystack_backup_default_objects_missing{backupclass="cozy-default"}` — how many of the objects the platform default backups depend on are absent: the credentials Secret, the Strategy CRs `cozy-default` routes to, and the Velero BSL. **This is the alert that would have caught the missing Strategy CRs**: it is non-zero whatever the HelmRelease's `Ready` condition says. Alert on `min_over_time(cozystack_backup_default_objects_missing[15m]) > 0`. +- `cozystack_backup_default_objects_check_errors_total{backupclass="cozy-default"}` — checks that could not reach a conclusion (an API error reading the source Secret, the BackupClass, or one of the routed objects). The gauge above is deliberately **not** written on those ticks, so that it does not flap on a transient API error — which means that while this counter climbs, the gauge is stale and a `0` on it proves nothing. Pair the two: `min_over_time(cozystack_backup_default_objects_missing[15m]) > 0 or rate(cozystack_backup_default_objects_check_errors_total[15m]) > 0`. +- `cozystack_backup_default_objects_force_reconciles_total{namespace,name}` — forced Helm upgrades issued, labelled by the release forced (this chart's own, or the bucket's `-system` release). A counter that keeps climbing means the forced render is not producing the objects (a missing CRD, for instance), which is a different problem from the install-time race. A suspended release is skipped before the patch and is **not** counted here, so a paused release cannot masquerade as a render that keeps failing — look for the `skipped forcing a suspended HelmRelease` log line instead. + ## Admin overrides for `cozy-default` `cozy-default` is rendered by the `backupstrategy-controller` chart and owned by Flux's helm-controller. **Direct `kubectl edit backupclass cozy-default` is overwritten on the next helm reconcile** — the same applies to its companion `strategy.backups.cozystack.io/*` CRs (`cozy-default-cnpg`, `cozy-default-etcd`, `cozy-default-mariadb`, `cozy-default-altinity`, `cozy-default-foundationdb`, the two `cozy-default-velero-*`). The supported override path is the `backupStorage` block on the **`platform` component** of the `cozystack.cozystack-platform` Package CR: diff --git a/docs/operations/seaweedfs-431-rename-recovery.md b/docs/operations/seaweedfs-431-rename-recovery.md index 06dd2c3422..f26a671ebc 100644 --- a/docs/operations/seaweedfs-431-rename-recovery.md +++ b/docs/operations/seaweedfs-431-rename-recovery.md @@ -55,6 +55,8 @@ hack/seaweedfs-naming-audit.sh # whole cluster hack/seaweedfs-naming-audit.sh tenant-foo # or named namespaces ``` +**Read the exit code, not just the table.** The audit fails closed: any error it cannot interpret — a kubectl call that fails, a Helm release payload it cannot decode — makes it print `FATAL` and exit non-zero, and the table it printed up to that point is incomplete. A non-zero exit means you do not yet know the state of the fleet, so none of the steps below may be taken on the strength of it. Only a zero exit means the table is the whole answer; an empty table with exit 0 is a genuinely clean fleet. + It mutates nothing. Earlier revisions of this runbook inlined the classification as a shell snippet here; it is a tested script now (`hack/seaweedfs-naming-audit.bats`), because it is what the chart's refusal hands you to and acting on it deletes PVCs. Two inline versions shipped wrong — one whose selector matched both generations at once and so inverted its own primary rule, one that could not see a long instance name at all — so it is not a snippet any more. It reports one class per SeaweedFS instance, matching exactly what the chart's guard decides: @@ -297,7 +299,7 @@ Do **not** start by deleting PVCs. If the duplicate ever served, they may hold o Never delete `data1-seaweedfs-volume-*` here — those are the live data PVCs of the authoritative set. (For an `S-damaged` tenant it is the other way round; that is Step 2a's job, and it has its own precondition check.) -5. **Re-run the audit.** The tenant must now read `L` (or `S`, if you are on the Step 2 path). Only then upgrade. +5. **Re-run the audit.** The tenant must now read `L` (or `S`, if you are on the Step 2 path), and the audit must exit zero — a `FATAL` here means the re-check never completed, not that the tenant is fine. Only then upgrade. ```sh hack/seaweedfs-naming-audit.sh "$ns" diff --git a/hack/lib/promoted-packages.sh b/hack/lib/promoted-packages.sh new file mode 100644 index 0000000000..e962eef761 --- /dev/null +++ b/hack/lib/promoted-packages.sh @@ -0,0 +1,25 @@ +# shellcheck shell=sh +# Shared definitions for the stable-candidate packages artifact. +# +# Sourced by hack/promote-packages-artifact.sh, which publishes the candidate, +# and hack/verify-promoted-packages.sh, which proves it. The publisher's whole +# reason to inspect the pin at all is to make the verifier's refusal land at +# dispatch time, before a registry write, two commits and a draft release exist +# — so the two have to be testing the same thing. Copied constants would drift +# apart silently and turn that dispatch-time refusal back into the late failure +# it was added to remove; they live here instead. +# +# Not a general library: promotion is the only caller, and both halves run from +# a `hack`-only sparse checkout of a trusted ref, so this file has to stay +# inside hack/ next to them. + +# The one OCI repository a packages candidate may ever be published to or read +# back from. Overridable only so the behavioural suites can point both halves +# at a fixture registry; production never sets it. +EXPECTED_PACKAGES_REPOSITORY="${EXPECTED_PACKAGES_REPOSITORY:-oci://ghcr.io/cozystack/cozystack/cozystack-packages}" + +# An immutable Flux OCI source pin, `digest=sha256:<64 lowercase hex>`. The +# tag form Flux also accepts is exactly what promotion must neither publish nor +# accept: a stable installer pinned by tag resolves to whatever that tag points +# at later, which is how a stable install ends up serving rc content (#3477). +PACKAGES_DIGEST_REF_PATTERN='^digest=sha256:[0-9a-f]{64}$' diff --git a/hack/overlay-main-images_test.bats b/hack/overlay-main-images_test.bats index 0f30265537..6953de9edb 100644 --- a/hack/overlay-main-images_test.bats +++ b/hack/overlay-main-images_test.bats @@ -147,3 +147,72 @@ grep -q 'present:main@sha256:bbbb' packages/apps/present/images/present.tag [ ! -e packages/apps/ghost/images/ghost.tag ] } + +# ── workflow wiring ───────────────────────────────────────────────────────── +# The script is only half the mechanism; WHICH artifact the workflow hands it +# decides which generation of images a PR gets. A release-line PR must read its +# own line's artifact — reading main's puts main's binaries against that line's +# charts, which failed install deterministically (#3437). + +@test "the overlay reads the artifact for the PR's own base branch" { + root=$(pwd) + wf="$root/.github/workflows/pull-requests.yaml" + [ -f "$wf" ] + + # Executable lines only: a comment mentioning the tag must not satisfy this. + code="$(grep -v '^[[:space:]]*#' "$wf")" + + block="$(printf '%s\n' "$code" | awk ' + $0 == " - name: Pull base-branch packages tree" { inside = 1; next } + /^ - name: / { inside = 0 } + inside')" + [ -n "$block" ] || { echo "the base-branch packages pull step is missing from $wf" >&2; exit 1; } + + # The artifact tag must come from the base branch, passed via env. + printf '%s\n' "$block" | grep -qF 'cozystack-packages:${BASE_REF}' || { + echo "the packages artifact tag is not the PR's base branch. Reading a fixed" >&2 + echo "tag (e.g. :main) hands a release-line PR another generation's images." >&2 + exit 1; } + printf '%s\n' "$block" | grep -qF 'BASE_REF: ${{ github.base_ref }}' || { + echo "BASE_REF is not wired to github.base_ref in the pull step's env." >&2; exit 1; } + + # A hardcoded :main anywhere in the two overlay steps defeats the point. + overlay="$(printf '%s\n' "$code" | awk ' + $0 == " - name: Overlay base-branch refs for unbuilt packages" { inside = 1; next } + /^ - name: / { inside = 0 } + inside')" + [ -n "$overlay" ] || { echo "the overlay step is missing from $wf" >&2; exit 1; } + printf '%s\n%s\n' "$block" "$overlay" | grep -qF 'cozystack-packages:main' && { + echo "an overlay step still pins cozystack-packages:main" >&2; exit 1; } + return 0 +} + +@test "every maintained release line publishes its own packages artifact" { + root=$(pwd) + wf="$root/.github/workflows/build-release.yaml" + [ -f "$wf" ] || { + echo "build-release.yaml is missing: without a per-line artifact the overlay" >&2 + echo "above degrades to a no-op on release branches, so their PRs test the" >&2 + echo "line's last release rather than its tip." >&2 + exit 1; } + + code="$(grep -v '^[[:space:]]*#' "$wf")" + + # Line branches only. The per-release and rc staging branches promote-rc.yaml + # and tags.yaml create (release-1.6.1, release-1.6.0-rc.4) must NOT trigger a + # full rebuild — their images come from the tag build. + printf '%s\n' "$code" | grep -qF "branches: ['release-[0-9]+.[0-9]+']" || { + echo "build-release.yaml does not restrict its trigger to release-." >&2; exit 1; } + + # The artifact and image tag must be the branch, or the overlay cannot find it. + printf '%s\n' "$code" | grep -qF 'IMAGE_TAG: ${{ github.ref_name }}' || { + echo "build-release.yaml does not tag images/artifact with the branch name" >&2; exit 1; } + + # It must not write the shared mode=max cache: that ref has exactly one + # serialized writer (build-main.yaml) so concurrent builds cannot race on the + # cache manifest. A line build can overlap a main build. + printf '%s\n' "$code" | grep -qF "WRITE_CACHE: '0'" || { + echo "build-release.yaml must set WRITE_CACHE: '0' — writing the shared" >&2 + echo ":buildcache ref races build-main.yaml on the cache manifest." >&2 + exit 1; } +} diff --git a/hack/promote-gate-contract.bats b/hack/promote-gate-contract.bats new file mode 100644 index 0000000000..c7e8b47454 --- /dev/null +++ b/hack/promote-gate-contract.bats @@ -0,0 +1,269 @@ +#!/usr/bin/env bats + +# Contract for the halves of the rc-E2E promotion gate that live on THIS branch. +# These tests intentionally pin executable/structural workflow lines, not prose: +# a commented-out gate, job, or step must never satisfy the contract. +# +# Scope, and why it is narrower than main's copy of this file. +# +# The gate itself is not here. `Promote RC` is a workflow_dispatch, so it runs +# from the ref the maintainer dispatches — main — and it is main's promote-rc.yaml +# that composes the expected job name, walks the Actions API for evidence, and +# refuses to promote without it. This branch's promote-rc.yaml predates all of +# that and is deliberately left alone: porting it would drag in a `parse` job, a +# `skip_e2e_gate` input, promote-time changelog and website-docs jobs, and the +# `labeled` pull_request trigger those depend on. main's copy of this file pins +# exactly those things, which is why it cannot be used verbatim here. +# +# What release-1.6 owns is the PRODUCER side of the same contract, and that is +# what these tests pin: +# +# * a tag push must actually run the full suite, under a job name whose +# rendered form is what the gate on main searches for. Workflows run from the +# ref they fire on, so only this file's tags.yaml can do that for an rc cut +# on this line; +# * the target base must carry the five files promote-rc.yaml's preflight +# demands before its first registry write, or the dispatch fails closed with +# "Target base 'release-1.6' lacks ''"; +# * the candidate verification in finalize must run before anything +# irreversible, because a guard that runs after the write-once stable tag +# exists cannot refuse it. +# +# No CI lane exercises tags.yaml (it runs only on tag pushes) and no lane +# exercises finalize's ordering, which is why these are pinned mechanically here. + +REPO_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:-$0}")/.." && pwd)" +PULL_REQUESTS="$REPO_ROOT/.github/workflows/pull-requests.yaml" +TAGS="$REPO_ROOT/.github/workflows/tags.yaml" +FINALIZE="$REPO_ROOT/.github/workflows/pull-requests-release.yaml" +E2E_TAG="$REPO_ROOT/.github/workflows/e2e-tag.yaml" + +job_block() { + awk -v job=" $1:" ' + $0 == job { inside = 1; next } + /^ [a-z0-9_-]+:$/ { inside = 0 } + inside' "$2" +} + +# Comment-stripping filter for the pins below. POSIX `grep` only: the unit-test +# runner has no ripgrep, and a missing filter used to be swallowed by `|| true`, +# silently reducing every pin to "0 matches" instead of failing on the real +# cause. grep exits 1 when nothing is selected (legitimate for an empty block) +# and 2 on an actual error. Do not read the `rc` check below as what catches +# that: code_lines is never the last stage of a pipeline in this file, so +# ordinary (non-pipefail) pipe semantics discard whatever it itself returns, +# and `hack/cozytest.sh`'s translator appends `return 0` to every line that is +# exactly `}`, code_lines' closing brace included, making its own exit status +# even less trustworthy as a signal. What catches a real grep error is the +# OUTPUT, and only for one class of pin: the error empties code_lines' stream, +# so a pin that requires something to be present in that stream fails on it. A +# pin demanding an absence stays blind, because an exact count of zero is what +# an emptied stream produces anyway, and what the pin was written to assert. +code_lines() { + local rc=0 + grep -v '^[[:space:]]*#' || rc=$? + [ "$rc" -le 1 ] +} + +# First line number of a named step inside an already-extracted job block, in +# code_lines' comment-stripped view. Top-level rather than nested inside a test +# so its closing brace is the only one the runner's translator has to rewrite. +step_line() { + printf '%s\n' "$2" | code_lines | grep -nF " - name: $1" \ + | awk -F: 'NR == 1 { print $1 }' +} + +# ── the producer half of the rc-E2E gate ───────────────────────────────────── + +@test "an rc tag push calls the full-suite e2e workflow" { + [ -f "$E2E_TAG" ] + + rc_e2e="$(job_block rc-e2e "$TAGS")" + [ -n "$rc_e2e" ] + + printf '%s\n' "$rc_e2e" | code_lines | grep -qF 'uses: ./.github/workflows/e2e-tag.yaml' + + # The tag under test must be the pushed ref. The gate correlates evidence by a + # job name that embeds the exact tag, so passing anything else here would + # produce a green run that no promotion can ever match to its rc. + printf '%s\n' "$rc_e2e" | code_lines | grep -qF 'tag: ${{ github.ref_name }}' + + # rc tags only. A stable tag push must not re-run the suite, and alpha/beta use + # e2e-tag.yaml's manual dispatch. + printf '%s\n' "$rc_e2e" | code_lines | grep -qF "contains(github.ref_name, '-rc.')" +} + +@test "rc-e2e grants the ceiling e2e-tag.yaml's jobs declare" { + rc_e2e="$(job_block rc-e2e "$TAGS")" + [ -n "$rc_e2e" ] + + # Every permission any job in the called workflow declares must be granted by + # the caller. GitHub validates that ceiling STATICALLY, when it creates the run, + # before this job's `if:` is evaluated — so a missing grant does not skip the + # rc lane, it fails the whole tags.yaml run for EVERY tag push, rc and stable + # alike, building nothing at all. Assert the pair in both files, so removing + # either side surfaces here instead of at the next release. + e2e_job="$(job_block e2e "$E2E_TAG")" + [ -n "$e2e_job" ] + printf '%s\n' "$e2e_job" | code_lines | grep -qF 'checks: write' + + printf '%s\n' "$rc_e2e" | code_lines | grep -qF 'contents: read' + printf '%s\n' "$rc_e2e" | code_lines | grep -qF 'checks: write' +} + +@test "the e2e job name is the exact string the promote gate searches for" { + # main's promote-rc.yaml builds `E2E ${rcTag} (full suite)` and matches a job + # whose name either equals it or ends in " / " plus it (the second form is how + # GitHub renders a called workflow's job: " / "). This + # branch supplies the called half, so the literal below is the contract — a + # rename on either side makes the gate fail closed and blocks promotion until + # someone notices. Cheap to pin, expensive to debug. + # + # -F is not optional: the pattern contains `${{`, and as a basic regular + # expression that silently matches nothing rather than erroring, which would + # turn this pin into a test that passes by finding its own subject absent. + count="$(code_lines < "$E2E_TAG" | grep -cF 'name: E2E ${{ inputs.tag }} (full suite)' || true)" + [ "${count:-0}" -eq 1 ] +} + +@test "the tag-push e2e lane runs the full chainsaw suite it claims to" { + e2e_job="$(job_block e2e "$E2E_TAG")" + [ -n "$e2e_job" ] + + # Empty CHAINSAW_SUITES is the testing Makefile's full-suite mode. A tag has no + # PR diff to scope with Test Impact Analysis, so a narrowed suite here would + # make the gate's evidence weaker than its name advertises. + printf '%s\n' "$e2e_job" | code_lines | grep -qF 'test-chainsaw CHAINSAW_SUITES=""' + + # The target and the sandbox binary the step above depends on. + grep -qE '^test-chainsaw' "$REPO_ROOT/packages/core/testing/Makefile" + grep -qF 'CHAINSAW_VERSION' "$REPO_ROOT/packages/core/testing/images/e2e-sandbox/Dockerfile" +} + +# ── the candidate-aware promotion pipeline this base must carry ────────────── + +@test "the base carries every file promote-rc.yaml's preflight demands" { + # promote-rc.yaml resolves the target base (release-X.Y when it exists) and + # reads each path below off it, refusing to promote unless the content contains + # the paired marker. It fails closed BEFORE the first registry write, so a base + # missing any of these cannot be promoted at all — which is exactly the state + # this branch was in. Keep the list in lockstep with the requiredBaseFiles array + # in main's promote-rc.yaml. + grep -qF 'verify-release-candidate:' "$PULL_REQUESTS" + grep -qF 'Verify stable packages candidate' "$FINALIZE" + grep -qF 'EXPECTED_PACKAGES_REPOSITORY' "$REPO_ROOT/hack/verify-promoted-packages.sh" + grep -qF 'collect_image_refs()' "$REPO_ROOT/hack/lib/image-refs.sh" + grep -qF 'PACKAGES_DIGEST_REF_PATTERN' "$REPO_ROOT/hack/lib/promoted-packages.sh" + + # The verifier sources both libraries by path relative to itself. A base + # carrying the script without them fails at the PR gate and again at finalize, + # after the candidate, the staging branch and the draft release already exist. + [ -x "$REPO_ROOT/hack/verify-promoted-packages.sh" ] + [ -f "$REPO_ROOT/hack/lib/promoted-packages.sh" ] + [ -f "$REPO_ROOT/hack/lib/image-refs.sh" ] +} + +@test "verify-release-candidate fires on the promote PR without needing a label" { + block="$(job_block verify-release-candidate "$PULL_REQUESTS")" + [ -n "$block" ] + + # This branch's on.pull_request.types has no `labeled` event, and the promote + # PR's `release` label is applied a moment AFTER the PR opens, so the `opened` + # payload carries no labels at all. A label-keyed guard would therefore never + # fire here. Pin the author + head-branch predicate that does, and pin the + # absence of the label dependency so nobody "restores" it from main and + # silently switches the job off. + printf '%s\n' "$block" | code_lines | grep -qF "github.event.pull_request.user.login == 'cozystack-ci[bot]'" + printf '%s\n' "$block" | code_lines | grep -qF "startsWith(github.head_ref, 'release-')" + + count="$(printf '%s\n' "$block" | code_lines | grep -cF "contains(github.event.pull_request.labels.*.name, 'release')" || true)" + [ "${count:-0}" -eq 0 ] + + # The verifier must come from the trusted base, not from the PR's own tree: a + # promote PR must not be able to weaken the guard that judges it. + printf '%s\n' "$block" | code_lines | grep -qF 'ref: ${{ github.event.pull_request.base.sha }}' + printf '%s\n' "$block" | code_lines | grep -qF '.release-tooling/hack/verify-promoted-packages.sh' +} + +@test "the candidate verification runs before every irreversible finalize step" { + block="$(job_block finalize "$FINALIZE")" + [ -n "$block" ] + + verify="$(step_line 'Verify stable packages candidate' "$block")" + [ -n "$verify" ] + + # Each of these creates or moves a name that cannot be taken back: the + # write-once stable git tag, the API submodule tag, the published release, the + # stable image tags (and :latest), and the stable installer chart. The + # verification is only a gate if it precedes all of them. + tag_step="$(step_line 'Create tag on merge commit (write-once)' "$block")" + [ -n "$tag_step" ] + [ "$verify" -lt "$tag_step" ] + + submodule_step="$(step_line 'Tag API submodule (write-once)' "$block")" + [ -n "$submodule_step" ] + [ "$verify" -lt "$submodule_step" ] + + publish_step="$(step_line 'Publish draft release' "$block")" + [ -n "$publish_step" ] + [ "$verify" -lt "$publish_step" ] + + retag_step="$(step_line 'Retag rc images to stable' "$block")" + [ -n "$retag_step" ] + [ "$verify" -lt "$retag_step" ] + + chart_step="$(step_line 'Publish stable cozy-installer chart' "$block")" + [ -n "$chart_step" ] + [ "$verify" -lt "$chart_step" ] + + # The toolchain and the registry login the verification needs must precede it + # too. They used to sit after "Publish draft release"; moving the verification + # up without them would make it fail on a missing `flux` instead of verifying. + toolchain_step="$(step_line 'Set up promotion toolchain (flux, skopeo, yq, helm)' "$block")" + [ -n "$toolchain_step" ] + [ "$toolchain_step" -lt "$verify" ] + + login_step="$(step_line 'Login to registry (GHCR)' "$block")" + [ -n "$login_step" ] + [ "$login_step" -lt "$verify" ] + + tooling_step="$(step_line 'Checkout release tooling from base' "$block")" + [ -n "$tooling_step" ] + [ "$tooling_step" -lt "$verify" ] +} + +@test "finalize checkout does not persist credentials so the app-token tag push triggers tags.yaml" { + block="$(job_block finalize "$FINALIZE")" + [ -n "$block" ] + checkout="$(printf '%s\n' "$block" | awk ' + /^ - name: Checkout repo$/ { inside = 1; next } + /^ - name: / { inside = 0 } + inside')" + [ -n "$checkout" ] + + # The one-line root-cause fix. Without persist-credentials:false the checkout + # persists GITHUB_TOKEN as http.extraheader, which silently defeats the app token + # each later `git remote set-url` injects onto the tag pushes — and a + # GITHUB_TOKEN-authenticated push creates no workflow run (anti-recursion), so + # tags.yaml's stable-tag backstops never fire (v1.6.0's tag never triggered it). + count="$(printf '%s\n' "$checkout" | code_lines | grep -cF 'persist-credentials: false' || true)" + [ "${count:-0}" -eq 1 ] +} + +# ── the tag-time changelog backstop ────────────────────────────────────────── + +@test "the tag-time changelog is validated and ported rather than regenerated" { + block="$(job_block generate-changelog "$TAGS")" + [ -n "$block" ] + + # The promote PR merges into release-1.6, so its reviewed changelog never + # reaches main and `exists` alone reads as "absent". at_tag is what stops a + # second AI run from overwriting an already-published release body. + printf '%s\n' "$block" | code_lines | grep -qF 'at_tag=true' + printf '%s\n' "$block" | code_lines | grep -qF "steps.check_changelog.outputs.at_tag == 'false'" + + # Neither a ported nor a generated file has been checked at this point, and the + # generating step is allowed to fail, so a truncated fragment must not pass. + printf '%s\n' "$block" | code_lines | grep -qF 'hack/validate-changelog.sh' + [ -x "$REPO_ROOT/hack/validate-changelog.sh" ] +} diff --git a/hack/promote-retag.sh b/hack/promote-retag.sh index 287bbf20a5..cbd57a323f 100755 --- a/hack/promote-retag.sh +++ b/hack/promote-retag.sh @@ -24,7 +24,7 @@ # files that covers), which is expected to be the promoted stable # digest-vendored tree (the release-X.Y.Z branch, whose digests are the rc's — # only the cosmetic tag string differs). -# Requires: yq (mikefarah), skopeo, and a registry login already done. +# Requires: yq (mikefarah), skopeo, sha256sum, and a registry login already done. # # The repo/tag split (ref_repo below) strips the :tag from the last path # component only, so registry hosts that carry a :port are preserved. @@ -49,6 +49,7 @@ command -v yq >/dev/null || { echo "yq (mikefarah) is required" >&2; exit 1; yq --version 2>&1 | grep -q mikefarah || { echo "yq (mikefarah) is required" >&2; exit 1; } # skopeo is only needed to actually copy; a --dry-run just prints the plan. [ "$DRY_RUN" -eq 1 ] || command -v skopeo >/dev/null || { echo "skopeo is required" >&2; exit 1; } +[ "$DRY_RUN" -eq 1 ] || command -v sha256sum >/dev/null || { echo "sha256sum is required" >&2; exit 1; } # Ref collection (which files are scanned, and the YAML shapes within them) is # shared with hack/nightly-mirror.sh and hack/promote-rewrite-tags.sh — see @@ -81,6 +82,47 @@ ref_repo() { } ref_digest() { printf '%s' "${1##*@}"; } # sha256:... +# manifest_digest — the digest of 's raw manifest, or empty output if +# the tag is PROVEN not to exist. Anything else is indeterminate and returns +# non-zero, which under `set -e` aborts the promotion before it writes. +# +# The three states matter because the caller turns "empty" into "not published, +# safe to copy over". Inferring that from a failure — or from a successful +# response with no bytes — is how a rate-limit or a proxy hiccup gets read as +# permission to overwrite released bytes. Absence has to be proven by the +# registry saying so. +manifest_digest() { + _ref="$1" + _manifest="$(mktemp)" + _mderr="$(mktemp)" + # Hash the raw bytes so this works for images and OCI artifacts alike. Files + # preserve inspect's status without relying on non-POSIX pipefail. + if skopeo inspect --raw "docker://$_ref" >"$_manifest" 2>"$_mderr"; then + if [ -s "$_manifest" ]; then + printf 'sha256:%s' "$(sha256sum "$_manifest" | cut -d' ' -f1)" + rm -f "$_manifest" "$_mderr" + return 0 + fi + # Exit 0 with no bytes — a registry answering 200 with an empty body. Not a + # digest (hashing nothing yields sha256:e3b0c442…, which looks real and + # belongs to no manifest) and not proof of absence either, so refuse to + # decide rather than let the caller copy over whatever is really there. + echo "::error::skopeo inspect of ${_ref} succeeded but returned no manifest bytes; refusing to decide whether that tag exists" >&2 + rm -f "$_manifest" "$_mderr" + return 1 + fi + # Non-zero. Only a registry that says the manifest is unknown proves absence; + # 429/5xx/auth/network failures all also exit non-zero with no bytes, and + # reading those as "not published" is exactly the fail-open this guards. + if grep -qiE 'manifest unknown|manifest not known|manifest for [^ ]* not found|not found|404' "$_mderr"; then + rm -f "$_manifest" "$_mderr" + return 0 + fi + echo "::error::skopeo inspect of ${_ref} failed and did not report the manifest as unknown, so it cannot be treated as unpublished: $(tr '\n' ' ' <"$_mderr")" >&2 + rm -f "$_manifest" "$_mderr" + return 1 +} + copy() { _src="$1"; _dst="$2" if [ "$DRY_RUN" -eq 1 ]; then @@ -128,13 +170,14 @@ echo "$refs" | while IFS= read -r ref; do repo="${ref%@*}" digest="${ref##*@}" echo "▸ ${repo} ${digest}" - # The stable tag is write-once at the image level: inspect the destination - # before copying. No-op if it already points at this rc digest (idempotent - # re-run), fail if it points elsewhere (a partial run or manual push already - # put different bytes there) rather than mutate released bytes. :latest is - # intentionally mutable and (re)pointed below only when MOVE_LATEST=1. + # The stable tag is write-once at the image level: resolve the destination's + # raw manifest digest before copying. No-op if it already points at this rc + # digest (idempotent re-run), fail if it points elsewhere (a partial run or + # manual push already put different bytes there) rather than mutate released + # bytes. :latest is intentionally mutable and (re)pointed below only when + # MOVE_LATEST=1. if [ "$DRY_RUN" -eq 0 ]; then - cur="$(skopeo inspect --format '{{.Digest}}' "docker://${repo}:${STABLE}" 2>/dev/null || echo '')" + cur="$(manifest_digest "${repo}:${STABLE}")" if [ -n "$cur" ] && [ "$cur" != "$digest" ]; then echo "::error::${repo}:${STABLE} already exists at '${cur}'; refusing to move it to '${digest}' (stable image tags are write-once)" >&2 exit 1 @@ -150,7 +193,7 @@ echo "$refs" | while IFS= read -r ref; do [ "$MOVE_LATEST" = "1" ] && copy "$ref" "${repo}:latest" # Verify the stable tag now resolves to the exact rc digest (skip in dry-run). if [ "$DRY_RUN" -eq 0 ]; then - got="$(skopeo inspect --format '{{.Digest}}' "docker://${repo}:${STABLE}" 2>/dev/null || echo '')" + got="$(manifest_digest "${repo}:${STABLE}")" if [ "$got" != "$digest" ]; then echo "::error::${repo}:${STABLE} resolved to '${got}', expected '${digest}'" >&2 exit 1 diff --git a/hack/promote-retag_test.bats b/hack/promote-retag_test.bats index b9901755ab..c2ac31113d 100644 --- a/hack/promote-retag_test.bats +++ b/hack/promote-retag_test.bats @@ -21,6 +21,54 @@ # (or `bats hack/promote-retag_test.bats` if the bats binary is # installed; cozytest.sh is the CI path.) +_make_registry_mocks() { + t="$1" + mkdir -p "$t/bin" + cat >"$t/bin/yq" <<'EOF' +#!/bin/sh +set -eu +if [ "${1:-}" = "--version" ]; then + echo 'yq (https://github.com/mikefarah/yq/) version v4.45.1' +else + printf '%s\n' "$MOCK_REF" +fi +EOF + cat >"$t/bin/skopeo" <<'EOF' +#!/bin/sh +set -eu +printf '%s\n' "$*" >>"$MOCK_SKOPEO_LOG" +case "$1" in + inspect) + [ "$2" = "--raw" ] + # An indeterminate failure: non-zero, no bytes, and the registry never says + # the manifest is unknown. Must NOT read as "tag absent". + if [ "${MOCK_TRANSIENT_ERROR:-0}" = "1" ]; then + echo "Error: reading manifest from docker://${3:-?}: received unexpected HTTP status: 429 Too Many Requests" >&2 + exit 1 + fi + if [ "$MOCK_MISSING_ONCE" = "1" ] && [ ! -f "$MOCK_STATE" ]; then + # Absence as a registry actually reports it — the script requires proof, + # not merely a non-zero exit. + echo "Error: reading manifest from docker://${3:-?}: manifest unknown" >&2 + exit 1 + fi + # A registry answering 200 with an empty body: exit 0, no bytes. + if [ "${MOCK_EMPTY_MANIFEST:-0}" = "1" ]; then + exit 0 + fi + printf '%s' "$MOCK_MANIFEST" + ;; + copy) + : >"$MOCK_STATE" + ;; + *) + exit 2 + ;; +esac +EOF + chmod +x "$t/bin/yq" "$t/bin/skopeo" +} + @test "dry-run over the real tree retags only cozystack-owned refs" { tmp=$(mktemp -d) trap 'rm -rf "$tmp"' EXIT @@ -160,3 +208,156 @@ grep -q 'cozystack/grafana:v9.9.9' "$tmp/out" grep -q 'cozystack/multus-cni:v9.9.9' "$tmp/out" } + +@test "raw manifest digest resolves for an OCI artifact" { + tmp=$(mktemp -d) + trap 'rm -rf "$tmp"' EXIT + _make_registry_mocks "$tmp" + + manifest='{"schemaVersion":2,"config":{"mediaType":"application/vnd.cncf.flux.config.v1+json","digest":"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"},"layers":[]}' + digest="sha256:$(printf '%s' "$manifest" | sha256sum | cut -d' ' -f1)" + ref="example.com/cozystack/cozystack-packages:v1.6.0-rc.1@${digest}" + + rc=0 + REGISTRY="example.com/cozystack" MOCK_REF="$ref" MOCK_MANIFEST="$manifest" \ + MOCK_MISSING_ONCE=0 MOCK_STATE="$tmp/state" MOCK_SKOPEO_LOG="$tmp/skopeo.log" \ + PATH="$tmp/bin:/usr/bin:/bin" hack/promote-retag.sh v1.6.0 \ + >"$tmp/out" 2>"$tmp/err" || rc=$? + if [ "$rc" -ne 0 ]; then + echo "promote-retag.sh exited $rc" >&2 + echo "--- stderr ---" >&2; cat "$tmp/err" >&2 + return "$rc" + fi + + grep -q "already at ${digest}; skipping stable copy" "$tmp/out" + [ "$(grep -c '^inspect --raw ' "$tmp/skopeo.log")" -eq 2 ] + ! grep -q '^copy ' "$tmp/skopeo.log" +} + +@test "post-copy raw manifest digest mismatch fails verification" { + tmp=$(mktemp -d) + trap 'rm -rf "$tmp"' EXIT + _make_registry_mocks "$tmp" + + manifest='{"schemaVersion":2,"config":{"mediaType":"application/vnd.cncf.flux.config.v1+json","digest":"sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"},"layers":[]}' + expected_manifest='{"schemaVersion":2,"config":{"mediaType":"application/vnd.cncf.flux.config.v1+json","digest":"sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"},"layers":[]}' + actual="sha256:$(printf '%s' "$manifest" | sha256sum | cut -d' ' -f1)" + expected="sha256:$(printf '%s' "$expected_manifest" | sha256sum | cut -d' ' -f1)" + ref="example.com/cozystack/cozystack-packages:v1.6.0-rc.1@${expected}" + + rc=0 + REGISTRY="example.com/cozystack" MOCK_REF="$ref" MOCK_MANIFEST="$manifest" \ + MOCK_MISSING_ONCE=1 MOCK_STATE="$tmp/state" MOCK_SKOPEO_LOG="$tmp/skopeo.log" \ + PATH="$tmp/bin:/usr/bin:/bin" hack/promote-retag.sh v1.6.0 \ + >"$tmp/out" 2>"$tmp/err" || rc=$? + + [ "$rc" -ne 0 ] + grep -q "resolved to '${actual}', expected '${expected}'" "$tmp/err" + grep -q '^copy --multi-arch all ' "$tmp/skopeo.log" + [ "$(grep -c '^inspect --raw ' "$tmp/skopeo.log")" -eq 2 ] +} + +@test "missing stable tag remains an empty digest and proceeds to copy" { + tmp=$(mktemp -d) + trap 'rm -rf "$tmp"' EXIT + _make_registry_mocks "$tmp" + + manifest='{"schemaVersion":2,"config":{"mediaType":"application/vnd.cncf.flux.config.v1+json","digest":"sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"},"layers":[]}' + digest="sha256:$(printf '%s' "$manifest" | sha256sum | cut -d' ' -f1)" + ref="example.com/cozystack/cozystack-packages:v1.6.0-rc.1@${digest}" + + rc=0 + REGISTRY="example.com/cozystack" MOCK_REF="$ref" MOCK_MANIFEST="$manifest" \ + MOCK_MISSING_ONCE=1 MOCK_STATE="$tmp/state" MOCK_SKOPEO_LOG="$tmp/skopeo.log" \ + PATH="$tmp/bin:/usr/bin:/bin" hack/promote-retag.sh v1.6.0 \ + >"$tmp/out" 2>"$tmp/err" || rc=$? + if [ "$rc" -ne 0 ]; then + echo "promote-retag.sh exited $rc" >&2 + echo "--- stderr ---" >&2; cat "$tmp/err" >&2 + return "$rc" + fi + + grep -q '^copy --multi-arch all ' "$tmp/skopeo.log" + [ "$(grep -c '^inspect --raw ' "$tmp/skopeo.log")" -eq 2 ] + grep -q 'Retagged image refs to v1.6.0' "$tmp/out" +} + +@test "existing stable tag at a different digest is refused, not moved" { + tmp=$(mktemp -d) + trap 'rm -rf "$tmp"' EXIT + _make_registry_mocks "$tmp" + + # The stable tag already exists (MOCK_MISSING_ONCE=0) but resolves to a + # manifest other than the rc's: released bytes must never be overwritten. + published='{"schemaVersion":2,"config":{"mediaType":"application/vnd.oci.image.config.v1+json","digest":"sha256:1111111111111111111111111111111111111111111111111111111111111111"},"layers":[]}' + rc_manifest='{"schemaVersion":2,"config":{"mediaType":"application/vnd.oci.image.config.v1+json","digest":"sha256:2222222222222222222222222222222222222222222222222222222222222222"},"layers":[]}' + published_digest="sha256:$(printf '%s' "$published" | sha256sum | cut -d' ' -f1)" + rc_digest="sha256:$(printf '%s' "$rc_manifest" | sha256sum | cut -d' ' -f1)" + ref="example.com/cozystack/cozystack-packages:v1.6.0-rc.1@${rc_digest}" + + rc=0 + REGISTRY="example.com/cozystack" MOCK_REF="$ref" MOCK_MANIFEST="$published" \ + MOCK_MISSING_ONCE=0 MOCK_STATE="$tmp/state" MOCK_SKOPEO_LOG="$tmp/skopeo.log" \ + PATH="$tmp/bin:/usr/bin:/bin" hack/promote-retag.sh v1.6.0 \ + >"$tmp/out" 2>"$tmp/err" || rc=$? + + [ "$rc" -ne 0 ] + grep -q "already exists at '${published_digest}'; refusing to move it to '${rc_digest}'" "$tmp/err" + # The refusal must happen BEFORE any write. + ! grep -q '^copy ' "$tmp/skopeo.log" + [ "$(grep -c '^inspect --raw ' "$tmp/skopeo.log")" -eq 1 ] +} + +@test "a zero-exit empty manifest refuses to decide and never writes" { + tmp=$(mktemp -d) + trap 'rm -rf "$tmp"' EXIT + _make_registry_mocks "$tmp" + + manifest='{"schemaVersion":2,"config":{"mediaType":"application/vnd.oci.image.config.v1+json","digest":"sha256:3333333333333333333333333333333333333333333333333333333333333333"},"layers":[]}' + digest="sha256:$(printf '%s' "$manifest" | sha256sum | cut -d' ' -f1)" + ref="example.com/cozystack/cozystack-packages:v1.6.0-rc.1@${digest}" + + rc=0 + REGISTRY="example.com/cozystack" MOCK_REF="$ref" MOCK_MANIFEST="$manifest" \ + MOCK_EMPTY_MANIFEST=1 MOCK_MISSING_ONCE=0 MOCK_STATE="$tmp/state" \ + MOCK_SKOPEO_LOG="$tmp/skopeo.log" \ + PATH="$tmp/bin:/usr/bin:/bin" hack/promote-retag.sh v1.6.0 \ + >"$tmp/out" 2>"$tmp/err" || rc=$? + + # A 200 with an empty body proves nothing: the tag may hold released bytes this + # promotion must not overwrite. So the script must abort BEFORE any copy — + # reading "no bytes" as "not published" is what would turn a proxy hiccup into + # an overwrite of a published stable tag. + [ "$rc" -ne 0 ] + grep -q 'returned no manifest bytes' "$tmp/err" + ! grep -q '^copy ' "$tmp/skopeo.log" + [ "$(grep -c '^inspect --raw ' "$tmp/skopeo.log")" -eq 1 ] + # The empty-input hash must never appear: that is the guard being absent. + ! grep -q 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855' "$tmp/err" +} + +@test "a transient registry failure is not read as an unpublished tag" { + tmp=$(mktemp -d) + trap 'rm -rf "$tmp"' EXIT + _make_registry_mocks "$tmp" + + manifest='{"schemaVersion":2,"config":{"mediaType":"application/vnd.oci.image.config.v1+json","digest":"sha256:4444444444444444444444444444444444444444444444444444444444444444"},"layers":[]}' + digest="sha256:$(printf '%s' "$manifest" | sha256sum | cut -d' ' -f1)" + ref="example.com/cozystack/cozystack-packages:v1.6.0-rc.1@${digest}" + + # 429 during finalize: non-zero exit, no bytes, no "manifest unknown". The old + # shape of this helper made that indistinguishable from an absent tag and + # proceeded to copy over it; finalize retags ~42 refs with no retry wrapper, so + # a single rate-limit was enough to reach that path. + rc=0 + REGISTRY="example.com/cozystack" MOCK_REF="$ref" MOCK_MANIFEST="$manifest" \ + MOCK_TRANSIENT_ERROR=1 MOCK_MISSING_ONCE=0 MOCK_STATE="$tmp/state" \ + MOCK_SKOPEO_LOG="$tmp/skopeo.log" \ + PATH="$tmp/bin:/usr/bin:/bin" hack/promote-retag.sh v1.6.0 \ + >"$tmp/out" 2>"$tmp/err" || rc=$? + + [ "$rc" -ne 0 ] + grep -q 'cannot be treated as unpublished' "$tmp/err" + grep -q '429' "$tmp/err" + ! grep -q '^copy ' "$tmp/skopeo.log" +} diff --git a/hack/seaweedfs-naming-audit.bats b/hack/seaweedfs-naming-audit.bats index eb6356e025..eb605c5297 100644 --- a/hack/seaweedfs-naming-audit.bats +++ b/hack/seaweedfs-naming-audit.bats @@ -138,3 +138,468 @@ EOF done < "$chart/rendered" rm -rf "$chart" } + +# ----------------------------------------------------------------------------- +# Fail-closed tests (issue #3431). +# +# The classification tests above mock nothing below the classifier. These drive +# the KUBECTL layer, where the fail-open bug lived: a kubectl call that failed +# used to return empty stdout, byte-identical to a genuinely clean fleet, so the +# audit printed an empty table and exited 0 -- the exact false "nothing to do" the +# operator is told to trust before deleting PVCs. They shim `kubectl` on PATH with +# a fake and assert the audit fails LOUDLY (non-zero exit + a FATAL naming the +# query) on any real error, while still treating a genuinely-absent object as +# clean. Two families: +# +# * enumerating LISTs (get ns / secret / pvc / sts) -- a failure is always fatal; +# * by-NAME GETs (a release secret in system_releases, a PVC/PV in pv_epoch) -- +# `--ignore-not-found` splits a real error (fatal) from a legitimate absence: +# an absent release revision is a clean skip; an absent PV degrades to the +# safe "cannot establish direction" note and must NOT become a deletion +# candidate (the range-narrowing flip Codex reproduced). +# +# The two golden tests guard the reverse: on a healthy cluster the output stays +# byte-for-byte what it was on origin/main, including the MIXED path that exercises +# every pv_epoch GET. +# +# The fake is a real executable on PATH, so it exercises the actual exit-status +# handling in run_kubectl and the propagation up through every caller -- a for +# loop over `$(...)` swallows the status, so this is where a regression would hide. + +# _release_blob [chart-name] -- the value kubectl returns for a Helm release +# secret's `.data.release`: base64(base64(gzip(json))). release_json base64-decodes +# twice and gunzips it, and system_releases reads the chart name from +# chart.metadata.name (as real Helm payloads carry it). Defaults to a cozy-seaweedfs +# release so the audit confirms the tenant; pass another chart name for a non- +# SeaweedFS release, or the literal EMPTY for a chartless {} payload. +# +# Three payload SHAPES beyond the default compact one, because the chart-name +# extraction is now fatal on a miss and every shape below is something a +# re-serializer or a user's values could produce: +# SPACED whitespace around the JSON punctuation (json.dumps' default) +# PRETTY indented and MULTI-LINE (jq . / yq -o=json), which a line-based +# matcher cannot read at all unless newlines are folded first +# DECOY a real cozy-seaweedfs chart PLUS a values subtree that spells +# chart.metadata.name with a different value. Helm marshals "config" +# (the values) AFTER "chart", so a last-match extraction returns the +# decoy and silently declares the tenant non-SeaweedFS. +_release_blob() { + _rb_name=${1:-cozy-seaweedfs} + case "$_rb_name" in + EMPTY) _rb_json='{}' ;; + SPACED) _rb_json='{"name": "seaweedfs-system", "chart": {"metadata": {"name": "cozy-seaweedfs", "version": "1.0.0"}}}' ;; + PRETTY) _rb_json='{ + "name": "seaweedfs-system", + "chart": { + "metadata": { + "name": "cozy-seaweedfs", + "version": "1.0.0" + } + } +}' ;; + DECOY) _rb_json='{"name":"seaweedfs-system","chart":{"metadata":{"name":"cozy-seaweedfs"}},"config":{"chart":{"metadata":{"name":"decoy-not-seaweedfs"}}}}' ;; + *) _rb_json='{"name":"seaweedfs-system","chart":{"metadata":{"name":"'"$_rb_name"'"}}}' ;; + esac + printf '%s' "$_rb_json" | gzip | base64 | base64 | tr -d '\n' +} + +# _write_fake_kubectl -- drop a fake kubectl +# into . +# one of ns | secretlist | secretget | secretget_absent | +# secret_missing_field | secret_bad_payload | pvclist | pvcget | +# sts | pvget | "" -- the single call to make behave badly. Real- +# error targets exit non-zero with a stderr message; secretget_absent +# mirrors an absent Secret (--ignore-not-found: exit 0, empty -o +# name); secret_missing_field / secret_bad_payload keep the Secret +# present but return an empty / undecodable .data.release payload. +# a PV name (pv-legacy|pv-legacy2|pv-renamed) to report as absent +# (exit 0, empty), exercising the "bound claim, PV gone" path. +# newline-separated `get pvc -o name` LIST output. +# It otherwise walks one confirmed seaweedfs-system tenant. The by-name PVC->PV and +# PV->timestamp maps are fixed here; timestamps are chosen so the two legacy PVs +# straddle the single renamed PV (true answer OVERLAP), which is what makes the +# range-narrowing flip observable. Kept POSIX and free of a column-0 `}` so +# cozytest.sh's awk converter passes the heredoc through untouched. +_write_fake_kubectl() { + # Grouped redirect (one open of the file). The closing brace is indented, so + # cozytest.sh's awk -- which only rewrites a `}` in column 0 -- leaves it and the + # heredoc alone. + { + printf '#!/bin/sh\n' + printf "FAIL='%s'\n" "$2" + printf "ABSENT_PV='%s'\n" "$3" + printf "BLOB='%s'\n" "$4" + printf "PVCS='%s'\n" "$5" + cat <<'FAKE' +verb=${1:-}; res=${2:-}; args="$*" +fail() { echo "fake kubectl: $1 (real error)" >&2; exit 1; } +# Anything this fake does not model must NOT look like a successful empty +# answer: that is the fail-open shape the audited script exists to reject, and +# it would let a new query added to the script pass these goldens unnoticed. +unmodelled() { echo "fake kubectl: unmodelled invocation: $args" >&2; exit 97; } +if [ "$verb $res" = "get ns" ]; then + [ "$FAIL" = ns ] && fail "get ns" + printf 'namespace/tenant-test\n'; exit 0 +fi +if [ "$verb $res" = "get secret" ]; then + case "$args" in + *sh.helm.release.v1*) + # release_json now asks two questions: existence (-o name) then payload + # (jsonpath .data.release). Mirror that split so absence, real error, and + # corrupt-payload are all reachable independently. + case "$args" in + *"-o name"*) + [ "$FAIL" = secretget ] && fail "release secret existence GET" + [ "$FAIL" = secretget_absent ] && exit 0 + printf 'secret/sh.helm.release.v1.seaweedfs-system.v1\n'; exit 0 ;; + *) + [ "$FAIL" = secret_missing_field ] && exit 0 + if [ "$FAIL" = secret_bad_payload ]; then printf '@@@not-base64@@@\n'; exit 0; fi + printf '%s\n' "$BLOB"; exit 0 ;; + esac ;; + *owner=helm*) printf '1\n'; exit 0 ;; + *) [ "$FAIL" = secretlist ] && fail "namespace secret LIST" + printf 'seaweedfs-system\n'; exit 0 ;; + esac +fi +if [ "$verb $res" = "get pvc" ]; then + case "$args" in + *"-o name"*) + [ "$FAIL" = pvclist ] && fail "pvc LIST" + [ -n "$PVCS" ] && printf '%s\n' "$PVCS" + exit 0 ;; + *data1-seaweedfs-system-volume-0*) [ "$FAIL" = pvcget ] && fail "pvc GET"; printf 'pv-renamed\n'; exit 0 ;; + *data1-seaweedfs-volume-1*) [ "$FAIL" = pvcget ] && fail "pvc GET"; printf 'pv-legacy2\n'; exit 0 ;; + *data1-seaweedfs-volume-0*) [ "$FAIL" = pvcget ] && fail "pvc GET"; printf 'pv-legacy\n'; exit 0 ;; + esac + unmodelled +fi +if [ "$verb $res" = "get sts" ]; then + [ "$FAIL" = sts ] && fail "sts LIST" + exit 0 +fi +if [ "$verb $res" = "get pv" ]; then + [ "$FAIL" = pvget ] && fail "get pv" + case "$args" in + *pv-legacy2*) [ "$ABSENT_PV" = pv-legacy2 ] && exit 0; printf '2099-01-01T00:00:00Z\n'; exit 0 ;; + *pv-legacy*) [ "$ABSENT_PV" = pv-legacy ] && exit 0; printf '2020-01-01T00:00:00Z\n'; exit 0 ;; + *pv-renamed*) [ "$ABSENT_PV" = pv-renamed ] && exit 0; printf '2020-06-01T00:00:00Z\n'; exit 0 ;; + esac + unmodelled +fi +unmodelled +FAKE + } > "$1/kubectl" + chmod +x "$1/kubectl" +} + +# _pvcs_mixed -- the `get pvc -o name` LIST for a MIXED tenant: two legacy claims +# and one release-named claim, whose PV vintages truly overlap. +_pvcs_mixed() { + printf '%s\n' \ + persistentvolumeclaim/data1-seaweedfs-volume-0 \ + persistentvolumeclaim/data1-seaweedfs-volume-1 \ + persistentvolumeclaim/data1-seaweedfs-system-volume-0 +} + +# _expected_L / _expected_mixed_overlap -- golden output built with the SAME printf +# contract the script uses, so a drift in row count, text, or padding fails the diff. +_expected_L() { + printf '%-24s %-14s %-8s %s\n' NAMESPACE RELEASE CLASS NOTE + printf '%-24s %-14s %-8s %s\n' --------- ------- ----- ---- + printf '%-24s %-14s %-8s %s\n' tenant-test seaweedfs-system L 'chart-named only; the upgrade adopts it, nothing to do' +} +_expected_mixed_overlap() { + printf '%-24s %-14s %-8s %s\n' NAMESPACE RELEASE CLASS NOTE + printf '%-24s %-14s %-8s %s\n' --------- ------- ----- ---- + printf '%-24s %-14s %-8s %s\n' tenant-test seaweedfs-system MIXED 'both generations; the chart REFUSES until one is removed' + printf '%-24s %-14s %-8s %s\n' '' '' '' 'PV vintages OVERLAP => no candidate. An interrupted Step 2 re-bind looks exactly like this (both generations on original PVs). Finish Step 2 if one is in progress; otherwise escalate. Do NOT run Step 2a.' + printf '%-24s %-14s %-8s %s\n' '' '' '' 'CANDIDATE ONLY: "original" does not mean the other set is EMPTY. A duplicate that' + printf '%-24s %-14s %-8s %s\n' '' '' '' 'served writes and later crashed looks identical here. Verify emptiness before deleting.' +} + +@test "fails closed when 'kubectl get ns' fails (whole-cluster mode)" { + # No namespace args -> main enumerates namespaces itself. If that LIST fails, + # the OLD code audited zero namespaces and still printed an empty clean table. + d=$(mktemp -d) + _write_fake_kubectl "$d" ns "" "$(_release_blob)" "" + rc=0 + out=$(PATH="$d:$PATH" SEAWEEDFS_AUDIT_LIB=0 sh "$PWD/hack/seaweedfs-naming-audit.sh" 2>&1) || rc=$? + rm -rf "$d" + echo "rc=$rc"; echo "$out" + [ "$rc" -ne 0 ] + printf '%s\n' "$out" | grep -q 'FATAL' + printf '%s\n' "$out" | grep -q 'list namespaces' +} + +@test "fails closed when the namespace-wide secret LIST fails" { + # system_releases enumerates Helm releases with a namespace-wide secret LIST -- + # the exact call that timed out in the field and reported a false clean. + d=$(mktemp -d) + _write_fake_kubectl "$d" secretlist "" "$(_release_blob)" "" + rc=0 + out=$(PATH="$d:$PATH" SEAWEEDFS_AUDIT_LIB=0 sh "$PWD/hack/seaweedfs-naming-audit.sh" tenant-test 2>&1) || rc=$? + rm -rf "$d" + echo "rc=$rc"; echo "$out" + [ "$rc" -ne 0 ] + printf '%s\n' "$out" | grep -q 'FATAL' + printf '%s\n' "$out" | grep -q 'list Helm release secrets' +} + +@test "fails closed when the by-name release-secret existence GET hits a real error" { + # Codex Finding 1: system_releases uses release_json to decide whether a release + # IS SeaweedFS. Both LISTs succeed, then a transient/forbidden by-name Secret GET + # used to leave `chart` empty -> the tenant was silently skipped -> exit 0, empty + # table. A real error here must now be fatal, not a false clean. + d=$(mktemp -d) + _write_fake_kubectl "$d" secretget "" "$(_release_blob)" "" + rc=0 + out=$(PATH="$d:$PATH" SEAWEEDFS_AUDIT_LIB=0 sh "$PWD/hack/seaweedfs-naming-audit.sh" tenant-test 2>&1) || rc=$? + rm -rf "$d" + echo "rc=$rc"; echo "$out" + [ "$rc" -ne 0 ] + printf '%s\n' "$out" | grep -q 'FATAL' + printf '%s\n' "$out" | grep -q 'Helm release secret' +} + +@test "an absent release-secret revision is a clean skip, not a failure" { + # The one legitimately-empty case for release_json: the revision secret does not + # exist (pruned by Helm history limit). The existence check (--ignore-not-found -o + # name) returns empty + exit 0, so the release is simply not confirmed as + # SeaweedFS. No error, no crash -- the audit completes and reports nothing here. + d=$(mktemp -d) + _write_fake_kubectl "$d" secretget_absent "" "$(_release_blob)" "" + rc=0 + out=$(PATH="$d:$PATH" SEAWEEDFS_AUDIT_LIB=0 sh "$PWD/hack/seaweedfs-naming-audit.sh" tenant-test 2>&1) || rc=$? + rm -rf "$d" + echo "rc=$rc"; echo "$out" + [ "$rc" -eq 0 ] + [ "$(printf '%s\n' "$out" | grep -c 'FATAL')" -eq 0 ] +} + +@test "fails closed when a present release secret has no .data.release payload" { + # Codex re-review round 2: the existence check passes (Secret EXISTS), but its + # .data.release field is empty/missing. That is CORRUPT state, not an absence -- + # and because the earlier fix used --ignore-not-found -o jsonpath, which returns + # empty+0 for BOTH, it used to read as a clean skip. It must be a loud stop. + d=$(mktemp -d) + _write_fake_kubectl "$d" secret_missing_field "" "$(_release_blob)" "" + rc=0 + out=$(PATH="$d:$PATH" SEAWEEDFS_AUDIT_LIB=0 sh "$PWD/hack/seaweedfs-naming-audit.sh" tenant-test 2>&1) || rc=$? + rm -rf "$d" + echo "rc=$rc"; echo "$out" + [ "$rc" -ne 0 ] + printf '%s\n' "$out" | grep -q 'FATAL' + printf '%s\n' "$out" | grep -q 'no .data.release payload' +} + +@test "fails closed when a present release secret has an undecodable payload" { + # Existence passes, .data.release is non-empty but is not base64(base64(gzip(...))), + # so every decode fails. The old `|| return 0` converted that decode failure into + # a clean 0 return -> silent skip. A corrupt payload must be fatal. + d=$(mktemp -d) + _write_fake_kubectl "$d" secret_bad_payload "" "$(_release_blob)" "" + rc=0 + out=$(PATH="$d:$PATH" SEAWEEDFS_AUDIT_LIB=0 sh "$PWD/hack/seaweedfs-naming-audit.sh" tenant-test 2>&1) || rc=$? + rm -rf "$d" + echo "rc=$rc"; echo "$out" + [ "$rc" -ne 0 ] + printf '%s\n' "$out" | grep -q 'FATAL' + printf '%s\n' "$out" | grep -q 'not decodable' +} + +@test "fails closed when a present release secret decodes to a chartless payload" { + # Codex round 3: the payload fetches and DECODES cleanly (valid JSON {}), so the + # decode guards all pass -- but it carries no chart name. system_releases then + # extracted chart='' and silently skipped the tenant -> exit 0, clean table. A + # decoded helm.sh/release.v1 release without a chart name is corrupt: fatal. + d=$(mktemp -d) + _write_fake_kubectl "$d" "" "" "$(_release_blob EMPTY)" "" + rc=0 + out=$(PATH="$d:$PATH" SEAWEEDFS_AUDIT_LIB=0 sh "$PWD/hack/seaweedfs-naming-audit.sh" tenant-test 2>&1) || rc=$? + rm -rf "$d" + echo "rc=$rc"; echo "$out" + [ "$rc" -ne 0 ] + printf '%s\n' "$out" | grep -q 'FATAL' + printf '%s\n' "$out" | grep -q 'could not read the chart name' + # The diagnostic must name the path it looked at, so a future Helm payload + # format change is diagnosable as such instead of reading as real corruption. + printf '%s\n' "$out" | grep -q '\.chart\.metadata\.name' +} + +@test "a present release secret for a non-SeaweedFS chart is a silent legitimate skip" { + # The counterpart the guard must NOT break: a real, well-formed release whose + # chart is simply not cozy-seaweedfs. It has a chart name (so it is not corrupt), + # but the wrong one, so it is filtered out exactly as before -- no row, no error. + d=$(mktemp -d) + _write_fake_kubectl "$d" "" "" "$(_release_blob cozy-postgres)" "" + rc=0 + out=$(PATH="$d:$PATH" SEAWEEDFS_AUDIT_LIB=0 sh "$PWD/hack/seaweedfs-naming-audit.sh" tenant-test 2>&1) || rc=$? + rm -rf "$d" + echo "rc=$rc"; echo "$out" + [ "$rc" -eq 0 ] + [ "$(printf '%s\n' "$out" | grep -c 'FATAL')" -eq 0 ] + # Not confirmed as SeaweedFS -> no classification row for the release. + [ "$(printf '%s\n' "$out" | grep -c 'seaweedfs-system')" -eq 0 ] +} + +@test "fails closed when 'kubectl get pvc' LIST fails" { + # The secret enumeration succeeds and confirms a seaweedfs-system tenant, so the + # audit reaches the per-namespace PVC LIST; a failure there must abort, not read + # as "this tenant has no claims". + d=$(mktemp -d) + _write_fake_kubectl "$d" pvclist "" "$(_release_blob)" "" + rc=0 + out=$(PATH="$d:$PATH" SEAWEEDFS_AUDIT_LIB=0 sh "$PWD/hack/seaweedfs-naming-audit.sh" tenant-test 2>&1) || rc=$? + rm -rf "$d" + echo "rc=$rc"; echo "$out" + [ "$rc" -ne 0 ] + printf '%s\n' "$out" | grep -q 'FATAL' + printf '%s\n' "$out" | grep -q 'list PVCs' +} + +@test "fails closed when 'kubectl get sts' LIST fails" { + # PVC LIST succeeds (empty), so the audit reaches the StatefulSet LIST; a + # failure there must abort rather than read as "no StatefulSets". + d=$(mktemp -d) + _write_fake_kubectl "$d" sts "" "$(_release_blob)" "" + rc=0 + out=$(PATH="$d:$PATH" SEAWEEDFS_AUDIT_LIB=0 sh "$PWD/hack/seaweedfs-naming-audit.sh" tenant-test 2>&1) || rc=$? + rm -rf "$d" + echo "rc=$rc"; echo "$out" + [ "$rc" -ne 0 ] + printf '%s\n' "$out" | grep -q 'FATAL' + printf '%s\n' "$out" | grep -q 'StatefulSets' +} + +@test "fails closed when the by-name PV GET hits a real error" { + # Codex Finding 2, error half: pv_epoch reads each bound PV's age by name. A real + # error (RBAC/timeout) must abort, not silently drop that PV from the range. + d=$(mktemp -d) + _write_fake_kubectl "$d" pvget "" "$(_release_blob)" "$(_pvcs_mixed)" + rc=0 + out=$(PATH="$d:$PATH" SEAWEEDFS_AUDIT_LIB=0 sh "$PWD/hack/seaweedfs-naming-audit.sh" tenant-test 2>&1) || rc=$? + rm -rf "$d" + echo "rc=$rc"; echo "$out" + [ "$rc" -ne 0 ] + printf '%s\n' "$out" | grep -q 'FATAL' + printf '%s\n' "$out" | grep -q "get PV '" +} + +@test "an absent PV degrades to the safe fallback, never a wrong candidate" { + # Codex Finding 2, absence half. True vintages: legacy PVs 2020 + 2099 straddle + # the single renamed PV 2020-06 => OVERLAP. Report the newest legacy PV (2099) as + # ABSENT: the observed legacy range collapses to {2020} and, unguarded, "every + # release-named PV is strictly newer" would fire -- naming the release-named set a + # deletion candidate on incomplete evidence. The generation must instead read as + # incomplete and fall to "cannot establish direction". + d=$(mktemp -d) + _write_fake_kubectl "$d" "" pv-legacy2 "$(_release_blob)" "$(_pvcs_mixed)" + rc=0 + out=$(PATH="$d:$PATH" SEAWEEDFS_AUDIT_LIB=0 sh "$PWD/hack/seaweedfs-naming-audit.sh" tenant-test 2>&1) || rc=$? + rm -rf "$d" + echo "rc=$rc"; echo "$out" + [ "$rc" -eq 0 ] + [ "$(printf '%s\n' "$out" | grep -c 'FATAL')" -eq 0 ] + printf '%s\n' "$out" | grep -q 'direction cannot be established from PV ages' + # The whole point: incomplete evidence must NOT be reported as a deletion candidate. + [ "$(printf '%s\n' "$out" | grep -c 'candidate duplicate')" -eq 0 ] +} + +@test "success path (L) is byte-identical to the expected fixture" { + # A single chart-named claim, no release-named one, no StatefulSets => L, adopt in + # place. Full-output compare, so an extra row / warning / duplicate line fails. + d=$(mktemp -d) + _write_fake_kubectl "$d" "" "" "$(_release_blob)" "persistentvolumeclaim/data1-seaweedfs-volume-0" + rc=0 + out=$(PATH="$d:$PATH" SEAWEEDFS_AUDIT_LIB=0 sh "$PWD/hack/seaweedfs-naming-audit.sh" tenant-test 2>&1) || rc=$? + printf '%s\n' "$out" > "$d/got" + _expected_L > "$d/want" + echo "rc=$rc"; echo "--- got ---"; cat "$d/got"; echo "--- want ---"; cat "$d/want" + [ "$rc" -eq 0 ] + diff "$d/want" "$d/got" + rm -rf "$d" +} + +@test "success path (MIXED/overlap) is byte-identical and exercises pv_epoch" { + # Both generations present with all PVs readable => the MIXED path runs every + # pv_epoch GET and lands on OVERLAP. Full-output compare against the golden. + d=$(mktemp -d) + _write_fake_kubectl "$d" "" "" "$(_release_blob)" "$(_pvcs_mixed)" + rc=0 + out=$(PATH="$d:$PATH" SEAWEEDFS_AUDIT_LIB=0 sh "$PWD/hack/seaweedfs-naming-audit.sh" tenant-test 2>&1) || rc=$? + printf '%s\n' "$out" > "$d/got" + _expected_mixed_overlap > "$d/want" + echo "rc=$rc"; echo "--- got ---"; cat "$d/got"; echo "--- want ---"; cat "$d/want" + [ "$rc" -eq 0 ] + diff "$d/want" "$d/got" + rm -rf "$d" +} + +@test "a whitespace-spaced Helm payload still classifies the tenant" { + # The chart-name read is FATAL on a miss, so any payload shape a re-serializer + # can produce must parse. Byte-compare against the same golden as the compact + # payload: the shape must make no difference to the report at all. + d=$(mktemp -d) + _write_fake_kubectl "$d" "" "" "$(_release_blob SPACED)" "persistentvolumeclaim/data1-seaweedfs-volume-0" + rc=0 + out=$(PATH="$d:$PATH" SEAWEEDFS_AUDIT_LIB=0 sh "$PWD/hack/seaweedfs-naming-audit.sh" tenant-test 2>&1) || rc=$? + printf '%s\n' "$out" > "$d/got" + _expected_L > "$d/want" + echo "rc=$rc"; echo "--- got ---"; cat "$d/got" + [ "$rc" -eq 0 ] + diff "$d/want" "$d/got" + rm -rf "$d" +} + +@test "a pretty-printed multi-line Helm payload still classifies the tenant" { + # Line-based sed/grep cannot see across newlines at all, so this shape is the + # one that fails hardest without the newline fold -- and jq/yq re-serialization + # is exactly how a payload would arrive indented. + d=$(mktemp -d) + _write_fake_kubectl "$d" "" "" "$(_release_blob PRETTY)" "persistentvolumeclaim/data1-seaweedfs-volume-0" + rc=0 + out=$(PATH="$d:$PATH" SEAWEEDFS_AUDIT_LIB=0 sh "$PWD/hack/seaweedfs-naming-audit.sh" tenant-test 2>&1) || rc=$? + printf '%s\n' "$out" > "$d/got" + _expected_L > "$d/want" + echo "rc=$rc"; echo "--- got ---"; cat "$d/got" + [ "$rc" -eq 0 ] + diff "$d/want" "$d/got" + rm -rf "$d" +} + +@test "a values subtree spelling chart.metadata.name cannot shadow the real chart" { + # Helm marshals "config" (the user's values) AFTER "chart", so a LAST-match + # extraction returns the decoy, the release reads as non-SeaweedFS, and the + # tenant vanishes from the report with exit 0 -- a false clean, the failure this + # whole script exists to prevent. First-match extraction is what stops it. + d=$(mktemp -d) + _write_fake_kubectl "$d" "" "" "$(_release_blob DECOY)" "persistentvolumeclaim/data1-seaweedfs-volume-0" + rc=0 + out=$(PATH="$d:$PATH" SEAWEEDFS_AUDIT_LIB=0 sh "$PWD/hack/seaweedfs-naming-audit.sh" tenant-test 2>&1) || rc=$? + printf '%s\n' "$out" > "$d/got" + _expected_L > "$d/want" + echo "rc=$rc"; echo "--- got ---"; cat "$d/got" + [ "$rc" -eq 0 ] + # The decoy name must never reach the report. + [ "$(printf '%s\n' "$out" | grep -c 'decoy-not-seaweedfs')" -eq 0 ] + diff "$d/want" "$d/got" + rm -rf "$d" +} + +@test "fails closed when the by-name PVC GET hits a real error" { + # pv_epoch resolves each claim's bound PV by name before reading the PV's age. + # A real error there (RBAC/timeout) must abort: silently dropping the claim + # shrinks the observed vintage range, which is what produces a wrong "candidate + # duplicate" verdict. The PV half of this pair was already covered; this is the + # PVC half, whose fake FAIL mode existed with no test behind it. + d=$(mktemp -d) + _write_fake_kubectl "$d" pvcget "" "$(_release_blob)" "$(_pvcs_mixed)" + rc=0 + out=$(PATH="$d:$PATH" SEAWEEDFS_AUDIT_LIB=0 sh "$PWD/hack/seaweedfs-naming-audit.sh" tenant-test 2>&1) || rc=$? + rm -rf "$d" + echo "rc=$rc"; echo "$out" + [ "$rc" -ne 0 ] + printf '%s\n' "$out" | grep -q 'FATAL' + printf '%s\n' "$out" | grep -q "get PVC '" +} diff --git a/hack/seaweedfs-naming-audit.sh b/hack/seaweedfs-naming-audit.sh index a1e28c4c41..b66e91939e 100755 --- a/hack/seaweedfs-naming-audit.sh +++ b/hack/seaweedfs-naming-audit.sh @@ -44,7 +44,15 @@ # # Usage: hack/seaweedfs-naming-audit.sh [namespace...] # KUBECONFIG= hack/seaweedfs-naming-audit.sh -# Exit: 0 always (audit only). Nothing is mutated. +# Exit: 0 the audit RAN TO COMPLETION; the table is the whole answer +# (an empty table then means a genuinely clean fleet). +# non-zero a kubectl query FAILED, so the audit is INCOMPLETE and the +# table must NOT be trusted -- a message naming the failed query +# is on stderr. This is fail-CLOSED by design (see run_kubectl): +# the output gates a destructive runbook step, so an unreachable +# API aborts loudly rather than printing an empty table that +# reads identically to "nothing to do". +# Nothing is ever mutated on either path -- every kubectl call is read-only. # # POSIX sh, deliberately. hack/seaweedfs-naming-audit.bats sources this file, and # cozytest.sh sources the converted test into its own /bin/sh — which is dash on @@ -55,6 +63,48 @@ # particular is not POSIX and dash 0.5.12 (Ubuntu) rejects it outright. set -u +# run_kubectl -- run ONE read-only kubectl query +# fail-CLOSED and print its stdout. This audit is the documented gate in front of +# a destructive runbook step (deleting a tenant's PVCs), so a query that FAILS +# must never be mistaken for a query that found nothing: an enumerating LIST +# returns empty stdout on a timeout or an RBAC denial, byte-identical to the same +# LIST succeeding on a genuinely clean fleet, and the old `2>/dev/null` on each +# call turned an unreachable API into a silent false "nothing to do". Every +# enumeration is routed through here so any failure is loud and terminal instead. +# +# On success prints kubectl's stdout verbatim -- which may legitimately be empty, +# because a LIST that matched nothing is honestly clean; keeping that case +# distinct from failure is the whole point. On a non-zero exit prints the failed +# query to stderr and returns that exit code; kubectl's own stderr stays on fd 2 +# and reaches the operator unfiltered. Callers MUST propagate the non-zero status +# (`|| return`/`|| exit`) -- a bare `for x in $(run_kubectl ...)` would swallow +# it, because a for-loop ignores the exit status of its word-list command, and an +# `exit` inside the `$(...)` subshell would only leave that subshell. +run_kubectl() { + _rk_what="$1"; shift + if _rk_out=$(kubectl "$@"); then + printf '%s' "$_rk_out" + return 0 + else + _rk_rc=$? + printf 'seaweedfs-naming-audit: FATAL: %s failed (kubectl %s -> exit %s). Refusing to report a clean fleet on an unreachable API.\n' \ + "$_rk_what" "$*" "$_rk_rc" >&2 + return "$_rk_rc" + fi +} + +# audit_fatal -- the counterpart to run_kubectl for corruption the API +# call itself did NOT report: the query succeeded, but what it returned is +# impossible for a healthy object (a helm.sh/release.v1 Secret with no decodable +# release payload). Same fail-CLOSED contract -- print a FATAL line naming the +# object to stderr and return non-zero; callers MUST propagate it. A silent skip +# here would be the identical false-clean this whole script guards against, just +# one field deeper than an unreachable API. +audit_fatal() { + printf 'seaweedfs-naming-audit: FATAL: %s Refusing to report a clean fleet on corrupt state.\n' "$1" >&2 + return 1 +} + # renamed_volume_prefix -- reconstruct the name 4.31 gives the volume # component of . Mirrors seaweedfs.fullname + seaweedfs.componentName # (see packages/system/seaweedfs/templates/_naming.tpl, which the chart's guard @@ -71,18 +121,51 @@ renamed_volume_prefix() { printf '%s-volume' "$(printf '%s' "$full" | cut -c1-56 | sed 's/-$//')" } -# release_json [rev] -- decode a Helm release secret. -# Helm stores base64(gzip(json)) in Secret.data.release, and Kubernetes base64s -# the data value again, hence the two decodes. +# release_json [rev] -- decode a Helm release secret's payload, or +# "" when that revision's secret does not exist. Helm stores base64(gzip(json)) in +# Secret.data.release, and Kubernetes base64s the data value again, hence the two +# decodes. +# +# This is a by-NAME GET, but it is NOT best-effort: system_releases decides whether +# a release IS SeaweedFS from the result, so a swallowed failure silently drops a +# real tenant -- the false-clean this script exists to prevent. EXISTENCE and +# EXTRACTION are therefore two separate questions, because `--ignore-not-found +# -o jsonpath` cannot tell them apart (it returns empty + exit 0 both for an absent +# Secret AND for a present Secret whose .data.release is missing): +# +# 1. Existence, via `--ignore-not-found -o name`. Empty + exit 0 = the revision +# was pruned by Helm's history limit -- a legitimate absence; return "" and +# let rev1_scheme / first_deployed treat it as "pruned". A real error (RBAC, +# timeout, apiserver down) stays non-zero and run_kubectl makes it fatal. +# 2. The Secret EXISTS, so its payload MUST decode. A helm.sh/release.v1 Secret +# with no .data.release, a payload that is not base64(base64(gzip(...))), or an +# empty JSON after decoding is CORRUPT state, not an absence -- and here that +# is safety-critical, so every such anomaly is audit_fatal, never a silent skip +# (contrast pv_epoch, where an empty field is a benign "no evidence"). +# +# The extraction GET drops --ignore-not-found deliberately: the Secret existed a +# moment ago, so a NotFound now is a mid-audit deletion race, and failing closed on +# it is correct. release_json() { - kubectl get secret -n "$1" "sh.helm.release.v1.$2.v${3:-1}" -o jsonpath='{.data.release}' 2>/dev/null \ - | base64 -d 2>/dev/null | base64 -d 2>/dev/null | gunzip 2>/dev/null + _rj_secret="sh.helm.release.v1.$2.v${3:-1}" + _rj_exists=$(run_kubectl "check Helm release secret '$_rj_secret' in namespace '$1'" \ + get secret -n "$1" "$_rj_secret" --ignore-not-found -o name) || return $? + [ -n "$_rj_exists" ] || return 0 + _rj_field=$(run_kubectl "read .data.release of secret '$_rj_secret' in namespace '$1'" \ + get secret -n "$1" "$_rj_secret" -o jsonpath='{.data.release}') || return $? + [ -n "$_rj_field" ] || { audit_fatal "secret '$_rj_secret' in namespace '$1' exists but has no .data.release payload (corrupt Helm release)."; return 1; } + _rj_json=$(printf '%s' "$_rj_field" | base64 -d 2>/dev/null | base64 -d 2>/dev/null | gunzip 2>/dev/null) \ + || { audit_fatal "secret '$_rj_secret' in namespace '$1': .data.release is not decodable base64(base64(gzip(json))) (corrupt Helm release)."; return 1; } + [ -n "$_rj_json" ] || { audit_fatal "secret '$_rj_secret' in namespace '$1': release payload decoded to nothing (corrupt Helm release)."; return 1; } + printf '%s' "$_rj_json" } # revisions -- retained revision numbers, oldest first. revisions() { - kubectl get secret -n "$1" -l "name=$2,owner=helm" \ - -o jsonpath='{range .items[*]}{.metadata.labels.version}{"\n"}{end}' 2>/dev/null | sort -n + _rev_out=$(run_kubectl "list Helm revision secrets for '$2' in namespace '$1'" \ + get secret -n "$1" -l "name=$2,owner=helm" \ + -o jsonpath='{range .items[*]}{.metadata.labels.version}{"\n"}{end}') || return $? + printf '%s\n' "$_rev_out" | sort -n } # system_releases -- the SeaweedFS -system Helm releases in a namespace. @@ -92,12 +175,51 @@ revisions() { # unrelated release in a namespace that happens to run SeaweedFS would be reported # as a SeaweedFS tenant. Confirm the chart. system_releases() { - for rel in $(kubectl get secret -n "$1" \ - -o jsonpath='{range .items[?(@.type=="helm.sh/release.v1")]}{.metadata.labels.name}{"\n"}{end}' 2>/dev/null \ - | grep -E -- '-system$' | sort -u); do - for rev in $(revisions "$1" "$rel"); do - chart=$(release_json "$1" "$rel" "$rev" | sed -n 's/.*"name":"\(cozy-seaweedfs\)".*/\1/p' | head -1) - if [ -n "$chart" ]; then printf '%s\n' "$rel"; fi + _sr_secrets=$(run_kubectl "list Helm release secrets in namespace '$1'" \ + get secret -n "$1" \ + -o jsonpath='{range .items[?(@.type=="helm.sh/release.v1")]}{.metadata.labels.name}{"\n"}{end}') || return $? + for rel in $(printf '%s\n' "$_sr_secrets" | grep -E -- '-system$' | sort -u); do + _sr_revs=$(revisions "$1" "$rel") || return $? + for rev in $_sr_revs; do + _sr_json=$(release_json "$1" "$rel" "$rev") || return $? + # Empty "" here means the revision secret was pruned (release_json's absence + # path) -- a legitimate skip. A NON-empty payload, however, was fetched and + # decoded successfully, so it MUST name its chart: a helm.sh/release.v1 release + # always carries chart.metadata.name. A payload without one ({} , or any valid + # JSON lacking it) is corrupt/unexpected, and since "no chart name" is + # indistinguishable downstream from "not SeaweedFS", silently skipping it is + # the very false-clean this guard exists to stop -- so it is FATAL. A present + # but different chart name is a real, non-SeaweedFS release and stays a + # legitimate skip, filtered exactly as before by the cozy-seaweedfs test. + # + # Two properties make this match trustworthy, and both are load-bearing now + # that a miss is FATAL: + # + # * FIRST match, not last. Helm's Release marshals "chart" before + # "config" (the user's values), so a values subtree that happens to + # spell chart.metadata.name sits LATER in the payload -- and a greedy + # `sed 's/.*"chart"...'` would return that decoy instead of the real + # chart. A wrong name reads downstream as "not SeaweedFS" and drops a + # real release from the report: the exact silent false-clean this guard + # exists to stop. `grep -o | head -1` takes the leftmost match. + # * The chart -> metadata -> name key path stays ADJACENT. A looser "any + # 'name' after 'metadata'" matches chart.templates[].name, which Helm + # serializes immediately after metadata on EVERY healthy release, so it + # would return a template path for every tenant. + # + # Newlines are folded first so a pretty-printed payload parses at all (sed + # and grep are line-based), and whitespace around the punctuation is + # tolerated. Failing loudly on a payload this cannot read is the safe + # direction -- over-strictness stops the runbook, over-looseness lets it + # delete data -- so the message says what shape was expected. + if [ -n "$_sr_json" ]; then + _sr_chart=$(printf '%s' "$_sr_json" | tr '\n' ' ' \ + | grep -o '"chart"[[:space:]]*:[[:space:]]*{[[:space:]]*"metadata"[[:space:]]*:[[:space:]]*{[[:space:]]*"name"[[:space:]]*:[[:space:]]*"[^"]*"' \ + | head -1 \ + | sed -n 's/.*"\([^"]*\)"$/\1/p') + [ -n "$_sr_chart" ] || { audit_fatal "could not read the chart name of secret 'sh.helm.release.v1.$rel.v$rev' in namespace '$1' at .chart.metadata.name (expected 'name' as metadata's first key; a corrupt Helm release, or a payload format this parser does not handle)."; return 1; } + if [ "$_sr_chart" = cozy-seaweedfs ]; then printf '%s\n' "$rel"; fi + fi break done done @@ -106,28 +228,61 @@ system_releases() { # first_deployed -- epoch seconds of the release's first install, # from any retained revision (the field is identical on all of them). first_deployed() { - for rev in $(revisions "$1" "$2"); do - ts=$(release_json "$1" "$2" "$rev" | sed -n 's/.*"first_deployed":"\([^"]*\)".*/\1/p' | head -1) + _fd_revs=$(revisions "$1" "$2") || return $? + for rev in $_fd_revs; do + _fd_json=$(release_json "$1" "$2" "$rev") || return $? + # Same extraction discipline as the chart name above: fold newlines so a + # pretty-printed payload parses at all, tolerate whitespace around the + # punctuation, and take the FIRST match (.info precedes .config, so a values + # subtree cannot shadow the real timestamp). A miss here is not fatal -- the + # caller degrades to the documented safe fallback -- but a WRONG timestamp + # would silently change a vintage verdict, which is worse than none. + ts=$(printf '%s' "$_fd_json" | tr '\n' ' ' \ + | grep -o '"first_deployed"[[:space:]]*:[[:space:]]*"[^"]*"' \ + | head -1 \ + | sed -n 's/.*"\([^"]*\)"$/\1/p') if [ -n "$ts" ]; then date -u -d "$(printf '%s' "$ts" | cut -c1-19)" +%s 2>/dev/null; return; fi done } # rev1_scheme -- "legacy" | "renamed" | "" (revision 1 pruned). rev1_scheme() { - m=$(release_json "$1" "$2" 1) + m=$(release_json "$1" "$2" 1) || return $? [ -n "$m" ] || return 0 if printf '%s' "$m" | grep -q 'name: seaweedfs-master'; then printf 'legacy' elif printf '%s' "$m" | grep -qE "name: $2(-seaweedfs)?-master"; then printf 'renamed' fi } -# pv_epoch -- creation epoch of the PV the claim is BOUND to. +# pv_epoch -- creation epoch of the PV the claim is BOUND to, or "" when +# there is no usable PV age. Two by-NAME GETs. +# +# A real API error must abort (an epoch silently missing from a range can invert +# the strict-newer comparison and name the WRONG deletion candidate), so both GETs +# go through run_kubectl. But UNLIKE release_json, an EMPTY field here is NOT +# corruption and is NOT fatal -- because the consequence differs. When release_json +# returns empty for a present Secret the tenant is dropped from the report: a +# false-clean. When pv_epoch returns "" the claim merely contributes no epoch, +# which marks its whole generation INCOMPLETE (see audit_ns) and forces the safe +# "direction cannot be established -> classify by hand" branch. That degradation is +# conservative by construction: it never yields a false-clean and never names a +# candidate. So the two empty-field cases are deliberately kept benign: +# * PVC .spec.volumeName empty -- a Pending / unbound claim, a routine state; +# * PV .metadata.creationTimestamp empty -- shouldn't happen (the apiserver always +# stamps it), but if it ever did the only effect is one missing epoch, i.e. the +# same safe incomplete fallback, so a hard stop is not warranted here. +# `--ignore-not-found` keeps a genuinely-absent PVC/PV (NotFound) on that same +# benign path rather than turning it into a run_kubectl failure, and the final +# `|| return 0` keeps an unparseable timestamp there too; only a real run_kubectl +# error propagates. pv_epoch() { - pv=$(kubectl get pvc -n "$1" "$2" -o jsonpath='{.spec.volumeName}' 2>/dev/null) + pv=$(run_kubectl "get PVC '$2' in namespace '$1'" \ + get pvc -n "$1" "$2" --ignore-not-found -o jsonpath='{.spec.volumeName}') || return $? [ -n "$pv" ] || return 0 - t=$(kubectl get pv "$pv" -o jsonpath='{.metadata.creationTimestamp}' 2>/dev/null) + t=$(run_kubectl "get PV '$pv' (bound by PVC '$2' in namespace '$1')" \ + get pv "$pv" --ignore-not-found -o jsonpath='{.metadata.creationTimestamp}') || return $? [ -n "$t" ] || return 0 - date -u -d "$(printf '%s' "$t" | cut -c1-19)" +%s 2>/dev/null + date -u -d "$(printf '%s' "$t" | cut -c1-19)" +%s 2>/dev/null || return 0 } # classify_mixed_direction -- direction of a MIXED @@ -153,17 +308,21 @@ classify_mixed_direction() { audit_ns() { ns="$1" - for rel in $(system_releases "$ns"); do + _rels=$(system_releases "$ns") || return $? + for rel in $_rels; do prefix=$(renamed_volume_prefix "$rel") legacy_pvcs=""; renamed_pvcs="" - for pvc in $(kubectl get pvc -n "$ns" -o name 2>/dev/null | sed 's|persistentvolumeclaim/||'); do + _pvcs=$(run_kubectl "list PVCs in namespace '$ns'" get pvc -n "$ns" -o name) || return $? + for pvc in $(printf '%s\n' "$_pvcs" | sed 's|persistentvolumeclaim/||'); do case "$pvc" in "data1-${prefix}"*) renamed_pvcs="$renamed_pvcs $pvc" ;; data1-seaweedfs-volume*) legacy_pvcs="$legacy_pvcs $pvc" ;; esac done legacy_sts=""; renamed_sts="" - for sts in $(kubectl get sts -n "$ns" -l app.kubernetes.io/name=seaweedfs -o name 2>/dev/null | sed 's|statefulset.apps/||'); do + _sts_list=$(run_kubectl "list SeaweedFS StatefulSets in namespace '$ns'" \ + get sts -n "$ns" -l app.kubernetes.io/name=seaweedfs -o name) || return $? + for sts in $(printf '%s\n' "$_sts_list" | sed 's|statefulset.apps/||'); do case "$sts" in "${prefix}"*) renamed_sts="$renamed_sts $sts" ;; seaweedfs-volume*) legacy_sts="$legacy_sts $sts" ;; @@ -184,21 +343,30 @@ audit_ns() { fi # Both generations. Report which is ORIGINAL from durable evidence. - fd=$(first_deployed "$ns" "$rel") - scheme=$(rev1_scheme "$ns" "$rel") + fd=$(first_deployed "$ns" "$rel") || return $? + scheme=$(rev1_scheme "$ns" "$rel") || return $? printf '%-24s %-14s %-8s %s\n' "$ns" "$rel" "MIXED" "both generations; the chart REFUSES until one is removed" if [ -n "$scheme" ]; then printf '%-24s %-14s %-8s revision 1 was installed with the %s names => the %s generation is ORIGINAL\n' \ "" "" "" "$scheme" "$scheme" fi - lmin=""; lmax=""; rmin=""; rmax="" + # The direction rule is "EVERY PV of one generation strictly newer than every + # PV of the other". That holds only if we saw EVERY bound PV: a claim whose PV + # age we could not read (unbound, or PV gone) leaves the range unbounded on one + # side, and a silently-narrowed range can flip OVERLAP into a confident (wrong) + # candidate. So a generation with any unreadable claim is INCOMPLETE + # and forces the safe "cannot establish" branch -- distinct from a real API + # error, which pv_epoch has already turned into a hard failure above. + lmin=""; lmax=""; rmin=""; rmax=""; l_incomplete=0; r_incomplete=0 for p in $legacy_pvcs; do - e=$(pv_epoch "$ns" "$p"); [ -n "$e" ] || continue + e=$(pv_epoch "$ns" "$p") || return $? + if [ -z "$e" ]; then l_incomplete=1; continue; fi { [ -z "$lmin" ] || [ "$e" -lt "$lmin" ]; } && lmin=$e { [ -z "$lmax" ] || [ "$e" -gt "$lmax" ]; } && lmax=$e done for p in $renamed_pvcs; do - e=$(pv_epoch "$ns" "$p"); [ -n "$e" ] || continue + e=$(pv_epoch "$ns" "$p") || return $? + if [ -z "$e" ]; then r_incomplete=1; continue; fi { [ -z "$rmin" ] || [ "$e" -lt "$rmin" ]; } && rmin=$e { [ -z "$rmax" ] || [ "$e" -gt "$rmax" ]; } && rmax=$e done @@ -206,7 +374,7 @@ audit_ns() { printf '%-24s %-14s %-8s oldest PV vs first_deployed: chart-named +%ss, release-named +%ss (context only, not the rule)\n' \ "" "" "" "$((lmin - fd))" "$((rmin - fd))" fi - if [ -n "$lmin" ] && [ -n "$rmin" ]; then + if [ -n "$lmin" ] && [ -n "$rmin" ] && [ "$l_incomplete" = 0 ] && [ "$r_incomplete" = 0 ]; then case $(classify_mixed_direction "$lmin" "$lmax" "$rmin" "$rmax") in legacy-original) printf '%-24s %-14s %-8s every release-named PV is strictly newer => chart-named is ORIGINAL, the release-named set is the candidate duplicate (Step 3)\n' "" "" "" ;; @@ -216,7 +384,7 @@ audit_ns() { printf '%-24s %-14s %-8s PV vintages OVERLAP => no candidate. An interrupted Step 2 re-bind looks exactly like this (both generations on original PVs). Finish Step 2 if one is in progress; otherwise escalate. Do NOT run Step 2a.\n' "" "" "" ;; esac else - printf '%-24s %-14s %-8s a generation has no bound PVs (Pending claims, or StatefulSets only) => direction cannot be established from PV ages. Resolve the Pending claims or classify by hand.\n' "" "" "" + printf '%-24s %-14s %-8s direction cannot be established from PV ages: a generation has no bound PVs (Pending/unbound claims, or StatefulSets only) or a bound PV age could not be read. Resolve those claims or classify by hand. Do NOT run Step 2a.\n' "" "" "" fi printf '%-24s %-14s %-8s CANDIDATE ONLY: "original" does not mean the other set is EMPTY. A duplicate that\n' "" "" "" printf '%-24s %-14s %-8s served writes and later crashed looks identical here. Verify emptiness before deleting.\n' "" "" "" @@ -227,9 +395,10 @@ main() { printf '%-24s %-14s %-8s %s\n' NAMESPACE RELEASE CLASS NOTE printf '%-24s %-14s %-8s %s\n' --------- ------- ----- ---- if [ "$#" -gt 0 ]; then - for ns in "$@"; do audit_ns "$ns"; done + for ns in "$@"; do audit_ns "$ns" || exit $?; done else - for ns in $(kubectl get ns -o name 2>/dev/null | sed 's|namespace/||'); do audit_ns "$ns"; done + _ns_list=$(run_kubectl 'list namespaces (whole-cluster mode)' get ns -o name) || exit $? + for ns in $(printf '%s\n' "$_ns_list" | sed 's|namespace/||'); do audit_ns "$ns" || exit $?; done fi } diff --git a/hack/validate-changelog.sh b/hack/validate-changelog.sh new file mode 100755 index 0000000000..117391be1c --- /dev/null +++ b/hack/validate-changelog.sh @@ -0,0 +1,62 @@ +#!/bin/sh +# Usage: hack/validate-changelog.sh +# +# Decides whether a changelog file is publishable as a GitHub Release body. +# +# promote-rc.yaml lets the AI generation step fail (a Copilot outage must never +# block a release), which means a run can die mid-stream and leave a fragment +# behind. Without this check that fragment would be committed, advertised as a +# valid changelog in the promote PR body, and published verbatim by finalize as +# the release notes — worse than having no changelog, because nobody is told. +# +# The assertions are deliberately narrow: only invariants every shipped changelog +# in docs/changelogs/ actually satisfies. Two forms exist and both are valid — +# minor releases use `# Cozystack vX.Y.Z`, patch releases use `# vX.Y.Z (date)` — +# so the header check accepts either. There is deliberately NO line-count floor: +# v1.5.1 is a complete, shipped, 19-line patch changelog, and a floor set above +# it would reject good output and mislabel it as a generation failure. +# +# The version is checked into the header, the release-link comment and the +# compare link, so a changelog generated for the wrong version (an rc string, or +# a stale version carried over from a retry) is caught rather than published. +# +# Exit 0 = publishable. Exit 1 = not publishable, with the reason on stderr. + +set -eu + +CL="${1:?usage: validate-changelog.sh }" +VERSION="${2:?usage: validate-changelog.sh }" + +fail() { + echo "$1" >&2 + exit 1 +} + +[ -f "$CL" ] || fail "file absent: $CL" +[ -s "$CL" ] || fail "file empty: $CL" +grep -q '[^[:space:]]' "$CL" || fail "file is only whitespace: $CL" + +# Both header conventions. \b is not portable in POSIX grep -E, so anchor on +# "end of line or a non-version character" instead: that keeps v1.5.1 from +# satisfying a check for v1.5.11. +grep -qE "^# (Cozystack )?v${VERSION}([^0-9.]|\$)" "$CL" \ + || fail "missing an H1 for v${VERSION}: expected '# Cozystack v${VERSION}' (minor) or '# v${VERSION} ()' (patch)" + +# Leading HTML comment pointing at the release. Present in every shipped +# changelog and the first thing a truncated write would still have, so on its +# own it is weak — it is the version check here that earns its keep. +grep -qF "releases/tag/v${VERSION}" "$CL" \ + || fail "missing the release-link comment for v${VERSION}" + +# A truncated stream loses the tail, so assert something that only exists at the +# end. The compare link must also name this version, catching a changelog +# generated against the rc tag or a stale version. +grep -qE "compare/.*\.\.\.v${VERSION}([^0-9.]|\$)" "$CL" \ + || fail "missing the 'Full Changelog' compare link ending in v${VERSION} (likely truncated, or generated for the wrong version)" + +# At least one section. Distinguishes a real changelog from a header plus a +# stub line, without assuming which sections a given release happens to have. +grep -qE '^## ' "$CL" \ + || fail "no '## ' sections — not a complete changelog" + +echo "OK: $CL is publishable as the v${VERSION} release body ($(wc -l < "$CL" | tr -d ' ') lines)" diff --git a/hack/verify-promoted-packages.sh b/hack/verify-promoted-packages.sh new file mode 100755 index 0000000000..26089e4188 --- /dev/null +++ b/hack/verify-promoted-packages.sh @@ -0,0 +1,267 @@ +#!/bin/sh +# Verify that the packages artifact pinned by a stable promotion is exactly the +# packages tree in the merge commit, apart from the artifact's impossible +# self-reference. This runs before finalize creates any stable git/release tag. +# +# Usage: hack/verify-promoted-packages.sh [root] +# X.Y.Z (without a leading v) +# [root] release packages tree; defaults to packages +# +# Environment: +# EXPECTED_PACKAGES_REPOSITORY exact trusted OCI repository; defaults to +# the public Cozystack release repository +# VERIFY_PACKAGES_WORKDIR caller-owned work directory; when set, the +# verifier neither creates nor removes it +set -eu + +STABLE_VERSION="${1:?usage: verify-promoted-packages.sh [root]}" +ROOT="${2:-packages}" + +# EXPECTED_PACKAGES_REPOSITORY and PACKAGES_DIGEST_REF_PATTERN, shared with the +# publisher so its dispatch-time preflight cannot drift from this guard. +# shellcheck source=hack/lib/promoted-packages.sh +. "$(dirname "$0")/lib/promoted-packages.sh" + +printf '%s\n' "$STABLE_VERSION" | grep -Eq '^[0-9]+\.[0-9]+\.[0-9]+$' \ + || { echo "stable-version '$STABLE_VERSION' must match X.Y.Z" >&2; exit 1; } +[ -d "$ROOT" ] || { echo "root '$ROOT' is not a directory" >&2; exit 1; } +VALUES="$ROOT/core/installer/values.yaml" +[ -f "$VALUES" ] || { echo "root '$ROOT' has no core/installer/values.yaml" >&2; exit 1; } + +command -v flux >/dev/null || { echo "flux is required" >&2; exit 1; } +command -v yq >/dev/null || { echo "yq (mikefarah) is required" >&2; exit 1; } +yq --version 2>&1 | grep -q mikefarah \ + || { echo "yq (mikefarah) is required" >&2; exit 1; } + +source_url="$(yq -e -r '.cozystackOperator.platformSourceUrl' "$VALUES")" +source_ref="$(yq -e -r '.cozystackOperator.platformSourceRef' "$VALUES")" +[ "$source_url" = "$EXPECTED_PACKAGES_REPOSITORY" ] \ + || { echo "platformSourceUrl '$source_url' must equal trusted repository '$EXPECTED_PACKAGES_REPOSITORY'" >&2; exit 1; } +printf '%s\n' "$source_ref" | grep -Eq "$PACKAGES_DIGEST_REF_PATTERN" \ + || { echo "platformSourceRef '$source_ref' is not an immutable digest" >&2; exit 1; } + +if [ -n "${VERIFY_PACKAGES_WORKDIR:-}" ]; then + tmp="$VERIFY_PACKAGES_WORKDIR" + [ ! -e "$tmp" ] || { echo "VERIFY_PACKAGES_WORKDIR '$tmp' already exists" >&2; exit 1; } + mkdir -p "$tmp" +else + tmp="$(mktemp -d)" + cleanup() { + case "$tmp" in + "${TMPDIR:-/tmp}"/*) rm -r -- "$tmp" ;; + *) echo "refusing to clean unexpected work directory '$tmp'" >&2 ;; + esac + } + trap cleanup EXIT HUP INT TERM +fi +artifact="$tmp/artifact" +mkdir -p "$artifact" +flux pull artifact "${source_url}@${source_ref#digest=}" --output "$artifact" + +ARTIFACT_VALUES="$artifact/core/installer/values.yaml" +[ -f "$ARTIFACT_VALUES" ] \ + || { echo "promoted artifact has no core/installer/values.yaml" >&2; exit 1; } + +# The candidate was pushed before its own digest was written into installer +# values, so its embedded platformSourceRef is the immutable rc artifact it was +# derived from. Pull that baseline now; comparing its normalized refs below is +# what proves promotion changed tag strings rather than container bytes. +rc_source_url="$(yq -e -r '.cozystackOperator.platformSourceUrl' "$ARTIFACT_VALUES")" +rc_source_ref="$(yq -e -r '.cozystackOperator.platformSourceRef' "$ARTIFACT_VALUES")" +[ "$rc_source_url" = "$EXPECTED_PACKAGES_REPOSITORY" ] \ + || { echo "candidate's original platformSourceUrl '$rc_source_url' must equal trusted repository '$EXPECTED_PACKAGES_REPOSITORY'" >&2; exit 1; } +printf '%s\n' "$rc_source_ref" | grep -Eq "$PACKAGES_DIGEST_REF_PATTERN" \ + || { echo "candidate's original platformSourceRef '$rc_source_ref' is not an immutable digest" >&2; exit 1; } + +# Promotion must have rewritten every version-line image tag in the artifact, +# not merely in git. Keep the match scoped to image-reference positions so an +# unrelated dependency version or historical prose cannot abort a release. +stable_esc="$(printf '%s' "$STABLE_VERSION" | sed 's/\./\\./g')" +scan_err="$tmp/rc-scan-err" +leftovers="$(find "$artifact" -type f ! -path '*/charts/*' ! -name '*.md' \ + -exec grep -lE -- \ + "(image|repository|tag)[\"']?:[^#]*${stable_esc}-rc\.[0-9]+|${stable_esc}-rc\.[0-9]+@sha256:" \ + {} + 2>"$scan_err" || true)" +# An unreadable file is NOT "a file with no match" — the silent-skip shape +# hack/promote-rewrite-tags.sh refuses by name, and the `|| true` here has to +# stay because `-exec … +` reports the legitimate "nothing matched" as failure +# too. That leaves grep's own diagnostics as the only signal that a file went +# unscanned, so treat any of them as a failed scan. Defence in depth: the +# per-file `cmp` below would catch a leftover rc string anyway, since the +# release tree has none. Cheap enough to not depend on that. +if [ -s "$scan_err" ]; then + echo "::error::could not scan the promoted artifact for rc image references:" >&2 + cat "$scan_err" >&2 + exit 1 +fi +if [ -n "$leftovers" ]; then + echo "::error::promoted packages artifact still carries rc image references:" >&2 + printf '%s\n' "$leftovers" >&2 + exit 1 +fi + +# Flux's default archive excludes these file classes and omits symlinks rather +# than following or storing them. Match that established RC artifact view here; +# changing it would be a separate packages-format migration. .build-revision is +# an ignored build nonce generated only by the rc image-packages target; it is +# not consumed at runtime. The installer values are compared separately below +# after normalizing the one impossible self-reference. +# +# This list is a RESTATEMENT of flux's built-in `excludeOCI` set, not a +# derivation of it — the CLI does not expose it — so it is coupled to a flux +# version. That version is pinned as FLUX_VERSION (2.8.6) in all three workflow +# steps that install the toolchain, and the promote-packages contract test pins +# that they agree. If flux ever changes the set, the symptom is a "contain +# different files" failure naming the newly included or excluded class, which +# is loud and lands before any stable name exists; the fix is to update this +# list and FLUX_VERSION together, in that order. +artifact_entries() { + ( + cd "$1" + find . -type f \ + ! -name '.gitignore' \ + ! -name '.gitmodules' \ + ! -name '.gitattributes' \ + ! -name '*.jpg' \ + ! -name '*.jpeg' \ + ! -name '*.gif' \ + ! -name '*.png' \ + ! -name '*.wmv' \ + ! -name '*.flv' \ + ! -name '*.tar.gz' \ + ! -name '*.zip' \ + ! -name '.build-revision' \ + ! -path './core/installer/values.yaml' \ + | LC_ALL=C sort + ) +} + +artifact_entries "$artifact" > "$tmp/artifact-files" +artifact_entries "$ROOT" > "$tmp/release-files" +if ! cmp -s "$tmp/artifact-files" "$tmp/release-files"; then + echo "::error::promoted artifact and release tree contain different files:" >&2 + diff -u "$tmp/artifact-files" "$tmp/release-files" >&2 || true + exit 1 +fi + +while IFS= read -r rel; do + rel="${rel#./}" + artifact_file="$artifact/$rel" + release_file="$ROOT/$rel" + if ! cmp -s "$artifact_file" "$release_file"; then + echo "::error::promoted artifact file differs from release tree: $rel" >&2 + exit 1 + fi + # Resolve each side to a value first. Written as one `&&`/`||` chain this + # reads like a symmetric difference and is not one: POSIX gives the two + # operators equal precedence and left associativity, so `A && B || C && D` + # groups as `((A && B) || C) && D` and the candidate-executable direction — + # the one that makes content executable inside the artifact the operator + # installs — is accepted silently. + if [ -x "$artifact_file" ]; then artifact_exec=1; else artifact_exec=0; fi + if [ -x "$release_file" ]; then release_exec=1; else release_exec=0; fi + if [ "$artifact_exec" -ne "$release_exec" ]; then + echo "::error::promoted artifact executable bit differs from release tree: $rel" >&2 + exit 1 + fi +done < "$tmp/artifact-files" + +normalize_installer() { + sed -E 's|^([[:space:]]*platformSourceRef:[[:space:]]*).*$|\1digest=sha256:SELF|' "$1" +} +normalize_installer "$ARTIFACT_VALUES" > "$tmp/artifact-installer" +normalize_installer "$VALUES" > "$tmp/release-installer" +if ! cmp -s "$tmp/artifact-installer" "$tmp/release-installer"; then + echo "::error::promoted artifact installer values differ beyond their self-reference" >&2 + diff -u "$tmp/artifact-installer" "$tmp/release-installer" >&2 || true + exit 1 +fi + +# Compare normalized repo@digest sets against the rc artifact as an explicit +# proof that promotion did not change any container bytes. The packages +# artifact's own digest is omitted because that is the intentional declarative +# content change this verification is proving. +# shellcheck source=hack/lib/image-refs.sh +. "$(dirname "$0")/lib/image-refs.sh" +normalized_refs() { + # Collect into a file rather than piping straight into the loop. As the head + # of a pipeline the collector's exit status is thrown away — the pipeline + # reports `sort -u`'s, POSIX sh has no `pipefail` and this script has to stay + # POSIX — so `set -e` never sees it and a failed or truncated collection + # becomes a smaller set that the comparison below happily matches. Same + # fail-open class as the rc-reference scan above, on the one guard that is + # supposed to prove the container bytes did not move across promotion. + _nr_raw="$tmp/refs-raw" + collect_image_refs "$1" > "$_nr_raw" + # A collection that comes back empty is that same failure wearing a zero exit + # status: two empty sets compare equal, so the proof passes having examined + # nothing at all. Every packages tree carries image references — none means + # the collector understood nothing it was given, not that there was nothing + # to find. + [ -s "$_nr_raw" ] \ + || { echo "::error::collected no image references from '$1'; cannot prove the container digests are unchanged" >&2; exit 1; } + while IFS= read -r raw; do + [ -n "$raw" ] || continue + without_digest="${raw%@*}" + digest="${raw##*@}" + image="${without_digest##*/}" + if [ "$without_digest" = "$image" ]; then + # No `/` anywhere ahead of the digest, so nothing in this ref names a + # registry host or a repository path — it is a bare scalar sitting in + # front of a digest, and the collector emits those by design. Shape 3 + # (`repository:` on one key, `tag: @sha256:` on another) is + # the dominant shape in the tree, and shape 1's recursive descent sees + # that `tag` scalar on its own, so every shape-3 map arrives here twice: + # once correctly joined, once as the bare tag. hack/lib/image-refs.sh + # documents that degeneracy and leaves host-less refs to its callers — + # promote-retag.sh drops them through its ownership filter, and this + # function had no equivalent. `${image%:*}` is then a no-op (a tag holds + # no colon) and the TAG arrives as the repository. Promotion rewrites + # exactly those tags, which is how a promotion that moved no container + # bytes at all reported six changed "repositories" whose digests were + # identical on both sides. + # + # Compare such a ref on its digest alone. The digest is the whole of the + # container identity a host-less ref carries, and it is the only part + # promotion must not move; the empty repository yields a leading `@`, + # which no real repository name can produce, so these cannot collide with + # a host-bearing entry. Where a repository name does exist behind one of + # them — kube-ovn's `global.images.kubeovn`, whose host lives in + # `global.registry.address` — the same image also arrives host-bearing + # from shape 4, so a repository change stays visible through that entry. + # + # Dropping host-less refs outright would be wrong rather than merely + # blunter: packages/system/kuberture/values.yaml carries an `image:` map + # with a `tag:` and no `repository:`, so shape 1 is the only rule that + # ever sees its digest. Skipping it would silently stop proving that + # digest is unchanged — a hole in the very guard this branch repairs. + # + # Do NOT reach for promote-retag.sh's `${REGISTRY}/` ownership filter to + # do this job. The verify job's REGISTRY names the private build registry + # while both artifacts under comparison live on the public one, so that + # filter drops every ref; and the emptiness guard above tests the RAW + # collection rather than the filtered set, so the proof would then pass + # by comparing two empty sets — failing open on the one invariant it + # exists to establish. + repo="" + else + repo="${without_digest%/*}/${image%:*}" + fi + case "$repo" in + */cozystack-packages) continue ;; + esac + printf '%s@%s\n' "$repo" "$digest" + done < "$_nr_raw" | LC_ALL=C sort -u +} +rc_artifact="$tmp/rc-artifact" +mkdir -p "$rc_artifact" +flux pull artifact "${rc_source_url}@${rc_source_ref#digest=}" --output "$rc_artifact" +normalized_refs "$rc_artifact" > "$tmp/rc-refs" +normalized_refs "$artifact" > "$tmp/artifact-refs" +if ! cmp -s "$tmp/rc-refs" "$tmp/artifact-refs"; then + echo "::error::promotion changed the container repository/digest set" >&2 + diff -u "$tmp/rc-refs" "$tmp/artifact-refs" >&2 || true + exit 1 +fi + +echo "Verified ${source_url}@${source_ref#digest=}: stable tags, identical package tree, and the rc artifact's container digests." diff --git a/hack/verify-promoted-packages_test.bats b/hack/verify-promoted-packages_test.bats new file mode 100644 index 0000000000..1f6bad367d --- /dev/null +++ b/hack/verify-promoted-packages_test.bats @@ -0,0 +1,472 @@ +#!/usr/bin/env bats +# Behavioural tests for the pre-publication packages artifact verification. +# Run with: hack/cozytest.sh hack/verify-promoted-packages_test.bats + +_test_workspace() { + if [ -n "${BATS_TEST_TMPDIR:-}" ]; then + printf '%s\n' "$BATS_TEST_TMPDIR" + else + printf '%s\n' "${tmp:?cozytest workspace is missing}" + fi +} + +_make_verify_fixture() { + t="$1" + mkdir -p "$t/bin" "$t/release/core/installer" "$t/release/system/dashboard" + mkdir -p "$t/artifact/core/installer" "$t/artifact/system/dashboard" + mkdir -p "$t/rc-artifact/core/installer" "$t/rc-artifact/system/dashboard" + cat > "$t/bin/flux" <<'EOF' +#!/bin/sh +set -eu +[ "$1" = "pull" ] +[ "$2" = "artifact" ] +shift 2 +ref="$1" +printf '%s\n' "$@" >> "$MOCK_FLUX_LOG" +out="" +while [ "$#" -gt 0 ]; do + if [ "$1" = "--output" ]; then + out="$2" + shift 2 + else + shift + fi +done +[ -n "$out" ] +case "$ref" in + *@sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb) + src="$MOCK_ARTIFACT" + ;; + *@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa) + src="$MOCK_RC_ARTIFACT" + ;; + *) + echo "unexpected artifact ref: $ref" >&2 + exit 1 + ;; +esac +cp -R "$src"/. "$out"/ +EOF + chmod +x "$t/bin/flux" + cat > "$t/release/core/installer/values.yaml" <<'EOF' +cozystackOperator: + platformSourceUrl: oci://ghcr.io/cozystack/cozystack/cozystack-packages + platformSourceRef: digest=sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb +EOF + cat > "$t/artifact/core/installer/values.yaml" <<'EOF' +cozystackOperator: + platformSourceUrl: oci://ghcr.io/cozystack/cozystack/cozystack-packages + platformSourceRef: digest=sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +EOF + image='ghcr.io/cozystack/cozystack/cozystack-ui:v9.9.9@sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc' + printf 'console:\n image: %s\n' "$image" > "$t/release/system/dashboard/values.yaml" + printf 'console:\n image: %s\n' "$image" > "$t/artifact/system/dashboard/values.yaml" + cp "$t/artifact/core/installer/values.yaml" "$t/rc-artifact/core/installer/values.yaml" + printf 'console:\n image: %s\n' \ + 'ghcr.io/cozystack/cozystack/cozystack-ui:v9.9.9-rc.3@sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc' \ + > "$t/rc-artifact/system/dashboard/values.yaml" + # Flux's established packages archive omits symlinks. The verifier must + # compare against that archive view rather than demanding this node back. + ln -s values.yaml "$t/release/system/dashboard/values-link.yaml" +} + +@test "accepts an identical candidate with only the self-reference changed" { + tmp="$(_test_workspace)/fixture" + _make_verify_fixture "$tmp" + + rc=0 + MOCK_ARTIFACT="$tmp/artifact" MOCK_RC_ARTIFACT="$tmp/rc-artifact" MOCK_FLUX_LOG="$tmp/flux.log" \ + VERIFY_PACKAGES_WORKDIR="$tmp/work" \ + PATH="$tmp/bin:$PATH" \ + hack/verify-promoted-packages.sh 9.9.9 "$tmp/release" \ + > "$tmp/out" 2> "$tmp/err" || rc=$? + if [ "$rc" -ne 0 ]; then + echo "verification exited $rc" >&2 + cat "$tmp/err" >&2 + return "$rc" + fi + + grep -qx 'oci://ghcr.io/cozystack/cozystack/cozystack-packages@sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb' "$tmp/flux.log" + grep -q "stable tags, identical package tree, and the rc artifact's container digests" "$tmp/out" +} + +@test "rejects an rc image reference inside the candidate" { + tmp="$(_test_workspace)/fixture" + _make_verify_fixture "$tmp" + sed 's/:v9.9.9@/:v9.9.9-rc.3@/' "$tmp/artifact/system/dashboard/values.yaml" \ + > "$tmp/rc-values.yaml" + mv "$tmp/rc-values.yaml" "$tmp/artifact/system/dashboard/values.yaml" + + rc=0 + MOCK_ARTIFACT="$tmp/artifact" MOCK_RC_ARTIFACT="$tmp/rc-artifact" MOCK_FLUX_LOG="$tmp/flux.log" \ + VERIFY_PACKAGES_WORKDIR="$tmp/work" \ + PATH="$tmp/bin:$PATH" \ + hack/verify-promoted-packages.sh 9.9.9 "$tmp/release" \ + > "$tmp/out" 2> "$tmp/err" || rc=$? + + [ "$rc" -ne 0 ] + grep -q 'still carries rc image references' "$tmp/err" +} + +# The scan's `|| true` is not optional — `find … -exec … +` reports the +# legitimate "nothing matched" as a failure too — which leaves grep's own +# diagnostics as the only remaining signal that a file went unscanned. Produce +# that signal the way an unreadable file produces it, without depending on the +# runner's uid: root reads a chmod 000 fixture anyway, so a permissions-based +# test would quietly stop testing anything wherever the suite runs as root. +# A grep on PATH answering the scan's invocation the way the real one answers a +# file it cannot open is the same injection idiom as _tooling_with_collector +# below, one dependency further out. +_grep_failing_the_rc_scan() { + real_grep="$(command -v grep)" + cat > "$1/grep" < . Every +# other grep in the verifier, and in the collector it sources, must behave. +if [ "\$1" = "-lE" ]; then + for _a in "\$@"; do _last="\$_a"; done + echo "grep: \$_last: Permission denied" >&2 + exit 2 +fi +exec $real_grep "\$@" +EOF + chmod +x "$1/grep" +} + +@test "refuses when the rc-reference scan could not read a file" { + tmp="$(_test_workspace)/fixture" + _make_verify_fixture "$tmp" + _grep_failing_the_rc_scan "$tmp/bin" + + rc=0 + MOCK_ARTIFACT="$tmp/artifact" MOCK_RC_ARTIFACT="$tmp/rc-artifact" MOCK_FLUX_LOG="$tmp/flux.log" \ + VERIFY_PACKAGES_WORKDIR="$tmp/work" \ + PATH="$tmp/bin:$PATH" \ + hack/verify-promoted-packages.sh 9.9.9 "$tmp/release" \ + > "$tmp/out" 2> "$tmp/err" || rc=$? + + [ "$rc" -ne 0 ] + grep -q 'could not scan the promoted artifact for rc image references' "$tmp/err" + # The diagnostic itself has to reach the log, or the failure names no file and + # nobody can tell which one went unscanned. + grep -q 'Permission denied' "$tmp/err" + # …and the verifier did not go on to report a proof it skipped part of. + # Counted rather than negated with `!`, which suppresses errexit and would + # pass regardless. + count="$(grep -c 'identical package tree' "$tmp/out" || true)" + [ "${count:-0}" -eq 0 ] +} + +# The threat this whole job exists for: someone edits packages/ on the release +# PR after promotion published the candidate, so the artifact the installer +# resolves is no longer the tree that was reviewed and tagged. A file present on +# BOTH sides with different bytes is the shape that takes, and only the per-file +# comparison catches it — the file-set compare sees the same names, the installer +# compare reads one file, and the rc-refs compare never looks at the release tree +# at all. The edit here is deliberately not an image reference, so no other leg +# can fire and claim the credit. +@test "rejects a file whose content changed on only one side" { + tmp="$(_test_workspace)/fixture" + _make_verify_fixture "$tmp" + printf ' replicas: 3\n' >> "$tmp/release/system/dashboard/values.yaml" + + rc=0 + MOCK_ARTIFACT="$tmp/artifact" MOCK_RC_ARTIFACT="$tmp/rc-artifact" MOCK_FLUX_LOG="$tmp/flux.log" \ + VERIFY_PACKAGES_WORKDIR="$tmp/work" \ + PATH="$tmp/bin:$PATH" \ + hack/verify-promoted-packages.sh 9.9.9 "$tmp/release" \ + > "$tmp/out" 2> "$tmp/err" || rc=$? + + [ "$rc" -ne 0 ] + grep -q 'promoted artifact file differs from release tree: system/dashboard/values.yaml' "$tmp/err" +} + +@test "rejects package content changed after the candidate was published" { + tmp="$(_test_workspace)/fixture" + _make_verify_fixture "$tmp" + printf 'changed: true\n' > "$tmp/release/system/dashboard/extra.yaml" + + rc=0 + MOCK_ARTIFACT="$tmp/artifact" MOCK_RC_ARTIFACT="$tmp/rc-artifact" MOCK_FLUX_LOG="$tmp/flux.log" \ + VERIFY_PACKAGES_WORKDIR="$tmp/work" \ + PATH="$tmp/bin:$PATH" \ + hack/verify-promoted-packages.sh 9.9.9 "$tmp/release" \ + > "$tmp/out" 2> "$tmp/err" || rc=$? + + [ "$rc" -ne 0 ] + grep -q 'contain different files' "$tmp/err" +} + +@test "rejects a candidate outside the trusted release repository" { + tmp="$(_test_workspace)/fixture" + _make_verify_fixture "$tmp" + sed 's#oci://ghcr.io/cozystack/cozystack/cozystack-packages#oci://example.invalid/cozystack-packages#' \ + "$tmp/release/core/installer/values.yaml" > "$tmp/untrusted-values.yaml" + mv "$tmp/untrusted-values.yaml" "$tmp/release/core/installer/values.yaml" + + rc=0 + MOCK_ARTIFACT="$tmp/artifact" MOCK_RC_ARTIFACT="$tmp/rc-artifact" MOCK_FLUX_LOG="$tmp/flux.log" \ + VERIFY_PACKAGES_WORKDIR="$tmp/work" \ + PATH="$tmp/bin:$PATH" \ + hack/verify-promoted-packages.sh 9.9.9 "$tmp/release" \ + > "$tmp/out" 2> "$tmp/err" || rc=$? + + [ "$rc" -ne 0 ] + grep -q 'must equal trusted repository' "$tmp/err" + [ ! -s "$tmp/flux.log" ] +} + +@test "rejects an embedded rc baseline outside the trusted release repository" { + tmp="$(_test_workspace)/fixture" + _make_verify_fixture "$tmp" + sed 's#oci://ghcr.io/cozystack/cozystack/cozystack-packages#oci://example.invalid/cozystack-packages#' \ + "$tmp/artifact/core/installer/values.yaml" > "$tmp/untrusted-values.yaml" + mv "$tmp/untrusted-values.yaml" "$tmp/artifact/core/installer/values.yaml" + + rc=0 + MOCK_ARTIFACT="$tmp/artifact" MOCK_RC_ARTIFACT="$tmp/rc-artifact" MOCK_FLUX_LOG="$tmp/flux.log" \ + VERIFY_PACKAGES_WORKDIR="$tmp/work" \ + PATH="$tmp/bin:$PATH" \ + hack/verify-promoted-packages.sh 9.9.9 "$tmp/release" \ + > "$tmp/out" 2> "$tmp/err" || rc=$? + + [ "$rc" -ne 0 ] + grep -q "candidate's original platformSourceUrl.*must equal trusted repository" "$tmp/err" +} + +# The mode check has to catch BOTH directions, and only one of them is the +# dangerous one. A candidate whose file is executable where the release tree's +# is not is content the merge reviewer never saw made runnable inside the +# artifact the operator installs — cmp(1) compares bytes and says nothing about +# it. The mirror case is the benign one, and it was the only one an earlier +# `&&`/`||` chain actually rejected: POSIX gives the two operators equal +# precedence and left associativity, so the guard grouped as +# `((A && B) || C) && D` and quietly accepted a candidate-only executable bit. +@test "rejects an executable bit set only in the candidate" { + tmp="$(_test_workspace)/fixture" + _make_verify_fixture "$tmp" + chmod +x "$tmp/artifact/system/dashboard/values.yaml" + + rc=0 + MOCK_ARTIFACT="$tmp/artifact" MOCK_RC_ARTIFACT="$tmp/rc-artifact" MOCK_FLUX_LOG="$tmp/flux.log" \ + VERIFY_PACKAGES_WORKDIR="$tmp/work" \ + PATH="$tmp/bin:$PATH" \ + hack/verify-promoted-packages.sh 9.9.9 "$tmp/release" \ + > "$tmp/out" 2> "$tmp/err" || rc=$? + + [ "$rc" -ne 0 ] + grep -q 'executable bit differs from release tree' "$tmp/err" +} + +@test "rejects an executable bit set only in the release tree" { + tmp="$(_test_workspace)/fixture" + _make_verify_fixture "$tmp" + chmod +x "$tmp/release/system/dashboard/values.yaml" + + rc=0 + MOCK_ARTIFACT="$tmp/artifact" MOCK_RC_ARTIFACT="$tmp/rc-artifact" MOCK_FLUX_LOG="$tmp/flux.log" \ + VERIFY_PACKAGES_WORKDIR="$tmp/work" \ + PATH="$tmp/bin:$PATH" \ + hack/verify-promoted-packages.sh 9.9.9 "$tmp/release" \ + > "$tmp/out" 2> "$tmp/err" || rc=$? + + [ "$rc" -ne 0 ] + grep -q 'executable bit differs from release tree' "$tmp/err" +} + +@test "rejects installer drift beyond the self-reference" { + tmp="$(_test_workspace)/fixture" + _make_verify_fixture "$tmp" + printf ' platformSourceSecret: changed-after-push\n' \ + >> "$tmp/release/core/installer/values.yaml" + + rc=0 + MOCK_ARTIFACT="$tmp/artifact" MOCK_RC_ARTIFACT="$tmp/rc-artifact" MOCK_FLUX_LOG="$tmp/flux.log" \ + VERIFY_PACKAGES_WORKDIR="$tmp/work" \ + PATH="$tmp/bin:$PATH" \ + hack/verify-promoted-packages.sh 9.9.9 "$tmp/release" \ + > "$tmp/out" 2> "$tmp/err" || rc=$? + + [ "$rc" -ne 0 ] + grep -q 'installer values differ beyond their self-reference' "$tmp/err" +} + +# Run the verifier from a copy whose sibling lib/ is ours. That is already how +# the workflows invoke it — `.release-tooling/hack/verify-promoted-packages.sh`, +# with its libraries resolved next to the script — so replacing one library is +# the faithful way to reach a failure inside it rather than a contrived one. +_tooling_with_collector() { + mkdir -p "$1/lib" + cp hack/verify-promoted-packages.sh "$1/" + cp hack/lib/promoted-packages.sh "$1/lib/" + cat > "$1/lib/image-refs.sh" <&2 + return 3' + + rc=0 + MOCK_ARTIFACT="$tmp/artifact" MOCK_RC_ARTIFACT="$tmp/rc-artifact" MOCK_FLUX_LOG="$tmp/flux.log" \ + VERIFY_PACKAGES_WORKDIR="$tmp/work" \ + PATH="$tmp/bin:$PATH" \ + "$tmp/tooling/verify-promoted-packages.sh" 9.9.9 "$tmp/release" \ + > "$tmp/out" 2> "$tmp/err" || rc=$? + + [ "$rc" -ne 0 ] + grep -q 'collector exploded' "$tmp/err" + # …and it did not also claim the proof it never completed. Counted rather + # than negated with `!`, which suppresses errexit and would pass regardless. + count="$(grep -c 'identical package tree' "$tmp/out" || true)" + [ "${count:-0}" -eq 0 ] +} + +@test "refuses when the collector reports no image references at all" { + tmp="$(_test_workspace)/fixture" + _make_verify_fixture "$tmp" + _tooling_with_collector "$tmp/tooling" ' return 0' + + rc=0 + MOCK_ARTIFACT="$tmp/artifact" MOCK_RC_ARTIFACT="$tmp/rc-artifact" MOCK_FLUX_LOG="$tmp/flux.log" \ + VERIFY_PACKAGES_WORKDIR="$tmp/work" \ + PATH="$tmp/bin:$PATH" \ + "$tmp/tooling/verify-promoted-packages.sh" 9.9.9 "$tmp/release" \ + > "$tmp/out" 2> "$tmp/err" || rc=$? + + [ "$rc" -ne 0 ] + grep -q 'collected no image references' "$tmp/err" +} + +@test "rejects a changed container digest even when candidate and merge tree match" { + tmp="$(_test_workspace)/fixture" + _make_verify_fixture "$tmp" + sed 's/cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc/dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd/' \ + "$tmp/artifact/system/dashboard/values.yaml" > "$tmp/changed-values.yaml" + mv "$tmp/changed-values.yaml" "$tmp/artifact/system/dashboard/values.yaml" + cp "$tmp/artifact/system/dashboard/values.yaml" "$tmp/release/system/dashboard/values.yaml" + + rc=0 + MOCK_ARTIFACT="$tmp/artifact" MOCK_RC_ARTIFACT="$tmp/rc-artifact" MOCK_FLUX_LOG="$tmp/flux.log" \ + VERIFY_PACKAGES_WORKDIR="$tmp/work" \ + PATH="$tmp/bin:$PATH" \ + hack/verify-promoted-packages.sh 9.9.9 "$tmp/release" \ + > "$tmp/out" 2> "$tmp/err" || rc=$? + + [ "$rc" -ne 0 ] + grep -q 'promotion changed the container repository/digest set' "$tmp/err" +} + +# Shape 3 — `repository:` on one key, `tag: @sha256:` on another — +# is the shape hack/lib/image-refs.sh itself calls the dominant one, and the one +# no fixture above builds. The collector emits TWO entries for such a map: the +# correctly joined repository@digest, plus shape 1's recursive scrape of the +# bare `tag` scalar, which names no repository at all. Promotion rewrites +# exactly that scalar, so normalization keeping it — treating the tag as the +# repository — makes every clean promotion look as though it moved container +# bytes. The single-string ref in _make_verify_fixture cannot reach that path: +# it carries a registry host, so it normalizes through the other branch and its +# own rc-to-stable rewrite passes either way. That is why the suite looked +# covered while the dominant shape went untested. +_write_split_map() { + cat > "$1" < "$tmp/out" 2> "$tmp/err" || rc=$? + if [ "$rc" -ne 0 ]; then + echo "verification exited $rc" >&2 + cat "$tmp/err" >&2 + return "$rc" + fi + + grep -q "stable tags, identical package tree, and the rc artifact's container digests" "$tmp/out" +} + +# The property a blunter fix would break. packages/system/kuberture/values.yaml +# carries an `image:` map with a `tag:` and no `repository:` — shape 2 wants a +# `digest` key and shape 3 a `repository` key, so shape 1's scrape of the bare +# tag scalar is the ONLY rule that ever sees that image's digest. Discarding +# host-less refs rather than comparing them on their digest would silently stop +# proving it unchanged, which is a hole in the same guard being repaired here. +# Asserted from the rejecting direction, because that is the direction able to +# go red: the digest moves and nothing else does, so the digest-set comparison +# is the only leg that can fire — artifact and release tree stay byte-identical, +# no rc string appears in the candidate, and the installer values match. +_write_repository_less_map() { + cat > "$1" < "$tmp/out" 2> "$tmp/err" || rc=$? + + [ "$rc" -ne 0 ] + grep -q 'promotion changed the container repository/digest set' "$tmp/err" + # The rejection has to come from the digest set and not from a neighbouring leg + # rejecting a tree this test never perturbed, or it would still pass with the + # digest dropped from the comparison entirely. Counted rather than negated with + # `!`, which suppresses errexit and would pass regardless. + for _other in 'contain different files' 'differs from release tree' \ + 'still carries rc image references' 'installer values differ'; do + count="$(grep -c "$_other" "$tmp/err" || true)" + [ "${count:-0}" -eq 0 ] + done +} diff --git a/internal/backupcontroller/cnpgstrategy_controller.go b/internal/backupcontroller/cnpgstrategy_controller.go index f2ee5f5c43..bd5f5981f0 100644 --- a/internal/backupcontroller/cnpgstrategy_controller.go +++ b/internal/backupcontroller/cnpgstrategy_controller.go @@ -436,8 +436,9 @@ func (r *BackupJobReconciler) applyClusterPluginBackup(ctx context.Context, name Controller: &controller, }} objStore.Spec = cnpgtypes.ObjectStoreSpec{ - Configuration: *buildBarmanObjectStore(t.BarmanObjectStore, ""), - RetentionPolicy: t.BarmanObjectStore.RetentionPolicy, + Configuration: *buildBarmanObjectStore(t.BarmanObjectStore, ""), + RetentionPolicy: t.BarmanObjectStore.RetentionPolicy, + InstanceSidecarConfiguration: barmanSidecarConfiguration(), } if err := r.Patch(ctx, objStore, client.Apply, client.FieldOwner(cnpgFieldManager), client.ForceOwnership); err != nil { return "", fmt.Errorf("apply ObjectStore %s/%s: %w", namespace, objStoreName, err) @@ -1391,6 +1392,24 @@ func buildBarmanPlugin(objectStoreName, serverName string) cnpgtypes.PluginConfi } } +// barmanSidecarConfiguration pins the barman-cloud sidecar's boto3 request +// checksum policy to "when_required". Since botocore ~1.36 the default +// (when_supported) attaches a flexible checksum to every PutObject, which +// non-AWS S3-compatible backends (Ceph RGW, and the platform's own default +// SeaweedFS system bucket) reject with "x-amz-content-sha256 must be +// UNSIGNED-PAYLOAD, ...". Compute a checksum only when required; AWS S3 +// accepts that too, so it is a safe default everywhere. This mirrors the same +// env set on the chart-rendered ObjectStores (packages/{apps/postgres, +// system/keycloak}/templates/db.yaml) and the etcd-operator fix (#342). +func barmanSidecarConfiguration() *cnpgtypes.InstanceSidecarConfiguration { + return &cnpgtypes.InstanceSidecarConfiguration{ + Env: []cnpgtypes.EnvVar{{ + Name: "AWS_REQUEST_CHECKSUM_CALCULATION", + Value: "when_required", + }}, + } +} + // buildBarmanObjectStore translates the typed strategy template into the // barman configuration shape shared by the deprecated // spec.backup.barmanObjectStore and the plugin's ObjectStore.spec.configuration. diff --git a/internal/backupcontroller/cnpgstrategy_controller_test.go b/internal/backupcontroller/cnpgstrategy_controller_test.go index 60707473b6..11ac2cf4b0 100644 --- a/internal/backupcontroller/cnpgstrategy_controller_test.go +++ b/internal/backupcontroller/cnpgstrategy_controller_test.go @@ -1464,6 +1464,17 @@ func TestApplyClusterPluginBackup_PatchesExistingCluster(t *testing.T) { if store.Spec.RetentionPolicy != "30d" { t.Errorf("ObjectStore retentionPolicy: got %q", store.Spec.RetentionPolicy) } + // The ObjectStore must pin the barman-cloud sidecar's S3 request-checksum + // policy to when_required, or uploads to non-AWS S3 gateways (Ceph RGW, the + // platform's own SeaweedFS system bucket) fail with an x-amz-content-sha256 + // InvalidArgument. The chart does not render this ObjectStore in the + // platform-managed useSystemBucket=true flow, so this Go path is the only + // place that sets it there. + sc := store.Spec.InstanceSidecarConfiguration + if sc == nil || len(sc.Env) != 1 || + sc.Env[0].Name != "AWS_REQUEST_CHECKSUM_CALCULATION" || sc.Env[0].Value != "when_required" { + t.Errorf("ObjectStore instanceSidecarConfiguration.env: got %+v, want [AWS_REQUEST_CHECKSUM_CALCULATION=when_required]", sc) + } // The ObjectStore must be owner-referenced to the Cluster so Kubernetes GC // removes it when the Cluster is deleted (no orphan in the platform flow, // where the chart does not render this ObjectStore). diff --git a/internal/backupcontroller/cnpgtypes/types.go b/internal/backupcontroller/cnpgtypes/types.go index bf2370ea83..3c03e70b42 100644 --- a/internal/backupcontroller/cnpgtypes/types.go +++ b/internal/backupcontroller/cnpgtypes/types.go @@ -208,6 +208,23 @@ type ObjectStoreSpec struct { // RetentionPolicy is a barman retention expression validated by the plugin // CRD against ^[1-9][0-9]*[dwm]$ (e.g. "30d"). RetentionPolicy string `json:"retentionPolicy,omitempty"` + // InstanceSidecarConfiguration passes settings to the barman-cloud sidecar + // that the plugin injects into the CNPG pods. Only the fields we set are + // modelled (see the upstream ObjectStore CRD for the full type). + InstanceSidecarConfiguration *InstanceSidecarConfiguration `json:"instanceSidecarConfiguration,omitempty"` +} + +// InstanceSidecarConfiguration is a minimal mirror of the barman-cloud +// ObjectStore's spec.instanceSidecarConfiguration (only spec.env). +type InstanceSidecarConfiguration struct { + Env []EnvVar `json:"env,omitempty"` +} + +// EnvVar is a minimal corev1.EnvVar (name/value only) — enough to pass a +// literal environment variable to the barman-cloud sidecar. +type EnvVar struct { + Name string `json:"name"` + Value string `json:"value,omitempty"` } type BackupStatus struct { diff --git a/internal/backupcontroller/cnpgtypes/zz_generated.deepcopy.go b/internal/backupcontroller/cnpgtypes/zz_generated.deepcopy.go index ad6e41864c..f6414fb1b5 100644 --- a/internal/backupcontroller/cnpgtypes/zz_generated.deepcopy.go +++ b/internal/backupcontroller/cnpgtypes/zz_generated.deepcopy.go @@ -239,6 +239,27 @@ func (in *ObjectStoreList) DeepCopyObject() runtime.Object { func (in *ObjectStoreSpec) DeepCopyInto(out *ObjectStoreSpec) { *out = *in in.Configuration.DeepCopyInto(&out.Configuration) + if in.InstanceSidecarConfiguration != nil { + out.InstanceSidecarConfiguration = new(InstanceSidecarConfiguration) + in.InstanceSidecarConfiguration.DeepCopyInto(out.InstanceSidecarConfiguration) + } +} + +func (in *InstanceSidecarConfiguration) DeepCopyInto(out *InstanceSidecarConfiguration) { + *out = *in + if in.Env != nil { + out.Env = make([]EnvVar, len(in.Env)) + copy(out.Env, in.Env) + } +} + +func (in *InstanceSidecarConfiguration) DeepCopy() *InstanceSidecarConfiguration { + if in == nil { + return nil + } + out := new(InstanceSidecarConfiguration) + in.DeepCopyInto(out) + return out } func (in *Backup) DeepCopy() *Backup { diff --git a/internal/backupcontroller/default_objects_gate.go b/internal/backupcontroller/default_objects_gate.go new file mode 100644 index 0000000000..023682036a --- /dev/null +++ b/internal/backupcontroller/default_objects_gate.go @@ -0,0 +1,552 @@ +package backupcontroller + +import ( + "context" + "errors" + "fmt" + "net/http" + "sort" + "time" + + "github.com/go-logr/logr" + "github.com/prometheus/client_golang/prometheus" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/rest" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/apiutil" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/metrics" + + backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1" +) + +// defaultObjectsMissing reports how many of the objects the platform default +// backups depend on are absent from the cluster: the bucket credentials +// Secret the projector reads, the Strategy CRs the BackupClass routes to, +// and the Velero BSL. It is the signal an operator alerts on: a non-zero +// value that does not return to zero means the platform default backups are +// not usable, whatever the HelmRelease's Ready condition says. +// +// It is only written when a check reached a conclusion. An API error mid-check +// leaves the previous value in place rather than flapping the alert, so the +// gauge alone cannot distinguish "healthy" from "not evaluated" — that is what +// defaultObjectsCheckErrors is for, and the runbook pairs the two. +var defaultObjectsMissing = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "cozystack_backup_default_objects_missing", + Help: "Number of objects the platform default backups depend on (bucket credentials Secret, BackupClass strategy CRs, Velero BSL) that do not exist in the cluster.", + }, + []string{"backupclass"}, +) + +// defaultObjectsForceReconciles counts the forced Helm upgrades the gate +// issued to materialise those objects. A counter that keeps climbing means +// the forced render is not producing the objects (e.g. a CRD is missing), +// which is a different failure than the install-time race this gate closes. +// A suspended release is NOT counted here: it is skipped before the patch, +// so a paused release cannot masquerade as a render that keeps failing. +var defaultObjectsForceReconciles = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "cozystack_backup_default_objects_force_reconciles_total", + Help: "Number of forced HelmRelease reconciles issued to create missing platform backup objects.", + }, + []string{"namespace", "name"}, +) + +// defaultObjectsCheckErrors counts the checks that could not reach a +// conclusion (API error reading the source Secret, the BackupClass, or one +// of the routed objects). While this climbs, defaultObjectsMissing is stale: +// alerting on the gauge alone would silently keep reporting the last known +// state, including a 0 that is no longer true. +var defaultObjectsCheckErrors = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "cozystack_backup_default_objects_check_errors_total", + Help: "Number of default backup object checks that failed before reaching a conclusion, leaving cozystack_backup_default_objects_missing stale.", + }, + []string{"backupclass"}, +) + +func init() { + metrics.Registry.MustRegister(defaultObjectsMissing, defaultObjectsForceReconciles, defaultObjectsCheckErrors) +} + +const ( + // strategyAPIGroup is the API group of the strategy.backups.cozystack.io + // CRs a BackupClass routes to. BackupClassStrategy.StrategyRef is a + // TypedLocalObjectReference and carries no version, so the group+kind + // are mapped to a resource through the RESTMapper. + strategyAPIGroup = "strategy.backups.cozystack.io" + + // defaultBackupStorageLocationName is the Velero BSL the cozy-default + // Velero strategies reference via storageLocation. It is rendered by the + // same lookup-gated template as the Strategy CRs, so it belongs to the + // same existence check. + defaultBackupStorageLocationName = "cozy-default" +) + +var backupStorageLocationGVR = schema.GroupVersionResource{ + Group: "velero.io", + Version: "v1", + Resource: "backupstoragelocations", +} + +// DefaultObjectsGate closes a race the chart's templates cannot close by +// themselves. +// +// The default Strategy CRs and the Velero BackupStorageLocation are +// Helm-templated behind a `lookup` of the BucketClaim the SAME chart +// creates, because the S3 bucket name is assigned by the COSI driver +// (`bucket-`) and is therefore unknowable at render time. On a +// fresh install the lookup is empty and those templates render nothing. +// helm-controller does not re-render a release that succeeded and whose +// chart and values did not change (drift detection is off on +// operator-generated HelmReleases), so the skip is PERMANENT: the objects +// are never created, and the only recovery is a forced Helm upgrade +// (reconcile.fluxcd.io/forceAt + requestedAt — a plain reconcile request +// does not re-render). +// +// This runnable performs that recovery automatically. Once the bucket name +// is resolvable — read from the same platform credentials Secret the +// projector already consumes, so no new dependency and no `lookup` — it +// checks that every object the platform BackupClass routes to actually +// exists, and forces one Helm upgrade when any is missing. A forced +// upgrade re-runs the lookups, so the gated templates materialise. +// +// It deliberately does not create the objects itself: their bodies are +// values-driven (endpoint, region, forcePathStyle, the Altinity strategy's +// whole PodTemplateSpec) and Helm remains their single owner. The gate only +// makes sure the render that produces them actually happens. +type DefaultObjectsGate struct { + // Client reads the BackupClass (cached) and the source credentials + // Secret (uncached — the manager disables the Secret cache). + Client client.Client + // Interface is a dynamic client used for the per-object existence + // checks and the HelmRelease annotation patch. Going through the + // dynamic client keeps these reads off the controller-runtime cache, + // so they need only `get` RBAC and start no cluster-wide informers. + dynamic.Interface + meta.RESTMapper + + // Config supplies the source credentials Secret coordinates. Reused + // verbatim from the credentials projector. + Config BackupCredentialsConfig + // BackupClassName is the platform BackupClass whose strategyRefs + // enumerate the objects that must exist. Empty disables the gate. + BackupClassName string + // HelmRelease is the release that renders those objects and is forced + // when they are missing. Empty name disables the gate. + HelmRelease types.NamespacedName + // CredentialsHelmRelease is the platform bucket's -system + // release, which renders the user-credentials Secret named by + // Config.SourceSecretName behind a `lookup` of the COSI Secret — the + // same permanent-skip trap as the Strategy CRs, one release earlier. + // Forcing it is what unblocks everything downstream, since the gate + // itself cannot resolve the bucket name without that Secret. + // + // Empty when the platform bucket is not provisioned by Cozystack + // (backupStorage.provisionBucket=false, external S3): the Secret is + // then admin-managed and no release renders it, so there is nothing + // to force. + CredentialsHelmRelease types.NamespacedName + // VeleroNamespace is where the cozy-default BackupStorageLocation + // lives. Empty skips the BSL check (velero.bslEnabled=false). + VeleroNamespace string + + // Period is the check interval. Defaults to 1 minute. + Period time.Duration + // MinForceInterval throttles forced upgrades so a condition the forced + // render cannot fix (missing CRD, wrong bucket) does not turn into a + // hot loop against helm-controller. Defaults to 5 minutes. + MinForceInterval time.Duration + + // now is a test seam for the throttle clock. + now func() time.Time + // lastForce and lastCredentialsForce throttle the two releases + // independently: the credentials release is forced during the window in + // which the objects release cannot even be evaluated, so sharing one + // timestamp would make the first force delay the second by + // MinForceInterval for no reason. + lastForce time.Time + lastCredentialsForce time.Time +} + +// errHelmReleaseSuspended reports that the target release has +// spec.suspend: true. helm-controller ignores reconcile.fluxcd.io/forceAt on +// a suspended release, so patching it would be a no-op repeated every +// MinForceInterval for as long as the suspension lasts. +var errHelmReleaseSuspended = errors.New("HelmRelease is suspended") + +var ( + _ manager.Runnable = (*DefaultObjectsGate)(nil) + _ manager.LeaderElectionRunnable = (*DefaultObjectsGate)(nil) +) + +// NeedLeaderElection keeps a single replica forcing the release. Two +// replicas racing the same annotation patch would double the Helm upgrades +// for no benefit. +func (g *DefaultObjectsGate) NeedLeaderElection() bool { return true } + +// SetupWithManager wires the dynamic client and the RESTMapper from the +// manager's rest.Config, mirroring RestoreJobReconciler, and registers the +// gate as a manager Runnable. +func (g *DefaultObjectsGate) SetupWithManager(mgr ctrl.Manager) error { + cfg := mgr.GetConfig() + var err error + if g.Interface, err = dynamic.NewForConfig(cfg); err != nil { + return err + } + var h *http.Client + if h, err = rest.HTTPClientFor(cfg); err != nil { + return err + } + if g.RESTMapper, err = apiutil.NewDynamicRESTMapper(cfg, h); err != nil { + return err + } + return mgr.Add(g) +} + +func (g *DefaultObjectsGate) Start(ctx context.Context) error { + logger := log.FromContext(ctx).WithName("default-objects-gate") + if g.BackupClassName == "" || g.HelmRelease.Name == "" || g.HelmRelease.Namespace == "" || !g.Config.IsEnabled() { + logger.V(1).Info("default backup objects gate disabled", + "backupClass", g.BackupClassName, + "helmRelease", g.HelmRelease.String(), + "credentialsConfigured", g.Config.IsEnabled()) + return nil + } + if g.Period == 0 { + g.Period = time.Minute + } + if g.MinForceInterval == 0 { + g.MinForceInterval = 5 * time.Minute + } + if g.now == nil { + g.now = time.Now + } + // force() logs the suspended-release skip from deep in the call stack; + // without this it would land on a bare logger with no name. + ctx = log.IntoContext(ctx, logger) + + tick := time.NewTicker(g.Period) + defer tick.Stop() + g.checkAndLog(ctx, logger) + for { + select { + case <-ctx.Done(): + return nil + case <-tick.C: + g.checkAndLog(ctx, logger) + } + } +} + +// checkTimeout bounds a single check. It stays below Period so a slow check +// cannot overlap the next tick, and is capped so a long Period does not mean a +// correspondingly long stall. +func (g *DefaultObjectsGate) checkTimeout() time.Duration { + const maxTimeout = 30 * time.Second + if g.Period <= 0 { + return maxTimeout + } + if half := g.Period / 2; half < maxTimeout { + return half + } + return maxTimeout +} + +func (g *DefaultObjectsGate) checkAndLog(ctx context.Context, logger logr.Logger) { + // Bound every tick. Check makes several sequential API calls (Secret get, + // BackupClass get, one Get per routed strategy, HelmRelease patch) and the + // manager context is only cancelled at shutdown, so an API server that + // stalls on any one of them would block here indefinitely. The loop that + // drives this is sequential, so that single stuck call would stop all + // later checks for the life of the pod: the recovery this gate exists to + // provide would go silent, with a stale gauge and no further log line, and + // only a restart would bring it back. + checkCtx, cancel := context.WithTimeout(ctx, g.checkTimeout()) + defer cancel() + missing, forced, err := g.Check(checkCtx) + switch { + case err != nil: + // Info, not Error: every branch here is either transient (the + // bucket is still being provisioned) or already exposed as a + // metric, and this runs every minute for the life of the cluster. + logger.Info("default backup objects check failed", "error", err.Error()) + case len(missing) > 0 && forced: + // The release named here is whichever one renders what is missing: + // the credentials Secret comes from the bucket's -system + // release, everything else from this chart's own. + logger.Info("forced a Helm upgrade to create missing default backup objects", + "helmRelease", g.forcedReleaseFor(missing).String(), "missing", missing) + case len(missing) > 0: + logger.Info("default backup objects still missing, no force issued on this tick", + "helmRelease", g.forcedReleaseFor(missing).String(), "missing", missing, + "minForceInterval", g.MinForceInterval.String()) + default: + logger.V(1).Info("all default backup objects present") + } +} + +// forcedReleaseFor reports which release renders the given missing set, for +// logging only. reconcileCredentials returns the credentials Secret as the +// sole missing object, and it is never mixed with the routed objects — +// resolving the bucket name is a precondition for checking those at all. +func (g *DefaultObjectsGate) forcedReleaseFor(missing []string) types.NamespacedName { + credsKey := fmt.Sprintf("Secret/%s", g.Config.SourceSecretName) + if len(missing) == 1 && missing[0] == credsKey && g.CredentialsHelmRelease.Name != "" { + return g.CredentialsHelmRelease + } + return g.HelmRelease +} + +// Check resolves the platform bucket name, verifies that every object the +// BackupClass routes to exists, and forces one Helm upgrade when any is +// missing. It returns the missing objects (formatted for logs) and whether +// a forced upgrade was issued on this call. +// +// While the source Secret is absent or carries no bucket name, the gate +// cannot evaluate the routed objects — their templates gate on the same +// unresolved bucket — so it instead forces the release that renders that +// Secret (see reconcileCredentials). The bucket name is read from the +// projector's source Secret rather than from the BucketClaim, so the gate +// needs no objectstorage RBAC and works for the external-S3 path too. +func (g *DefaultObjectsGate) Check(ctx context.Context) ([]string, bool, error) { + src := &corev1.Secret{} + if err := g.Client.Get(ctx, types.NamespacedName{Namespace: g.Config.SourceNamespace, Name: g.Config.SourceSecretName}, src); err != nil { + if apierrors.IsNotFound(err) { + // The Secret the projector reads does not exist. On a fresh + // install that is the bootstrap window; months later it is the + // permanent skip this gate exists to repair, and the two are + // indistinguishable from here — so treat both the same way and + // let the throttle bound the cost. + return g.reconcileCredentials(ctx) + } + defaultObjectsCheckErrors.WithLabelValues(g.BackupClassName).Inc() + return nil, false, fmt.Errorf("get source credentials Secret %s/%s: %w", g.Config.SourceNamespace, g.Config.SourceSecretName, err) + } + creds, err := parseSourceSecret(src) + if err != nil { + defaultObjectsCheckErrors.WithLabelValues(g.BackupClassName).Inc() + return nil, false, err + } + if creds.bucket == "" { + // The Secret exists but carries no bucket name: a partially rendered + // or hand-written Secret. Same treatment — a re-render of the + // producing release is the only thing that can complete it. + return g.reconcileCredentials(ctx) + } + + backupClass := &backupsv1alpha1.BackupClass{} + if err := g.Client.Get(ctx, client.ObjectKey{Name: g.BackupClassName}, backupClass); err != nil { + defaultObjectsCheckErrors.WithLabelValues(g.BackupClassName).Inc() + return nil, false, fmt.Errorf("get BackupClass %s: %w", g.BackupClassName, err) + } + + missing, err := g.missingObjects(ctx, backupClass) + if err != nil { + defaultObjectsCheckErrors.WithLabelValues(g.BackupClassName).Inc() + return nil, false, err + } + defaultObjectsMissing.WithLabelValues(g.BackupClassName).Set(float64(len(missing))) + if len(missing) == 0 { + return nil, false, nil + } + + return g.force(ctx, g.HelmRelease, &g.lastForce, missing) +} + +// reconcileCredentials handles the one object the gate cannot check the same +// way as the others: the bucket user-credentials Secret it reads to resolve +// the bucket name in the first place. +// +// That Secret is rendered by the platform bucket's -system release +// behind a `lookup` of the COSI Secret, so it is subject to the identical +// permanent-skip trap — and when it is missing, nothing downstream can +// resolve: the projector has no source, every Strategy CR and the Velero BSL +// stay gated off, and migration 50 cannot find its snapshot target. +// +// The chart cannot repair this itself. Making the render `fail` instead of +// skip would abort the whole -system release — the other users' +// Secrets and the bucket UI Deployment/Service/Ingress with it — and one +// user whose BucketAccess never provisions would park the release in Failed, +// blocking every later upgrade of that bucket with no per-bucket way out. So +// the repair belongs here, where it is per-object and costs a throttled +// annotation patch: exactly the mechanism the gate already applies to the +// Strategy CRs, one release earlier in the chain. +// +// Forcing that release cannot deadlock its own precondition: the BucketClaim +// and the BucketAccess whose COSI Secret the lookup reads are rendered +// unconditionally by the PARENT release (packages/apps/bucket/templates/ +// bucketclaim.yaml), not by the one being forced. +func (g *DefaultObjectsGate) reconcileCredentials(ctx context.Context) ([]string, bool, error) { + missing := []string{fmt.Sprintf("Secret/%s", g.Config.SourceSecretName)} + defaultObjectsMissing.WithLabelValues(g.BackupClassName).Set(float64(len(missing))) + if g.CredentialsHelmRelease.Name == "" || g.CredentialsHelmRelease.Namespace == "" { + // External S3: the Secret is admin-managed and no release renders + // it. Report it missing, force nothing. + return missing, false, nil + } + return g.force(ctx, g.CredentialsHelmRelease, &g.lastCredentialsForce, missing) +} + +// force issues at most one throttled forced Helm upgrade of target and +// reports whether it went through. last is the caller's throttle timestamp, +// advanced on every outcome that must not be retried immediately. +func (g *DefaultObjectsGate) force(ctx context.Context, target types.NamespacedName, last *time.Time, missing []string) ([]string, bool, error) { + now := g.now() + if !last.IsZero() && now.Sub(*last) < g.MinForceInterval { + return missing, false, nil + } + if err := g.forceHelmRelease(ctx, target, now); err != nil { + switch { + case apierrors.IsNotFound(err): + // Not a Flux-managed install (a plain `helm install` for local + // development). There is nothing to force; stay quiet rather + // than logging an error every tick. + *last = now + return missing, false, nil + case errors.Is(err, errHelmReleaseSuspended): + // Deliberate operator state (cozyhr suspend, maintenance). The + // annotations would be ignored, so skip the patch entirely and + // leave the force counter alone — a climbing counter must keep + // meaning "the forced render is not producing the objects". + log.FromContext(ctx).Info("skipped forcing a suspended HelmRelease; missing default backup objects will not be repaired until it is resumed", + "helmRelease", target.String(), "missing", missing) + *last = now + return missing, false, nil + } + return missing, false, fmt.Errorf("force HelmRelease %s: %w", target.String(), err) + } + *last = now + defaultObjectsForceReconciles.WithLabelValues(target.Namespace, target.Name).Inc() + return missing, true, nil +} + +// missingObjects returns a sorted, de-duplicated list of the +// "/" objects the BackupClass references that do not exist, +// plus the Velero BSL when a Velero namespace is configured. +// +// An unmappable group/kind (the CRD is not installed) is NOT reported as +// missing: no Helm re-render can create an object whose CRD is absent, and +// counting it would keep the gate forcing upgrades forever. +func (g *DefaultObjectsGate) missingObjects(ctx context.Context, backupClass *backupsv1alpha1.BackupClass) ([]string, error) { + seen := map[string]struct{}{} + var missing []string + + for _, strategy := range backupClass.Spec.Strategies { + name := strategy.StrategyRef.Name + kind := strategy.StrategyRef.Kind + if name == "" || kind == "" { + continue + } + group := strategyAPIGroup + if strategy.StrategyRef.APIGroup != nil && *strategy.StrategyRef.APIGroup != "" { + group = *strategy.StrategyRef.APIGroup + } + // When the Velero BSL is disabled (velero.bslEnabled=false, surfaced + // here as an empty VeleroNamespace) the chart gates the Velero + // Strategy CRs off the SAME flag, so they are never rendered. The + // BackupClass still routes to them unconditionally, so counting them + // as missing would force a Helm upgrade every MinForceInterval + // forever against a render that can never produce them. + if g.VeleroNamespace == "" && group == strategyAPIGroup && kind == "Velero" { + continue + } + key := fmt.Sprintf("%s/%s", kind, name) + if _, dup := seen[key]; dup { + continue + } + seen[key] = struct{}{} + + mapping, err := g.RESTMapping(schema.GroupKind{Group: group, Kind: kind}) + if err != nil { + if meta.IsNoMatchError(err) { + continue + } + return nil, fmt.Errorf("map %s/%s: %w", group, kind, err) + } + // Strategy CRs are cluster-scoped; Namespace("") is correct for + // both scopes on the dynamic client. + _, err = g.Resource(mapping.Resource).Get(ctx, name, metav1.GetOptions{}) + switch { + case err == nil: + case apierrors.IsNotFound(err): + missing = append(missing, key) + default: + return nil, fmt.Errorf("get %s %s: %w", kind, name, err) + } + } + + if g.VeleroNamespace != "" { + // Resolve through the RESTMapper, like the strategy loop above, so a + // genuinely absent Velero API is a NoMatch we skip rather than a + // NotFound we count. The dynamic client's Get bypasses the mapper and + // returns a plain 404 for an unserved group, which IsNotFound would + // catch first, making the "CRDs absent" branch dead code and looping + // the gate forever if Velero were removed while bslEnabled=true. + mapping, err := g.RESTMapping(schema.GroupKind{Group: backupStorageLocationGVR.Group, Kind: "BackupStorageLocation"}) + switch { + case err == nil: + _, getErr := g.Resource(mapping.Resource).Namespace(g.VeleroNamespace).Get(ctx, defaultBackupStorageLocationName, metav1.GetOptions{}) + switch { + case getErr == nil: + case apierrors.IsNotFound(getErr): + missing = append(missing, fmt.Sprintf("BackupStorageLocation/%s", defaultBackupStorageLocationName)) + default: + return nil, fmt.Errorf("get BackupStorageLocation %s: %w", defaultBackupStorageLocationName, getErr) + } + case meta.IsNoMatchError(err): + // Velero API not served (e.g. Velero uninstalled after bootstrap): + // no Helm re-render can create the BSL, so it is not "missing". + default: + return nil, fmt.Errorf("map BackupStorageLocation: %w", err) + } + } + + sort.Strings(missing) + return missing, nil +} + +// forceHelmRelease stamps BOTH reconcile.fluxcd.io/forceAt and +// reconcile.fluxcd.io/requestedAt. requestedAt alone only asks +// helm-controller to reconcile, which is a no-op for a release whose chart +// and values are unchanged — it is forceAt that makes it run a real Helm +// upgrade, and forceAt is only honoured together with requestedAt. +// +// A point Get precedes the patch so a suspended release is skipped rather +// than re-stamped forever: helm-controller ignores both annotations while +// spec.suspend is true, and `cozyhr suspend` sets exactly that. Without the +// check the gate would patch every MinForceInterval for the whole suspension +// and inflate the force counter, which the runbook reads as a render that +// keeps failing — the wrong diagnosis. The Get needs no RBAC beyond the +// `get` on helmreleases the chart already grants. +func (g *DefaultObjectsGate) forceHelmRelease(ctx context.Context, target types.NamespacedName, now time.Time) error { + hr, err := g.Resource(helmReleaseGVR).Namespace(target.Namespace).Get(ctx, target.Name, metav1.GetOptions{}) + if err != nil { + return err + } + // A malformed spec.suspend (wrong type) is reported by NestedBool as an + // error; treat it as not suspended and let the patch proceed, rather + // than silently disabling the repair on a field we could not parse. + if suspended, found, err := unstructured.NestedBool(hr.Object, "spec", "suspend"); err == nil && found && suspended { + return errHelmReleaseSuspended + } + + stamp := now.UTC().Format(time.RFC3339Nano) + patch := fmt.Sprintf( + `{"metadata":{"annotations":{"reconcile.fluxcd.io/forceAt":%q,"reconcile.fluxcd.io/requestedAt":%q}}}`, + stamp, stamp, + ) + _, err = g.Resource(helmReleaseGVR).Namespace(target.Namespace). + Patch(ctx, target.Name, types.MergePatchType, []byte(patch), metav1.PatchOptions{}) + return err +} diff --git a/internal/backupcontroller/default_objects_gate_test.go b/internal/backupcontroller/default_objects_gate_test.go new file mode 100644 index 0000000000..d04de2b665 --- /dev/null +++ b/internal/backupcontroller/default_objects_gate_test.go @@ -0,0 +1,748 @@ +package backupcontroller + +import ( + "context" + "testing" + "time" + + "github.com/prometheus/client_golang/prometheus/testutil" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + dynamicfake "k8s.io/client-go/dynamic/fake" + k8stesting "k8s.io/client-go/testing" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + backupsv1alpha1 "github.com/cozystack/cozystack/api/backups/v1alpha1" +) + +// gateGVKs are the kinds the gate touches in these tests: the +// strategy.backups.cozystack.io CRs cozy-default routes to, the Velero BSL, +// and the HelmRelease it patches. The RESTMapper and the dynamic fake's +// resource registry are both derived from this one list so the plural the +// gate resolves and the plural the fake serves cannot drift apart. +var gateGVKs = []schema.GroupVersionKind{ + {Group: strategyAPIGroup, Version: "v1alpha1", Kind: "CNPG"}, + {Group: strategyAPIGroup, Version: "v1alpha1", Kind: "MariaDB"}, + {Group: strategyAPIGroup, Version: "v1alpha1", Kind: "Etcd"}, + {Group: strategyAPIGroup, Version: "v1alpha1", Kind: "Altinity"}, + {Group: strategyAPIGroup, Version: "v1alpha1", Kind: "Velero"}, + {Group: "velero.io", Version: "v1", Kind: "BackupStorageLocation"}, + {Group: "helm.toolkit.fluxcd.io", Version: "v2", Kind: "HelmRelease"}, +} + +func gateScheme() *runtime.Scheme { + scheme := runtime.NewScheme() + _ = corev1.AddToScheme(scheme) + _ = backupsv1alpha1.AddToScheme(scheme) + return scheme +} + +func gateListKinds() map[schema.GroupVersionResource]string { + out := map[schema.GroupVersionResource]string{} + for _, gvk := range gateGVKs { + gvr, _ := meta.UnsafeGuessKindToResource(gvk) + out[gvr] = gvk.Kind + "List" + } + return out +} + +// gateRESTMapper resolves group+kind (the only coordinates a +// BackupClass strategyRef carries) to a resource, which is what the gate +// does in production through the dynamic RESTMapper. The group versions must +// be passed as defaults: RESTMapping is called without a version. +func gateRESTMapper() meta.RESTMapper { + groupVersions := map[schema.GroupVersion]struct{}{} + for _, gvk := range gateGVKs { + groupVersions[gvk.GroupVersion()] = struct{}{} + } + var gvs []schema.GroupVersion + for gv := range groupVersions { + gvs = append(gvs, gv) + } + m := meta.NewDefaultRESTMapper(gvs) + for _, gvk := range gateGVKs { + scope := meta.RESTScopeRoot + if gvk.Kind == "BackupStorageLocation" || gvk.Kind == "HelmRelease" { + scope = meta.RESTScopeNamespace + } + m.Add(gvk, scope) + } + return m +} + +func apiGroup(s string) *string { return &s } + +// cozyDefaultBackupClass mirrors the routes the chart's +// backupclass-default.yaml renders unconditionally: it is the manifest of +// what must exist, which is exactly why the gate reads it instead of +// hard-coding a list of Strategy names. +func cozyDefaultBackupClass() *backupsv1alpha1.BackupClass { + ref := func(kind, name string) backupsv1alpha1.BackupClassStrategy { + return backupsv1alpha1.BackupClassStrategy{ + Application: backupsv1alpha1.ApplicationSelector{ + APIGroup: apiGroup("apps.cozystack.io"), + Kind: kind + "App", + }, + StrategyRef: corev1.TypedLocalObjectReference{ + APIGroup: apiGroup(strategyAPIGroup), + Kind: kind, + Name: name, + }, + } + } + return &backupsv1alpha1.BackupClass{ + ObjectMeta: metav1.ObjectMeta{Name: "cozy-default"}, + Spec: backupsv1alpha1.BackupClassSpec{ + Strategies: []backupsv1alpha1.BackupClassStrategy{ + ref("CNPG", "cozy-default-cnpg"), + ref("MariaDB", "cozy-default-mariadb"), + ref("Etcd", "cozy-default-etcd"), + ref("Altinity", "cozy-default-altinity"), + ref("Velero", "cozy-default-velero-vminstance"), + ref("Velero", "cozy-default-velero-vmdisk"), + }, + }, + } +} + +func strategyObject(kind, name string) *unstructured.Unstructured { + u := &unstructured.Unstructured{} + u.SetAPIVersion(strategyAPIGroup + "/v1alpha1") + u.SetKind(kind) + u.SetName(name) + return u +} + +func helmReleaseObject() *unstructured.Unstructured { + u := &unstructured.Unstructured{} + u.SetAPIVersion("helm.toolkit.fluxcd.io/v2") + u.SetKind("HelmRelease") + u.SetNamespace("cozy-backup-controller") + u.SetName("backupstrategy-controller") + return u +} + +// credentialsHelmReleaseObject is the platform bucket's -system +// release: the one that renders the user-credentials Secret the gate reads +// to resolve the bucket name, behind its own install-time lookup. +func credentialsHelmReleaseObject() *unstructured.Unstructured { + u := &unstructured.Unstructured{} + u.SetAPIVersion("helm.toolkit.fluxcd.io/v2") + u.SetKind("HelmRelease") + u.SetNamespace("tenant-root") + u.SetName("bucket-cozy-backups-system") + return u +} + +func suspended(u *unstructured.Unstructured) *unstructured.Unstructured { + _ = unstructured.SetNestedField(u.Object, true, "spec", "suspend") + return u +} + +func newGate(t *testing.T, ctrlObjs []client.Object, dynObjs ...runtime.Object) (*DefaultObjectsGate, *dynamicfake.FakeDynamicClient) { + t.Helper() + dyn := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(runtime.NewScheme(), gateListKinds(), dynObjs...) + g := &DefaultObjectsGate{ + Client: fake.NewClientBuilder().WithScheme(gateScheme()).WithObjects(ctrlObjs...).Build(), + Config: BackupCredentialsConfig{ + SourceNamespace: "tenant-root", + SourceSecretName: "bucket-cozy-backups-system-credentials", + TargetSecretName: "cozy-backups-creds", + }, + BackupClassName: "cozy-default", + HelmRelease: types.NamespacedName{ + Namespace: "cozy-backup-controller", + Name: "backupstrategy-controller", + }, + CredentialsHelmRelease: types.NamespacedName{ + Namespace: "tenant-root", + Name: "bucket-cozy-backups-system", + }, + VeleroNamespace: "cozy-velero", + MinForceInterval: 5 * time.Minute, + now: func() time.Time { return time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) }, + } + g.Interface = dyn + g.RESTMapper = gateRESTMapper() + return g, dyn +} + +func sourceSecret(bucket string) *corev1.Secret { + data := map[string][]byte{ + "accessKey": []byte("AK"), + "secretKey": []byte("SK"), + "endpoint": []byte("s3.example.com"), + } + if bucket != "" { + data["bucketName"] = []byte(bucket) + } + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Namespace: "tenant-root", Name: "bucket-cozy-backups-system-credentials"}, + Data: data, + } +} + +func bslObject() *unstructured.Unstructured { + u := &unstructured.Unstructured{} + u.SetAPIVersion("velero.io/v1") + u.SetKind("BackupStorageLocation") + u.SetNamespace("cozy-velero") + u.SetName(defaultBackupStorageLocationName) + return u +} + +func forceAnnotations(t *testing.T, dyn *dynamicfake.FakeDynamicClient) map[string]string { + t.Helper() + return annotationsOf(t, dyn, "cozy-backup-controller", "backupstrategy-controller") +} + +func credentialsForceAnnotations(t *testing.T, dyn *dynamicfake.FakeDynamicClient) map[string]string { + t.Helper() + return annotationsOf(t, dyn, "tenant-root", "bucket-cozy-backups-system") +} + +func annotationsOf(t *testing.T, dyn *dynamicfake.FakeDynamicClient, namespace, name string) map[string]string { + t.Helper() + hr, err := dyn.Resource(helmReleaseGVR).Namespace(namespace). + Get(context.Background(), name, metav1.GetOptions{}) + if err != nil { + t.Fatalf("get HelmRelease %s/%s: %v", namespace, name, err) + } + return hr.GetAnnotations() +} + +// TestCheckForcesWhenObjectsMissing is the bug this gate exists for: the +// bucket name resolves, so a Helm re-render WOULD produce the gated +// templates, but the objects are absent because the install-time lookup was +// empty and helm-controller never re-rendered. The gate must force a real +// Helm upgrade — BOTH annotations, because requestedAt alone is a no-op for +// an unchanged release. +func TestCheckForcesWhenObjectsMissing(t *testing.T) { + g, dyn := newGate(t, + []client.Object{sourceSecret("bucket-1a2b"), cozyDefaultBackupClass()}, + helmReleaseObject(), + ) + + missing, forced, err := g.Check(context.Background()) + if err != nil { + t.Fatalf("Check: %v", err) + } + if !forced { + t.Fatal("expected a forced Helm upgrade when default objects are missing") + } + // 6 strategyRefs (Velero twice under different names) + the BSL. + if len(missing) != 7 { + t.Fatalf("missing = %v, want 7 entries", missing) + } + ann := forceAnnotations(t, dyn) + forceAt, ok := ann["reconcile.fluxcd.io/forceAt"] + if !ok || forceAt == "" { + t.Errorf("reconcile.fluxcd.io/forceAt not set: %v", ann) + } + requestedAt, ok := ann["reconcile.fluxcd.io/requestedAt"] + if !ok || requestedAt == "" { + t.Errorf("reconcile.fluxcd.io/requestedAt not set: %v", ann) + } + if forceAt != requestedAt { + t.Errorf("forceAt (%q) and requestedAt (%q) must carry the same stamp", forceAt, requestedAt) + } +} + +// TestCheckNoopWhenAllPresent pins the steady state: no forced upgrades +// once the objects exist, otherwise the gate would rewrite the annotation +// forever and re-run a Helm upgrade every MinForceInterval for the life of +// the cluster. +func TestCheckNoopWhenAllPresent(t *testing.T) { + g, dyn := newGate(t, + []client.Object{sourceSecret("bucket-1a2b"), cozyDefaultBackupClass()}, + helmReleaseObject(), + strategyObject("CNPG", "cozy-default-cnpg"), + strategyObject("MariaDB", "cozy-default-mariadb"), + strategyObject("Etcd", "cozy-default-etcd"), + strategyObject("Altinity", "cozy-default-altinity"), + strategyObject("Velero", "cozy-default-velero-vminstance"), + strategyObject("Velero", "cozy-default-velero-vmdisk"), + bslObject(), + ) + + missing, forced, err := g.Check(context.Background()) + if err != nil { + t.Fatalf("Check: %v", err) + } + if len(missing) != 0 || forced { + t.Fatalf("missing = %v, forced = %v, want none", missing, forced) + } + if ann := forceAnnotations(t, dyn); len(ann) != 0 { + t.Errorf("HelmRelease annotated in the steady state: %v", ann) + } +} + +// TestCheckForcesCredentialsReleaseWhenSourceSecretAbsent is the second half +// of the bug. The Secret the gate reads to resolve the bucket name is itself +// rendered behind an install-time lookup, by the bucket's -system +// release — so it is subject to the identical permanent skip, one release +// earlier. While it is absent nothing downstream can resolve: no projected +// credentials, no Strategy CRs, no Velero, and migration 50 has no snapshot +// target. Forcing THIS chart's release would be useless (its own lookups +// depend on that Secret); the bucket release is the one to force. +func TestCheckForcesCredentialsReleaseWhenSourceSecretAbsent(t *testing.T) { + g, dyn := newGate(t, + []client.Object{cozyDefaultBackupClass()}, + helmReleaseObject(), + credentialsHelmReleaseObject(), + ) + + missing, forced, err := g.Check(context.Background()) + if err != nil { + t.Fatalf("Check: %v", err) + } + if !forced { + t.Fatal("expected the bucket release to be forced while the credentials Secret is absent") + } + if len(missing) != 1 || missing[0] != "Secret/bucket-cozy-backups-system-credentials" { + t.Fatalf("missing = %v, want the credentials Secret", missing) + } + ann := credentialsForceAnnotations(t, dyn) + if ann["reconcile.fluxcd.io/forceAt"] == "" || ann["reconcile.fluxcd.io/requestedAt"] == "" { + t.Errorf("bucket release not forced with both annotations: %v", ann) + } + // This chart's own release must be left alone: its templates gate on the + // same unresolved bucket, so forcing it would burn a Helm upgrade that + // re-runs the same empty lookup. + if ann := forceAnnotations(t, dyn); len(ann) != 0 { + t.Errorf("own HelmRelease forced before the bucket resolved: %v", ann) + } +} + +// TestCheckForcesCredentialsReleaseWhenBucketNameEmpty pins the same +// treatment one state later: the Secret exists but carries no bucket name (a +// partial render, or a hand-written Secret missing the key). Only a +// re-render of the producing release can complete it. +func TestCheckForcesCredentialsReleaseWhenBucketNameEmpty(t *testing.T) { + g, dyn := newGate(t, + []client.Object{sourceSecret(""), cozyDefaultBackupClass()}, + helmReleaseObject(), + credentialsHelmReleaseObject(), + ) + + missing, forced, err := g.Check(context.Background()) + if err != nil { + t.Fatalf("Check: %v", err) + } + if !forced || len(missing) != 1 { + t.Fatalf("missing = %v, forced = %v; want the credentials Secret forced", missing, forced) + } + if ann := credentialsForceAnnotations(t, dyn); ann["reconcile.fluxcd.io/forceAt"] == "" { + t.Errorf("bucket release not forced: %v", ann) + } +} + +// TestCheckReportsCredentialsSecretWithoutBucketRelease covers external S3 +// (backupStorage.provisionBucket=false): the Secret is admin-managed and no +// release renders it, so there is nothing to force — but it must still be +// reported missing, because that is the state an operator has to alert on. +func TestCheckReportsCredentialsSecretWithoutBucketRelease(t *testing.T) { + g, dyn := newGate(t, + []client.Object{cozyDefaultBackupClass()}, + helmReleaseObject(), + credentialsHelmReleaseObject(), + ) + g.CredentialsHelmRelease = types.NamespacedName{} + + missing, forced, err := g.Check(context.Background()) + if err != nil { + t.Fatalf("Check: %v", err) + } + if forced { + t.Error("forced a release on the external-S3 path, where none renders the Secret") + } + if len(missing) != 1 { + t.Fatalf("missing = %v, want the credentials Secret reported", missing) + } + if ann := credentialsForceAnnotations(t, dyn); len(ann) != 0 { + t.Errorf("bucket release annotated with no coordinates configured: %v", ann) + } +} + +// TestCheckThrottlesTheTwoReleasesIndependently pins that forcing the bucket +// release does not eat the objects release's throttle budget. The two happen +// in sequence on a real bootstrap — credentials first, then the objects the +// resolved bucket unblocks — so a shared timestamp would delay the second +// force by a full MinForceInterval for no reason. +func TestCheckThrottlesTheTwoReleasesIndependently(t *testing.T) { + base := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) + g, dyn := newGate(t, + []client.Object{cozyDefaultBackupClass()}, + helmReleaseObject(), + credentialsHelmReleaseObject(), + ) + g.now = func() time.Time { return base } + + // No source Secret yet: the bucket release is forced. + if _, forced, err := g.Check(context.Background()); err != nil || !forced { + t.Fatalf("first Check: forced = %v, err = %v; want the bucket release forced", forced, err) + } + + // A minute later the Secret lands. The objects are still missing, and + // this force must go through despite being well inside MinForceInterval + // of the previous one. + g.now = func() time.Time { return base.Add(time.Minute) } + if err := g.Client.Create(context.Background(), sourceSecret("bucket-1a2b")); err != nil { + t.Fatalf("create source Secret: %v", err) + } + missing, forced, err := g.Check(context.Background()) + if err != nil { + t.Fatalf("second Check: %v", err) + } + if !forced { + t.Fatal("objects release throttled by the earlier credentials force") + } + if len(missing) != 7 { + t.Fatalf("missing = %v, want the 6 strategies + the BSL", missing) + } + if ann := forceAnnotations(t, dyn); ann["reconcile.fluxcd.io/forceAt"] == "" { + t.Errorf("own HelmRelease not forced: %v", ann) + } +} + +// TestForceSkipsSuspendedHelmRelease pins the suspend guard. helm-controller +// ignores forceAt/requestedAt while spec.suspend is true (`cozyhr suspend` is +// a standard dev workflow), so patching anyway would re-stamp every +// MinForceInterval for the whole suspension and climb +// cozystack_backup_default_objects_force_reconciles_total — which the runbook +// reads as "the forced render is not producing the objects", the wrong +// diagnosis. The objects must still be reported missing. +func TestForceSkipsSuspendedHelmRelease(t *testing.T) { + g, dyn := newGate(t, + []client.Object{sourceSecret("bucket-1a2b"), cozyDefaultBackupClass()}, + suspended(helmReleaseObject()), + ) + + missing, forced, err := g.Check(context.Background()) + if err != nil { + t.Fatalf("Check: %v", err) + } + if forced { + t.Error("forced reported true for a suspended HelmRelease that ignores the annotations") + } + if len(missing) != 7 { + t.Fatalf("missing = %v, want the objects still reported", missing) + } + if ann := forceAnnotations(t, dyn); len(ann) != 0 { + t.Errorf("suspended HelmRelease annotated: %v", ann) + } +} + +// TestForceSkipsSuspendedCredentialsHelmRelease pins the same guard on the +// bucket release, which an operator is far more likely to have suspended: +// it lives in a tenant namespace and is not part of this chart. +func TestForceSkipsSuspendedCredentialsHelmRelease(t *testing.T) { + g, dyn := newGate(t, + []client.Object{cozyDefaultBackupClass()}, + helmReleaseObject(), + suspended(credentialsHelmReleaseObject()), + ) + + missing, forced, err := g.Check(context.Background()) + if err != nil { + t.Fatalf("Check: %v", err) + } + if forced { + t.Error("forced reported true for a suspended bucket HelmRelease") + } + if len(missing) != 1 { + t.Fatalf("missing = %v, want the credentials Secret still reported", missing) + } + if ann := credentialsForceAnnotations(t, dyn); len(ann) != 0 { + t.Errorf("suspended bucket HelmRelease annotated: %v", ann) + } +} + +// TestCheckTimeoutStaysBelowPeriod pins the bound on a single check. The ticker +// loop is sequential and the manager context is only cancelled at shutdown, so +// an unbounded check that hangs on a stalled API call stops every later check +// for the life of the pod. +func TestCheckTimeoutStaysBelowPeriod(t *testing.T) { + for _, tc := range []struct { + name string + period time.Duration + want time.Duration + }{ + {"unset falls back to the cap", 0, 30 * time.Second}, + {"default period halves", time.Minute, 30 * time.Second}, + {"short period halves", 10 * time.Second, 5 * time.Second}, + {"long period is capped", time.Hour, 30 * time.Second}, + } { + t.Run(tc.name, func(t *testing.T) { + g := &DefaultObjectsGate{Period: tc.period} + got := g.checkTimeout() + if got != tc.want { + t.Fatalf("checkTimeout() = %v, want %v", got, tc.want) + } + if tc.period > 0 && got >= tc.period { + t.Fatalf("checkTimeout() = %v, must stay below Period %v", got, tc.period) + } + }) + } +} + +// TestCheckThrottlesRepeatedForces pins the throttle. A condition a +// re-render cannot fix (missing CRD, unreachable bucket) must not turn into +// a hot loop of Helm upgrades against helm-controller. +func TestCheckThrottlesRepeatedForces(t *testing.T) { + g, _ := newGate(t, + []client.Object{sourceSecret("bucket-1a2b"), cozyDefaultBackupClass()}, + helmReleaseObject(), + ) + base := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) + g.now = func() time.Time { return base } + + if _, forced, err := g.Check(context.Background()); err != nil || !forced { + t.Fatalf("first Check: forced = %v, err = %v; want forced", forced, err) + } + + // Well inside MinForceInterval: still missing, but no second upgrade. + g.now = func() time.Time { return base.Add(time.Minute) } + missing, forced, err := g.Check(context.Background()) + if err != nil { + t.Fatalf("second Check: %v", err) + } + if forced { + t.Error("second Check forced inside MinForceInterval") + } + if len(missing) == 0 { + t.Error("second Check should still report the objects as missing") + } + + // Past MinForceInterval: force again, because the objects are still gone. + g.now = func() time.Time { return base.Add(6 * time.Minute) } + if _, forced, err := g.Check(context.Background()); err != nil || !forced { + t.Fatalf("third Check: forced = %v, err = %v; want forced", forced, err) + } +} + +// TestMissingObjectsIgnoresUnmappedKinds pins that a strategyRef whose CRD +// is not installed is not counted as missing. No Helm re-render can create +// such an object, so counting it would force upgrades forever. +func TestMissingObjectsIgnoresUnmappedKinds(t *testing.T) { + bc := cozyDefaultBackupClass() + bc.Spec.Strategies = append(bc.Spec.Strategies, backupsv1alpha1.BackupClassStrategy{ + Application: backupsv1alpha1.ApplicationSelector{Kind: "Nonexistent"}, + StrategyRef: corev1.TypedLocalObjectReference{ + APIGroup: apiGroup(strategyAPIGroup), + Kind: "NotInstalled", + Name: "cozy-default-notinstalled", + }, + }) + g, _ := newGate(t, []client.Object{sourceSecret("b"), bc}, + helmReleaseObject(), + strategyObject("CNPG", "cozy-default-cnpg"), + strategyObject("MariaDB", "cozy-default-mariadb"), + strategyObject("Etcd", "cozy-default-etcd"), + strategyObject("Altinity", "cozy-default-altinity"), + strategyObject("Velero", "cozy-default-velero-vminstance"), + strategyObject("Velero", "cozy-default-velero-vmdisk"), + bslObject(), + ) + + missing, err := g.missingObjects(context.Background(), bc) + if err != nil { + t.Fatalf("missingObjects: %v", err) + } + if len(missing) != 0 { + t.Fatalf("missing = %v, want none (unmapped kind must be ignored)", missing) + } +} + +// TestMissingObjectsSkipsVeleroWhenNamespaceEmpty covers +// velero.bslEnabled=false: the chart gates BOTH the BSL and the Velero +// Strategy CRs off the same flag, so neither is rendered and their absence +// is not a defect. The BackupClass still routes to the Velero strategies +// unconditionally, which is the trap: without the skip the gate would count +// the never-rendered Velero CRs as missing and force a Helm upgrade every +// MinForceInterval forever. The Velero objects are deliberately NOT +// pre-created here, because with the BSL disabled the chart never renders +// them on a real cluster. +func TestMissingObjectsSkipsVeleroWhenNamespaceEmpty(t *testing.T) { + bc := cozyDefaultBackupClass() + g, _ := newGate(t, []client.Object{sourceSecret("b"), bc}, + helmReleaseObject(), + strategyObject("CNPG", "cozy-default-cnpg"), + strategyObject("MariaDB", "cozy-default-mariadb"), + strategyObject("Etcd", "cozy-default-etcd"), + strategyObject("Altinity", "cozy-default-altinity"), + ) + g.VeleroNamespace = "" + + missing, err := g.missingObjects(context.Background(), bc) + if err != nil { + t.Fatalf("missingObjects: %v", err) + } + if len(missing) != 0 { + t.Fatalf("missing = %v, want none when Velero is disabled", missing) + } +} + +// TestMissingObjectsSkipsBSLWhenVeleroAPIAbsent covers Velero being removed +// after bootstrap while bslEnabled=true: the velero.io API is no longer +// served, so the BSL cannot be re-rendered and must not be counted as +// missing. Resolving through the RESTMapper turns that into a NoMatch skip; +// the previous hardcoded-GVR Get returned a NotFound that would have looped +// the gate forever. +func TestMissingObjectsSkipsBSLWhenVeleroAPIAbsent(t *testing.T) { + bc := cozyDefaultBackupClass() + g, _ := newGate(t, []client.Object{sourceSecret("b"), bc}, + helmReleaseObject(), + strategyObject("CNPG", "cozy-default-cnpg"), + strategyObject("MariaDB", "cozy-default-mariadb"), + strategyObject("Etcd", "cozy-default-etcd"), + strategyObject("Altinity", "cozy-default-altinity"), + strategyObject("Velero", "cozy-default-velero-vminstance"), + strategyObject("Velero", "cozy-default-velero-vmdisk"), + ) + // Rebuild the RESTMapper without BackupStorageLocation: the velero.io API + // is not served on this cluster. + gvset := map[schema.GroupVersion]struct{}{} + var kept []schema.GroupVersionKind + for _, gvk := range gateGVKs { + if gvk.Kind == "BackupStorageLocation" { + continue + } + gvset[gvk.GroupVersion()] = struct{}{} + kept = append(kept, gvk) + } + var gvs []schema.GroupVersion + for gv := range gvset { + gvs = append(gvs, gv) + } + m := meta.NewDefaultRESTMapper(gvs) + for _, gvk := range kept { + scope := meta.RESTScopeRoot + if gvk.Kind == "HelmRelease" { + scope = meta.RESTScopeNamespace + } + m.Add(gvk, scope) + } + g.RESTMapper = m + + missing, err := g.missingObjects(context.Background(), bc) + if err != nil { + t.Fatalf("missingObjects: %v", err) + } + if len(missing) != 0 { + t.Fatalf("missing = %v, want none when the Velero API is absent", missing) + } +} + +// TestCheckSurfacesPatchFailure pins that a failed force is reported rather +// than silently recorded as done — and that lastForce is NOT advanced, so +// the next tick retries instead of waiting out MinForceInterval. +func TestCheckSurfacesPatchFailure(t *testing.T) { + g, dyn := newGate(t, + []client.Object{sourceSecret("bucket-1a2b"), cozyDefaultBackupClass()}, + helmReleaseObject(), + ) + dyn.PrependReactor("patch", "helmreleases", func(k8stesting.Action) (bool, runtime.Object, error) { + return true, nil, apierrors.NewForbidden(schema.GroupResource{Group: "helm.toolkit.fluxcd.io", Resource: "helmreleases"}, "backupstrategy-controller", nil) + }) + + if _, forced, err := g.Check(context.Background()); err == nil || forced { + t.Fatalf("forced = %v, err = %v; want an error and forced=false", forced, err) + } + if !g.lastForce.IsZero() { + t.Error("lastForce advanced despite the patch failing; the next tick would be throttled") + } +} + +// TestMissingGaugeCoversTheUnresolvedPath pins the alerting contract. The +// gauge used to be written only after the bucket resolved, so the state the +// gate exists to catch — no credentials Secret, therefore no strategies — +// reported 0 or nothing at all, and an alert on it never fired. Deleting the +// source Secret later (credential rotation) had the same effect: the gauge +// froze at its last value. +func TestMissingGaugeCoversTheUnresolvedPath(t *testing.T) { + defaultObjectsMissing.Reset() + g, _ := newGate(t, + []client.Object{cozyDefaultBackupClass()}, + helmReleaseObject(), + credentialsHelmReleaseObject(), + ) + + if _, _, err := g.Check(context.Background()); err != nil { + t.Fatalf("Check: %v", err) + } + if got := testutil.ToFloat64(defaultObjectsMissing.WithLabelValues("cozy-default")); got != 1 { + t.Fatalf("cozystack_backup_default_objects_missing = %v, want 1 while the credentials Secret is absent", got) + } +} + +// TestCheckErrorsCounterMarksTheGaugeStale pins the other half of that +// contract. On an API error the gauge deliberately keeps its last value +// rather than flapping the alert, which means the gauge alone cannot tell +// "healthy" from "not evaluated". The errors counter is what closes that +// gap, and the runbook alerts on both. +func TestCheckErrorsCounterMarksTheGaugeStale(t *testing.T) { + defaultObjectsMissing.Reset() + defaultObjectsCheckErrors.Reset() + // No BackupClass: the check cannot enumerate what must exist. + g, _ := newGate(t, + []client.Object{sourceSecret("bucket-1a2b")}, + helmReleaseObject(), + ) + + if _, _, err := g.Check(context.Background()); err == nil { + t.Fatal("Check succeeded with no BackupClass, want an error") + } + if got := testutil.ToFloat64(defaultObjectsCheckErrors.WithLabelValues("cozy-default")); got != 1 { + t.Fatalf("cozystack_backup_default_objects_check_errors_total = %v, want 1", got) + } + if got := testutil.ToFloat64(defaultObjectsMissing.WithLabelValues("cozy-default")); got != 0 { + t.Fatalf("gauge = %v, want it left untouched by a failed check", got) + } +} + +// TestStartDisabledWithoutHelmRelease pins the opt-out: without release +// coordinates the runnable must return immediately instead of ticking with +// a nil target. +func TestStartDisabledWithoutHelmRelease(t *testing.T) { + g, _ := newGate(t, []client.Object{sourceSecret("b"), cozyDefaultBackupClass()}, helmReleaseObject()) + g.HelmRelease = types.NamespacedName{} + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + done := make(chan error, 1) + go func() { done <- g.Start(ctx) }() + select { + case err := <-done: + if err != nil { + t.Fatalf("Start: %v", err) + } + case <-ctx.Done(): + t.Fatal("Start did not return immediately when disabled") + } +} + +// TestCheckToleratesAbsentHelmRelease covers a plain `helm install` (local +// development): there is no HelmRelease to force, which is not an error to +// report every tick. +func TestCheckToleratesAbsentHelmRelease(t *testing.T) { + g, _ := newGate(t, []client.Object{sourceSecret("bucket-1a2b"), cozyDefaultBackupClass()}) + + missing, forced, err := g.Check(context.Background()) + if err != nil { + t.Fatalf("Check: %v", err) + } + if forced { + t.Error("forced reported true with no HelmRelease to patch") + } + if len(missing) == 0 { + t.Error("objects should still be reported as missing") + } +} diff --git a/internal/fluxshardoperator/provisioner.go b/internal/fluxshardoperator/provisioner.go index a0acb9400c..1a0066d59d 100644 --- a/internal/fluxshardoperator/provisioner.go +++ b/internal/fluxshardoperator/provisioner.go @@ -263,7 +263,15 @@ func mergeResourceList(dst *corev1.ResourceList, overrides corev1.ResourceList) // - the required podAntiAffinity cloned from flux-aio (which keeps its own // replicas off one node) is dropped, since it targets // app.kubernetes.io/name=flux and would otherwise leave every shard -// Pending on a single-node cluster. +// Pending on a single-node cluster; +// - the corporate-proxy env inherited from flux-aio is dropped, in both the +// upper- and lower-case spellings (HTTP_PROXY/HTTPS_PROXY/NO_PROXY): a +// standalone shard needs no external egress, and behind an unreachable +// proxy a stalled startup call leaves the manager never serving /healthz, +// so the liveness probe crashloops the pod; +// - a startupProbe is derived from the liveness handler (generous failure +// budget, liveness handler and TimeoutSeconds inherited) so a slow but +// progressing start is not killed by the short inherited liveness window. func BuildShardDeployment(flux *appsv1.Deployment, idx int, cfg *Config) (*appsv1.Deployment, error) { var src *corev1.Container for i := range flux.Spec.Template.Spec.Containers { @@ -316,6 +324,24 @@ func BuildShardDeployment(flux *appsv1.Deployment, idx int, cfg *Config) (*appsv // is not hostNetwork, so it must fall back to the in-cluster defaults. case "KUBERNETES_SERVICE_HOST", "KUBERNETES_SERVICE_PORT": continue + // Corporate-proxy env inherited from flux-aio. A standalone shard needs + // no external egress: source-controller does artifact fetching and + // cosign/TUF verification, and every hop the shard makes is in-cluster + // (artifacts advertise as flux.$(RUNTIME_NAMESPACE).svc, guest-cluster + // apiservers are reached over .svc kubeconfigs). NO_PROXY=.svc covers + // those, but not the management apiserver: the KUBERNETES_SERVICE_HOST + // case above makes the shard fall back to the kubelet-injected + // ClusterIP, a bare IP that no .svc suffix matches, so that startup + // call is the one that stalls through an unreachable proxy and leaves + // the manager never serving /healthz until the liveness probe + // crashloops the pod. Dropping the proxy env (and the then-pointless + // NO_PROXY) keeps every hop direct. A HelmRelease targeting a remote + // cluster via spec.kubeConfig reachable only through the proxy is the + // one theoretical exception; cozystack guest-cluster apiservers are + // in-cluster, so this does not apply here. + case "HTTP_PROXY", "HTTPS_PROXY", "NO_PROXY", + "http_proxy", "https_proxy", "no_proxy": + continue } env = append(env, e) } @@ -327,6 +353,25 @@ func BuildShardDeployment(flux *appsv1.Deployment, idx int, cfg *Config) (*appsv mergeResourceList(&hc.Resources.Requests, cfg.ShardResources.Requests) mergeResourceList(&hc.Resources.Limits, cfg.ShardResources.Limits) + // Guard startup with a startupProbe derived from the liveness handler. + // Without it the inherited ~30s liveness window kills a controller that is + // still syncing caches (a slow start on a large cluster, or a transient + // dependency), turning any slow start into an unrecoverable crashloop. The + // generous startup budget defers liveness until the manager is serving, + // then liveness still catches a wedged running pod. Only the budget fields + // are normalised; the liveness handler and its TimeoutSeconds are inherited, + // so a controller whose /healthz is slow under load keeps the same tolerance + // at startup as at runtime (overriding TimeoutSeconds down could make the + // startup probe stricter than liveness and recreate the crashloop). + if hc.LivenessProbe != nil && hc.StartupProbe == nil { + sp := hc.LivenessProbe.DeepCopy() + sp.InitialDelaySeconds = 0 + sp.PeriodSeconds = 10 + sp.SuccessThreshold = 1 + sp.FailureThreshold = 30 + hc.StartupProbe = sp + } + mounted := map[string]bool{} for _, m := range hc.VolumeMounts { mounted[m.Name] = true diff --git a/internal/fluxshardoperator/provisioner_test.go b/internal/fluxshardoperator/provisioner_test.go index 678f93833c..96ad34ff00 100644 --- a/internal/fluxshardoperator/provisioner_test.go +++ b/internal/fluxshardoperator/provisioner_test.go @@ -8,6 +8,7 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" ) // fluxAIODeployment models the relevant shape of the flux-aio "flux" @@ -92,8 +93,31 @@ func fluxAIODeployment() *appsv1.Deployment { FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.namespace"}, }}, {Name: "TUF_ROOT", Value: "/tmp/.sigstore"}, + // Corporate-proxy env present on flux-aio in + // proxied installs (operator/patch level, not the + // installer, which injects only KUBERNETES_SERVICE_*); + // harmless there, hangs a standalone shard at startup. + {Name: "HTTP_PROXY", Value: "http://proxy.example:3128"}, + {Name: "HTTPS_PROXY", Value: "http://proxy.example:3128"}, + {Name: "NO_PROXY", Value: ".svc"}, + {Name: "http_proxy", Value: "http://proxy.example:3128"}, + {Name: "https_proxy", Value: "http://proxy.example:3128"}, + {Name: "no_proxy", Value: ".svc"}, }, VolumeMounts: []corev1.VolumeMount{{Name: "tmp", MountPath: "/tmp"}}, + // flux-aio ships helm-controller with a bare httpGet + // liveness probe: no timing fields at all, so the + // kube-apiserver defaults (~30s window) apply. The + // startupProbe must inherit this handler and its unset + // TimeoutSeconds and only normalise the startup budget. + LivenessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/healthz", + Port: intstr.FromString("healthz-hc"), + }, + }, + }, }, {Name: "notification-controller", Image: "ghcr.io/fluxcd/notification-controller:v1.8.0"}, }, @@ -114,7 +138,18 @@ func TestBuildShardDeployment(t *testing.T) { }, } - dep, err := BuildShardDeployment(fluxAIODeployment(), 2, cfg) + flux := fluxAIODeployment() + // Capture the source liveness probe verbatim so the startupProbe assertions + // can check inheritance against what flux-aio actually ships, rather than + // against a hardcoded value baked into the test. + var srcLiveness *corev1.Probe + for i := range flux.Spec.Template.Spec.Containers { + if flux.Spec.Template.Spec.Containers[i].Name == "helm-controller" { + srcLiveness = flux.Spec.Template.Spec.Containers[i].LivenessProbe.DeepCopy() + } + } + + dep, err := BuildShardDeployment(flux, 2, cfg) if err != nil { t.Fatal(err) } @@ -188,6 +223,11 @@ func TestBuildShardDeployment(t *testing.T) { // A non-hostNetwork pod dialing the node-local KubePrism endpoint // crashloops on "dial tcp [::1]:7445: connect: connection refused". t.Fatalf("node-local apiserver endpoint env leaked through: %s", e.Name) + case "HTTP_PROXY", "HTTPS_PROXY", "NO_PROXY", + "http_proxy", "https_proxy", "no_proxy": + // A standalone shard behind an unreachable proxy hangs at startup + // and crashloops before it ever serves /healthz. + t.Fatalf("corporate-proxy env leaked through: %s", e.Name) } } envNames := make([]string, 0, len(hc.Env)) @@ -201,6 +241,42 @@ func TestBuildShardDeployment(t *testing.T) { if hc.Resources.Limits.Memory().String() != "1Gi" { t.Fatalf("resources override not applied: %v", hc.Resources) } + + if hc.StartupProbe == nil { + t.Fatal("startupProbe must be added so a slow start is not liveness-killed into a crashloop") + } + // Handler inherited from liveness verbatim. + if hc.StartupProbe.HTTPGet == nil || hc.StartupProbe.HTTPGet.Path != "/healthz" { + t.Fatalf("startupProbe must reuse the liveness /healthz handler: %+v", hc.StartupProbe) + } + // Assert the complete normalised startup budget, not just FailureThreshold: + // a threshold check alone still passes if PeriodSeconds later regresses to 1, + // which would silently restore a short crashloop window. + if hc.StartupProbe.InitialDelaySeconds != 0 || + hc.StartupProbe.PeriodSeconds != 10 || + hc.StartupProbe.SuccessThreshold != 1 || + hc.StartupProbe.FailureThreshold != 30 { + t.Fatalf("startupProbe budget not normalised to the expected contract "+ + "(delay=0 period=10 success=1 failure=30): %+v", hc.StartupProbe) + } + // TimeoutSeconds is inherited from the source liveness probe, never forced. + // flux-aio ships a bare probe (TimeoutSeconds 0), so a stray + // sp.TimeoutSeconds = N would diverge from the source and fail here. + if hc.StartupProbe.TimeoutSeconds != srcLiveness.TimeoutSeconds { + t.Fatalf("startupProbe must inherit the liveness TimeoutSeconds (%d), got %d", + srcLiveness.TimeoutSeconds, hc.StartupProbe.TimeoutSeconds) + } + // The startupProbe must be a DeepCopy of the liveness probe, never an alias. + // If it aliased, normalising the startup FailureThreshold to 30 would also + // stamp 30 onto liveness, producing exactly the never-failing liveness probe + // this change exists to avoid. + if hc.LivenessProbe == hc.StartupProbe { + t.Fatal("startupProbe must be a DeepCopy of liveness, not an alias sharing its backing probe") + } + if hc.LivenessProbe.FailureThreshold != srcLiveness.FailureThreshold { + t.Fatalf("liveness FailureThreshold was clobbered by the startup budget "+ + "(alias regression): source=%d live=%d", srcLiveness.FailureThreshold, hc.LivenessProbe.FailureThreshold) + } } func TestBuildShardDeploymentInheritsResourcesWhenUnset(t *testing.T) { diff --git a/packages/apps/clickhouse/images/altinity-clickhouse-backup.tag b/packages/apps/clickhouse/images/altinity-clickhouse-backup.tag index 73c1764be0..0a693fdc87 100644 --- a/packages/apps/clickhouse/images/altinity-clickhouse-backup.tag +++ b/packages/apps/clickhouse/images/altinity-clickhouse-backup.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/altinity-clickhouse-backup:v1.6.0@sha256:e890d78cd83968bdff9a36c9ca4312afc05e138af486e476da0a935f81f9b7a7 +ghcr.io/cozystack/cozystack/altinity-clickhouse-backup:v1.6.2@sha256:e890d78cd83968bdff9a36c9ca4312afc05e138af486e476da0a935f81f9b7a7 diff --git a/packages/apps/clickhouse/images/clickhouse-backup.tag b/packages/apps/clickhouse/images/clickhouse-backup.tag index 2ba1bf36f4..216a99c319 100644 --- a/packages/apps/clickhouse/images/clickhouse-backup.tag +++ b/packages/apps/clickhouse/images/clickhouse-backup.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/clickhouse-backup:v1.6.0@sha256:8efef62d59fb82d39544823f81a797c2b9ac130a50241fee48e701161ce2670d +ghcr.io/cozystack/cozystack/clickhouse-backup:v1.6.2@sha256:8efef62d59fb82d39544823f81a797c2b9ac130a50241fee48e701161ce2670d diff --git a/packages/apps/http-cache/images/nginx-cache.tag b/packages/apps/http-cache/images/nginx-cache.tag index 8f54917c2f..d25e1e3445 100644 --- a/packages/apps/http-cache/images/nginx-cache.tag +++ b/packages/apps/http-cache/images/nginx-cache.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/nginx-cache:v1.6.0@sha256:bd042a8a9789e8b21b12cb266e390484a3a48f4af6ca6e2d2864a8a83c74bd25 +ghcr.io/cozystack/cozystack/nginx-cache:v1.6.2@sha256:bd042a8a9789e8b21b12cb266e390484a3a48f4af6ca6e2d2864a8a83c74bd25 diff --git a/packages/apps/kubernetes/images/cluster-autoscaler.tag b/packages/apps/kubernetes/images/cluster-autoscaler.tag index c9a4369e23..83233f03dc 100644 --- a/packages/apps/kubernetes/images/cluster-autoscaler.tag +++ b/packages/apps/kubernetes/images/cluster-autoscaler.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/cluster-autoscaler:v1.6.0@sha256:d188f395a31e37ea09bfa91e45d7ce1606b7459192a437fa66bb760ffb5f17d2 +ghcr.io/cozystack/cozystack/cluster-autoscaler:v1.6.2@sha256:e77ff02b328e4e119439efd1edd3aa71e9325fabf10a4a84b03a87d12d90b308 diff --git a/packages/apps/kubernetes/images/kubevirt-cloud-provider.tag b/packages/apps/kubernetes/images/kubevirt-cloud-provider.tag index 4c0395201a..4629ad3a7f 100644 --- a/packages/apps/kubernetes/images/kubevirt-cloud-provider.tag +++ b/packages/apps/kubernetes/images/kubevirt-cloud-provider.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/kubevirt-cloud-provider:v1.6.0@sha256:f005b99041d191eed9dde98f9f6691ba2d0cbfa1b3b0b2fe4b32cc0fdba74acf +ghcr.io/cozystack/cozystack/kubevirt-cloud-provider:v1.6.2@sha256:f005b99041d191eed9dde98f9f6691ba2d0cbfa1b3b0b2fe4b32cc0fdba74acf diff --git a/packages/apps/kubernetes/images/kubevirt-csi-driver.tag b/packages/apps/kubernetes/images/kubevirt-csi-driver.tag index cd94f98ea7..22fea6235c 100644 --- a/packages/apps/kubernetes/images/kubevirt-csi-driver.tag +++ b/packages/apps/kubernetes/images/kubevirt-csi-driver.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/kubevirt-csi-driver:v1.6.0@sha256:d6326e343c7932ce3d375c6742b9b9745090a4a26bf5bbc5ad19f5504ea93d65 +ghcr.io/cozystack/cozystack/kubevirt-csi-driver:v1.6.2@sha256:0fe6808c5857493a0d7e98344a2bcdfc3fea34917386fc654c03328fc11afe2e diff --git a/packages/apps/kubernetes/images/talos-csr-signer.tag b/packages/apps/kubernetes/images/talos-csr-signer.tag index 48d18c892d..0e6f67d82d 100644 --- a/packages/apps/kubernetes/images/talos-csr-signer.tag +++ b/packages/apps/kubernetes/images/talos-csr-signer.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/talos-csr-signer:v1.6.0@sha256:85b41e16e5e40398b1272fe0a69de72e205532c8034d0e6a9fc4a1d510f17818 +ghcr.io/cozystack/cozystack/talos-csr-signer:v1.6.2@sha256:7051e90245cb446ce4deb857d4337e2fd69ce84436243ba60939d374defc2a07 diff --git a/packages/apps/kubernetes/templates/talos/talos-reconcile-job.yaml b/packages/apps/kubernetes/templates/talos/talos-reconcile-job.yaml index 9e2ed31448..8422f81aa4 100644 --- a/packages/apps/kubernetes/templates/talos/talos-reconcile-job.yaml +++ b/packages/apps/kubernetes/templates/talos/talos-reconcile-job.yaml @@ -485,7 +485,14 @@ spec: *management* cluster-domain, sourced from .Values._cluster (populated by cozystack-controller from the platform config). */}} {{- $mgmtClusterDomain := (index .Values._cluster "cluster-domain") | default "cozy.local" }} -{{- range $groupName, $group := .Values.nodeGroups }} +{{- /* Iterate the EFFECTIVE node groups (the kubernetes.nodeGroups helper), + never the raw .Values.nodeGroups map. The built-in md0 group lives in + the helper's else-branch, so a cluster that supplies no nodeGroups has + an empty raw map: ranging over it renders zero Jobs, leaving the md0 + MachineDeployment below pointing at a TalosConfigTemplate that only + this Job ever creates. CAPI then blocks every Machine the autoscaler + adds. Must stay in lockstep with the loop in cluster.yaml. */}} +{{- range $groupName, $group := (include "kubernetes.nodeGroups" $ | fromYaml) }} {{- /* Mirror the kubelet-reservation computation from cluster.yaml so the worker machineconfig the Job applies carries the same numbers the MachineDeployment validator was happy with. Auto-computed diff --git a/packages/apps/kubernetes/tests/talos_reconcile_nodegroups_test.yaml b/packages/apps/kubernetes/tests/talos_reconcile_nodegroups_test.yaml new file mode 100644 index 0000000000..03a6f13bcc --- /dev/null +++ b/packages/apps/kubernetes/tests/talos_reconcile_nodegroups_test.yaml @@ -0,0 +1,131 @@ +suite: talos-reconcile Job covers every effective node group + +# Regression guard for cozystack/cozystack#3504. The Job in +# templates/talos/talos-reconcile-job.yaml is the only producer of the +# TalosConfigTemplate that each worker MachineDeployment names in +# `spec.template.spec.bootstrap.configRef` (cluster.yaml renders the reference +# but not the object). Its loop must therefore iterate the same effective group +# set as the MachineDeployment loop — the kubernetes.nodeGroups helper — not the +# raw .Values.nodeGroups map. When the raw map was read, a cluster that supplied +# no nodeGroups got the helper's built-in md0 MachineDeployment and zero Jobs, so +# every Machine the autoscaler added to md0 blocked forever on a +# TalosConfigTemplate nothing would ever create, and the KamajiControlPlane +# certSANs patch the same Job performs never ran either. +# +# The two cases below pin both halves of the helper contract: the default group +# gets a Job (the bug), and a user-supplied group set stays authoritative — md0 +# must NOT reappear alongside it, which is the removability #2936 introduced. +# +# Suite-level `templates` is load-bearing: it limits the render to this one +# template. Declaring `templates` per test does not filter the render, and the +# unrelated `lookup` in templates/helmreleases/csi.yaml then fails the suite. + +templates: + - templates/talos/talos-reconcile-job.yaml + +tests: + - it: renders a Job for the built-in md0 group when nodeGroups is empty + release: + name: test-k8s + namespace: tenant-test + set: + _namespace: + etcd: etcd + ingress: nginx + host: example.com + _cluster: + cluster-domain: cozy.local + # nodeGroups intentionally left at the chart default ({}) so the helper + # emits the built-in md0 — the case that rendered no Job at all. + asserts: + # 6 scaffolding documents (ServiceAccount, Role, CiliumNetworkPolicy, + # RoleBinding, ClusterRole, ClusterRoleBinding) plus one Job for md0. + # Before the fix this was 6: the Job was missing entirely. + - hasDocuments: + count: 7 + - isKind: + of: Job + documentIndex: 6 + - matchRegex: + path: metadata.name + pattern: ^test-k8s-talos-reconcile-md0-[0-9a-f]{6}$ + documentIndex: 6 + # The Job applies TalosConfigTemplate ${RELEASE}-${GROUP_NAME}, so + # GROUP_NAME is what ties it to the MachineDeployment's configRef name + # (test-k8s-md0). A Job for the wrong group name would satisfy the + # document count above while leaving the MachineDeployment stuck. + - contains: + path: spec.template.spec.containers[0].env + content: + name: GROUP_NAME + value: md0 + documentIndex: 6 + - matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: 'kind: TalosConfigTemplate' + documentIndex: 6 + # Secondary effect of the missing Job: the KamajiControlPlane certSANs + # patch that adds the live apiserver Service ClusterIP never ran. + - matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: 'patch kamajicontrolplane' + documentIndex: 6 + + - it: user-supplied node groups stay authoritative — one Job each, no md0 + release: + name: test-k8s + namespace: tenant-test + set: + _namespace: + etcd: etcd + ingress: nginx + host: example.com + _cluster: + cluster-domain: cozy.local + nodeGroups: + worker0: + minReplicas: 1 + maxReplicas: 3 + instanceType: u1.medium + diskSize: 20Gi + storageClass: "" + roles: + - ingress-nginx + resources: {} + gpus: [] + kubelet: {} + worker1: + minReplicas: 0 + maxReplicas: 5 + instanceType: u1.medium + diskSize: 20Gi + storageClass: "" + roles: [] + resources: {} + gpus: [] + kubelet: {} + asserts: + # 6 scaffolding documents + exactly two Jobs. A fix that unconditionally + # merged md0 into the loop would render three and fail here. + - hasDocuments: + count: 8 + - matchRegex: + path: metadata.name + pattern: ^test-k8s-talos-reconcile-worker0-[0-9a-f]{6}$ + documentIndex: 6 + - matchRegex: + path: metadata.name + pattern: ^test-k8s-talos-reconcile-worker1-[0-9a-f]{6}$ + documentIndex: 7 + - contains: + path: spec.template.spec.containers[0].env + content: + name: GROUP_NAME + value: worker0 + documentIndex: 6 + - contains: + path: spec.template.spec.containers[0].env + content: + name: GROUP_NAME + value: worker1 + documentIndex: 7 diff --git a/packages/apps/mariadb/images/mariadb-backup.tag b/packages/apps/mariadb/images/mariadb-backup.tag index 3dafe6c5e1..cc10075498 100644 --- a/packages/apps/mariadb/images/mariadb-backup.tag +++ b/packages/apps/mariadb/images/mariadb-backup.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/mariadb-backup:v1.6.0@sha256:1de944b5c4fbeef94004b8620b0865aa7886a692dacb23d50c8f9b31fbeece07 +ghcr.io/cozystack/cozystack/mariadb-backup:v1.6.2@sha256:1de944b5c4fbeef94004b8620b0865aa7886a692dacb23d50c8f9b31fbeece07 diff --git a/packages/apps/postgres/templates/db.yaml b/packages/apps/postgres/templates/db.yaml index a59b137d15..b03c322f6a 100644 --- a/packages/apps/postgres/templates/db.yaml +++ b/packages/apps/postgres/templates/db.yaml @@ -262,6 +262,9 @@ spec: {{- if .Values.backup.retentionPolicy }} retentionPolicy: {{ .Values.backup.retentionPolicy | quote }} {{- end }} + # Pin the barman-cloud sidecar's S3 request checksum to when_required + # (rationale in cozy-lib.barman.checksumSidecarConfiguration). + {{- include "cozy-lib.barman.checksumSidecarConfiguration" . | nindent 2 }} configuration: destinationPath: {{ $destinationPath }} endpointURL: {{ $endpointURL }} @@ -288,6 +291,8 @@ kind: ObjectStore metadata: name: {{ .Release.Name }}-recovery spec: + # Same S3 request-checksum pin as the backup ObjectStore above. + {{- include "cozy-lib.barman.checksumSidecarConfiguration" . | nindent 2 }} configuration: destinationPath: {{ $destinationPath }} endpointURL: {{ $endpointURL }} diff --git a/packages/apps/postgres/tests/backup_storage_test.yaml b/packages/apps/postgres/tests/backup_storage_test.yaml index 0a25143138..72274cf4d7 100644 --- a/packages/apps/postgres/tests/backup_storage_test.yaml +++ b/packages/apps/postgres/tests/backup_storage_test.yaml @@ -97,6 +97,22 @@ tests: documentSelector: path: kind value: ObjectStore + # The barman-cloud sidecar's S3 request checksum is pinned to when_required + # (non-AWS S3 gateways reject the botocore default flexible checksum). + - equal: + path: spec.instanceSidecarConfiguration.env[0].name + value: AWS_REQUEST_CHECKSUM_CALCULATION + template: templates/db.yaml + documentSelector: + path: kind + value: ObjectStore + - equal: + path: spec.instanceSidecarConfiguration.env[0].value + value: when_required + template: templates/db.yaml + documentSelector: + path: kind + value: ObjectStore - hasDocuments: count: 1 template: templates/backup-secret.yaml @@ -204,6 +220,21 @@ tests: documentSelector: path: metadata.name value: pg-test-recovery + # The non-AWS S3 request-checksum pin is applied to the recovery ObjectStore too. + - equal: + path: spec.instanceSidecarConfiguration.env[0].name + value: AWS_REQUEST_CHECKSUM_CALCULATION + template: templates/db.yaml + documentSelector: + path: metadata.name + value: pg-test-recovery + - equal: + path: spec.instanceSidecarConfiguration.env[0].value + value: when_required + template: templates/db.yaml + documentSelector: + path: metadata.name + value: pg-test-recovery - it: "bootstrap recovery without serverName: plugin serverName defaults to oldName (native parity)" # Native CNPG defaulted an absent serverName to the externalClusters entry diff --git a/packages/core/installer/values.yaml b/packages/core/installer/values.yaml index 6817be8c0b..52a1cbb237 100644 --- a/packages/core/installer/values.yaml +++ b/packages/core/installer/values.yaml @@ -14,9 +14,9 @@ cozystackOperator: # to the value baked at build time. The Makefile sets this for the # kubectl-apply release artifacts (_out/assets/cozystack-operator-*.yaml). platformVersion: "" - image: ghcr.io/cozystack/cozystack/cozystack-operator:v1.6.0@sha256:93b305f78823b070c59ca631b213e39bd9d8a256d5c520f156a03f5f72f57a5d + image: ghcr.io/cozystack/cozystack/cozystack-operator:v1.6.2@sha256:41ea5ca9c6a7d471105920b09fc8ed68279b8ca0dbffe7f839dccd530b42e4c7 platformSourceUrl: 'oci://ghcr.io/cozystack/cozystack/cozystack-packages' - platformSourceRef: 'digest=sha256:bf68208730860fa8e47f378a0260e79a0262e0783fa63ba4f7f97f57a48b2d23' + platformSourceRef: 'digest=sha256:251626e80efad3e87561b138e03301d8b63c15a42895359716aa18802b04fed5' # When non-empty, overrides the operator's --helmrelease-interval flag # (operator default: 5m). E2E sets this to 30s; production should leave empty. helmReleaseInterval: "" diff --git a/packages/core/platform/sources/velero.yaml b/packages/core/platform/sources/velero.yaml index 2374e84733..acfb4d2bdd 100644 --- a/packages/core/platform/sources/velero.yaml +++ b/packages/core/platform/sources/velero.yaml @@ -21,3 +21,4 @@ spec: privileged: true namespace: cozy-velero releaseName: velero + upgradeCRDs: CreateReplace diff --git a/packages/core/platform/templates/bundles/iaas.yaml b/packages/core/platform/templates/bundles/iaas.yaml index f2c2c8b728..25215c321a 100644 --- a/packages/core/platform/templates/bundles/iaas.yaml +++ b/packages/core/platform/templates/bundles/iaas.yaml @@ -157,7 +157,6 @@ {{include "cozystack.platform.package.default" (list "cozystack.capi-provider-infra-kubevirt" $) }} {{include "cozystack.platform.package.default" (list "cozystack.bucket-application" $) }} {{include "cozystack.platform.package" (list "cozystack.kubernetes-application" "kubevirt" $) }} -{{include "cozystack.platform.package" (list "cozystack.kubernetes-nodes-application" "kubevirt" $) }} {{include "cozystack.platform.package" (list "cozystack.virtualprivatecloud-application" "kubevirt" $) }} {{include "cozystack.platform.package" (list "cozystack.vm-disk-application" "kubevirt" $) }} {{include "cozystack.platform.package" (list "cozystack.vm-instance-application" "kubevirt" $) }} diff --git a/packages/core/platform/tests/sources_velero_crds_test.yaml b/packages/core/platform/tests/sources_velero_crds_test.yaml new file mode 100644 index 0000000000..a48ad53c87 --- /dev/null +++ b/packages/core/platform/tests/sources_velero_crds_test.yaml @@ -0,0 +1,22 @@ +suite: velero applies CRD updates on upgrade +# velero evolves its CRD schema between versions (v1.18 adds the Queued and +# ReadyToStart backup phases). Helm never upgrades CRDs shipped in crds/, and +# the velero package disables the chart's upgrade-crds Job, so the PackageSource +# must opt into CreateReplace; without it the server binary upgrades while the +# live CRDs stay at first-install state, the apiserver rejects the new phases, +# and backups sit in New while the HelmRelease stays green. Pin the policy so an +# upgrade cannot silently regress. +templates: + - templates/sources.yaml +release: + name: cozystack + namespace: cozy-system +tests: + - it: velero PackageSource sets upgradeCRDs to CreateReplace + documentSelector: + path: metadata.name + value: cozystack.velero + asserts: + - equal: + path: spec.variants[0].components[0].install.upgradeCRDs + value: CreateReplace diff --git a/packages/core/platform/values.yaml b/packages/core/platform/values.yaml index 69c0de5596..4c01da3403 100644 --- a/packages/core/platform/values.yaml +++ b/packages/core/platform/values.yaml @@ -13,7 +13,7 @@ migrations: # below therefore lags targetVersion in-tree by design; run-migrations.sh # refuses to advance past a migration file missing from the image (exit 1), so # a stale pin fails loudly rather than silently skipping the etcd adoption. - image: ghcr.io/cozystack/cozystack/platform-migrations:v1.6.0@sha256:6777042e3e9c6bad76e7d96fb64baa115061975f39061cb3ede84e21d0e2213f + image: ghcr.io/cozystack/cozystack/platform-migrations:v1.6.2@sha256:a34e7156a52dc55b27662917e5f9d5bbea671188d8db233296d0a98c800dc6c6 targetVersion: 54 # Adopt legacy etcd clusters onto the v1alpha2 operator WITHOUT the mandatory # pre-adoption safety snapshot (migration 50). Leave false unless migration 50 diff --git a/packages/core/testing/values.yaml b/packages/core/testing/values.yaml index 7e3c1739ae..ffc3cb2818 100644 --- a/packages/core/testing/values.yaml +++ b/packages/core/testing/values.yaml @@ -1,2 +1,2 @@ e2e: - image: ghcr.io/cozystack/cozystack/e2e-sandbox:v1.6.0@sha256:a9980e8c48d6e50ed2b5d8245a4e0aae00970777b6e2255577fdb66035a44c68 + image: ghcr.io/cozystack/cozystack/e2e-sandbox:v1.6.2@sha256:6f9f059305b057dce4dc0c68e82b86fe5573ecae40795f997b70db2f7e16f82d diff --git a/packages/extra/bootbox/images/matchbox.tag b/packages/extra/bootbox/images/matchbox.tag index 5d086145d8..b7f6101120 100644 --- a/packages/extra/bootbox/images/matchbox.tag +++ b/packages/extra/bootbox/images/matchbox.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/matchbox:v1.6.0@sha256:7805917462ad1288272a6324c0f32a4f9a250e1723a2759f7afebce181400f96 +ghcr.io/cozystack/cozystack/matchbox:v1.6.2@sha256:d3db5e7b5eb146447c3be22845bc49e7322641a617d60dcd7803f502f0e93bb7 diff --git a/packages/extra/seaweedfs/images/objectstorage-sidecar.tag b/packages/extra/seaweedfs/images/objectstorage-sidecar.tag index a3cd453be9..c9e15b1c33 100644 --- a/packages/extra/seaweedfs/images/objectstorage-sidecar.tag +++ b/packages/extra/seaweedfs/images/objectstorage-sidecar.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/objectstorage-sidecar:v1.6.0@sha256:4d665becd3399c2fe1e8a66e15d14d16e4a5a75395fb0ae65cff46679a8989f1 +ghcr.io/cozystack/cozystack/objectstorage-sidecar:v1.6.2@sha256:ef3c154c1a6dd1ac9fa1b0763a2bd23707d68f2f16f16a97ebf331f3646b53ce diff --git a/packages/library/cozy-lib/templates/_barman.tpl b/packages/library/cozy-lib/templates/_barman.tpl new file mode 100644 index 0000000000..e398598feb --- /dev/null +++ b/packages/library/cozy-lib/templates/_barman.tpl @@ -0,0 +1,24 @@ +{{- /* +cozy-lib.barman.checksumSidecarConfiguration renders a barman-cloud ObjectStore +`spec.instanceSidecarConfiguration` that pins the sidecar's boto3 request-checksum +policy to when_required. + +Since botocore ~1.36 (early 2025) the default RequestChecksumCalculation is +when_supported, which attaches a flexible checksum (the x-amz-content-sha256 +header) to every PutObject. Non-AWS S3-compatible backends (Ceph RADOS Gateway, +the platform's own SeaweedFS system bucket, some MinIO/Cloudflare R2 builds) +reject it with "InvalidArgument: x-amz-content-sha256 must be UNSIGNED-PAYLOAD, +...", so every backup/WAL-archive upload fails against them. when_required +computes a checksum only when the operation mandates one; AWS S3 accepts that on +a plain PutObject too, so it is a safe default everywhere. + +Emit under an ObjectStore `spec:` with `{{- include "cozy-lib.barman.checksumSidecarConfiguration" . | nindent 2 }}`. +The Go-driven platform (useSystemBucket=true) ObjectStore sets the same env in +internal/backupcontroller/cnpgstrategy_controller.go (barmanSidecarConfiguration). +*/ -}} +{{- define "cozy-lib.barman.checksumSidecarConfiguration" -}} +instanceSidecarConfiguration: + env: + - name: AWS_REQUEST_CHECKSUM_CALCULATION + value: when_required +{{- end -}} diff --git a/packages/system/backup-controller/values.yaml b/packages/system/backup-controller/values.yaml index a62272f454..483d9eb5e5 100644 --- a/packages/system/backup-controller/values.yaml +++ b/packages/system/backup-controller/values.yaml @@ -1,5 +1,5 @@ backupController: - image: "ghcr.io/cozystack/cozystack/backup-controller:v1.6.0@sha256:fc4571354413d86ed11fa38dee8fd3d2d879e11c48d7d92c28630b2d54a2bcbb" + image: "ghcr.io/cozystack/cozystack/backup-controller:v1.6.2@sha256:0774d068befce98dd067cb6239e5d37cb0b03b3d17a429e80caa46787d7f3726" replicas: 2 debug: false metrics: diff --git a/packages/system/backupstrategy-controller/templates/_helpers.tpl b/packages/system/backupstrategy-controller/templates/_helpers.tpl index 0739e420dd..a9f8165b80 100644 --- a/packages/system/backupstrategy-controller/templates/_helpers.tpl +++ b/packages/system/backupstrategy-controller/templates/_helpers.tpl @@ -27,9 +27,22 @@ cozystack.bucket-application + cozystack.objectstorage-controller ensures the controllers exist before this chart installs, but the BucketClaim status is reconciled asynchronously, so the first render - sees an unpopulated status and skips. Flux re-renders the - HelmRelease on its next reconcile (spec.interval); once COSI has - populated status.bucketName, the gated templates materialise. + sees an unpopulated status and skips. + + The render CANNOT fail here instead: this chart is the producer of + the very Bucket the lookup reads, so a failed render would never + apply templates/bucket.yaml and the condition could never resolve. + + Nor does Flux repair the skip on its own. helm-controller does not + re-render a release whose chart and values did not change — the + interval reconcile is a no-op for a healthy release, and drift + detection is off on operator-generated HelmReleases — so the skip is + PERMANENT, not a bootstrap window. Convergence is driven instead by + the controller's DefaultObjectsGate + (internal/backupcontroller/default_objects_gate.go), which forces a + real Helm upgrade (reconcile.fluxcd.io/forceAt + requestedAt) once + the bucket name is resolvable and any gated object is missing. See + backupStorage.reconcileDefaultObjects in values.yaml. - bucketNameOverride set → bypass the lookup and use it directly. This is the escape hatch for offline `helm template` / `--dry-run` renders (CI / local diffs), where lookup returns nil and no apiserver is @@ -53,11 +66,13 @@ {{- end -}} {{/* When neither path produces a value, emit the empty string. Strategy/BSL templates that include this helper must gate - themselves on a non-empty result and skip rendering until Flux - re-reconciles the HelmRelease (driven by spec.interval) once the - BucketClaim's COSI-assigned status.bucketName is populated. The - accompanying templates/bucket.yaml ALWAYS renders so the - BucketClaim CAN come into existence even on the first install. */}} + themselves on a non-empty result and skip rendering until a real + Helm upgrade re-runs this lookup with the BucketClaim's + COSI-assigned status.bucketName populated. That upgrade is forced by + the controller's DefaultObjectsGate — it does NOT happen on the + HelmRelease's interval reconcile. The accompanying + templates/bucket.yaml ALWAYS renders so the BucketClaim CAN come + into existence even on the first install. */}} {{- end -}} {{- end -}} diff --git a/packages/system/backupstrategy-controller/templates/backupclass-default.yaml b/packages/system/backupstrategy-controller/templates/backupclass-default.yaml index f635c69c89..366b34ad36 100644 --- a/packages/system/backupstrategy-controller/templates/backupclass-default.yaml +++ b/packages/system/backupstrategy-controller/templates/backupclass-default.yaml @@ -6,9 +6,15 @@ from Day 1; the Strategy CRs it references stay gated on a populated $bucketName because they DO need the COSI-assigned bucket name embedded in their template. A BackupJob fired during the bootstrap - window surfaces a transient Ready=False/StrategyNotReady in the - driver and self-heals once Flux re-renders the chart with the - populated BucketClaim status. + window surfaces a transient Ready=False/StrategyNotReady in the driver. + + It does NOT self-heal on a Flux reconcile: helm-controller re-renders + only when the chart or values change, so the gated Strategy CRs would + stay absent forever. The controller's DefaultObjectsGate reads THIS + object's strategyRefs as the manifest of what must exist and forces the + real Helm upgrade that materialises them (see + backupStorage.reconcileDefaultObjects). Adding a route here therefore + also adds the referenced Strategy CR to that existence check. */}} apiVersion: backups.cozystack.io/v1alpha1 kind: BackupClass diff --git a/packages/system/backupstrategy-controller/templates/deployment.yaml b/packages/system/backupstrategy-controller/templates/deployment.yaml index 7c89f416fa..f35e97661a 100644 --- a/packages/system/backupstrategy-controller/templates/deployment.yaml +++ b/packages/system/backupstrategy-controller/templates/deployment.yaml @@ -50,6 +50,49 @@ spec: value: {{ .Values.backupStorage.forcePathStyle | quote }} - name: BACKUP_STORAGE_SYSTEM_NAMESPACES value: {{ .Values.backupStorage.systemNamespaces | join "," | quote }} + {{- if .Values.backupStorage.reconcileDefaultObjects }} + # DefaultObjectsGate: the Strategy CRs and the Velero BSL are gated + # on a `lookup` of the BucketClaim this same chart creates, so a + # fresh install renders them empty — permanently, because + # helm-controller does not re-render an unchanged, successful + # release. The gate detects the missing objects once the bucket name + # is resolvable and forces exactly one real Helm upgrade on THIS + # release, which is the only thing that re-runs the lookups. + - name: BACKUP_DEFAULT_OBJECTS_BACKUPCLASS + value: cozy-default + - name: BACKUP_DEFAULT_OBJECTS_HELMRELEASE_NAME + value: {{ .Release.Name | quote }} + - name: BACKUP_DEFAULT_OBJECTS_HELMRELEASE_NAMESPACE + value: {{ .Release.Namespace | quote }} + {{- if .Values.backupStorage.provisionBucket }} + # The credentials Secret named by BACKUP_STORAGE_SECRET_NAME is + # itself rendered behind an install-time `lookup` — of the COSI + # Secret, by packages/system/bucket/templates/user-credentials.yaml — + # so it is subject to the same permanent skip, one release earlier in + # the chain. While it is missing the projector has no source, every + # Strategy CR and the Velero BSL stay gated off, and migration 50 + # cannot resolve its snapshot target. These coordinates let the gate + # force the release that renders it. + # + # The name is the Bucket CR name carrying the bucket-rd release + # prefix (packages/system/bucket-rd/cozyrds/bucket.yaml) plus the + # "-system" suffix the parent chart appends + # (packages/apps/bucket/templates/helmrelease.yaml), which is exactly + # BACKUP_STORAGE_SECRET_NAME minus "-credentials". tests/ + # default_objects_gate_test.yaml asserts the two stay in lockstep. + # + # Omitted on provisionBucket=false (external S3): the Secret is + # admin-managed there and no release renders it. + - name: BACKUP_CREDENTIALS_HELMRELEASE_NAME + value: {{ printf "bucket-%s-system" .Values.backupStorage.bucketName | quote }} + - name: BACKUP_CREDENTIALS_HELMRELEASE_NAMESPACE + value: {{ .Values.backupStorage.namespace | quote }} + {{- end }} + {{- if .Values.velero.bslEnabled }} + - name: BACKUP_DEFAULT_OBJECTS_VELERO_NAMESPACE + value: {{ .Values.velero.namespace | quote }} + {{- end }} + {{- end }} ports: {{- if .Values.backupStrategyController.metrics.enabled }} - name: metrics diff --git a/packages/system/backupstrategy-controller/templates/rbac.yaml b/packages/system/backupstrategy-controller/templates/rbac.yaml index 24d9f8d8ad..b61c7eb566 100644 --- a/packages/system/backupstrategy-controller/templates/rbac.yaml +++ b/packages/system/backupstrategy-controller/templates/rbac.yaml @@ -53,10 +53,15 @@ rules: - apiGroups: ["cdi.kubevirt.io"] resources: ["datavolumes"] verbs: ["delete"] -# HelmReleases: pre-restore suspends HRs; post-restore rename creates new HR and deletes old +# HelmReleases: pre-restore suspends HRs; post-restore rename creates new HR +# and deletes old. patch additionally lets the DefaultObjectsGate stamp +# reconcile.fluxcd.io/forceAt + requestedAt on THIS chart's own release to +# force the Helm upgrade that materialises the lookup-gated default backup +# objects (no list/watch: every access is a point Get through the dynamic +# client, so no cluster-wide HelmRelease informer is started). - apiGroups: ["helm.toolkit.fluxcd.io"] resources: ["helmreleases"] - verbs: ["get", "create", "update", "delete"] + verbs: ["get", "create", "update", "patch", "delete"] # PVCs and PVs: pre-restore renames PVCs (delete old, create new) and patches PV reclaim policy - apiGroups: [""] resources: ["persistentvolumeclaims"] @@ -68,6 +73,13 @@ rules: - apiGroups: ["velero.io"] resources: ["backups", "restores", "datauploads"] verbs: ["create", "get", "list", "watch", "update", "patch", "delete", "deletecollection"] +# Velero BackupStorageLocation: the DefaultObjectsGate checks that the +# cozy-default BSL — rendered by the same lookup-gated template as the +# Strategy CRs — actually exists. Read-only, and a point Get through the +# dynamic client, so `get` alone is enough (no informer, no list/watch). +- apiGroups: ["velero.io"] + resources: ["backupstoragelocations"] + verbs: ["get"] # Velero DeleteBackupRequest: used by BackupReconciler to delete Velero backups with their storage data - apiGroups: ["velero.io"] resources: ["deletebackuprequests"] diff --git a/packages/system/backupstrategy-controller/tests/default_objects_gate_test.yaml b/packages/system/backupstrategy-controller/tests/default_objects_gate_test.yaml new file mode 100644 index 0000000000..976fee1667 --- /dev/null +++ b/packages/system/backupstrategy-controller/tests/default_objects_gate_test.yaml @@ -0,0 +1,156 @@ +suite: DefaultObjectsGate wiring — the release the gate forces, and its RBAC + +# The Strategy CRs and the Velero BSL are rendered behind a `lookup` of the +# BucketClaim this same chart creates, so a fresh install renders them empty +# and helm-controller never re-renders (unchanged chart + values, drift +# detection off) — the skip is permanent. The controller's DefaultObjectsGate +# repairs that by forcing a real Helm upgrade on THIS release, so the release +# coordinates it is given must be this chart's own release, and it must hold +# the verbs to do it. + +templates: + - templates/deployment.yaml + - templates/rbac.yaml + +release: + name: backupstrategy-controller + namespace: cozy-backup-controller + +tests: + - it: "gate targets this chart's own HelmRelease and the cozy-default BackupClass" + template: templates/deployment.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_DEFAULT_OBJECTS_HELMRELEASE_NAME + value: backupstrategy-controller + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_DEFAULT_OBJECTS_HELMRELEASE_NAMESPACE + value: cozy-backup-controller + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_DEFAULT_OBJECTS_BACKUPCLASS + value: cozy-default + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_DEFAULT_OBJECTS_VELERO_NAMESPACE + value: cozy-velero + + # The credentials Secret is rendered one release earlier, by the platform + # bucket's -system release, behind the same kind of install-time + # lookup. Its name is assembled from the bucket-rd release prefix and the + # "-system" suffix the parent bucket chart appends, and it must stay in + # lockstep with systemSecretName: the gate resolves the bucket name from + # that Secret and forces exactly this release when it is absent. Two + # independently written strings, so pin the relationship, not just the value. + - it: "gate targets the bucket release that renders the credentials Secret" + template: templates/deployment.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_CREDENTIALS_HELMRELEASE_NAME + value: bucket-cozy-backups-system + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_CREDENTIALS_HELMRELEASE_NAMESPACE + value: tenant-root + # BACKUP_STORAGE_SECRET_NAME is exactly the release name above with + # "-credentials" appended — that is what makes forcing this release the + # thing that creates that Secret. + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_SECRET_NAME + value: bucket-cozy-backups-system-credentials + + - it: "a renamed bucket moves both the forced release and the Secret together" + template: templates/deployment.yaml + set: + backupStorage.bucketName: platform-backups + backupStorage.systemSecretName: bucket-platform-backups-system-credentials + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_CREDENTIALS_HELMRELEASE_NAME + value: bucket-platform-backups-system + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_STORAGE_SECRET_NAME + value: bucket-platform-backups-system-credentials + + # External S3: the Secret is admin-managed and no release renders it, so + # there is nothing to force. Handing the gate stale coordinates would make + # it patch a HelmRelease that does not exist every MinForceInterval. + - it: "provisionBucket=false: no bucket release to force" + template: templates/deployment.yaml + set: + backupStorage.provisionBucket: false + asserts: + - notContains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_CREDENTIALS_HELMRELEASE_NAME + value: bucket-cozy-backups-system + - notContains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_CREDENTIALS_HELMRELEASE_NAMESPACE + value: tenant-root + + - it: "velero.bslEnabled=false: the BSL is not rendered, so the gate must not look for it" + template: templates/deployment.yaml + set: + velero.bslEnabled: false + asserts: + - notContains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_DEFAULT_OBJECTS_VELERO_NAMESPACE + value: cozy-velero + # The rest of the gate stays wired: the Strategy CRs still need it. + - contains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_DEFAULT_OBJECTS_HELMRELEASE_NAME + value: backupstrategy-controller + + - it: "reconcileDefaultObjects=false: the gate is fully unwired (main.go disables it on an empty release name)" + template: templates/deployment.yaml + set: + backupStorage.reconcileDefaultObjects: false + asserts: + - notContains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_DEFAULT_OBJECTS_HELMRELEASE_NAME + value: backupstrategy-controller + - notContains: + path: spec.template.spec.containers[0].env + content: + name: BACKUP_DEFAULT_OBJECTS_BACKUPCLASS + value: cozy-default + + - it: "ClusterRole grants patch on helmreleases (forceAt stamp) and get on backupstoragelocations" + template: templates/rbac.yaml + asserts: + - contains: + path: rules + content: + apiGroups: ["helm.toolkit.fluxcd.io"] + resources: ["helmreleases"] + verbs: ["get", "create", "update", "patch", "delete"] + - contains: + path: rules + content: + apiGroups: ["velero.io"] + resources: ["backupstoragelocations"] + verbs: ["get"] diff --git a/packages/system/backupstrategy-controller/values.yaml b/packages/system/backupstrategy-controller/values.yaml index 225a8f8d14..de005bac46 100644 --- a/packages/system/backupstrategy-controller/values.yaml +++ b/packages/system/backupstrategy-controller/values.yaml @@ -1,5 +1,5 @@ backupStrategyController: - image: "ghcr.io/cozystack/cozystack/backupstrategy-controller:v1.6.0@sha256:e8bf0521d1d1d8b90ebe66e9013010887033ba2653901799d57f2df47c9d879e" + image: "ghcr.io/cozystack/cozystack/backupstrategy-controller:v1.6.2@sha256:390caeb8cabf80dfe9bcd9f38b4ed97f0e71c22af36496fb744cdc50bb9501fc" # chBackupClientImage is the image rendered into the Altinity strategy # Pod: it drives clickhouse-backup's HTTP API via curl + jq. # @@ -35,7 +35,7 @@ backupStrategyController: # retag both to the same stable tag, and the write-once guard in # hack/promote-retag.sh fails the release. hack/image-pin-consistency.bats # asserts no repository is pinned at more than one digest. - chBackupClientImage: "ghcr.io/cozystack/cozystack/platform-migrations:v1.6.0@sha256:6777042e3e9c6bad76e7d96fb64baa115061975f39061cb3ede84e21d0e2213f" + chBackupClientImage: "ghcr.io/cozystack/cozystack/platform-migrations:v1.6.2@sha256:a34e7156a52dc55b27662917e5f9d5bbea671188d8db233296d0a98c800dc6c6" replicas: 2 debug: false metrics: @@ -118,6 +118,40 @@ backupStorage: # rendering. Tenants are projected lazily during BackupJob reconcile. systemNamespaces: - cozy-velero + # reconcileDefaultObjects enables the controller's DefaultObjectsGate. + # + # The Strategy CRs and the Velero BSL below are Helm-templated behind a + # `lookup` of the BucketClaim THIS chart creates, because the S3 bucket + # name is assigned by the COSI driver (bucket-) and cannot be + # computed at render time. On a fresh install the lookup is empty and those + # templates render nothing — and helm-controller does not re-render a + # release whose chart and values did not change (drift detection is off on + # operator-generated HelmReleases), so the skip is PERMANENT. Clusters have + # run for months with only the BackupClass present and no Strategy CRs at + # all; recovery needed a hand-stamped reconcile.fluxcd.io/forceAt + + # requestedAt, because a plain reconcile request does not re-render. + # + # The systemSecretName Secret above sits one release earlier in the same + # trap: packages/system/bucket/templates/user-credentials.yaml renders it + # behind a `lookup` of the COSI Secret, and skips it just as permanently. + # Without it the projector has no source at all, so nothing downstream can + # resolve — and neither can this gate, which reads the bucket name from it. + # + # With this enabled the controller repairs both, in order: while the + # credentials Secret is missing it forces the bucket's -system + # release; once the bucket name resolves it checks that every object + # cozy-default routes to exists and forces THIS release when any is + # missing. Both are throttled, independently, and a no-op in the steady + # state. A suspended release is skipped rather than re-stamped forever. The + # gate does NOT create the objects itself: their bodies are values-driven + # and Helm stays their single owner. + # + # Turn this off only if you manage the default Strategy CRs out of band and + # accept that a missed install-time render is never repaired. Watch + # cozystack_backup_default_objects_missing (paired with + # cozystack_backup_default_objects_check_errors_total, which marks the + # gauge stale) if you do. + reconcileDefaultObjects: true # velero — BSL rendering. The cozy-default BackupStorageLocation is # created in cozy-velero by templates/velero-bsl.yaml so endpoint/bucket/ # region come from the same backupStorage block used by Strategy CRs and @@ -129,12 +163,17 @@ backupStorage: # Velero CRDs are guaranteed to exist by the time this chart installs. # The BSL template additionally gates itself on a populated BucketClaim # status via the bucketName helper, so on a bootstrap install the BSL -# materialises only after the platform Bucket reconciles. Setting this +# materialises only after the platform Bucket reconciles AND the +# DefaultObjectsGate has forced a real Helm upgrade (an interval reconcile +# alone does not re-render). Setting this # to false is an explicit opt-out for clusters that disable VM backups -# entirely and want to avoid the cluster-default BSL — in that case the -# cozy-default-velero-vminstance / -vmdisk strategy CRs still ship but -# their `storageLocation: cozy-default` reference will not resolve, so -# VM BackupJobs against cozy-default will fail. +# entirely and want to avoid the cluster-default BSL. In that case the +# cozy-default-velero-vminstance / -vmdisk Strategy CRs are gated off the +# same flag and are NOT rendered either, so the DefaultObjectsGate does +# not expect them (it would otherwise force a Helm upgrade forever against +# a render that can never produce them). VM BackupJobs against cozy-default +# still fail, because the BackupClass keeps routing VMInstance / VMDisk to +# those absent strategies. velero: bslEnabled: true namespace: cozy-velero diff --git a/packages/system/bucket/images/s3manager.tag b/packages/system/bucket/images/s3manager.tag index b5234c0e78..0fb589ef4d 100644 --- a/packages/system/bucket/images/s3manager.tag +++ b/packages/system/bucket/images/s3manager.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/s3manager:v1.6.0@sha256:73665aaf7f5406b4a1e456d14060fe874e0ae2a436ba7fd3058fa73abff8a927 +ghcr.io/cozystack/cozystack/s3manager:v1.6.2@sha256:73665aaf7f5406b4a1e456d14060fe874e0ae2a436ba7fd3058fa73abff8a927 diff --git a/packages/system/bucket/templates/user-credentials.yaml b/packages/system/bucket/templates/user-credentials.yaml index da65cf94a4..6c2a14bd10 100644 --- a/packages/system/bucket/templates/user-credentials.yaml +++ b/packages/system/bucket/templates/user-credentials.yaml @@ -1,3 +1,39 @@ +{{/* + The human-friendly per-user credentials Secret is derived from the COSI + Secret the objectstorage sidecar writes for the matching BucketAccess + (packages/apps/bucket/templates/bucketclaim.yaml). That Secret carries a + single BucketInfo JSON document with the driver-assigned bucket name and + the S3 keys — none of it is knowable at render time, so a `lookup` is + unavoidable here. + + The skip below is NOT self-healing, contrary to what this file used to + imply. helm-controller does not re-render a release whose chart and values + did not change, and drift detection is off on operator-generated + HelmReleases, so a lookup that comes back empty at install time is skipped + PERMANENTLY: the Secret is never created, and every consumer that reads it + stays broken until somebody forces a real Helm upgrade + (reconcile.fluxcd.io/forceAt + requestedAt — a plain `flux reconcile` does + not re-render). + + Failing the render instead is tempting and wrong. Helm cannot render a + partial set, so a `fail` inside this range aborts the WHOLE + -system release — the other users' Secrets, and the bucket UI + Deployment/Service/Ingress/HTTPRoute with them. One declared user whose + BucketAccess never provisions (a misconfigured bucketAccessClassName, a + COSI failure) would then take down every other user of that bucket, and on + an already-installed release it would park it in Failed and block every + later upgrade, with no per-bucket way out. + + So the repair lives outside the chart, where it is per-object and costs a + throttled annotation patch: backupstrategy-controller's DefaultObjectsGate + (internal/backupcontroller/default_objects_gate.go) forces this release + when the platform bucket's credentials Secret is absent. That covers the + cluster-breaking case — the cozy-backups Secret feeds the credentials + projector, hence every default BackupClass strategy, Velero, and migration + 50. Tenant buckets keep the pre-existing behaviour; generalising the gate + to every bucket needs a controller that owns them and is tracked + separately. +*/}} {{- range $name, $user := .Values.users }} {{- $secretName := printf "%s-%s" $.Values.bucketName $name }} {{- $existingSecret := lookup "v1" "Secret" $.Release.Namespace $secretName }} diff --git a/packages/system/bucket/tests/user_credentials_test.yaml b/packages/system/bucket/tests/user_credentials_test.yaml new file mode 100644 index 0000000000..c0c4a16344 --- /dev/null +++ b/packages/system/bucket/tests/user_credentials_test.yaml @@ -0,0 +1,80 @@ +suite: an unresolvable user must not take the rest of the release down + +# The --credentials Secret is derived from the COSI Secret the +# objectstorage sidecar writes, which is only knowable through a `lookup`. +# helm-unittest renders with no live cluster, so lookup always returns nil — +# exactly the pre-reconcile install window these cases pin. +# +# The skip is not self-healing (helm-controller does not re-render an +# unchanged release), and the repair for the platform bucket lives in +# backupstrategy-controller's DefaultObjectsGate, which forces this release. +# +# What these cases guard is the shape of that decision: the render must stay +# partial, never fatal. A `fail` here cannot be scoped to one user — Helm +# renders all or nothing — so it would take the other users' Secrets and the +# bucket UI down with it, and park an installed release in Failed with no +# per-bucket way out. +# +# _namespace / _cluster are set in every case because helm-unittest renders +# the whole chart for each one, not only the templates it asserts on: the +# ingress template dereferences both and would abort the render otherwise. + +release: + name: bucket-cozy-backups + namespace: tenant-root + +set: &base + bucketName: bucket-cozy-backups + _namespace: + host: example.org + ingress: tenant-root + gateway: "" + _cluster: + issuer-name: letsencrypt-prod + solver: http01 + +tests: + - it: "emits no Secret while a declared user's COSI Secret does not exist yet" + templates: + - templates/user-credentials.yaml + set: + <<: *base + users: + system: + readonly: false + asserts: + - hasDocuments: + count: 0 + + - it: "no declared users: nothing to render" + templates: + - templates/user-credentials.yaml + set: + <<: *base + users: {} + asserts: + - hasDocuments: + count: 0 + + # The regression guard. With every user unresolvable — the multi-user + # partial-failure case — the rest of the release must still render. If a + # `fail` is ever reintroduced in user-credentials.yaml, Helm aborts the + # whole chart and this case fails on the aborted render, not on an + # assertion. + - it: "the bucket UI still renders when no user's COSI Secret resolves" + templates: + - templates/service.yaml + set: + <<: *base + users: + system: + readonly: false + alice: + readonly: true + asserts: + - equal: + path: kind + value: Service + - equal: + path: metadata.name + value: bucket-cozy-backups-ui diff --git a/packages/system/capi-providers-cpprovider/files/components.gz b/packages/system/capi-providers-cpprovider/files/components.gz index b96f984eb0..7fa19e3df7 100644 Binary files a/packages/system/capi-providers-cpprovider/files/components.gz and b/packages/system/capi-providers-cpprovider/files/components.gz differ diff --git a/packages/system/capi-providers-cpprovider/files/control-plane-components.yaml b/packages/system/capi-providers-cpprovider/files/control-plane-components.yaml index bfd7e9128e..a118301052 100644 --- a/packages/system/capi-providers-cpprovider/files/control-plane-components.yaml +++ b/packages/system/capi-providers-cpprovider/files/control-plane-components.yaml @@ -16292,7 +16292,7 @@ spec: - --dynamic-infrastructure-clusters=${CACPPK_INFRASTRUCTURE_CLUSTERS:= } command: - /manager - image: ghcr.io/cozystack/cozystack/cluster-api-control-plane-provider-kamaji:v0.19.0-cozystack.0@sha256:a0b8367f374432a5fa0b12018a35b0e4828466f53eb6de263d90c3408d9fd07d + image: ghcr.io/cozystack/cozystack/cluster-api-control-plane-provider-kamaji:v0.19.0-cozystack.0@sha256:e996a072eee1c74d71421ad82a8f898b38e9fe73fbe3a1308e3170399d863bc4 livenessProbe: httpGet: path: /healthz diff --git a/packages/system/capi-providers-cpprovider/images/cluster-api-control-plane-provider-kamaji.tag b/packages/system/capi-providers-cpprovider/images/cluster-api-control-plane-provider-kamaji.tag index a32ca90854..8e804938d0 100644 --- a/packages/system/capi-providers-cpprovider/images/cluster-api-control-plane-provider-kamaji.tag +++ b/packages/system/capi-providers-cpprovider/images/cluster-api-control-plane-provider-kamaji.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/cluster-api-control-plane-provider-kamaji:v0.19.0-cozystack.0@sha256:a0b8367f374432a5fa0b12018a35b0e4828466f53eb6de263d90c3408d9fd07d +ghcr.io/cozystack/cozystack/cluster-api-control-plane-provider-kamaji:v0.19.0-cozystack.0@sha256:e996a072eee1c74d71421ad82a8f898b38e9fe73fbe3a1308e3170399d863bc4 diff --git a/packages/system/cilium/values.yaml b/packages/system/cilium/values.yaml index 698eb32dff..dd60e1a88f 100644 --- a/packages/system/cilium/values.yaml +++ b/packages/system/cilium/values.yaml @@ -23,7 +23,7 @@ cilium: mode: "kubernetes" image: repository: ghcr.io/cozystack/cozystack/cilium - tag: v1.6.0 + tag: v1.6.2 digest: "sha256:136a7dfff4d6adcd448424c2354144160028a8e96eb6558b917fd3ee6d7b39fe" envoy: enabled: true diff --git a/packages/system/cozystack-api/values.yaml b/packages/system/cozystack-api/values.yaml index 3192b2fa8c..99bba399b4 100644 --- a/packages/system/cozystack-api/values.yaml +++ b/packages/system/cozystack-api/values.yaml @@ -1,3 +1,3 @@ cozystackAPI: - image: ghcr.io/cozystack/cozystack/cozystack-api:v1.6.0@sha256:c4d374a9bc5e6f6e66d24b29396f1eae9d7f06fd268ff767d05c45ca35e84e2e + image: ghcr.io/cozystack/cozystack/cozystack-api:v1.6.2@sha256:8f412e6332e96370c678c776f663ff81b813bd84ed5b4303ecbd6a47dbba9a01 replicas: 2 diff --git a/packages/system/cozystack-basics/templates/gateway-hostname-policy.yaml b/packages/system/cozystack-basics/templates/gateway-hostname-policy.yaml index 193ac9efab..f048a4fd10 100644 --- a/packages/system/cozystack-basics/templates/gateway-hostname-policy.yaml +++ b/packages/system/cozystack-basics/templates/gateway-hostname-policy.yaml @@ -1,3 +1,5 @@ +{{- /* Render only where the ValidatingAdmissionPolicy API is served (GA since Kubernetes 1.30; the management cluster requires 1.33+). Same guard as packages/core/platform/templates/deletion-protection.yaml. */}} +{{- if .Capabilities.APIVersions.Has "admissionregistration.k8s.io/v1/ValidatingAdmissionPolicy" }} --- apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingAdmissionPolicy @@ -154,3 +156,4 @@ metadata: spec: policyName: cozystack-namespace-host-label-policy validationActions: [Deny] +{{- end }} diff --git a/packages/system/cozystack-basics/templates/ingress-hostname-policy.yaml b/packages/system/cozystack-basics/templates/ingress-hostname-policy.yaml index 828a49115d..aa4b15261f 100644 --- a/packages/system/cozystack-basics/templates/ingress-hostname-policy.yaml +++ b/packages/system/cozystack-basics/templates/ingress-hostname-policy.yaml @@ -57,7 +57,8 @@ labelling yet — in which case the caller should retry. */}} {{- $rootHost := index .Values._cluster "root-host" }} -{{- if $rootHost }} +{{- /* Also gate on the ValidatingAdmissionPolicy API (GA since Kubernetes 1.30; the management cluster requires 1.33+), so a cluster without it does not get an unrenderable resource. Same guard as packages/core/platform/templates/deletion-protection.yaml. */}} +{{- if and $rootHost (.Capabilities.APIVersions.Has "admissionregistration.k8s.io/v1/ValidatingAdmissionPolicy") }} {{- $A := `namespaceObject.metadata.labels["namespace.cozystack.io/host"]` }} {{- $celValidator := printf `(namespaceObject == null || !has(namespaceObject.metadata.labels) || !("namespace.cozystack.io/host" in namespaceObject.metadata.labels)) ? false : (!has(object.spec.defaultBackend) && (!has(object.spec.rules) || object.spec.rules.all(r, has(r.host) && r.host != "" && (r.host == %s || r.host.endsWith("." + %s) || (!r.host.startsWith("*.") && !(r.host == %q || r.host.endsWith(%q)))))) && (!has(object.spec.tls) || object.spec.tls.all(t, !has(t.hosts) || t.hosts.all(h, h != "" && (h == %s || h.endsWith("." + %s) || (!h.startsWith("*.") && !(h == %q || h.endsWith(%q))))))))` $A $A $rootHost (printf ".%s" $rootHost) $A $A $rootHost (printf ".%s" $rootHost) -}} --- diff --git a/packages/system/cozystack-basics/templates/route-hostname-policy.yaml b/packages/system/cozystack-basics/templates/route-hostname-policy.yaml index cde87d2494..ec39b80942 100644 --- a/packages/system/cozystack-basics/templates/route-hostname-policy.yaml +++ b/packages/system/cozystack-basics/templates/route-hostname-policy.yaml @@ -36,6 +36,8 @@ finished labelling yet — in which case the caller should retry. */}} {{- $celValidator := `(namespaceObject == null || !has(namespaceObject.metadata.labels) || !("namespace.cozystack.io/host" in namespaceObject.metadata.labels)) ? false : (!has(object.spec.hostnames) || object.spec.hostnames.all(h, h == namespaceObject.metadata.labels["namespace.cozystack.io/host"] || h.endsWith("." + namespaceObject.metadata.labels["namespace.cozystack.io/host"])))` -}} +{{- /* Render only where the ValidatingAdmissionPolicy API is served (GA since Kubernetes 1.30; the management cluster requires 1.33+). Same guard as packages/core/platform/templates/deletion-protection.yaml. */}} +{{- if .Capabilities.APIVersions.Has "admissionregistration.k8s.io/v1/ValidatingAdmissionPolicy" }} --- apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingAdmissionPolicy @@ -104,3 +106,4 @@ metadata: spec: policyName: cozystack-route-hostname-policy-tls validationActions: [Deny] +{{- end }} diff --git a/packages/system/cozystack-basics/tests/gateway-hostname-policy_test.yaml b/packages/system/cozystack-basics/tests/gateway-hostname-policy_test.yaml index 50940dee43..80749c4813 100644 --- a/packages/system/cozystack-basics/tests/gateway-hostname-policy_test.yaml +++ b/packages/system/cozystack-basics/tests/gateway-hostname-policy_test.yaml @@ -19,6 +19,13 @@ release: # Both gates are dropped; inheritance now flows via the label # selector on Gateway.spec.listeners[].allowedRoutes. +# The template now gates on the ValidatingAdmissionPolicy API (GA since +# Kubernetes 1.30); helm-unittest's default capability set omits it, so every +# rendering test declares it, exactly as the platform deletion-protection suite. +capabilities: + apiVersions: + - admissionregistration.k8s.io/v1/ValidatingAdmissionPolicy + tests: - it: renders 3 VAPs + 3 Bindings (6 documents total) asserts: diff --git a/packages/system/cozystack-basics/tests/hostname-policies-capability-gate_test.yaml b/packages/system/cozystack-basics/tests/hostname-policies-capability-gate_test.yaml new file mode 100644 index 0000000000..0c119c63d1 --- /dev/null +++ b/packages/system/cozystack-basics/tests/hostname-policies-capability-gate_test.yaml @@ -0,0 +1,30 @@ +suite: cozystack-basics hostname VAP policies are gated on the ValidatingAdmissionPolicy API +templates: + - templates/route-hostname-policy.yaml + - templates/gateway-hostname-policy.yaml + - templates/ingress-hostname-policy.yaml +release: + name: cozystack-basics + namespace: cozy-system +# root-host is set so the ingress policy's own value gate passes; with it set, +# the only reason any of these three templates render nothing is the missing +# ValidatingAdmissionPolicy API declared below. helm-unittest's per-test +# capabilities merge rather than replace the suite's, so the absent case is a +# separate suite pinned at suite level (mirrors the platform notes gate suite). +set: + _cluster: + root-host: example.org +capabilities: + apiVersions: [] +tests: + - it: none of the hostname VAP policies render when the API is unavailable + asserts: + - hasDocuments: + count: 0 + template: templates/route-hostname-policy.yaml + - hasDocuments: + count: 0 + template: templates/gateway-hostname-policy.yaml + - hasDocuments: + count: 0 + template: templates/ingress-hostname-policy.yaml diff --git a/packages/system/cozystack-basics/tests/ingress-hostname-policy_test.yaml b/packages/system/cozystack-basics/tests/ingress-hostname-policy_test.yaml index 80afe64834..407407095a 100644 --- a/packages/system/cozystack-basics/tests/ingress-hostname-policy_test.yaml +++ b/packages/system/cozystack-basics/tests/ingress-hostname-policy_test.yaml @@ -13,6 +13,13 @@ set: _cluster: root-host: example.org +# The template now also gates on the ValidatingAdmissionPolicy API (GA since +# Kubernetes 1.30); helm-unittest's default capability set omits it, so every +# rendering test declares it, exactly as the platform deletion-protection suite. +capabilities: + apiVersions: + - admissionregistration.k8s.io/v1/ValidatingAdmissionPolicy + tests: - it: renders ValidatingAdmissionPolicy + Binding for legacy Ingress # Legacy Ingress (networking.k8s.io/v1) is the default ingress path diff --git a/packages/system/cozystack-basics/tests/route-hostname-policy_test.yaml b/packages/system/cozystack-basics/tests/route-hostname-policy_test.yaml index 36f87c7a47..16ef82a67a 100644 --- a/packages/system/cozystack-basics/tests/route-hostname-policy_test.yaml +++ b/packages/system/cozystack-basics/tests/route-hostname-policy_test.yaml @@ -6,6 +6,13 @@ release: name: cozystack-basics namespace: cozy-system +# The template now gates on the ValidatingAdmissionPolicy API (GA since +# Kubernetes 1.30); helm-unittest's default capability set omits it, so every +# rendering test declares it, exactly as the platform deletion-protection suite. +capabilities: + apiVersions: + - admissionregistration.k8s.io/v1/ValidatingAdmissionPolicy + tests: - it: renders ValidatingAdmissionPolicy + Binding for HTTPRoute and TLSRoute (Layer 7) # Layer 7 (route-hostname VAP) is the only VAP rendered by THIS diff --git a/packages/system/cozystack-controller/values.yaml b/packages/system/cozystack-controller/values.yaml index 1cac9e6c56..5219a55d1a 100644 --- a/packages/system/cozystack-controller/values.yaml +++ b/packages/system/cozystack-controller/values.yaml @@ -1,4 +1,4 @@ cozystackController: - image: ghcr.io/cozystack/cozystack/cozystack-controller:v1.6.0@sha256:b01f6fe6e8fe6426b112fbda2b4abac946b4e45cd118137c9001c9e6ee784b3d + image: ghcr.io/cozystack/cozystack/cozystack-controller:v1.6.2@sha256:fd0cc8e40293f52ea12ac8a7933dc0104a828045f9d7a73b3a38095da86c40f7 debug: false disableTelemetry: false diff --git a/packages/system/dashboard/values.yaml b/packages/system/dashboard/values.yaml index c4210359cd..427df90411 100644 --- a/packages/system/dashboard/values.yaml +++ b/packages/system/dashboard/values.yaml @@ -1,4 +1,4 @@ console: - image: ghcr.io/cozystack/cozystack/cozystack-ui:v1.6.0@sha256:ad0e132ce506bbc9e0f6a5dfb94311c9bbc08da3d8eca84eb0e527ba6c1a8897 + image: ghcr.io/cozystack/cozystack/cozystack-ui:v1.6.2@sha256:051282dbc216ed4f7b237ee93f172a5c8f9302e549805b70373c849c7dea6c7b tokenProxy: - image: ghcr.io/cozystack/cozystack/token-proxy:v1.6.0@sha256:97124c012246c33cdcac60e8f884097b48c0081624716fe8161efc2b3ceeb2a8 + image: ghcr.io/cozystack/cozystack/token-proxy:v1.6.2@sha256:97124c012246c33cdcac60e8f884097b48c0081624716fe8161efc2b3ceeb2a8 diff --git a/packages/system/etcd-operator-crds/Makefile b/packages/system/etcd-operator-crds/Makefile index e7a6e765c5..330bf9e23e 100644 --- a/packages/system/etcd-operator-crds/Makefile +++ b/packages/system/etcd-operator-crds/Makefile @@ -4,7 +4,7 @@ export NAMESPACE=cozy-etcd-operator include ../../../hack/package.mk # Pinned ref of github.com/cozystack/etcd-operator to vendor CRDs from. -ETCD_OPERATOR_REF ?= v0.5.3 +ETCD_OPERATOR_REF ?= v0.5.4 # controller-gen writes the CRDs to charts/etcd-operator/crd-bases/ upstream (there # is no config/crd kustomization). Vendor each verbatim and stamp diff --git a/packages/system/etcd-operator-crds/templates/etcdmembers.yaml b/packages/system/etcd-operator-crds/templates/etcdmembers.yaml index 5aa09d51e6..7b1d85ade8 100644 --- a/packages/system/etcd-operator-crds/templates/etcdmembers.yaml +++ b/packages/system/etcd-operator-crds/templates/etcdmembers.yaml @@ -1101,11 +1101,10 @@ spec: replicas: default: 1 description: |- - Replicas exists only because the PodDisruptionBudget controller - traverses Pods' controllerRef looking for /scale on the parent and - fails closed ("does not implement the scale subresource") if it - isn't there. Each EtcdMember represents exactly one Pod; this field - is locked to 1 by validation and cannot be tuned. + Replicas backs the /scale subresource. The operator's own PDB + (integer minAvailable) never resolves scale; kept because + maxUnavailable or percentage budgets over member Pods fail + without it. Locked to 1: an EtcdMember is exactly one Pod. format: int32 maximum: 1 minimum: 1 @@ -1677,15 +1676,14 @@ spec: replicas: description: |- Replicas exposes via /scale "this EtcdMember owns 1 Pod if it has - a PodName, 0 otherwise". Required by the PodDisruptionBudget - controller to derive expectedPods for the cluster's PDB — without - /scale on the Pod controller-ref it sets the PDB to SyncFailed. + a PodName, 0 otherwise". Unused by the operator's own PDB; + scale-resolving budgets go SyncFailed without it. format: int32 type: integer selector: description: |- - Selector exposes the label-selector that matches this member's Pod - via /scale (consumed by the PDB controller; not user-facing). + Selector exposes the label-selector matching this member's Pod via + /scale (for scale-resolving disruption budgets; not user-facing). type: string version: description: |- diff --git a/packages/system/etcd-operator/Chart.yaml b/packages/system/etcd-operator/Chart.yaml index c943505c8b..11f70e5987 100644 --- a/packages/system/etcd-operator/Chart.yaml +++ b/packages/system/etcd-operator/Chart.yaml @@ -3,4 +3,4 @@ name: cozy-etcd-operator description: Cozystack etcd-operator (etcd-operator.cozystack.io/v1alpha2) controller-manager type: application version: 0.0.0 # Placeholder, the actual version will be automatically set during the build process -appVersion: v0.5.3 +appVersion: v0.5.4 diff --git a/packages/system/etcd-operator/Makefile b/packages/system/etcd-operator/Makefile index c7d4764e85..6660aa144d 100644 --- a/packages/system/etcd-operator/Makefile +++ b/packages/system/etcd-operator/Makefile @@ -8,7 +8,7 @@ include ../../../hack/package.mk # domain, VPA), so only the RBAC role is re-vendored; refresh templates/rbac.yaml # by hand from config/rbac/role.yaml at this ref. CRDs live in the sibling # etcd-operator-crds package. -ETCD_OPERATOR_REF ?= v0.5.3 +ETCD_OPERATOR_REF ?= v0.5.4 update: @echo "etcd-operator is a cozystack-authored chart; refresh templates/rbac.yaml" diff --git a/packages/system/etcd-operator/tests/deployment_test.yaml b/packages/system/etcd-operator/tests/deployment_test.yaml index a91d207d3d..893bb8506d 100644 --- a/packages/system/etcd-operator/tests/deployment_test.yaml +++ b/packages/system/etcd-operator/tests/deployment_test.yaml @@ -20,12 +20,12 @@ tests: asserts: - equal: path: spec.template.spec.containers[0].image - value: ghcr.io/cozystack/etcd-operator:v0.5.3 + value: ghcr.io/cozystack/etcd-operator:v0.5.4 - contains: path: spec.template.spec.containers[0].env content: name: OPERATOR_IMAGE - value: ghcr.io/cozystack/etcd-operator:v0.5.3 + value: ghcr.io/cozystack/etcd-operator:v0.5.4 any: true - it: "image.tag overrides the appVersion default for both the container and the agent env" diff --git a/packages/system/flux-shard-operator/README.md b/packages/system/flux-shard-operator/README.md index 660a6542b6..9f06cb9b6a 100644 --- a/packages/system/flux-shard-operator/README.md +++ b/packages/system/flux-shard-operator/README.md @@ -6,7 +6,7 @@ Spreads tenant HelmReleases across multiple helm-controller shards so a noisy te The operator has three parts, all served by one Deployment (leader-elected controllers, webhook on every replica): -1. **Shard runtime.** Reconciles `shardCount` helm-controller Deployments (`helm-controller-shard`, `--watch-label-selector=sharding.fluxcd.io/key=shard`) in the flux namespace, cloned from the flux-aio `flux` Deployment's helm-controller container and sanitised (no host networking, no localhost cross-container wiring). The helm-controller image and feature-gates are inherited from flux-aio automatically. Deployments beyond `shardCount` are pruned once they drain, and the legacy hand-rolled `flux-tenants` Deployment is retired once no HelmRelease carries `sharding.fluxcd.io/key=tenants`. +1. **Shard runtime.** Reconciles `shardCount` helm-controller Deployments (`helm-controller-shard`, `--watch-label-selector=sharding.fluxcd.io/key=shard`) in the flux namespace, cloned from the flux-aio `flux` Deployment's helm-controller container and sanitised (no host networking, no localhost cross-container wiring, no inherited corporate-proxy env, and a startupProbe guarding a slow start). The helm-controller image and feature-gates are inherited from flux-aio automatically. Deployments beyond `shardCount` are pruned once they drain, and the legacy hand-rolled `flux-tenants` Deployment is retired once no HelmRelease carries `sharding.fluxcd.io/key=tenants`. 2. **Placement controller.** Owns the tenant→shard assignment. The unit of placement is the tenant: all HelmReleases of one tenant (parent `tenant-` plus everything in namespace `tenant-`) carry the same shard label, so a noisy tenant's blast radius is bounded to its shard's co-residents. Tenants are distributed greedy least-loaded, weighted by HelmRelease count (N tenants over N shards land exactly 1 per shard). The assignment is recorded as the `internal.cozystack.io/flux-shard` label on the tenant namespace; HelmRelease labels remain the source of truth on restarts. Moves are paced and deleting tenants are never moved. Watches are metadata-only, so the controller does not decode the helm-controller status-patch firehose. diff --git a/packages/system/flux-shard-operator/values.yaml b/packages/system/flux-shard-operator/values.yaml index eb5bc5f122..cf89e440bd 100644 --- a/packages/system/flux-shard-operator/values.yaml +++ b/packages/system/flux-shard-operator/values.yaml @@ -1,5 +1,5 @@ fluxShardOperator: - image: ghcr.io/cozystack/cozystack/flux-shard-operator:v1.6.0@sha256:daf002af34671371646185d4135c5096f8aba9518f132089bd5d6fbe7d9d06e7 + image: ghcr.io/cozystack/cozystack/flux-shard-operator:v1.6.2@sha256:b59ec5d228575a0aee678b95b352270acf6020fb80caf9636ed690225b7858d7 debug: false replicas: 2 ## Number of helm-controller shards to provision and distribute tenants diff --git a/packages/system/grafana-operator/images/grafana-dashboards.tag b/packages/system/grafana-operator/images/grafana-dashboards.tag index 412303f2be..337aa6b017 100644 --- a/packages/system/grafana-operator/images/grafana-dashboards.tag +++ b/packages/system/grafana-operator/images/grafana-dashboards.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/grafana-dashboards:v1.6.0@sha256:cb22416b34d4385e4df58314de390920af805c03358fe00734c8fa44ba505fe9 +ghcr.io/cozystack/cozystack/grafana-dashboards:v1.6.2@sha256:cf8880fc5615e8c5309b31ad2624af206158958cf839d7f51dbf043ff7ceef21 diff --git a/packages/system/kamaji/values.yaml b/packages/system/kamaji/values.yaml index 7146b1328c..be97fa5c0a 100644 --- a/packages/system/kamaji/values.yaml +++ b/packages/system/kamaji/values.yaml @@ -30,7 +30,7 @@ kamaji: topologyKey: kubernetes.io/hostname image: pullPolicy: IfNotPresent - tag: v1.6.0@sha256:7f7751b3d01b256621a131c217fc90464cd51d3d07a6d870c3d8f28784745293 + tag: v1.6.2@sha256:dd81f5b76208e11665f2d2eed00396653c32182079ee53f44169074ac35d62ed repository: ghcr.io/cozystack/cozystack/kamaji resources: limits: @@ -63,4 +63,4 @@ kamaji: periodSeconds: 10 timeoutSeconds: 1 extraArgs: - - --migrate-image=ghcr.io/cozystack/cozystack/kamaji:v1.6.0@sha256:7f7751b3d01b256621a131c217fc90464cd51d3d07a6d870c3d8f28784745293 + - --migrate-image=ghcr.io/cozystack/cozystack/kamaji:v1.6.2@sha256:dd81f5b76208e11665f2d2eed00396653c32182079ee53f44169074ac35d62ed diff --git a/packages/system/keycloak-configure/templates/delete.yaml b/packages/system/keycloak-configure/templates/delete.yaml index 99abd6e67e..f499c77f6d 100644 --- a/packages/system/keycloak-configure/templates/delete.yaml +++ b/packages/system/keycloak-configure/templates/delete.yaml @@ -39,7 +39,7 @@ spec: done done - kubectl patch hr keycloak-configure -n cozy-system --type=merge -p '{"metadata":{"finalizers":[]}}' + kubectl patch hr {{ .Release.Name }} -n {{ .Release.Namespace }} --type=merge -p '{"metadata":{"finalizers":[]}}' --- diff --git a/packages/system/keycloak-configure/tests/delete_test.yaml b/packages/system/keycloak-configure/tests/delete_test.yaml new file mode 100644 index 0000000000..9e740d5b83 --- /dev/null +++ b/packages/system/keycloak-configure/tests/delete_test.yaml @@ -0,0 +1,58 @@ +suite: flux teardown job patches the HelmRelease in the release namespace + +# The pre-delete teardown Job clears the HelmRelease finalizers as its last +# step. Its RBAC (Role + RoleBinding, resourceNames: [.Release.Name]) is created +# in .Release.Namespace, so the kubectl patch MUST target that same namespace and +# release name. A hardcoded namespace that differs from the install namespace +# makes the ServiceAccount Forbidden to patch the HelmRelease, the Job retries +# forever, and the Helm release wedges in "uninstalling". +release: + name: keycloak-configure + namespace: cozy-keycloak +tests: + - it: teardown patches the HelmRelease using the release name and namespace + template: templates/delete.yaml + documentSelector: + path: kind + value: Job + set: + _cluster: + root-host: example.com + kube-root-ca: "" + asserts: + - matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: kubectl patch hr keycloak-configure -n cozy-keycloak + + - it: teardown never targets a namespace other than the release namespace + template: templates/delete.yaml + documentSelector: + path: kind + value: Job + set: + _cluster: + root-host: example.com + kube-root-ca: "" + asserts: + - notMatchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: patch hr \S+ -n cozy-system + + # Prove the patch is parameterized on .Release.Name / .Release.Namespace + # rather than matching the default install namespace by coincidence. + - it: teardown tracks a non-default release name and namespace + template: templates/delete.yaml + documentSelector: + path: kind + value: Job + release: + name: custom-release + namespace: custom-ns + set: + _cluster: + root-host: example.com + kube-root-ca: "" + asserts: + - matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: kubectl patch hr custom-release -n custom-ns diff --git a/packages/system/keycloak/templates/db.yaml b/packages/system/keycloak/templates/db.yaml index 8b8b328234..26c25fa360 100644 --- a/packages/system/keycloak/templates/db.yaml +++ b/packages/system/keycloak/templates/db.yaml @@ -51,6 +51,9 @@ spec: {{- if .Values.backup.retentionPolicy }} retentionPolicy: {{ .Values.backup.retentionPolicy | quote }} {{- end }} + # Pin the barman-cloud sidecar's S3 request checksum to when_required + # (rationale in cozy-lib.barman.checksumSidecarConfiguration). + {{- include "cozy-lib.barman.checksumSidecarConfiguration" . | nindent 2 }} configuration: destinationPath: {{ .Values.backup.destinationPath | quote }} {{- with .Values.backup.endpointURL }} diff --git a/packages/system/keycloak/tests/db_backup_test.yaml b/packages/system/keycloak/tests/db_backup_test.yaml new file mode 100644 index 0000000000..052d34eff6 --- /dev/null +++ b/packages/system/keycloak/tests/db_backup_test.yaml @@ -0,0 +1,40 @@ +suite: keycloak DB backup ObjectStore (barman-cloud plugin) +templates: + - templates/db.yaml +release: + name: keycloak + namespace: cozy-keycloak +tests: + - it: "backup enabled: renders a barman-cloud ObjectStore that pins the sidecar S3 request checksum to when_required" + set: + _cluster: + root-host: example.org + cluster-domain: cozy.local + backup: + enabled: true + destinationPath: s3://aenix/keycloak-db/ + endpointURL: https://s3.frcloud.kz + existingSecretName: keycloak-db-backup-s3-creds + asserts: + # A barman-cloud ObjectStore is rendered for the Keycloak DB. + - equal: + path: metadata.name + value: keycloak-db + documentSelector: + path: kind + value: ObjectStore + # Non-AWS S3 gateways (Ceph RGW, the platform's SeaweedFS system bucket) + # reject botocore's default flexible checksum; the sidecar must request + # one only when required. + - equal: + path: spec.instanceSidecarConfiguration.env[0].name + value: AWS_REQUEST_CHECKSUM_CALCULATION + documentSelector: + path: kind + value: ObjectStore + - equal: + path: spec.instanceSidecarConfiguration.env[0].value + value: when_required + documentSelector: + path: kind + value: ObjectStore diff --git a/packages/system/kubeovn-plunger/values.yaml b/packages/system/kubeovn-plunger/values.yaml index 9610d7cfd9..8e5242c743 100644 --- a/packages/system/kubeovn-plunger/values.yaml +++ b/packages/system/kubeovn-plunger/values.yaml @@ -1,4 +1,4 @@ portSecurity: true routes: "" -image: ghcr.io/cozystack/cozystack/kubeovn-plunger:v1.6.0@sha256:0d2a42d5aa05f73e9a5bab2ffbcf20c64e2925b7b5f50d7b39b7d366a1d8f3d6 +image: ghcr.io/cozystack/cozystack/kubeovn-plunger:v1.6.2@sha256:6f38f72c16f86a5937630ba58ee42afca2fd8f14dfe45bb79670f9a4a8447b60 ovnCentralName: ovn-central diff --git a/packages/system/kubeovn-webhook/images/kubeovn-webhook/cert.go b/packages/system/kubeovn-webhook/images/kubeovn-webhook/cert.go new file mode 100644 index 0000000000..1fcbf573b4 --- /dev/null +++ b/packages/system/kubeovn-webhook/images/kubeovn-webhook/cert.go @@ -0,0 +1,45 @@ +package main + +import "crypto/tls" + +// newReloadingTLSConfig returns a tls.Config that reloads the certificate and +// key from disk on every TLS handshake, so a cert-manager renewal of the mounted +// Secret is picked up by the running process without a restart. +// +// Without this, the key pair is read exactly once at startup and cached for the +// lifetime of the process: after the certificate expires (roughly one year after +// install) the pod keeps presenting the expired certificate, the kube-apiserver's +// TLS call to the webhook fails, and because the MutatingWebhookConfiguration uses +// failurePolicy: Fail, every pod creation in tenant namespaces is rejected. +// +// tls.Config.GetCertificate is invoked on every handshake whenever Certificates is +// left unset, so re-reading the files there is enough; no watcher, mtime tracking +// or cache is needed: +// - The cert/key files are mounted from a plain Secret volume with no subPath. +// Kubernetes' atomic writer swaps the whole ..data directory via a single +// symlink flip, so a reader always sees a complete old or complete new +// generation of both files, never a torn mid-write mixture. A per-handshake +// LoadX509KeyPair therefore cannot observe a partial renewal. +// - The per-handshake cost (a few KB of file I/O plus a PEM/key parse) is +// negligible next to the asymmetric crypto the handshake already performs. +// - The webhook configures no mTLS (no ClientCAs), so GetCertificate's +// limitation of not refreshing client CA pools does not apply. +// +// If the mounted files genuinely become unreadable (Secret deleted, permissions +// broken: an operator error, not a normal renewal) the handshake fails loudly +// rather than silently serving a stale certificate. +func newReloadingTLSConfig(certFile, keyFile string) (*tls.Config, error) { + // Fail fast at startup if the initial key pair is missing or malformed. + if _, err := tls.LoadX509KeyPair(certFile, keyFile); err != nil { + return nil, err + } + return &tls.Config{ + GetCertificate: func(*tls.ClientHelloInfo) (*tls.Certificate, error) { + cert, err := tls.LoadX509KeyPair(certFile, keyFile) + if err != nil { + return nil, err + } + return &cert, nil + }, + }, nil +} diff --git a/packages/system/kubeovn-webhook/images/kubeovn-webhook/cert_test.go b/packages/system/kubeovn-webhook/images/kubeovn-webhook/cert_test.go new file mode 100644 index 0000000000..2d39fe5999 --- /dev/null +++ b/packages/system/kubeovn-webhook/images/kubeovn-webhook/cert_test.go @@ -0,0 +1,139 @@ +package main + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "math/big" + "os" + "path/filepath" + "testing" + "time" +) + +// genSelfSigned returns a PEM-encoded self-signed cert/key pair carrying the given +// serial number, so tests can tell two generations of the certificate apart. +func genSelfSigned(t *testing.T, serial int64) (certPEM, keyPEM []byte) { + t.Helper() + + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatalf("generate key: %v", err) + } + + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(serial), + Subject: pkix.Name{CommonName: "kube-ovn-webhook-test"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + DNSNames: []string{"localhost"}, + BasicConstraintsValid: true, + } + + der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) + if err != nil { + t.Fatalf("create certificate: %v", err) + } + keyDER, err := x509.MarshalPKCS8PrivateKey(key) + if err != nil { + t.Fatalf("marshal key: %v", err) + } + + certPEM = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}) + keyPEM = pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: keyDER}) + return certPEM, keyPEM +} + +// writeKeyPair writes the cert/key files and sets their mtime, mirroring how +// cert-manager replaces the mounted Secret on renewal. +func writeKeyPair(t *testing.T, certFile, keyFile string, certPEM, keyPEM []byte, mtime time.Time) { + t.Helper() + if err := os.WriteFile(certFile, certPEM, 0o600); err != nil { + t.Fatalf("write cert: %v", err) + } + if err := os.WriteFile(keyFile, keyPEM, 0o600); err != nil { + t.Fatalf("write key: %v", err) + } + setModTime(t, certFile, mtime) + setModTime(t, keyFile, mtime) +} + +func setModTime(t *testing.T, name string, mtime time.Time) { + t.Helper() + if err := os.Chtimes(name, mtime, mtime); err != nil { + t.Fatalf("chtimes %s: %v", name, err) + } +} + +// servedSerial completes a TLS handshake against addr and returns the serial number +// of the leaf certificate the server actually presented. +func servedSerial(t *testing.T, addr string) int64 { + t.Helper() + conn, err := tls.Dial("tcp", addr, &tls.Config{InsecureSkipVerify: true}) //nolint:gosec // test-only, inspecting the served cert + if err != nil { + t.Fatalf("dial: %v", err) + } + defer conn.Close() + certs := conn.ConnectionState().PeerCertificates + if len(certs) == 0 { + t.Fatalf("server presented no certificate") + } + return certs[0].SerialNumber.Int64() +} + +// TestReloadingTLSConfigServesRenewedCertificate is the core regression test: it fails +// against the old code that loads the key pair once into tls.Config.Certificates, and +// passes once the certificate is served via a reloading GetCertificate callback. +func TestReloadingTLSConfigServesRenewedCertificate(t *testing.T) { + dir := t.TempDir() + certFile := filepath.Join(dir, "tls.crt") + keyFile := filepath.Join(dir, "tls.key") + + certA, keyA := genSelfSigned(t, 1) + writeKeyPair(t, certFile, keyFile, certA, keyA, time.Now().Add(-2*time.Second)) + + tlsConfig, err := newReloadingTLSConfig(certFile, keyFile) + if err != nil { + t.Fatalf("newReloadingTLSConfig: %v", err) + } + + ln, err := tls.Listen("tcp", "127.0.0.1:0", tlsConfig) + if err != nil { + t.Fatalf("listen: %v", err) + } + defer ln.Close() + go func() { + for { + conn, err := ln.Accept() + if err != nil { + return + } + go func() { + if tc, ok := conn.(*tls.Conn); ok { + _ = tc.Handshake() + } + conn.Close() + }() + } + }() + + addr := ln.Addr().String() + + if got := servedSerial(t, addr); got != 1 { + t.Fatalf("before renewal: expected serial 1, got %d", got) + } + + // cert-manager renews the Secret: the mounted files are replaced in place. + certB, keyB := genSelfSigned(t, 2) + writeKeyPair(t, certFile, keyFile, certB, keyB, time.Now().Add(2*time.Second)) + + if got := servedSerial(t, addr); got != 2 { + t.Fatalf("after renewal: expected renewed serial 2, got %d (certificate was not reloaded)", got) + } +} diff --git a/packages/system/kubeovn-webhook/images/kubeovn-webhook/main.go b/packages/system/kubeovn-webhook/images/kubeovn-webhook/main.go index 70185961d7..7f1d8ec7ab 100644 --- a/packages/system/kubeovn-webhook/images/kubeovn-webhook/main.go +++ b/packages/system/kubeovn-webhook/images/kubeovn-webhook/main.go @@ -1,10 +1,10 @@ package main import ( - "crypto/tls" "flag" "log" "net/http" + "time" ) var ( @@ -28,17 +28,19 @@ func main() { mux := http.NewServeMux() mux.HandleFunc("/mutate-pods", HandleMutatePods) - tlsCert, err := tls.LoadX509KeyPair(tlsCertFile, tlsKeyFile) + tlsConfig, err := newReloadingTLSConfig(tlsCertFile, tlsKeyFile) if err != nil { log.Fatalf("Failed to load key pair: %v", err) } server := &http.Server{ - Addr: ":8443", - TLSConfig: &tls.Config{ - Certificates: []tls.Certificate{tlsCert}, - }, - Handler: mux, + Addr: ":8443", + TLSConfig: tlsConfig, + Handler: mux, + ReadHeaderTimeout: 10 * time.Second, + ReadTimeout: 30 * time.Second, + WriteTimeout: 30 * time.Second, + IdleTimeout: 60 * time.Second, } log.Printf("Starting webhook server on %s", server.Addr) diff --git a/packages/system/kubeovn-webhook/templates/certmanager.yaml b/packages/system/kubeovn-webhook/templates/certmanager.yaml index f8eee740e4..cb2770360a 100644 --- a/packages/system/kubeovn-webhook/templates/certmanager.yaml +++ b/packages/system/kubeovn-webhook/templates/certmanager.yaml @@ -38,7 +38,7 @@ metadata: spec: secretName: {{ include "namespace-annotation-webhook.fullname" . }}-tls duration: 8760h - renewBefore: 24h + renewBefore: 720h issuerRef: name: {{ include "namespace-annotation-webhook.fullname" . }}-ca-issuer commonName: {{ include "namespace-annotation-webhook.fullname" . }}-tls diff --git a/packages/system/kubeovn-webhook/values.yaml b/packages/system/kubeovn-webhook/values.yaml index a26fbd34ad..134fed945d 100644 --- a/packages/system/kubeovn-webhook/values.yaml +++ b/packages/system/kubeovn-webhook/values.yaml @@ -1,3 +1,3 @@ portSecurity: true routes: "" -image: ghcr.io/cozystack/cozystack/kubeovn-webhook:v1.6.0@sha256:593c324a38db59495497c5b68b90cad25dfe6753c73b7a287b3280e26f70a15f +image: ghcr.io/cozystack/cozystack/kubeovn-webhook:v1.6.2@sha256:8abc089c4469b6572d7eb6cba1bd87528dc1298101a2250d4b351d6ba0ca0620 diff --git a/packages/system/kubevirt-csi-node/values.yaml b/packages/system/kubevirt-csi-node/values.yaml index a14e51bbdd..563fd45d8a 100644 --- a/packages/system/kubevirt-csi-node/values.yaml +++ b/packages/system/kubevirt-csi-node/values.yaml @@ -9,4 +9,4 @@ storageClasses: infraStorageClass: replicated default: false csiDriver: - image: ghcr.io/cozystack/cozystack/kubevirt-csi-driver:v1.6.0@sha256:d6326e343c7932ce3d375c6742b9b9745090a4a26bf5bbc5ad19f5504ea93d65 + image: ghcr.io/cozystack/cozystack/kubevirt-csi-driver:v1.6.2@sha256:0fe6808c5857493a0d7e98344a2bcdfc3fea34917386fc654c03328fc11afe2e diff --git a/packages/system/lineage-controller-webhook/values.yaml b/packages/system/lineage-controller-webhook/values.yaml index 0fea7f3104..7c56993732 100644 --- a/packages/system/lineage-controller-webhook/values.yaml +++ b/packages/system/lineage-controller-webhook/values.yaml @@ -1,5 +1,5 @@ lineageControllerWebhook: - image: ghcr.io/cozystack/cozystack/lineage-controller-webhook:v1.6.0@sha256:3a5b4951a78d507eb847f4f3311052810dc658fd7c85c49eeb32ac5469cb39a2 + image: ghcr.io/cozystack/cozystack/lineage-controller-webhook:v1.6.2@sha256:454e3da9468e578bf627210ea1ff664cf58b1f618edeaa158f52c9a930a44794 debug: false replicas: 2 # DEPRECATED. Injects KUBERNETES_SERVICE_HOST=status.hostIP and diff --git a/packages/system/linstor-gui/values.yaml b/packages/system/linstor-gui/values.yaml index 61c12a7ebc..8f9b888933 100644 --- a/packages/system/linstor-gui/values.yaml +++ b/packages/system/linstor-gui/values.yaml @@ -3,7 +3,7 @@ ## @param image.tag LINSTOR GUI container image tag (digest recommended) image: repository: ghcr.io/cozystack/cozystack/linstor-gui - tag: v1.6.0@sha256:6e11829de86709f5e21636cdd1a1ada1239a7dcc1741f91e9d95d9f6ade603a2 + tag: v1.6.2@sha256:6e11829de86709f5e21636cdd1a1ada1239a7dcc1741f91e9d95d9f6ade603a2 ## @section Deployment ## @param replicas Number of linstor-gui replicas replicas: 1 diff --git a/packages/system/linstor/values.yaml b/packages/system/linstor/values.yaml index 2395d2fabe..ee9884e2b9 100644 --- a/packages/system/linstor/values.yaml +++ b/packages/system/linstor/values.yaml @@ -1,7 +1,7 @@ piraeusServer: image: repository: ghcr.io/cozystack/cozystack/piraeus-server - tag: v1.6.0@sha256:70ddd1a66fbc1ea58078f4645fc572ab8e2e189dbb4d9a5f9ca43d5663106483 + tag: v1.6.2@sha256:82d2234181ac8d5942e57286101b242a08fbcc49a9bf37d5df529921624fdca9 # Talos-specific workarounds (disable for generic Linux like Ubuntu/Debian) talos: enabled: true @@ -13,7 +13,7 @@ linstor: linstorCSI: image: repository: ghcr.io/cozystack/cozystack/linstor-csi - tag: v1.6.0@sha256:ae854423de62e94fb89b59d92c20da8939196e643e83b2d60d9f6915d5b0e55c + tag: v1.6.2@sha256:3228487ae861ddf02cac984289f8c377fc9e66a71fb3c387e9af9f72c1dbc78b controller: # Resource requests for the linstor-controller JVM. Requests (not limits) are # set on purpose: a CPU request guarantees CPU shares so the JVM can answer its diff --git a/packages/system/metallb/values.yaml b/packages/system/metallb/values.yaml index 60d31c6cdc..1ad3a0b295 100644 --- a/packages/system/metallb/values.yaml +++ b/packages/system/metallb/values.yaml @@ -37,11 +37,11 @@ metallb: controller: image: repository: ghcr.io/cozystack/cozystack/metallb-controller - tag: v1.6.0@sha256:cb7321c5674509048dfbea7938378b84013d1370eca5ace7f9665fc79b4a8a05 + tag: v1.6.2@sha256:9d8ba76cdb9c7c6221334ad05d706dee22b138b3e90c1fe8fc884925b7480c02 speaker: image: repository: ghcr.io/cozystack/cozystack/metallb-speaker - tag: v1.6.0@sha256:61ab7e2381f0897607b15df7cb5ba4d51a947c261c1c197fcda3b79c7a280016 + tag: v1.6.2@sha256:87df3c82d0b6ea223b26fd5d6fbba6e940c13e56418dfc1cd90863d145795be9 # The vendored metallb chart's values.yaml leaves `frr-k8s.prometheus:` as # a YAML block containing only commented examples — it parses to null. # Helm-controller's deep-merge then writes that null over the frr-k8s diff --git a/packages/system/monitoring/images/grafana.tag b/packages/system/monitoring/images/grafana.tag index 7675921997..b36fb89286 100644 --- a/packages/system/monitoring/images/grafana.tag +++ b/packages/system/monitoring/images/grafana.tag @@ -1 +1 @@ -ghcr.io/cozystack/cozystack/grafana:v1.6.0@sha256:d29c2306b96dfc47aab05e6afd1bce581c552f02d022084c11737632d8839029 +ghcr.io/cozystack/cozystack/grafana:v1.6.2@sha256:63c90beb8bf31f49820f529c74ebeb9e8be10acb02c45ac517844aecdfd1a4f2 diff --git a/packages/system/multus/templates/multus-daemonset-thick.yml b/packages/system/multus/templates/multus-daemonset-thick.yml index 11451961b6..56684711c3 100644 --- a/packages/system/multus/templates/multus-daemonset-thick.yml +++ b/packages/system/multus/templates/multus-daemonset-thick.yml @@ -156,7 +156,7 @@ spec: serviceAccountName: multus containers: - name: kube-multus - image: ghcr.io/cozystack/cozystack/multus-cni:v1.6.0@sha256:0a92e861ead179dd6b46f475fd8878d44b3cfa40ce4d7a8ff92f790d54250d25 + image: ghcr.io/cozystack/cozystack/multus-cni:v1.6.2@sha256:f94fdd68d674a1f8c8a2b45fa8fd4b478fc632e66fa147aea73a6b60b721920c command: [ "/usr/src/multus-cni/bin/multus-daemon" ] resources: requests: @@ -202,7 +202,7 @@ spec: fieldPath: spec.nodeName initContainers: - name: install-multus-binary - image: ghcr.io/cozystack/cozystack/multus-cni:v1.6.0@sha256:0a92e861ead179dd6b46f475fd8878d44b3cfa40ce4d7a8ff92f790d54250d25 + image: ghcr.io/cozystack/cozystack/multus-cni:v1.6.2@sha256:f94fdd68d674a1f8c8a2b45fa8fd4b478fc632e66fa147aea73a6b60b721920c command: - "/usr/src/multus-cni/bin/install_multus" - "-d" diff --git a/packages/system/objectstorage-controller/values.yaml b/packages/system/objectstorage-controller/values.yaml index be5263fdee..42e116a8c8 100644 --- a/packages/system/objectstorage-controller/values.yaml +++ b/packages/system/objectstorage-controller/values.yaml @@ -1,3 +1,3 @@ objectstorage: controller: - image: "ghcr.io/cozystack/cozystack/objectstorage-controller:v1.6.0@sha256:9c284ef4f2fbbf5ee3ccd75f86ec6c993dc60aa673c6df3f9bd15d21a7e8485c" + image: "ghcr.io/cozystack/cozystack/objectstorage-controller:v1.6.2@sha256:0d9ff6c2a3453fdc7e72894bbfdd3e5a292be78342c7c5352f655c976c3b7f0b" diff --git a/packages/system/postgres-operator/Makefile b/packages/system/postgres-operator/Makefile index 9b8f454597..9df330da39 100644 --- a/packages/system/postgres-operator/Makefile +++ b/packages/system/postgres-operator/Makefile @@ -12,5 +12,6 @@ update: helm repo update cnpg helm pull cnpg/cloudnative-pg --untar --untardir charts --version 0.27.1 rm -rf charts/cloudnative-pg/charts + patch --no-backup-if-mismatch -p4 < patches/cloudnative-pg-1.28.2.patch helm pull cnpg/plugin-barman-cloud --untar --untardir charts --version 0.7.0 rm -rf charts/plugin-barman-cloud/charts diff --git a/packages/system/postgres-operator/charts/cloudnative-pg/Chart.yaml b/packages/system/postgres-operator/charts/cloudnative-pg/Chart.yaml index 8ea6169ef8..62490487d8 100644 --- a/packages/system/postgres-operator/charts/cloudnative-pg/Chart.yaml +++ b/packages/system/postgres-operator/charts/cloudnative-pg/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v2 -appVersion: 1.28.1 +appVersion: 1.28.2 dependencies: - alias: monitoring condition: monitoring.grafanaDashboard.create diff --git a/packages/system/postgres-operator/charts/cloudnative-pg/templates/crds/crds.yaml b/packages/system/postgres-operator/charts/cloudnative-pg/templates/crds/crds.yaml index 7982b5e122..1b040e0bc1 100644 --- a/packages/system/postgres-operator/charts/cloudnative-pg/templates/crds/crds.yaml +++ b/packages/system/postgres-operator/charts/cloudnative-pg/templates/crds/crds.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.20.0 + controller-gen.kubebuilder.io/version: v0.20.1 helm.sh/resource-policy: keep name: backups.postgresql.cnpg.io spec: @@ -466,7 +466,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.20.0 + controller-gen.kubebuilder.io/version: v0.20.1 helm.sh/resource-policy: keep name: clusterimagecatalogs.postgresql.cnpg.io spec: @@ -515,6 +515,112 @@ spec: items: description: CatalogImage defines the image and major version properties: + extensions: + description: The configuration of the extensions to be added + items: + description: |- + ExtensionConfiguration is the configuration used to add + PostgreSQL extensions to the Cluster. + properties: + bin_path: + description: |- + A list of directories within the image to be appended to the + PostgreSQL process's `PATH` environment variable. + items: + type: string + type: array + dynamic_library_path: + description: |- + The list of directories inside the image which should be added to dynamic_library_path. + If not defined, defaults to "/lib". + items: + type: string + type: array + env: + description: |- + Env is a list of custom environment variables to be set in the + PostgreSQL process for this extension. It is the responsibility of the + cluster administrator to ensure the variables are correct for the + specific extension. Note that changes to these variables require + a manual cluster restart to take effect. + items: + description: |- + ExtensionEnvVar defines an environment variable for a specific extension + image volume. + properties: + name: + description: |- + Name of the environment variable to be injected into the + PostgreSQL process. + minLength: 1 + pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ + type: string + value: + description: |- + Value of the environment variable. CloudNativePG performs a direct + replacement of this value, with support for placeholder expansion. + The ${`image_root`} placeholder resolves to the absolute mount path + of the extension's volume (e.g., `/extensions/my-extension`). This + is particularly useful for allowing applications or libraries to + locate specific directories within the mounted image. + Unrecognized placeholders are rejected. To include a literal ${...} + in the value, escape it as $${...}. + minLength: 1 + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + extension_control_path: + description: |- + The list of directories inside the image which should be added to extension_control_path. + If not defined, defaults to "/share". + items: + type: string + type: array + image: + description: The image containing the extension. + properties: + pullPolicy: + description: |- + Policy for pulling OCI objects. Possible values are: + Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. + Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. + IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + type: string + reference: + description: |- + Required: Image or artifact reference to be used. + Behaves in the same way as pod.spec.containers[*].image. + Pull secrets will be assembled in the same way as for the container image by looking up node credentials, SA image pull secrets, and pod spec image pull secrets. + More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management to default or override + container images in workload controllers like Deployments and StatefulSets. + type: string + type: object + ld_library_path: + description: The list of directories inside the image + which should be added to ld_library_path. + items: + type: string + type: array + name: + description: The name of the extension, required + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9_]*[a-z0-9])?$ + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map image: description: The image reference type: string @@ -548,7 +654,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.20.0 + controller-gen.kubebuilder.io/version: v0.20.1 helm.sh/resource-policy: keep name: clusters.postgresql.cnpg.io spec: @@ -4913,6 +5019,13 @@ spec: ExtensionConfiguration is the configuration used to add PostgreSQL extensions to the Cluster. properties: + bin_path: + description: |- + A list of directories within the image to be appended to the + PostgreSQL process's `PATH` environment variable. + items: + type: string + type: array dynamic_library_path: description: |- The list of directories inside the image which should be added to dynamic_library_path. @@ -4920,6 +5033,45 @@ spec: items: type: string type: array + env: + description: |- + Env is a list of custom environment variables to be set in the + PostgreSQL process for this extension. It is the responsibility of the + cluster administrator to ensure the variables are correct for the + specific extension. Note that changes to these variables require + a manual cluster restart to take effect. + items: + description: |- + ExtensionEnvVar defines an environment variable for a specific extension + image volume. + properties: + name: + description: |- + Name of the environment variable to be injected into the + PostgreSQL process. + minLength: 1 + pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ + type: string + value: + description: |- + Value of the environment variable. CloudNativePG performs a direct + replacement of this value, with support for placeholder expansion. + The ${`image_root`} placeholder resolves to the absolute mount path + of the extension's volume (e.g., `/extensions/my-extension`). This + is particularly useful for allowing applications or libraries to + locate specific directories within the mounted image. + Unrecognized placeholders are rejected. To include a literal ${...} + in the value, escape it as $${...}. + minLength: 1 + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map extension_control_path: description: |- The list of directories inside the image which should be added to extension_control_path. @@ -4928,7 +5080,7 @@ spec: type: string type: array image: - description: The image containing the extension, required + description: The image containing the extension. properties: pullPolicy: description: |- @@ -4948,9 +5100,6 @@ spec: container images in workload controllers like Deployments and StatefulSets. type: string type: object - x-kubernetes-validations: - - message: An image reference is required - rule: has(self.reference) ld_library_path: description: The list of directories inside the image which should be added to ld_library_path. @@ -4963,7 +5112,6 @@ spec: pattern: ^[a-z0-9]([-a-z0-9_]*[a-z0-9])?$ type: string required: - - image - name type: object type: array @@ -7787,7 +7935,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.20.0 + controller-gen.kubebuilder.io/version: v0.20.1 helm.sh/resource-policy: keep name: databases.postgresql.cnpg.io spec: @@ -8382,7 +8530,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.20.0 + controller-gen.kubebuilder.io/version: v0.20.1 helm.sh/resource-policy: keep name: failoverquorums.postgresql.cnpg.io spec: @@ -8460,7 +8608,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.20.0 + controller-gen.kubebuilder.io/version: v0.20.1 helm.sh/resource-policy: keep name: imagecatalogs.postgresql.cnpg.io spec: @@ -8508,6 +8656,112 @@ spec: items: description: CatalogImage defines the image and major version properties: + extensions: + description: The configuration of the extensions to be added + items: + description: |- + ExtensionConfiguration is the configuration used to add + PostgreSQL extensions to the Cluster. + properties: + bin_path: + description: |- + A list of directories within the image to be appended to the + PostgreSQL process's `PATH` environment variable. + items: + type: string + type: array + dynamic_library_path: + description: |- + The list of directories inside the image which should be added to dynamic_library_path. + If not defined, defaults to "/lib". + items: + type: string + type: array + env: + description: |- + Env is a list of custom environment variables to be set in the + PostgreSQL process for this extension. It is the responsibility of the + cluster administrator to ensure the variables are correct for the + specific extension. Note that changes to these variables require + a manual cluster restart to take effect. + items: + description: |- + ExtensionEnvVar defines an environment variable for a specific extension + image volume. + properties: + name: + description: |- + Name of the environment variable to be injected into the + PostgreSQL process. + minLength: 1 + pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ + type: string + value: + description: |- + Value of the environment variable. CloudNativePG performs a direct + replacement of this value, with support for placeholder expansion. + The ${`image_root`} placeholder resolves to the absolute mount path + of the extension's volume (e.g., `/extensions/my-extension`). This + is particularly useful for allowing applications or libraries to + locate specific directories within the mounted image. + Unrecognized placeholders are rejected. To include a literal ${...} + in the value, escape it as $${...}. + minLength: 1 + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + extension_control_path: + description: |- + The list of directories inside the image which should be added to extension_control_path. + If not defined, defaults to "/share". + items: + type: string + type: array + image: + description: The image containing the extension. + properties: + pullPolicy: + description: |- + Policy for pulling OCI objects. Possible values are: + Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. + Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. + IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. + Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. + type: string + reference: + description: |- + Required: Image or artifact reference to be used. + Behaves in the same way as pod.spec.containers[*].image. + Pull secrets will be assembled in the same way as for the container image by looking up node credentials, SA image pull secrets, and pod spec image pull secrets. + More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config management to default or override + container images in workload controllers like Deployments and StatefulSets. + type: string + type: object + ld_library_path: + description: The list of directories inside the image + which should be added to ld_library_path. + items: + type: string + type: array + name: + description: The name of the extension, required + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9_]*[a-z0-9])?$ + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map image: description: The image reference type: string @@ -8541,7 +8795,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.20.0 + controller-gen.kubebuilder.io/version: v0.20.1 helm.sh/resource-policy: keep name: poolers.postgresql.cnpg.io spec: @@ -17910,7 +18164,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.20.0 + controller-gen.kubebuilder.io/version: v0.20.1 helm.sh/resource-policy: keep name: publications.postgresql.cnpg.io spec: @@ -18106,7 +18360,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.20.0 + controller-gen.kubebuilder.io/version: v0.20.1 helm.sh/resource-policy: keep name: scheduledbackups.postgresql.cnpg.io spec: @@ -18298,7 +18552,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.20.0 + controller-gen.kubebuilder.io/version: v0.20.1 helm.sh/resource-policy: keep name: subscriptions.postgresql.cnpg.io spec: diff --git a/packages/system/postgres-operator/patches/cloudnative-pg-1.28.2.patch b/packages/system/postgres-operator/patches/cloudnative-pg-1.28.2.patch new file mode 100644 index 0000000000..d535809e80 --- /dev/null +++ b/packages/system/postgres-operator/patches/cloudnative-pg-1.28.2.patch @@ -0,0 +1,414 @@ +--- a/packages/system/postgres-operator/charts/cloudnative-pg/Chart.yaml ++++ b/packages/system/postgres-operator/charts/cloudnative-pg/Chart.yaml +@@ -1,5 +1,5 @@ + apiVersion: v2 +-appVersion: 1.28.1 ++appVersion: 1.28.2 + dependencies: + - alias: monitoring + condition: monitoring.grafanaDashboard.create +--- a/packages/system/postgres-operator/charts/cloudnative-pg/templates/crds/crds.yaml ++++ b/packages/system/postgres-operator/charts/cloudnative-pg/templates/crds/crds.yaml +@@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 + kind: CustomResourceDefinition + metadata: + annotations: +- controller-gen.kubebuilder.io/version: v0.20.0 ++ controller-gen.kubebuilder.io/version: v0.20.1 + helm.sh/resource-policy: keep + name: backups.postgresql.cnpg.io + spec: +@@ -466,7 +466,7 @@ apiVersion: apiextensions.k8s.io/v1 + kind: CustomResourceDefinition + metadata: + annotations: +- controller-gen.kubebuilder.io/version: v0.20.0 ++ controller-gen.kubebuilder.io/version: v0.20.1 + helm.sh/resource-policy: keep + name: clusterimagecatalogs.postgresql.cnpg.io + spec: +@@ -515,6 +515,112 @@ spec: + items: + description: CatalogImage defines the image and major version + properties: ++ extensions: ++ description: The configuration of the extensions to be added ++ items: ++ description: |- ++ ExtensionConfiguration is the configuration used to add ++ PostgreSQL extensions to the Cluster. ++ properties: ++ bin_path: ++ description: |- ++ A list of directories within the image to be appended to the ++ PostgreSQL process's `PATH` environment variable. ++ items: ++ type: string ++ type: array ++ dynamic_library_path: ++ description: |- ++ The list of directories inside the image which should be added to dynamic_library_path. ++ If not defined, defaults to "/lib". ++ items: ++ type: string ++ type: array ++ env: ++ description: |- ++ Env is a list of custom environment variables to be set in the ++ PostgreSQL process for this extension. It is the responsibility of the ++ cluster administrator to ensure the variables are correct for the ++ specific extension. Note that changes to these variables require ++ a manual cluster restart to take effect. ++ items: ++ description: |- ++ ExtensionEnvVar defines an environment variable for a specific extension ++ image volume. ++ properties: ++ name: ++ description: |- ++ Name of the environment variable to be injected into the ++ PostgreSQL process. ++ minLength: 1 ++ pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ ++ type: string ++ value: ++ description: |- ++ Value of the environment variable. CloudNativePG performs a direct ++ replacement of this value, with support for placeholder expansion. ++ The ${`image_root`} placeholder resolves to the absolute mount path ++ of the extension's volume (e.g., `/extensions/my-extension`). This ++ is particularly useful for allowing applications or libraries to ++ locate specific directories within the mounted image. ++ Unrecognized placeholders are rejected. To include a literal ${...} ++ in the value, escape it as $${...}. ++ minLength: 1 ++ type: string ++ required: ++ - name ++ - value ++ type: object ++ type: array ++ x-kubernetes-list-map-keys: ++ - name ++ x-kubernetes-list-type: map ++ extension_control_path: ++ description: |- ++ The list of directories inside the image which should be added to extension_control_path. ++ If not defined, defaults to "/share". ++ items: ++ type: string ++ type: array ++ image: ++ description: The image containing the extension. ++ properties: ++ pullPolicy: ++ description: |- ++ Policy for pulling OCI objects. Possible values are: ++ Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. ++ Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. ++ IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. ++ Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. ++ type: string ++ reference: ++ description: |- ++ Required: Image or artifact reference to be used. ++ Behaves in the same way as pod.spec.containers[*].image. ++ Pull secrets will be assembled in the same way as for the container image by looking up node credentials, SA image pull secrets, and pod spec image pull secrets. ++ More info: https://kubernetes.io/docs/concepts/containers/images ++ This field is optional to allow higher level config management to default or override ++ container images in workload controllers like Deployments and StatefulSets. ++ type: string ++ type: object ++ ld_library_path: ++ description: The list of directories inside the image ++ which should be added to ld_library_path. ++ items: ++ type: string ++ type: array ++ name: ++ description: The name of the extension, required ++ minLength: 1 ++ pattern: ^[a-z0-9]([-a-z0-9_]*[a-z0-9])?$ ++ type: string ++ required: ++ - name ++ type: object ++ type: array ++ x-kubernetes-list-map-keys: ++ - name ++ x-kubernetes-list-type: map + image: + description: The image reference + type: string +@@ -548,7 +654,7 @@ apiVersion: apiextensions.k8s.io/v1 + kind: CustomResourceDefinition + metadata: + annotations: +- controller-gen.kubebuilder.io/version: v0.20.0 ++ controller-gen.kubebuilder.io/version: v0.20.1 + helm.sh/resource-policy: keep + name: clusters.postgresql.cnpg.io + spec: +@@ -4913,6 +5019,13 @@ spec: + ExtensionConfiguration is the configuration used to add + PostgreSQL extensions to the Cluster. + properties: ++ bin_path: ++ description: |- ++ A list of directories within the image to be appended to the ++ PostgreSQL process's `PATH` environment variable. ++ items: ++ type: string ++ type: array + dynamic_library_path: + description: |- + The list of directories inside the image which should be added to dynamic_library_path. +@@ -4920,6 +5033,45 @@ spec: + items: + type: string + type: array ++ env: ++ description: |- ++ Env is a list of custom environment variables to be set in the ++ PostgreSQL process for this extension. It is the responsibility of the ++ cluster administrator to ensure the variables are correct for the ++ specific extension. Note that changes to these variables require ++ a manual cluster restart to take effect. ++ items: ++ description: |- ++ ExtensionEnvVar defines an environment variable for a specific extension ++ image volume. ++ properties: ++ name: ++ description: |- ++ Name of the environment variable to be injected into the ++ PostgreSQL process. ++ minLength: 1 ++ pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ ++ type: string ++ value: ++ description: |- ++ Value of the environment variable. CloudNativePG performs a direct ++ replacement of this value, with support for placeholder expansion. ++ The ${`image_root`} placeholder resolves to the absolute mount path ++ of the extension's volume (e.g., `/extensions/my-extension`). This ++ is particularly useful for allowing applications or libraries to ++ locate specific directories within the mounted image. ++ Unrecognized placeholders are rejected. To include a literal ${...} ++ in the value, escape it as $${...}. ++ minLength: 1 ++ type: string ++ required: ++ - name ++ - value ++ type: object ++ type: array ++ x-kubernetes-list-map-keys: ++ - name ++ x-kubernetes-list-type: map + extension_control_path: + description: |- + The list of directories inside the image which should be added to extension_control_path. +@@ -4928,7 +5080,7 @@ spec: + type: string + type: array + image: +- description: The image containing the extension, required ++ description: The image containing the extension. + properties: + pullPolicy: + description: |- +@@ -4948,9 +5100,6 @@ spec: + container images in workload controllers like Deployments and StatefulSets. + type: string + type: object +- x-kubernetes-validations: +- - message: An image reference is required +- rule: has(self.reference) + ld_library_path: + description: The list of directories inside the image which + should be added to ld_library_path. +@@ -4963,7 +5112,6 @@ spec: + pattern: ^[a-z0-9]([-a-z0-9_]*[a-z0-9])?$ + type: string + required: +- - image + - name + type: object + type: array +@@ -7787,7 +7935,7 @@ apiVersion: apiextensions.k8s.io/v1 + kind: CustomResourceDefinition + metadata: + annotations: +- controller-gen.kubebuilder.io/version: v0.20.0 ++ controller-gen.kubebuilder.io/version: v0.20.1 + helm.sh/resource-policy: keep + name: databases.postgresql.cnpg.io + spec: +@@ -8382,7 +8530,7 @@ apiVersion: apiextensions.k8s.io/v1 + kind: CustomResourceDefinition + metadata: + annotations: +- controller-gen.kubebuilder.io/version: v0.20.0 ++ controller-gen.kubebuilder.io/version: v0.20.1 + helm.sh/resource-policy: keep + name: failoverquorums.postgresql.cnpg.io + spec: +@@ -8460,7 +8608,7 @@ apiVersion: apiextensions.k8s.io/v1 + kind: CustomResourceDefinition + metadata: + annotations: +- controller-gen.kubebuilder.io/version: v0.20.0 ++ controller-gen.kubebuilder.io/version: v0.20.1 + helm.sh/resource-policy: keep + name: imagecatalogs.postgresql.cnpg.io + spec: +@@ -8508,6 +8656,112 @@ spec: + items: + description: CatalogImage defines the image and major version + properties: ++ extensions: ++ description: The configuration of the extensions to be added ++ items: ++ description: |- ++ ExtensionConfiguration is the configuration used to add ++ PostgreSQL extensions to the Cluster. ++ properties: ++ bin_path: ++ description: |- ++ A list of directories within the image to be appended to the ++ PostgreSQL process's `PATH` environment variable. ++ items: ++ type: string ++ type: array ++ dynamic_library_path: ++ description: |- ++ The list of directories inside the image which should be added to dynamic_library_path. ++ If not defined, defaults to "/lib". ++ items: ++ type: string ++ type: array ++ env: ++ description: |- ++ Env is a list of custom environment variables to be set in the ++ PostgreSQL process for this extension. It is the responsibility of the ++ cluster administrator to ensure the variables are correct for the ++ specific extension. Note that changes to these variables require ++ a manual cluster restart to take effect. ++ items: ++ description: |- ++ ExtensionEnvVar defines an environment variable for a specific extension ++ image volume. ++ properties: ++ name: ++ description: |- ++ Name of the environment variable to be injected into the ++ PostgreSQL process. ++ minLength: 1 ++ pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ ++ type: string ++ value: ++ description: |- ++ Value of the environment variable. CloudNativePG performs a direct ++ replacement of this value, with support for placeholder expansion. ++ The ${`image_root`} placeholder resolves to the absolute mount path ++ of the extension's volume (e.g., `/extensions/my-extension`). This ++ is particularly useful for allowing applications or libraries to ++ locate specific directories within the mounted image. ++ Unrecognized placeholders are rejected. To include a literal ${...} ++ in the value, escape it as $${...}. ++ minLength: 1 ++ type: string ++ required: ++ - name ++ - value ++ type: object ++ type: array ++ x-kubernetes-list-map-keys: ++ - name ++ x-kubernetes-list-type: map ++ extension_control_path: ++ description: |- ++ The list of directories inside the image which should be added to extension_control_path. ++ If not defined, defaults to "/share". ++ items: ++ type: string ++ type: array ++ image: ++ description: The image containing the extension. ++ properties: ++ pullPolicy: ++ description: |- ++ Policy for pulling OCI objects. Possible values are: ++ Always: the kubelet always attempts to pull the reference. Container creation will fail If the pull fails. ++ Never: the kubelet never pulls the reference and only uses a local image or artifact. Container creation will fail if the reference isn't present. ++ IfNotPresent: the kubelet pulls if the reference isn't already present on disk. Container creation will fail if the reference isn't present and the pull fails. ++ Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. ++ type: string ++ reference: ++ description: |- ++ Required: Image or artifact reference to be used. ++ Behaves in the same way as pod.spec.containers[*].image. ++ Pull secrets will be assembled in the same way as for the container image by looking up node credentials, SA image pull secrets, and pod spec image pull secrets. ++ More info: https://kubernetes.io/docs/concepts/containers/images ++ This field is optional to allow higher level config management to default or override ++ container images in workload controllers like Deployments and StatefulSets. ++ type: string ++ type: object ++ ld_library_path: ++ description: The list of directories inside the image ++ which should be added to ld_library_path. ++ items: ++ type: string ++ type: array ++ name: ++ description: The name of the extension, required ++ minLength: 1 ++ pattern: ^[a-z0-9]([-a-z0-9_]*[a-z0-9])?$ ++ type: string ++ required: ++ - name ++ type: object ++ type: array ++ x-kubernetes-list-map-keys: ++ - name ++ x-kubernetes-list-type: map + image: + description: The image reference + type: string +@@ -8541,7 +8795,7 @@ apiVersion: apiextensions.k8s.io/v1 + kind: CustomResourceDefinition + metadata: + annotations: +- controller-gen.kubebuilder.io/version: v0.20.0 ++ controller-gen.kubebuilder.io/version: v0.20.1 + helm.sh/resource-policy: keep + name: poolers.postgresql.cnpg.io + spec: +@@ -17910,7 +18164,7 @@ apiVersion: apiextensions.k8s.io/v1 + kind: CustomResourceDefinition + metadata: + annotations: +- controller-gen.kubebuilder.io/version: v0.20.0 ++ controller-gen.kubebuilder.io/version: v0.20.1 + helm.sh/resource-policy: keep + name: publications.postgresql.cnpg.io + spec: +@@ -18106,7 +18360,7 @@ apiVersion: apiextensions.k8s.io/v1 + kind: CustomResourceDefinition + metadata: + annotations: +- controller-gen.kubebuilder.io/version: v0.20.0 ++ controller-gen.kubebuilder.io/version: v0.20.1 + helm.sh/resource-policy: keep + name: scheduledbackups.postgresql.cnpg.io + spec: +@@ -18298,7 +18552,7 @@ apiVersion: apiextensions.k8s.io/v1 + kind: CustomResourceDefinition + metadata: + annotations: +- controller-gen.kubebuilder.io/version: v0.20.0 ++ controller-gen.kubebuilder.io/version: v0.20.1 + helm.sh/resource-policy: keep + name: subscriptions.postgresql.cnpg.io + spec: diff --git a/packages/system/postgres-operator/tests/cnpg-version_test.yaml b/packages/system/postgres-operator/tests/cnpg-version_test.yaml new file mode 100644 index 0000000000..eeb22077fa --- /dev/null +++ b/packages/system/postgres-operator/tests/cnpg-version_test.yaml @@ -0,0 +1,38 @@ +suite: CNPG operator and CRDs stay aligned + +templates: + - charts/cloudnative-pg/templates/deployment.yaml + - charts/cloudnative-pg/templates/rbac.yaml + - charts/cloudnative-pg/templates/config.yaml + - charts/cloudnative-pg/templates/monitoring-configmap.yaml + - charts/cloudnative-pg/templates/crds/crds.yaml + +release: + name: postgres-operator + namespace: cozy-postgres-operator + +tests: + - it: runs the CNPG 1.28.2 operator image + template: charts/cloudnative-pg/templates/deployment.yaml + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: ghcr.io/cloudnative-pg/cloudnative-pg:1.28.2 + - equal: + path: spec.template.spec.containers[0].env[0].value + value: ghcr.io/cloudnative-pg/cloudnative-pg:1.28.2 + - equal: + path: metadata.labels["app.kubernetes.io/version"] + value: 1.28.2 + + - it: ships the CNPG 1.28.2 CRDs patched in lockstep with the operator + template: charts/cloudnative-pg/templates/crds/crds.yaml + asserts: + - documentIndex: 0 + equal: + path: metadata.annotations["controller-gen.kubebuilder.io/version"] + value: v0.20.1 + - documentIndex: 0 + equal: + path: spec.versions[0].schema.openAPIV3Schema.properties.status.properties.instanceID.properties.sessionID.type + value: string diff --git a/packages/system/postgres-operator/values.yaml b/packages/system/postgres-operator/values.yaml index 1e3fa9c118..a2211c6d03 100644 --- a/packages/system/postgres-operator/values.yaml +++ b/packages/system/postgres-operator/values.yaml @@ -1,13 +1,17 @@ cloudnative-pg: crds: create: true - # Image tag intentionally left to the chart's appVersion (1.28.1). Do NOT pin - # image.tag to a version the vendored chart's CRDs don't match: an operator - # newer than its CRDs writes status fields the CRD prunes (e.g. - # status.instanceID.sessionID, added in CNPG 1.27.3), which makes the operator - # believe the instance manager restarted and fail EVERY backup cluster-wide - # ("instance manager was restarted during backup"). Bump the chart version in - # the Makefile to move the operator + CRDs together. + # The vendored chart is 0.27.1 (appVersion 1.28.1). patches/cloudnative-pg-1.28.2.patch, + # applied by `make update`, raises the operator image AND its CRDs together to 1.28.2 for the + # PVC resize-deadlock fix (cloudnative-pg#9980 / #9981), which ships in the 1.28.2 operator + # binary. Upstream publishes a chart only per minor .0/.1, so there is no 1.28.2 chart to bump + # to on the 1.28 line without jumping a minor - hence the patch keeps operator and CRDs aligned. + # + # Do NOT pin image.tag here: the tag follows the chart's appVersion, so operator and CRDs stay + # in lockstep. Pinning image.tag to a version the CRDs do not match makes the operator write + # status fields the CRD prunes (e.g. status.instanceID.sessionID, added in CNPG 1.27.3), + # failing every backup cluster-wide. To move the operator + CRDs, edit the patch (or bump the + # chart version in the Makefile). # CloudNativePG Barman Cloud Plugin (github.com/cloudnative-pg/plugin-barman-cloud). # Native spec.backup.barmanObjectStore is deprecated in CNPG 1.27 and removed in 1.29, diff --git a/packages/system/seaweedfs/values.yaml b/packages/system/seaweedfs/values.yaml index d3a545aed2..7462cc5794 100644 --- a/packages/system/seaweedfs/values.yaml +++ b/packages/system/seaweedfs/values.yaml @@ -259,7 +259,7 @@ seaweedfs: bucketClassName: "seaweedfs" region: "" sidecar: - image: "ghcr.io/cozystack/cozystack/objectstorage-sidecar:v1.6.0@sha256:4d665becd3399c2fe1e8a66e15d14d16e4a5a75395fb0ae65cff46679a8989f1" + image: "ghcr.io/cozystack/cozystack/objectstorage-sidecar:v1.6.2@sha256:ef3c154c1a6dd1ac9fa1b0763a2bd23707d68f2f16f16a97ebf331f3646b53ce" certificates: commonName: "SeaweedFS CA" ipAddresses: [] diff --git a/packages/system/securitygroup-controller/values.yaml b/packages/system/securitygroup-controller/values.yaml index a57cef7881..3cb5dcfe89 100644 --- a/packages/system/securitygroup-controller/values.yaml +++ b/packages/system/securitygroup-controller/values.yaml @@ -1,5 +1,5 @@ securityGroupController: - image: "ghcr.io/cozystack/cozystack/securitygroup-controller:v1.6.0@sha256:09ee1d8fbfacc5b4ac24d0efbc8313bb0ad0e5507a149235894ac7d4b5539232" + image: "ghcr.io/cozystack/cozystack/securitygroup-controller:v1.6.2@sha256:b1c7153a93344318ec96363c6802e20a93c559ac0586d4ea6eabd8d95baa324b" replicas: 2 debug: false resources: diff --git a/packages/system/velero/tests/velero_test.yaml b/packages/system/velero/tests/velero_test.yaml index 182bd4027c..44177de012 100644 --- a/packages/system/velero/tests/velero_test.yaml +++ b/packages/system/velero/tests/velero_test.yaml @@ -26,11 +26,10 @@ tests: path: spec.template.spec.initContainers[1].image value: quay.io/kubevirt/kubevirt-velero-plugin:v0.9.0 - # Cozystack disables the upgrade-crds Job (upgradeCRDs: false) because CRDs - # ship via the chart's crds/ directory, making the Job redundant. (The old - # kubectl-image-compatibility rationale is obsolete in velero 12.x, where the - # Job now runs the velero image natively.) Pin that invariant — it silently - # regressed once. + # Cozystack disables the upgrade-crds Job (upgradeCRDs: false): CRD lifecycle + # is owned by the generated HelmRelease (upgradeCRDs: CreateReplace on the + # velero PackageSource), making the Job redundant. Pin that invariant — it + # silently regressed once. - it: does not emit the upgrade-crds Job when upgradeCRDs is disabled template: charts/velero/templates/upgrade-crds/upgrade-crds.yaml asserts: diff --git a/packages/system/velero/values.yaml b/packages/system/velero/values.yaml index 07476feed1..a2c54e615b 100644 --- a/packages/system/velero/values.yaml +++ b/packages/system/velero/values.yaml @@ -1,8 +1,8 @@ velero: - # Disable the upgrade-crds Job: CRDs ship via the chart's crds/ directory, so - # the Job is redundant. (The old kubectl-image-compatibility rationale is - # obsolete in velero 12.x — the Job now runs the velero image natively, not a - # kubectl one — but it stays disabled because cozystack manages CRDs via crds/.) + # Disable the upgrade-crds Job: CRD lifecycle is owned by the generated + # HelmRelease instead (upgradeCRDs: CreateReplace on the velero PackageSource), + # which applies the chart's crds/ directory on install and on every upgrade, + # so the Job is redundant. upgradeCRDs: false initContainers: - name: velero-plugin-for-aws diff --git a/pkg/apis/core/v1alpha1/doc.go b/pkg/apis/core/v1alpha1/doc.go index 15b059bed7..c074ab0537 100644 --- a/pkg/apis/core/v1alpha1/doc.go +++ b/pkg/apis/core/v1alpha1/doc.go @@ -15,6 +15,7 @@ limitations under the License. */ // +k8s:openapi-gen=true +// +k8s:openapi-model-package=com.github.cozystack.cozystack.pkg.apis.core.v1alpha1 // +k8s:deepcopy-gen=package // +k8s:conversion-gen=github.com/cozystack/cozystack/pkg/apis/core // +k8s:conversion-gen=k8s.io/apiextensions-apiserver/pkg/apis/apiextensions diff --git a/pkg/apis/core/v1alpha1/model_name.go b/pkg/apis/core/v1alpha1/model_name.go new file mode 100644 index 0000000000..581dca1d2a --- /dev/null +++ b/pkg/apis/core/v1alpha1/model_name.go @@ -0,0 +1,100 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +// OpenAPIModelName pins the OpenAPI model name for the core.cozystack.io types. +// This is hand-written (not generated) on purpose. +// +// Without it, openapi-gen keys the generated definition map on the Go import +// path ("github.com/cozystack/cozystack/pkg/apis/core/v1alpha1.Option"), and +// since Kubernetes 0.35 the apiserver's DefinitionNamer.GetDefinitionName +// returns whatever name it is given verbatim instead of converting it to the +// "friendly" reversed-path form. The published document then keys the definition +// on the raw path while every $ref to it is JSON pointer escaped +// ("#/definitions/github.com~1cozystack~1…v1alpha1.OptionSpec", ~1 being a +// slash), and clients resolve a $ref by trimming "#/definitions/" without +// unescaping. The two spellings never meet, so the reference dangles and +// client-side validation fails on every resource in the group, not only the one +// named in the error: +// +// error validating data: SchemaError(…core/v1alpha1.Option.spec): +// unknown model in reference: "github.com~1cozystack~1…v1alpha1.OptionSpec" +// +// Declaring the dotted name here makes GetCanonicalTypeName, +// Scheme.ToOpenAPIDefinitionName and the generated openapi map key all agree on +// a name with no slash in it, so nothing needs escaping and every $ref resolves. +// The same agreement is what lets DefinitionNamer attach the +// x-kubernetes-group-version-kind extension, which server-side apply needs to +// resolve a kind (see the apps package, where its absence broke SSA instead). +// +// code-generator's openapi-gen could emit these via --output-model-name-file, +// but that also rewrites zz_generated.model_name.go in the read-only apimachinery +// / apiextensions module-cache packages the shared gen_openapi helper always +// passes as inputs, which fails on any consumer (including CI) that vendors deps +// from the module cache. Keeping the methods here sidesteps that. +// +// The +k8s:openapi-model-package marker in doc.go makes openapi-gen emit +// Type{}.OpenAPIModelName() for every type in this package it generates a schema +// or a reference for, so a new core.cozystack.io type without a method here +// fails the build rather than silently reintroducing a Go-path name. The +// returned string must be the dotted form Scheme.ToOpenAPIDefinitionName derives +// from the type's Go package path, which is the marker's value plus the type +// name. See https://github.com/cozystack/cozystack/issues/3806. + +func (in Option) OpenAPIModelName() string { + return "com.github.cozystack.cozystack.pkg.apis.core.v1alpha1.Option" +} + +func (in OptionItem) OpenAPIModelName() string { + return "com.github.cozystack.cozystack.pkg.apis.core.v1alpha1.OptionItem" +} + +func (in OptionList) OpenAPIModelName() string { + return "com.github.cozystack.cozystack.pkg.apis.core.v1alpha1.OptionList" +} + +func (in OptionSpec) OpenAPIModelName() string { + return "com.github.cozystack.cozystack.pkg.apis.core.v1alpha1.OptionSpec" +} + +func (in TenantModule) OpenAPIModelName() string { + return "com.github.cozystack.cozystack.pkg.apis.core.v1alpha1.TenantModule" +} + +func (in TenantModuleList) OpenAPIModelName() string { + return "com.github.cozystack.cozystack.pkg.apis.core.v1alpha1.TenantModuleList" +} + +func (in TenantModuleStatus) OpenAPIModelName() string { + return "com.github.cozystack.cozystack.pkg.apis.core.v1alpha1.TenantModuleStatus" +} + +func (in TenantNamespace) OpenAPIModelName() string { + return "com.github.cozystack.cozystack.pkg.apis.core.v1alpha1.TenantNamespace" +} + +func (in TenantNamespaceList) OpenAPIModelName() string { + return "com.github.cozystack.cozystack.pkg.apis.core.v1alpha1.TenantNamespaceList" +} + +func (in TenantSecret) OpenAPIModelName() string { + return "com.github.cozystack.cozystack.pkg.apis.core.v1alpha1.TenantSecret" +} + +func (in TenantSecretList) OpenAPIModelName() string { + return "com.github.cozystack.cozystack.pkg.apis.core.v1alpha1.TenantSecretList" +} diff --git a/pkg/apis/sdn/v1alpha1/doc.go b/pkg/apis/sdn/v1alpha1/doc.go index 53aa095976..d4f632e15e 100644 --- a/pkg/apis/sdn/v1alpha1/doc.go +++ b/pkg/apis/sdn/v1alpha1/doc.go @@ -15,6 +15,7 @@ limitations under the License. */ // +k8s:openapi-gen=true +// +k8s:openapi-model-package=com.github.cozystack.cozystack.pkg.apis.sdn.v1alpha1 // +k8s:deepcopy-gen=package // +k8s:conversion-gen=github.com/cozystack/cozystack/pkg/apis/sdn // +k8s:conversion-gen=k8s.io/apiextensions-apiserver/pkg/apis/apiextensions diff --git a/pkg/apis/sdn/v1alpha1/model_name.go b/pkg/apis/sdn/v1alpha1/model_name.go new file mode 100644 index 0000000000..8ed9eccc52 --- /dev/null +++ b/pkg/apis/sdn/v1alpha1/model_name.go @@ -0,0 +1,92 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +// OpenAPIModelName pins the OpenAPI model name for the sdn.cozystack.io types. +// This is hand-written (not generated) on purpose. +// +// Without it, openapi-gen keys the generated definition map on the Go import +// path ("github.com/cozystack/cozystack/pkg/apis/sdn/v1alpha1.SecurityGroup"), +// and since Kubernetes 0.35 the apiserver's DefinitionNamer.GetDefinitionName +// returns whatever name it is given verbatim instead of converting it to the +// "friendly" reversed-path form. The published document then keys the definition +// on the raw path while every $ref to it is JSON pointer escaped +// ("#/definitions/github.com~1cozystack~1…v1alpha1.SecurityGroupSpec", ~1 being +// a slash), and clients resolve a $ref by trimming "#/definitions/" without +// unescaping. The two spellings never meet, so the reference dangles and +// client-side validation fails on every resource in the group, not only the one +// named in the error: +// +// error validating data: SchemaError(…sdn/v1alpha1.SecurityGroup.spec): +// unknown model in reference: "github.com~1cozystack~1…v1alpha1.SecurityGroupSpec" +// +// Declaring the dotted name here makes GetCanonicalTypeName, +// Scheme.ToOpenAPIDefinitionName and the generated openapi map key all agree on +// a name with no slash in it, so nothing needs escaping and every $ref resolves. +// The same agreement is what lets DefinitionNamer attach the +// x-kubernetes-group-version-kind extension, which server-side apply needs to +// resolve a kind (see the apps package, where its absence broke SSA instead). +// +// code-generator's openapi-gen could emit these via --output-model-name-file, +// but that also rewrites zz_generated.model_name.go in the read-only apimachinery +// / apiextensions module-cache packages the shared gen_openapi helper always +// passes as inputs, which fails on any consumer (including CI) that vendors deps +// from the module cache. Keeping the methods here sidesteps that. +// +// The +k8s:openapi-model-package marker in doc.go makes openapi-gen emit +// Type{}.OpenAPIModelName() for every type in this package it generates a schema +// or a reference for, so a new sdn.cozystack.io type without a method here fails +// the build rather than silently reintroducing a Go-path name. The returned +// string must be the dotted form Scheme.ToOpenAPIDefinitionName derives from the +// type's Go package path, which is the marker's value plus the type name. See +// https://github.com/cozystack/cozystack/issues/3806. + +func (in ApplicationReference) OpenAPIModelName() string { + return "com.github.cozystack.cozystack.pkg.apis.sdn.v1alpha1.ApplicationReference" +} + +func (in EgressRule) OpenAPIModelName() string { + return "com.github.cozystack.cozystack.pkg.apis.sdn.v1alpha1.EgressRule" +} + +func (in FQDNSelector) OpenAPIModelName() string { + return "com.github.cozystack.cozystack.pkg.apis.sdn.v1alpha1.FQDNSelector" +} + +func (in IngressRule) OpenAPIModelName() string { + return "com.github.cozystack.cozystack.pkg.apis.sdn.v1alpha1.IngressRule" +} + +func (in PortProtocol) OpenAPIModelName() string { + return "com.github.cozystack.cozystack.pkg.apis.sdn.v1alpha1.PortProtocol" +} + +func (in PortRule) OpenAPIModelName() string { + return "com.github.cozystack.cozystack.pkg.apis.sdn.v1alpha1.PortRule" +} + +func (in SecurityGroup) OpenAPIModelName() string { + return "com.github.cozystack.cozystack.pkg.apis.sdn.v1alpha1.SecurityGroup" +} + +func (in SecurityGroupList) OpenAPIModelName() string { + return "com.github.cozystack.cozystack.pkg.apis.sdn.v1alpha1.SecurityGroupList" +} + +func (in SecurityGroupSpec) OpenAPIModelName() string { + return "com.github.cozystack.cozystack.pkg.apis.sdn.v1alpha1.SecurityGroupSpec" +} diff --git a/pkg/generated/openapi/definitions_test.go b/pkg/generated/openapi/definitions_test.go new file mode 100644 index 0000000000..9440a7283a --- /dev/null +++ b/pkg/generated/openapi/definitions_test.go @@ -0,0 +1,157 @@ +/* +Copyright 2026 The Cozystack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package openapi + +import ( + "sort" + "strings" + "testing" + + "k8s.io/kube-openapi/pkg/common" + "k8s.io/kube-openapi/pkg/validation/spec" +) + +// The two tests below guard the published OpenAPI document against the class of +// defect in https://github.com/cozystack/cozystack/issues/3806: a definition +// name that is a Go import path rather than a dotted Kubernetes model name. +// +// Since Kubernetes 0.35 the apiserver's DefinitionNamer.GetDefinitionName +// returns the model name it was given verbatim — it no longer converts the Go +// import-path form into the "friendly" reversed-path form — so whatever name +// this generated map uses becomes both the published definition key and, JSON +// pointer escaped, the $ref that points at it. A name containing "/" therefore +// ships as a definition keyed on the raw path while every reference to it is +// spelled with "~1" in place of each slash, and clients resolve a $ref by +// trimming "#/definitions/" without unescaping (kube-openapi +// pkg/util/proto/document.go). The two spellings never meet, so the reference +// dangles and client-side validation fails on every resource of the group: +// +// error validating data: SchemaError(…core/v1alpha1.Option.spec): +// unknown model in reference: "github.com~1cozystack~1…v1alpha1.OptionSpec" +// +// The fix for a group is the +k8s:openapi-model-package marker in its doc.go +// plus an OpenAPIModelName method per type (see each package's model_name.go). +// Nothing asserted that, which is why the core and sdn groups shipped broken in +// v1.6.0 and v1.6.1. These tests assert the invariant rather than today's set of +// names, so a future group that omits the marker fails here instead of in a +// cluster. + +// buildDefinitions builds the generated definition map, recording every model +// name the generated code passes to the reference callback. Refs are built +// exactly as kube-openapi's builder builds them, so the recorded name and the +// emitted $ref differ in the same way they do in the served document. +func buildDefinitions() (defs map[string]common.OpenAPIDefinition, refNames []string) { + defs = GetOpenAPIDefinitions(func(path string) spec.Ref { + refNames = append(refNames, path) + return spec.MustCreateRef("#/definitions/" + common.EscapeJsonPointer(path)) + }) + return defs, refNames +} + +// sortedKeys returns the definition names in a stable order so failures are +// reproducible rather than map-iteration ordered. +func sortedKeys(defs map[string]common.OpenAPIDefinition) []string { + out := make([]string, 0, len(defs)) + for name := range defs { + out = append(out, name) + } + sort.Strings(out) + return out +} + +// TestDefinitionNamesAreDottedModelNames asserts that no published definition +// name contains a slash. A slash-bearing name is a Go import path that escaped +// as-is, which means the group it belongs to is missing its +// +k8s:openapi-model-package marker and its OpenAPIModelName methods. +func TestDefinitionNamesAreDottedModelNames(t *testing.T) { + defs, _ := buildDefinitions() + + // Guard against the assertion going vacuous: the map must be populated and + // must actually cover cozystack's own types, not only the vendored + // apimachinery and apiextensions models that already declare their names. + if len(defs) == 0 { + t.Fatal("GetOpenAPIDefinitions returned no definitions; this test is asserting nothing") + } + own := 0 + for _, name := range sortedKeys(defs) { + if strings.Contains(name, "cozystack") { + own++ + } + } + if own == 0 { + t.Fatal("no cozystack-owned definitions found; this test is asserting nothing") + } + + for _, name := range sortedKeys(defs) { + if strings.Contains(name, "/") { + t.Errorf("definition %q is a Go import path, not a dotted model name: "+ + "every $ref to it is escaped with ~1 and will not resolve. Add "+ + "+k8s:openapi-model-package= to that package's doc.go and an "+ + "OpenAPIModelName method for the type in its model_name.go, then re-run make generate", + name) + } + } +} + +// TestDefinitionRefsResolve asserts that every $ref the generated document +// emits resolves to a published definition, resolving it the way a client does: +// trim "#/definitions/" and look the remainder up verbatim, with no +// unescaping. This is the failure the user sees, so it is checked directly +// rather than only through the slash-freedom proxy above; it also catches a +// group whose types disagree with each other (some named, some not) and a ref +// to a model that is not published at all. +func TestDefinitionRefsResolve(t *testing.T) { + defs, refNames := buildDefinitions() + + if len(refNames) == 0 { + t.Fatal("generated definitions emitted no $ref; this test is asserting nothing") + } + + // Index the published definitions under the key a client sees, which is the + // definition name exactly as published. + published := make(map[string]struct{}, len(defs)) + for name := range defs { + published[name] = struct{}{} + } + + seen := make(map[string]struct{}, len(refNames)) + for _, modelName := range refNames { + // The served $ref, built the way kube-openapi's builder builds it. + ref := "#/definitions/" + common.EscapeJsonPointer(modelName) + // The lookup a client performs on it. + resolved := strings.TrimPrefix(ref, "#/definitions/") + if _, ok := published[resolved]; ok { + continue + } + if _, dup := seen[resolved]; dup { + continue + } + seen[resolved] = struct{}{} + t.Errorf("$ref %q does not resolve: no definition is published under %q (model name %q)", + ref, resolved, modelName) + } + + // Dependencies must agree with the refs: the generated code lists them from + // the same set, and consumers walk them to build the transitive closure. + for _, name := range sortedKeys(defs) { + for _, dep := range defs[name].Dependencies { + if _, ok := published[dep]; !ok { + t.Errorf("definition %q declares dependency %q, which is not published", name, dep) + } + } + } +} diff --git a/pkg/generated/openapi/zz_generated.openapi.go b/pkg/generated/openapi/zz_generated.openapi.go index f8d6548197..b50e629be5 100644 --- a/pkg/generated/openapi/zz_generated.openapi.go +++ b/pkg/generated/openapi/zz_generated.openapi.go @@ -23,6 +23,8 @@ package openapi import ( v1alpha1 "github.com/cozystack/cozystack/pkg/apis/apps/v1alpha1" + corev1alpha1 "github.com/cozystack/cozystack/pkg/apis/core/v1alpha1" + sdnv1alpha1 "github.com/cozystack/cozystack/pkg/apis/sdn/v1alpha1" v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" resource "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -34,110 +36,110 @@ import ( func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenAPIDefinition { return map[string]common.OpenAPIDefinition{ - v1alpha1.Application{}.OpenAPIModelName(): schema_pkg_apis_apps_v1alpha1_Application(ref), - v1alpha1.ApplicationList{}.OpenAPIModelName(): schema_pkg_apis_apps_v1alpha1_ApplicationList(ref), - v1alpha1.ApplicationStatus{}.OpenAPIModelName(): schema_pkg_apis_apps_v1alpha1_ApplicationStatus(ref), - "github.com/cozystack/cozystack/pkg/apis/core/v1alpha1.Option": schema_pkg_apis_core_v1alpha1_Option(ref), - "github.com/cozystack/cozystack/pkg/apis/core/v1alpha1.OptionItem": schema_pkg_apis_core_v1alpha1_OptionItem(ref), - "github.com/cozystack/cozystack/pkg/apis/core/v1alpha1.OptionList": schema_pkg_apis_core_v1alpha1_OptionList(ref), - "github.com/cozystack/cozystack/pkg/apis/core/v1alpha1.OptionSpec": schema_pkg_apis_core_v1alpha1_OptionSpec(ref), - "github.com/cozystack/cozystack/pkg/apis/core/v1alpha1.TenantModule": schema_pkg_apis_core_v1alpha1_TenantModule(ref), - "github.com/cozystack/cozystack/pkg/apis/core/v1alpha1.TenantModuleList": schema_pkg_apis_core_v1alpha1_TenantModuleList(ref), - "github.com/cozystack/cozystack/pkg/apis/core/v1alpha1.TenantModuleStatus": schema_pkg_apis_core_v1alpha1_TenantModuleStatus(ref), - "github.com/cozystack/cozystack/pkg/apis/core/v1alpha1.TenantNamespace": schema_pkg_apis_core_v1alpha1_TenantNamespace(ref), - "github.com/cozystack/cozystack/pkg/apis/core/v1alpha1.TenantNamespaceList": schema_pkg_apis_core_v1alpha1_TenantNamespaceList(ref), - "github.com/cozystack/cozystack/pkg/apis/core/v1alpha1.TenantSecret": schema_pkg_apis_core_v1alpha1_TenantSecret(ref), - "github.com/cozystack/cozystack/pkg/apis/core/v1alpha1.TenantSecretList": schema_pkg_apis_core_v1alpha1_TenantSecretList(ref), - "github.com/cozystack/cozystack/pkg/apis/sdn/v1alpha1.ApplicationReference": schema_pkg_apis_sdn_v1alpha1_ApplicationReference(ref), - "github.com/cozystack/cozystack/pkg/apis/sdn/v1alpha1.EgressRule": schema_pkg_apis_sdn_v1alpha1_EgressRule(ref), - "github.com/cozystack/cozystack/pkg/apis/sdn/v1alpha1.FQDNSelector": schema_pkg_apis_sdn_v1alpha1_FQDNSelector(ref), - "github.com/cozystack/cozystack/pkg/apis/sdn/v1alpha1.IngressRule": schema_pkg_apis_sdn_v1alpha1_IngressRule(ref), - "github.com/cozystack/cozystack/pkg/apis/sdn/v1alpha1.PortProtocol": schema_pkg_apis_sdn_v1alpha1_PortProtocol(ref), - "github.com/cozystack/cozystack/pkg/apis/sdn/v1alpha1.PortRule": schema_pkg_apis_sdn_v1alpha1_PortRule(ref), - "github.com/cozystack/cozystack/pkg/apis/sdn/v1alpha1.SecurityGroup": schema_pkg_apis_sdn_v1alpha1_SecurityGroup(ref), - "github.com/cozystack/cozystack/pkg/apis/sdn/v1alpha1.SecurityGroupList": schema_pkg_apis_sdn_v1alpha1_SecurityGroupList(ref), - "github.com/cozystack/cozystack/pkg/apis/sdn/v1alpha1.SecurityGroupSpec": schema_pkg_apis_sdn_v1alpha1_SecurityGroupSpec(ref), - v1.ConversionRequest{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_ConversionRequest(ref), - v1.ConversionResponse{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_ConversionResponse(ref), - v1.ConversionReview{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_ConversionReview(ref), - v1.CustomResourceColumnDefinition{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_CustomResourceColumnDefinition(ref), - v1.CustomResourceConversion{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_CustomResourceConversion(ref), - v1.CustomResourceDefinition{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_CustomResourceDefinition(ref), - v1.CustomResourceDefinitionCondition{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_CustomResourceDefinitionCondition(ref), - v1.CustomResourceDefinitionList{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_CustomResourceDefinitionList(ref), - v1.CustomResourceDefinitionNames{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_CustomResourceDefinitionNames(ref), - v1.CustomResourceDefinitionSpec{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_CustomResourceDefinitionSpec(ref), - v1.CustomResourceDefinitionStatus{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_CustomResourceDefinitionStatus(ref), - v1.CustomResourceDefinitionVersion{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_CustomResourceDefinitionVersion(ref), - v1.CustomResourceSubresourceScale{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_CustomResourceSubresourceScale(ref), - v1.CustomResourceSubresourceStatus{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_CustomResourceSubresourceStatus(ref), - v1.CustomResourceSubresources{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_CustomResourceSubresources(ref), - v1.CustomResourceValidation{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_CustomResourceValidation(ref), - v1.ExternalDocumentation{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_ExternalDocumentation(ref), - v1.JSON{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_JSON(ref), - v1.JSONSchemaProps{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_JSONSchemaProps(ref), - v1.JSONSchemaPropsOrArray{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_JSONSchemaPropsOrArray(ref), - v1.JSONSchemaPropsOrBool{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_JSONSchemaPropsOrBool(ref), - v1.JSONSchemaPropsOrStringArray{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_JSONSchemaPropsOrStringArray(ref), - v1.SelectableField{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_SelectableField(ref), - v1.ServiceReference{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_ServiceReference(ref), - v1.ValidationRule{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_ValidationRule(ref), - v1.WebhookClientConfig{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_WebhookClientConfig(ref), - v1.WebhookConversion{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_WebhookConversion(ref), - resource.Quantity{}.OpenAPIModelName(): schema_apimachinery_pkg_api_resource_Quantity(ref), - metav1.APIGroup{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_APIGroup(ref), - metav1.APIGroupList{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_APIGroupList(ref), - metav1.APIResource{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_APIResource(ref), - metav1.APIResourceList{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_APIResourceList(ref), - metav1.APIVersions{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_APIVersions(ref), - metav1.ApplyOptions{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_ApplyOptions(ref), - metav1.Condition{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_Condition(ref), - metav1.CreateOptions{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_CreateOptions(ref), - metav1.DeleteOptions{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_DeleteOptions(ref), - metav1.Duration{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_Duration(ref), - metav1.FieldSelectorRequirement{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_FieldSelectorRequirement(ref), - metav1.FieldsV1{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_FieldsV1(ref), - metav1.GetOptions{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_GetOptions(ref), - metav1.GroupKind{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_GroupKind(ref), - metav1.GroupResource{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_GroupResource(ref), - metav1.GroupVersion{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_GroupVersion(ref), - metav1.GroupVersionForDiscovery{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_GroupVersionForDiscovery(ref), - metav1.GroupVersionKind{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_GroupVersionKind(ref), - metav1.GroupVersionResource{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_GroupVersionResource(ref), - metav1.InternalEvent{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_InternalEvent(ref), - metav1.LabelSelector{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_LabelSelector(ref), - metav1.LabelSelectorRequirement{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_LabelSelectorRequirement(ref), - metav1.List{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_List(ref), - metav1.ListMeta{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_ListMeta(ref), - metav1.ListOptions{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_ListOptions(ref), - metav1.ManagedFieldsEntry{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_ManagedFieldsEntry(ref), - metav1.MicroTime{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_MicroTime(ref), - metav1.ObjectMeta{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_ObjectMeta(ref), - metav1.OwnerReference{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_OwnerReference(ref), - metav1.PartialObjectMetadata{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_PartialObjectMetadata(ref), - metav1.PartialObjectMetadataList{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_PartialObjectMetadataList(ref), - metav1.Patch{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_Patch(ref), - metav1.PatchOptions{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_PatchOptions(ref), - metav1.Preconditions{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_Preconditions(ref), - metav1.RootPaths{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_RootPaths(ref), - metav1.ServerAddressByClientCIDR{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_ServerAddressByClientCIDR(ref), - metav1.Status{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_Status(ref), - metav1.StatusCause{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_StatusCause(ref), - metav1.StatusDetails{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_StatusDetails(ref), - metav1.Table{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_Table(ref), - metav1.TableColumnDefinition{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_TableColumnDefinition(ref), - metav1.TableOptions{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_TableOptions(ref), - metav1.TableRow{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_TableRow(ref), - metav1.TableRowCondition{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_TableRowCondition(ref), - metav1.Time{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_Time(ref), - metav1.Timestamp{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_Timestamp(ref), - metav1.TypeMeta{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_TypeMeta(ref), - metav1.UpdateOptions{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_UpdateOptions(ref), - metav1.WatchEvent{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_WatchEvent(ref), - runtime.RawExtension{}.OpenAPIModelName(): schema_k8sio_apimachinery_pkg_runtime_RawExtension(ref), - runtime.TypeMeta{}.OpenAPIModelName(): schema_k8sio_apimachinery_pkg_runtime_TypeMeta(ref), - runtime.Unknown{}.OpenAPIModelName(): schema_k8sio_apimachinery_pkg_runtime_Unknown(ref), - version.Info{}.OpenAPIModelName(): schema_k8sio_apimachinery_pkg_version_Info(ref), + v1alpha1.Application{}.OpenAPIModelName(): schema_pkg_apis_apps_v1alpha1_Application(ref), + v1alpha1.ApplicationList{}.OpenAPIModelName(): schema_pkg_apis_apps_v1alpha1_ApplicationList(ref), + v1alpha1.ApplicationStatus{}.OpenAPIModelName(): schema_pkg_apis_apps_v1alpha1_ApplicationStatus(ref), + corev1alpha1.Option{}.OpenAPIModelName(): schema_pkg_apis_core_v1alpha1_Option(ref), + corev1alpha1.OptionItem{}.OpenAPIModelName(): schema_pkg_apis_core_v1alpha1_OptionItem(ref), + corev1alpha1.OptionList{}.OpenAPIModelName(): schema_pkg_apis_core_v1alpha1_OptionList(ref), + corev1alpha1.OptionSpec{}.OpenAPIModelName(): schema_pkg_apis_core_v1alpha1_OptionSpec(ref), + corev1alpha1.TenantModule{}.OpenAPIModelName(): schema_pkg_apis_core_v1alpha1_TenantModule(ref), + corev1alpha1.TenantModuleList{}.OpenAPIModelName(): schema_pkg_apis_core_v1alpha1_TenantModuleList(ref), + corev1alpha1.TenantModuleStatus{}.OpenAPIModelName(): schema_pkg_apis_core_v1alpha1_TenantModuleStatus(ref), + corev1alpha1.TenantNamespace{}.OpenAPIModelName(): schema_pkg_apis_core_v1alpha1_TenantNamespace(ref), + corev1alpha1.TenantNamespaceList{}.OpenAPIModelName(): schema_pkg_apis_core_v1alpha1_TenantNamespaceList(ref), + corev1alpha1.TenantSecret{}.OpenAPIModelName(): schema_pkg_apis_core_v1alpha1_TenantSecret(ref), + corev1alpha1.TenantSecretList{}.OpenAPIModelName(): schema_pkg_apis_core_v1alpha1_TenantSecretList(ref), + sdnv1alpha1.ApplicationReference{}.OpenAPIModelName(): schema_pkg_apis_sdn_v1alpha1_ApplicationReference(ref), + sdnv1alpha1.EgressRule{}.OpenAPIModelName(): schema_pkg_apis_sdn_v1alpha1_EgressRule(ref), + sdnv1alpha1.FQDNSelector{}.OpenAPIModelName(): schema_pkg_apis_sdn_v1alpha1_FQDNSelector(ref), + sdnv1alpha1.IngressRule{}.OpenAPIModelName(): schema_pkg_apis_sdn_v1alpha1_IngressRule(ref), + sdnv1alpha1.PortProtocol{}.OpenAPIModelName(): schema_pkg_apis_sdn_v1alpha1_PortProtocol(ref), + sdnv1alpha1.PortRule{}.OpenAPIModelName(): schema_pkg_apis_sdn_v1alpha1_PortRule(ref), + sdnv1alpha1.SecurityGroup{}.OpenAPIModelName(): schema_pkg_apis_sdn_v1alpha1_SecurityGroup(ref), + sdnv1alpha1.SecurityGroupList{}.OpenAPIModelName(): schema_pkg_apis_sdn_v1alpha1_SecurityGroupList(ref), + sdnv1alpha1.SecurityGroupSpec{}.OpenAPIModelName(): schema_pkg_apis_sdn_v1alpha1_SecurityGroupSpec(ref), + v1.ConversionRequest{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_ConversionRequest(ref), + v1.ConversionResponse{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_ConversionResponse(ref), + v1.ConversionReview{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_ConversionReview(ref), + v1.CustomResourceColumnDefinition{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_CustomResourceColumnDefinition(ref), + v1.CustomResourceConversion{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_CustomResourceConversion(ref), + v1.CustomResourceDefinition{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_CustomResourceDefinition(ref), + v1.CustomResourceDefinitionCondition{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_CustomResourceDefinitionCondition(ref), + v1.CustomResourceDefinitionList{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_CustomResourceDefinitionList(ref), + v1.CustomResourceDefinitionNames{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_CustomResourceDefinitionNames(ref), + v1.CustomResourceDefinitionSpec{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_CustomResourceDefinitionSpec(ref), + v1.CustomResourceDefinitionStatus{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_CustomResourceDefinitionStatus(ref), + v1.CustomResourceDefinitionVersion{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_CustomResourceDefinitionVersion(ref), + v1.CustomResourceSubresourceScale{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_CustomResourceSubresourceScale(ref), + v1.CustomResourceSubresourceStatus{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_CustomResourceSubresourceStatus(ref), + v1.CustomResourceSubresources{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_CustomResourceSubresources(ref), + v1.CustomResourceValidation{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_CustomResourceValidation(ref), + v1.ExternalDocumentation{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_ExternalDocumentation(ref), + v1.JSON{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_JSON(ref), + v1.JSONSchemaProps{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_JSONSchemaProps(ref), + v1.JSONSchemaPropsOrArray{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_JSONSchemaPropsOrArray(ref), + v1.JSONSchemaPropsOrBool{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_JSONSchemaPropsOrBool(ref), + v1.JSONSchemaPropsOrStringArray{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_JSONSchemaPropsOrStringArray(ref), + v1.SelectableField{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_SelectableField(ref), + v1.ServiceReference{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_ServiceReference(ref), + v1.ValidationRule{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_ValidationRule(ref), + v1.WebhookClientConfig{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_WebhookClientConfig(ref), + v1.WebhookConversion{}.OpenAPIModelName(): schema_pkg_apis_apiextensions_v1_WebhookConversion(ref), + resource.Quantity{}.OpenAPIModelName(): schema_apimachinery_pkg_api_resource_Quantity(ref), + metav1.APIGroup{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_APIGroup(ref), + metav1.APIGroupList{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_APIGroupList(ref), + metav1.APIResource{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_APIResource(ref), + metav1.APIResourceList{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_APIResourceList(ref), + metav1.APIVersions{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_APIVersions(ref), + metav1.ApplyOptions{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_ApplyOptions(ref), + metav1.Condition{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_Condition(ref), + metav1.CreateOptions{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_CreateOptions(ref), + metav1.DeleteOptions{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_DeleteOptions(ref), + metav1.Duration{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_Duration(ref), + metav1.FieldSelectorRequirement{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_FieldSelectorRequirement(ref), + metav1.FieldsV1{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_FieldsV1(ref), + metav1.GetOptions{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_GetOptions(ref), + metav1.GroupKind{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_GroupKind(ref), + metav1.GroupResource{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_GroupResource(ref), + metav1.GroupVersion{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_GroupVersion(ref), + metav1.GroupVersionForDiscovery{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_GroupVersionForDiscovery(ref), + metav1.GroupVersionKind{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_GroupVersionKind(ref), + metav1.GroupVersionResource{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_GroupVersionResource(ref), + metav1.InternalEvent{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_InternalEvent(ref), + metav1.LabelSelector{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_LabelSelector(ref), + metav1.LabelSelectorRequirement{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_LabelSelectorRequirement(ref), + metav1.List{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_List(ref), + metav1.ListMeta{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_ListMeta(ref), + metav1.ListOptions{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_ListOptions(ref), + metav1.ManagedFieldsEntry{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_ManagedFieldsEntry(ref), + metav1.MicroTime{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_MicroTime(ref), + metav1.ObjectMeta{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_ObjectMeta(ref), + metav1.OwnerReference{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_OwnerReference(ref), + metav1.PartialObjectMetadata{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_PartialObjectMetadata(ref), + metav1.PartialObjectMetadataList{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_PartialObjectMetadataList(ref), + metav1.Patch{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_Patch(ref), + metav1.PatchOptions{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_PatchOptions(ref), + metav1.Preconditions{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_Preconditions(ref), + metav1.RootPaths{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_RootPaths(ref), + metav1.ServerAddressByClientCIDR{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_ServerAddressByClientCIDR(ref), + metav1.Status{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_Status(ref), + metav1.StatusCause{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_StatusCause(ref), + metav1.StatusDetails{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_StatusDetails(ref), + metav1.Table{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_Table(ref), + metav1.TableColumnDefinition{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_TableColumnDefinition(ref), + metav1.TableOptions{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_TableOptions(ref), + metav1.TableRow{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_TableRow(ref), + metav1.TableRowCondition{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_TableRowCondition(ref), + metav1.Time{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_Time(ref), + metav1.Timestamp{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_Timestamp(ref), + metav1.TypeMeta{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_TypeMeta(ref), + metav1.UpdateOptions{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_UpdateOptions(ref), + metav1.WatchEvent{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_WatchEvent(ref), + runtime.RawExtension{}.OpenAPIModelName(): schema_k8sio_apimachinery_pkg_runtime_RawExtension(ref), + runtime.TypeMeta{}.OpenAPIModelName(): schema_k8sio_apimachinery_pkg_runtime_TypeMeta(ref), + runtime.Unknown{}.OpenAPIModelName(): schema_k8sio_apimachinery_pkg_runtime_Unknown(ref), + version.Info{}.OpenAPIModelName(): schema_k8sio_apimachinery_pkg_version_Info(ref), } } @@ -321,14 +323,14 @@ func schema_pkg_apis_core_v1alpha1_Option(ref common.ReferenceCallback) common.O "spec": { SchemaProps: spec.SchemaProps{ Default: map[string]interface{}{}, - Ref: ref("github.com/cozystack/cozystack/pkg/apis/core/v1alpha1.OptionSpec"), + Ref: ref(corev1alpha1.OptionSpec{}.OpenAPIModelName()), }, }, }, }, }, Dependencies: []string{ - "github.com/cozystack/cozystack/pkg/apis/core/v1alpha1.OptionSpec", metav1.ObjectMeta{}.OpenAPIModelName()}, + corev1alpha1.OptionSpec{}.OpenAPIModelName(), metav1.ObjectMeta{}.OpenAPIModelName()}, } } @@ -408,7 +410,7 @@ func schema_pkg_apis_core_v1alpha1_OptionList(ref common.ReferenceCallback) comm Schema: &spec.Schema{ SchemaProps: spec.SchemaProps{ Default: map[string]interface{}{}, - Ref: ref("github.com/cozystack/cozystack/pkg/apis/core/v1alpha1.Option"), + Ref: ref(corev1alpha1.Option{}.OpenAPIModelName()), }, }, }, @@ -419,7 +421,7 @@ func schema_pkg_apis_core_v1alpha1_OptionList(ref common.ReferenceCallback) comm }, }, Dependencies: []string{ - "github.com/cozystack/cozystack/pkg/apis/core/v1alpha1.Option", metav1.ListMeta{}.OpenAPIModelName()}, + corev1alpha1.Option{}.OpenAPIModelName(), metav1.ListMeta{}.OpenAPIModelName()}, } } @@ -437,7 +439,7 @@ func schema_pkg_apis_core_v1alpha1_OptionSpec(ref common.ReferenceCallback) comm Schema: &spec.Schema{ SchemaProps: spec.SchemaProps{ Default: map[string]interface{}{}, - Ref: ref("github.com/cozystack/cozystack/pkg/apis/core/v1alpha1.OptionItem"), + Ref: ref(corev1alpha1.OptionItem{}.OpenAPIModelName()), }, }, }, @@ -447,7 +449,7 @@ func schema_pkg_apis_core_v1alpha1_OptionSpec(ref common.ReferenceCallback) comm }, }, Dependencies: []string{ - "github.com/cozystack/cozystack/pkg/apis/core/v1alpha1.OptionItem"}, + corev1alpha1.OptionItem{}.OpenAPIModelName()}, } } @@ -489,14 +491,14 @@ func schema_pkg_apis_core_v1alpha1_TenantModule(ref common.ReferenceCallback) co SchemaProps: spec.SchemaProps{ Description: "Status contains the module status", Default: map[string]interface{}{}, - Ref: ref("github.com/cozystack/cozystack/pkg/apis/core/v1alpha1.TenantModuleStatus"), + Ref: ref(corev1alpha1.TenantModuleStatus{}.OpenAPIModelName()), }, }, }, }, }, Dependencies: []string{ - "github.com/cozystack/cozystack/pkg/apis/core/v1alpha1.TenantModuleStatus", metav1.ObjectMeta{}.OpenAPIModelName()}, + corev1alpha1.TenantModuleStatus{}.OpenAPIModelName(), metav1.ObjectMeta{}.OpenAPIModelName()}, } } @@ -534,7 +536,7 @@ func schema_pkg_apis_core_v1alpha1_TenantModuleList(ref common.ReferenceCallback Schema: &spec.Schema{ SchemaProps: spec.SchemaProps{ Default: map[string]interface{}{}, - Ref: ref("github.com/cozystack/cozystack/pkg/apis/core/v1alpha1.TenantModule"), + Ref: ref(corev1alpha1.TenantModule{}.OpenAPIModelName()), }, }, }, @@ -545,7 +547,7 @@ func schema_pkg_apis_core_v1alpha1_TenantModuleList(ref common.ReferenceCallback }, }, Dependencies: []string{ - "github.com/cozystack/cozystack/pkg/apis/core/v1alpha1.TenantModule", metav1.ListMeta{}.OpenAPIModelName()}, + corev1alpha1.TenantModule{}.OpenAPIModelName(), metav1.ListMeta{}.OpenAPIModelName()}, } } @@ -654,7 +656,7 @@ func schema_pkg_apis_core_v1alpha1_TenantNamespaceList(ref common.ReferenceCallb Schema: &spec.Schema{ SchemaProps: spec.SchemaProps{ Default: map[string]interface{}{}, - Ref: ref("github.com/cozystack/cozystack/pkg/apis/core/v1alpha1.TenantNamespace"), + Ref: ref(corev1alpha1.TenantNamespace{}.OpenAPIModelName()), }, }, }, @@ -665,7 +667,7 @@ func schema_pkg_apis_core_v1alpha1_TenantNamespaceList(ref common.ReferenceCallb }, }, Dependencies: []string{ - "github.com/cozystack/cozystack/pkg/apis/core/v1alpha1.TenantNamespace", metav1.ListMeta{}.OpenAPIModelName()}, + corev1alpha1.TenantNamespace{}.OpenAPIModelName(), metav1.ListMeta{}.OpenAPIModelName()}, } } @@ -772,7 +774,7 @@ func schema_pkg_apis_core_v1alpha1_TenantSecretList(ref common.ReferenceCallback Schema: &spec.Schema{ SchemaProps: spec.SchemaProps{ Default: map[string]interface{}{}, - Ref: ref("github.com/cozystack/cozystack/pkg/apis/core/v1alpha1.TenantSecret"), + Ref: ref(corev1alpha1.TenantSecret{}.OpenAPIModelName()), }, }, }, @@ -783,7 +785,7 @@ func schema_pkg_apis_core_v1alpha1_TenantSecretList(ref common.ReferenceCallback }, }, Dependencies: []string{ - "github.com/cozystack/cozystack/pkg/apis/core/v1alpha1.TenantSecret", metav1.ListMeta{}.OpenAPIModelName()}, + corev1alpha1.TenantSecret{}.OpenAPIModelName(), metav1.ListMeta{}.OpenAPIModelName()}, } } @@ -839,7 +841,7 @@ func schema_pkg_apis_sdn_v1alpha1_EgressRule(ref common.ReferenceCallback) commo Schema: &spec.Schema{ SchemaProps: spec.SchemaProps{ Default: map[string]interface{}{}, - Ref: ref("github.com/cozystack/cozystack/pkg/apis/sdn/v1alpha1.ApplicationReference"), + Ref: ref(sdnv1alpha1.ApplicationReference{}.OpenAPIModelName()), }, }, }, @@ -883,7 +885,7 @@ func schema_pkg_apis_sdn_v1alpha1_EgressRule(ref common.ReferenceCallback) commo Schema: &spec.Schema{ SchemaProps: spec.SchemaProps{ Default: map[string]interface{}{}, - Ref: ref("github.com/cozystack/cozystack/pkg/apis/sdn/v1alpha1.FQDNSelector"), + Ref: ref(sdnv1alpha1.FQDNSelector{}.OpenAPIModelName()), }, }, }, @@ -897,7 +899,7 @@ func schema_pkg_apis_sdn_v1alpha1_EgressRule(ref common.ReferenceCallback) commo Schema: &spec.Schema{ SchemaProps: spec.SchemaProps{ Default: map[string]interface{}{}, - Ref: ref("github.com/cozystack/cozystack/pkg/apis/sdn/v1alpha1.PortRule"), + Ref: ref(sdnv1alpha1.PortRule{}.OpenAPIModelName()), }, }, }, @@ -907,7 +909,7 @@ func schema_pkg_apis_sdn_v1alpha1_EgressRule(ref common.ReferenceCallback) commo }, }, Dependencies: []string{ - "github.com/cozystack/cozystack/pkg/apis/sdn/v1alpha1.ApplicationReference", "github.com/cozystack/cozystack/pkg/apis/sdn/v1alpha1.FQDNSelector", "github.com/cozystack/cozystack/pkg/apis/sdn/v1alpha1.PortRule"}, + sdnv1alpha1.ApplicationReference{}.OpenAPIModelName(), sdnv1alpha1.FQDNSelector{}.OpenAPIModelName(), sdnv1alpha1.PortRule{}.OpenAPIModelName()}, } } @@ -953,7 +955,7 @@ func schema_pkg_apis_sdn_v1alpha1_IngressRule(ref common.ReferenceCallback) comm Schema: &spec.Schema{ SchemaProps: spec.SchemaProps{ Default: map[string]interface{}{}, - Ref: ref("github.com/cozystack/cozystack/pkg/apis/sdn/v1alpha1.ApplicationReference"), + Ref: ref(sdnv1alpha1.ApplicationReference{}.OpenAPIModelName()), }, }, }, @@ -997,7 +999,7 @@ func schema_pkg_apis_sdn_v1alpha1_IngressRule(ref common.ReferenceCallback) comm Schema: &spec.Schema{ SchemaProps: spec.SchemaProps{ Default: map[string]interface{}{}, - Ref: ref("github.com/cozystack/cozystack/pkg/apis/sdn/v1alpha1.PortRule"), + Ref: ref(sdnv1alpha1.PortRule{}.OpenAPIModelName()), }, }, }, @@ -1007,7 +1009,7 @@ func schema_pkg_apis_sdn_v1alpha1_IngressRule(ref common.ReferenceCallback) comm }, }, Dependencies: []string{ - "github.com/cozystack/cozystack/pkg/apis/sdn/v1alpha1.ApplicationReference", "github.com/cozystack/cozystack/pkg/apis/sdn/v1alpha1.PortRule"}, + sdnv1alpha1.ApplicationReference{}.OpenAPIModelName(), sdnv1alpha1.PortRule{}.OpenAPIModelName()}, } } @@ -1053,7 +1055,7 @@ func schema_pkg_apis_sdn_v1alpha1_PortRule(ref common.ReferenceCallback) common. Schema: &spec.Schema{ SchemaProps: spec.SchemaProps{ Default: map[string]interface{}{}, - Ref: ref("github.com/cozystack/cozystack/pkg/apis/sdn/v1alpha1.PortProtocol"), + Ref: ref(sdnv1alpha1.PortProtocol{}.OpenAPIModelName()), }, }, }, @@ -1063,7 +1065,7 @@ func schema_pkg_apis_sdn_v1alpha1_PortRule(ref common.ReferenceCallback) common. }, }, Dependencies: []string{ - "github.com/cozystack/cozystack/pkg/apis/sdn/v1alpha1.PortProtocol"}, + sdnv1alpha1.PortProtocol{}.OpenAPIModelName()}, } } @@ -1098,14 +1100,14 @@ func schema_pkg_apis_sdn_v1alpha1_SecurityGroup(ref common.ReferenceCallback) co SchemaProps: spec.SchemaProps{ Description: "Spec describes the applications this SecurityGroup attaches to and the traffic it allows.", Default: map[string]interface{}{}, - Ref: ref("github.com/cozystack/cozystack/pkg/apis/sdn/v1alpha1.SecurityGroupSpec"), + Ref: ref(sdnv1alpha1.SecurityGroupSpec{}.OpenAPIModelName()), }, }, }, }, }, Dependencies: []string{ - "github.com/cozystack/cozystack/pkg/apis/sdn/v1alpha1.SecurityGroupSpec", metav1.ObjectMeta{}.OpenAPIModelName()}, + sdnv1alpha1.SecurityGroupSpec{}.OpenAPIModelName(), metav1.ObjectMeta{}.OpenAPIModelName()}, } } @@ -1143,7 +1145,7 @@ func schema_pkg_apis_sdn_v1alpha1_SecurityGroupList(ref common.ReferenceCallback Schema: &spec.Schema{ SchemaProps: spec.SchemaProps{ Default: map[string]interface{}{}, - Ref: ref("github.com/cozystack/cozystack/pkg/apis/sdn/v1alpha1.SecurityGroup"), + Ref: ref(sdnv1alpha1.SecurityGroup{}.OpenAPIModelName()), }, }, }, @@ -1154,7 +1156,7 @@ func schema_pkg_apis_sdn_v1alpha1_SecurityGroupList(ref common.ReferenceCallback }, }, Dependencies: []string{ - "github.com/cozystack/cozystack/pkg/apis/sdn/v1alpha1.SecurityGroup", metav1.ListMeta{}.OpenAPIModelName()}, + sdnv1alpha1.SecurityGroup{}.OpenAPIModelName(), metav1.ListMeta{}.OpenAPIModelName()}, } } @@ -1173,7 +1175,7 @@ func schema_pkg_apis_sdn_v1alpha1_SecurityGroupSpec(ref common.ReferenceCallback Schema: &spec.Schema{ SchemaProps: spec.SchemaProps{ Default: map[string]interface{}{}, - Ref: ref("github.com/cozystack/cozystack/pkg/apis/sdn/v1alpha1.ApplicationReference"), + Ref: ref(sdnv1alpha1.ApplicationReference{}.OpenAPIModelName()), }, }, }, @@ -1187,7 +1189,7 @@ func schema_pkg_apis_sdn_v1alpha1_SecurityGroupSpec(ref common.ReferenceCallback Schema: &spec.Schema{ SchemaProps: spec.SchemaProps{ Default: map[string]interface{}{}, - Ref: ref("github.com/cozystack/cozystack/pkg/apis/sdn/v1alpha1.IngressRule"), + Ref: ref(sdnv1alpha1.IngressRule{}.OpenAPIModelName()), }, }, }, @@ -1201,7 +1203,7 @@ func schema_pkg_apis_sdn_v1alpha1_SecurityGroupSpec(ref common.ReferenceCallback Schema: &spec.Schema{ SchemaProps: spec.SchemaProps{ Default: map[string]interface{}{}, - Ref: ref("github.com/cozystack/cozystack/pkg/apis/sdn/v1alpha1.EgressRule"), + Ref: ref(sdnv1alpha1.EgressRule{}.OpenAPIModelName()), }, }, }, @@ -1211,7 +1213,7 @@ func schema_pkg_apis_sdn_v1alpha1_SecurityGroupSpec(ref common.ReferenceCallback }, }, Dependencies: []string{ - "github.com/cozystack/cozystack/pkg/apis/sdn/v1alpha1.ApplicationReference", "github.com/cozystack/cozystack/pkg/apis/sdn/v1alpha1.EgressRule", "github.com/cozystack/cozystack/pkg/apis/sdn/v1alpha1.IngressRule"}, + sdnv1alpha1.ApplicationReference{}.OpenAPIModelName(), sdnv1alpha1.EgressRule{}.OpenAPIModelName(), sdnv1alpha1.IngressRule{}.OpenAPIModelName()}, } }